├── .github ├── dependabot.yml └── workflows │ ├── automerge.yml │ ├── codeql.yml │ └── go.yml ├── .gitignore ├── .gitmodules ├── CHANGELOG.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE.md ├── Makefile ├── README.md ├── apiprov.go ├── channelid.go ├── errors.go ├── filc ├── Makefile ├── README.md ├── cmds.go ├── disk.go ├── flags.go ├── main.go ├── parsing.go ├── printing.go └── retrieval.go ├── filclient.go ├── filclient_test.go ├── go.mod ├── go.sum ├── keystore └── keystore.go ├── libp2ptransfermgr.go ├── libp2ptransfermgr_test.go ├── msgpusher.go ├── rep ├── event.go ├── publisher.go └── publisher_test.go └── retrievehelper ├── params.go └── selector.go /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: gomod 4 | directory: / 5 | schedule: 6 | interval: daily 7 | target-branch: "master" 8 | labels: 9 | - "gomod dependencies" 10 | -------------------------------------------------------------------------------- /.github/workflows/automerge.yml: -------------------------------------------------------------------------------- 1 | # .github/workflows/automerge.yml 2 | 3 | name: Dependabot auto-merge 4 | 5 | on: pull_request 6 | 7 | permissions: 8 | contents: write 9 | pull-requests: write 10 | 11 | jobs: 12 | dependabot: 13 | runs-on: ubuntu-latest 14 | if: ${{ github.actor == 'dependabot[bot]' }} 15 | steps: 16 | - name: Enable auto-merge for Dependabot PRs 17 | run: gh pr merge --auto --merge "$PR_URL" 18 | env: 19 | PR_URL: ${{ github.event.pull_request.html_url }} 20 | # GitHub provides this variable in the CI env. You don't 21 | # need to add anything to the secrets vault. 22 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 23 | - name: Approve a PR 24 | run: gh pr review --approve "$PR_URL" 25 | env: 26 | PR_URL: ${{github.event.pull_request.html_url}} 27 | GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} 28 | -------------------------------------------------------------------------------- /.github/workflows/codeql.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ "master" ] 17 | tags: 18 | - '*' 19 | pull_request: 20 | # The branches below must be a subset of the branches above 21 | branches: [ "master" ] 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: [ 'go' ] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 37 | # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support 38 | 39 | steps: 40 | - name: Checkout repository 41 | uses: actions/checkout@v3 42 | 43 | # Initializes the CodeQL tools for scanning. 44 | - name: Initialize CodeQL 45 | uses: github/codeql-action/init@v2 46 | with: 47 | languages: ${{ matrix.language }} 48 | # If you wish to specify custom queries, you can do so here or in a config file. 49 | # By default, queries listed here will override any specified in a config file. 50 | # Prefix the list here with "+" to use these queries and those in the config file. 51 | 52 | # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs 53 | # queries: security-extended,security-and-quality 54 | 55 | 56 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 57 | # If this step fails, then you should remove it and run the build manually (see below) 58 | - name: Autobuild 59 | uses: github/codeql-action/autobuild@v2 60 | 61 | # ℹ️ Command-line programs to run using the OS shell. 62 | # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun 63 | 64 | # If the Autobuild fails above, remove it and uncomment the following three lines. 65 | # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. 66 | 67 | # - run: | 68 | # echo "Run, Build Application using script" 69 | # ./location_of_script_within_repo/buildscript.sh 70 | 71 | - name: Perform CodeQL Analysis 72 | uses: github/codeql-action/analyze@v2 73 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | branches: [ "master" ] 6 | tags: 7 | - '*' 8 | pull_request: 9 | branches: [ "master" ] 10 | 11 | jobs: 12 | 13 | build: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@v2 17 | 18 | - name: Set up Go 19 | uses: actions/setup-go@v2 20 | with: 21 | go-version: 1.18 22 | 23 | - name: Set up Rust 24 | uses: actions-rs/toolchain@v1 25 | with: 26 | toolchain: stable 27 | default: true 28 | 29 | - name: Install dependencies 30 | run: | 31 | sudo apt update 32 | sudo apt install hwloc libhwloc-dev jq ocl-icd-opencl-dev 33 | - name: Build 34 | run: make clean all 35 | 36 | - name: Test 37 | run: go test -v ./... -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build/.* 2 | 3 | .idea 4 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "extern/filecoin-ffi"] 2 | path = extern/filecoin-ffi 3 | url = https://github.com/filecoin-project/filecoin-ffi 4 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | All notable changes to this project will be documented in this file. 3 | 4 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 5 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 6 | 7 | ## [v0.2.0] - 2022-08-23 8 | ### Added 9 | - LICENSE.md 10 | - CONTRIBUTING.md 11 | - CI Dockerfile and GitHub workflows 12 | 13 | ### Changed 14 | - Bump Lotus to v1.17.0 15 | - Bump Boost to v1.3.1 16 | - Filc blockstore is now stored with flatfs instead of lmdb 17 | - Old blockstores will no longer work, use [this migration 18 | tool](https://github.com/elijaharita/migrate-lmdb-flatfs) or simply clear 19 | the old blockstore `filc clear-blockstore` 20 | 21 | ### Fixed 22 | - Protect libp2p connections while querying 23 | - Filctl label string conversion 24 | 25 | ## [v0.1.0] - 2022-08-05 26 | ### Added 27 | - First release -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | Please update the changelog with your modifications under the `## [Unreleased]` 3 | section. If it doesn't exist, add it. You can use previous releases as a style 4 | reference, or check [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) for 5 | the full guide. -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.17-stretch AS builder 2 | USER root 3 | RUN apt-get update && \ 4 | apt-get install -y wget jq hwloc ocl-icd-opencl-dev git libhwloc-dev pkg-config make && \ 5 | apt-get install -y cargo 6 | WORKDIR /app/ 7 | 8 | RUN curl https://sh.rustup.rs -sSf | bash -s -- -y 9 | ENV PATH="/root/.cargo/bin:${PATH}" 10 | RUN cargo --help 11 | ARG TAG=${TAG} 12 | RUN echo ${TAG} 13 | RUN git clone https://github.com/application-research/filclient . && \ 14 | git pull && \ 15 | git fetch --all --tags && \ 16 | git checkout ${TAG} && \ 17 | make 18 | USER 1001 -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | The contents of this repository are Copyright (c) corresponding authors and 2 | contributors, licensed under the `Permissive License Stack` meaning either of: 3 | 4 | - Apache-2.0 Software License: https://www.apache.org/licenses/LICENSE-2.0 5 | ([...4tr2kfsq](https://dweb.link/ipfs/bafkreiankqxazcae4onkp436wag2lj3ccso4nawxqkkfckd6cg4tr2kfsq)) 6 | 7 | - MIT Software License: https://opensource.org/licenses/MIT 8 | ([...vljevcba](https://dweb.link/ipfs/bafkreiepofszg4gfe2gzuhojmksgemsub2h4uy2gewdnr35kswvljevcba)) 9 | 10 | You may not use the contents of this repository except in compliance 11 | with one of the listed Licenses. For an extended clarification of the 12 | intent behind the choice of Licensing please refer to 13 | https://protocol.ai/blog/announcing-the-permissive-license-stack/ 14 | 15 | Unless required by applicable law or agreed to in writing, software 16 | distributed under the terms listed in this notice is distributed on 17 | an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 18 | either express or implied. See each License for the specific language 19 | governing permissions and limitations under that License. 20 | 21 | 22 | 23 | `SPDX-License-Identifier: Apache-2.0 OR MIT` 24 | 25 | Verbatim copies of both licenses are included below: 26 | 27 |
Apache-2.0 Software License 28 | 29 | ``` 30 | Apache License 31 | Version 2.0, January 2004 32 | http://www.apache.org/licenses/ 33 | 34 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 35 | 36 | 1. Definitions. 37 | 38 | "License" shall mean the terms and conditions for use, reproduction, 39 | and distribution as defined by Sections 1 through 9 of this document. 40 | 41 | "Licensor" shall mean the copyright owner or entity authorized by 42 | the copyright owner that is granting the License. 43 | 44 | "Legal Entity" shall mean the union of the acting entity and all 45 | other entities that control, are controlled by, or are under common 46 | control with that entity. For the purposes of this definition, 47 | "control" means (i) the power, direct or indirect, to cause the 48 | direction or management of such entity, whether by contract or 49 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 50 | outstanding shares, or (iii) beneficial ownership of such entity. 51 | 52 | "You" (or "Your") shall mean an individual or Legal Entity 53 | exercising permissions granted by this License. 54 | 55 | "Source" form shall mean the preferred form for making modifications, 56 | including but not limited to software source code, documentation 57 | source, and configuration files. 58 | 59 | "Object" form shall mean any form resulting from mechanical 60 | transformation or translation of a Source form, including but 61 | not limited to compiled object code, generated documentation, 62 | and conversions to other media types. 63 | 64 | "Work" shall mean the work of authorship, whether in Source or 65 | Object form, made available under the License, as indicated by a 66 | copyright notice that is included in or attached to the work 67 | (an example is provided in the Appendix below). 68 | 69 | "Derivative Works" shall mean any work, whether in Source or Object 70 | form, that is based on (or derived from) the Work and for which the 71 | editorial revisions, annotations, elaborations, or other modifications 72 | represent, as a whole, an original work of authorship. For the purposes 73 | of this License, Derivative Works shall not include works that remain 74 | separable from, or merely link (or bind by name) to the interfaces of, 75 | the Work and Derivative Works thereof. 76 | 77 | "Contribution" shall mean any work of authorship, including 78 | the original version of the Work and any modifications or additions 79 | to that Work or Derivative Works thereof, that is intentionally 80 | submitted to Licensor for inclusion in the Work by the copyright owner 81 | or by an individual or Legal Entity authorized to submit on behalf of 82 | the copyright owner. For the purposes of this definition, "submitted" 83 | means any form of electronic, verbal, or written communication sent 84 | to the Licensor or its representatives, including but not limited to 85 | communication on electronic mailing lists, source code control systems, 86 | and issue tracking systems that are managed by, or on behalf of, the 87 | Licensor for the purpose of discussing and improving the Work, but 88 | excluding communication that is conspicuously marked or otherwise 89 | designated in writing by the copyright owner as "Not a Contribution." 90 | 91 | "Contributor" shall mean Licensor and any individual or Legal Entity 92 | on behalf of whom a Contribution has been received by Licensor and 93 | subsequently incorporated within the Work. 94 | 95 | 2. Grant of Copyright License. Subject to the terms and conditions of 96 | this License, each Contributor hereby grants to You a perpetual, 97 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 98 | copyright license to reproduce, prepare Derivative Works of, 99 | publicly display, publicly perform, sublicense, and distribute the 100 | Work and such Derivative Works in Source or Object form. 101 | 102 | 3. Grant of Patent License. Subject to the terms and conditions of 103 | this License, each Contributor hereby grants to You a perpetual, 104 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 105 | (except as stated in this section) patent license to make, have made, 106 | use, offer to sell, sell, import, and otherwise transfer the Work, 107 | where such license applies only to those patent claims licensable 108 | by such Contributor that are necessarily infringed by their 109 | Contribution(s) alone or by combination of their Contribution(s) 110 | with the Work to which such Contribution(s) was submitted. If You 111 | institute patent litigation against any entity (including a 112 | cross-claim or counterclaim in a lawsuit) alleging that the Work 113 | or a Contribution incorporated within the Work constitutes direct 114 | or contributory patent infringement, then any patent licenses 115 | granted to You under this License for that Work shall terminate 116 | as of the date such litigation is filed. 117 | 118 | 4. Redistribution. You may reproduce and distribute copies of the 119 | Work or Derivative Works thereof in any medium, with or without 120 | modifications, and in Source or Object form, provided that You 121 | meet the following conditions: 122 | 123 | (a) You must give any other recipients of the Work or 124 | Derivative Works a copy of this License; and 125 | 126 | (b) You must cause any modified files to carry prominent notices 127 | stating that You changed the files; and 128 | 129 | (c) You must retain, in the Source form of any Derivative Works 130 | that You distribute, all copyright, patent, trademark, and 131 | attribution notices from the Source form of the Work, 132 | excluding those notices that do not pertain to any part of 133 | the Derivative Works; and 134 | 135 | (d) If the Work includes a "NOTICE" text file as part of its 136 | distribution, then any Derivative Works that You distribute must 137 | include a readable copy of the attribution notices contained 138 | within such NOTICE file, excluding those notices that do not 139 | pertain to any part of the Derivative Works, in at least one 140 | of the following places: within a NOTICE text file distributed 141 | as part of the Derivative Works; within the Source form or 142 | documentation, if provided along with the Derivative Works; or, 143 | within a display generated by the Derivative Works, if and 144 | wherever such third-party notices normally appear. The contents 145 | of the NOTICE file are for informational purposes only and 146 | do not modify the License. You may add Your own attribution 147 | notices within Derivative Works that You distribute, alongside 148 | or as an addendum to the NOTICE text from the Work, provided 149 | that such additional attribution notices cannot be construed 150 | as modifying the License. 151 | 152 | You may add Your own copyright statement to Your modifications and 153 | may provide additional or different license terms and conditions 154 | for use, reproduction, or distribution of Your modifications, or 155 | for any such Derivative Works as a whole, provided Your use, 156 | reproduction, and distribution of the Work otherwise complies with 157 | the conditions stated in this License. 158 | 159 | 5. Submission of Contributions. Unless You explicitly state otherwise, 160 | any Contribution intentionally submitted for inclusion in the Work 161 | by You to the Licensor shall be under the terms and conditions of 162 | this License, without any additional terms or conditions. 163 | Notwithstanding the above, nothing herein shall supersede or modify 164 | the terms of any separate license agreement you may have executed 165 | with Licensor regarding such Contributions. 166 | 167 | 6. Trademarks. This License does not grant permission to use the trade 168 | names, trademarks, service marks, or product names of the Licensor, 169 | except as required for reasonable and customary use in describing the 170 | origin of the Work and reproducing the content of the NOTICE file. 171 | 172 | 7. Disclaimer of Warranty. Unless required by applicable law or 173 | agreed to in writing, Licensor provides the Work (and each 174 | Contributor provides its Contributions) on an "AS IS" BASIS, 175 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 176 | implied, including, without limitation, any warranties or conditions 177 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 178 | PARTICULAR PURPOSE. You are solely responsible for determining the 179 | appropriateness of using or redistributing the Work and assume any 180 | risks associated with Your exercise of permissions under this License. 181 | 182 | 8. Limitation of Liability. In no event and under no legal theory, 183 | whether in tort (including negligence), contract, or otherwise, 184 | unless required by applicable law (such as deliberate and grossly 185 | negligent acts) or agreed to in writing, shall any Contributor be 186 | liable to You for damages, including any direct, indirect, special, 187 | incidental, or consequential damages of any character arising as a 188 | result of this License or out of the use or inability to use the 189 | Work (including but not limited to damages for loss of goodwill, 190 | work stoppage, computer failure or malfunction, or any and all 191 | other commercial damages or losses), even if such Contributor 192 | has been advised of the possibility of such damages. 193 | 194 | 9. Accepting Warranty or Additional Liability. While redistributing 195 | the Work or Derivative Works thereof, You may choose to offer, 196 | and charge a fee for, acceptance of support, warranty, indemnity, 197 | or other liability obligations and/or rights consistent with this 198 | License. However, in accepting such obligations, You may act only 199 | on Your own behalf and on Your sole responsibility, not on behalf 200 | of any other Contributor, and only if You agree to indemnify, 201 | defend, and hold each Contributor harmless for any liability 202 | incurred by, or claims asserted against, such Contributor by reason 203 | of your accepting any such warranty or additional liability. 204 | 205 | END OF TERMS AND CONDITIONS 206 | ``` 207 | 208 |
209 | 210 |
MIT Software License 211 | 212 | ``` 213 | Permission is hereby granted, free of charge, to any person obtaining a copy 214 | of this software and associated documentation files (the "Software"), to deal 215 | in the Software without restriction, including without limitation the rights 216 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 217 | copies of the Software, and to permit persons to whom the Software is 218 | furnished to do so, subject to the following conditions: 219 | 220 | The above copyright notice and this permission notice shall be included in 221 | all copies or substantial portions of the Software. 222 | 223 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 224 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 225 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 226 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 227 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 228 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 229 | THE SOFTWARE. 230 | ``` 231 | 232 |
233 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SHELL=/usr/bin/env bash 2 | 3 | GO_BUILD_IMAGE?=golang:1.18 4 | COMMIT := $(shell git rev-parse --short=8 HEAD) 5 | 6 | # GITVERSION is the nearest tag plus number of commits and short form of most recent commit since the tag, if any 7 | GITVERSION=$(shell git describe --always --tag --dirty) 8 | 9 | unexport GOFLAGS 10 | 11 | CLEAN:= 12 | BINS:= 13 | 14 | GOFLAGS:= 15 | 16 | .PHONY: all 17 | all: build 18 | 19 | ## FFI 20 | 21 | FFI_PATH:=extern/filecoin-ffi/ 22 | FFI_DEPS:=.install-filcrypto 23 | FFI_DEPS:=$(addprefix $(FFI_PATH),$(FFI_DEPS)) 24 | 25 | $(FFI_DEPS): build/.filecoin-install ; 26 | 27 | build/.filecoin-install: $(FFI_PATH) 28 | @mkdir -p build 29 | $(MAKE) -C $(FFI_PATH) $(FFI_DEPS:$(FFI_PATH)%=%) 30 | @touch $@ 31 | 32 | MODULES+=$(FFI_PATH) 33 | BUILD_DEPS+=build/.filecoin-install 34 | CLEAN+=build/.filecoin-install 35 | 36 | ffi-version-check: 37 | @[[ "$$(awk '/const Version/{print $$5}' extern/filecoin-ffi/version.go)" -eq 3 ]] || (echo "FFI version mismatch, update submodules"; exit 1) 38 | BUILD_DEPS+=ffi-version-check 39 | 40 | .PHONY: ffi-version-check 41 | 42 | $(MODULES): build/.update-modules ; 43 | # dummy file that marks the last time modules were updated 44 | build/.update-modules: 45 | git submodule update --init --recursive 46 | @mkdir -p build 47 | touch $@ 48 | 49 | CLEAN+=build/.update-modules 50 | 51 | # Once filclient has it's own version cmd add this back in 52 | #ldflags=-X=github.com/application-research/filclient/version.GitVersion=$(GITVERSION) 53 | #ifneq ($(strip $(LDFLAGS)),) 54 | # ldflags+=-extldflags=$(LDFLAGS) 55 | #endif 56 | #GOFLAGS+=-ldflags="$(ldflags)" 57 | 58 | .PHONY: build 59 | build: deps filclient 60 | 61 | .PHONY: deps 62 | deps: $(BUILD_DEPS) 63 | 64 | .PHONY: filclient 65 | filclient: 66 | go build 67 | 68 | .PHONY: filc 69 | filc: filclient 70 | make -C filc 71 | 72 | .PHONY: clean 73 | clean: 74 | rm -rf $(CLEAN) $(BINS) 75 | make -C filc clean 76 | 77 | .PHONY: dist-clean 78 | dist-clean: 79 | git clean -xdff 80 | git submodule deinit --all -f 81 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # filclient 2 | 3 | A standalone client library that enables users to interact with the Filecoin storage network. The types of interactions available are listed below, under "Features". 4 | 5 | This functionality is accomplished by allowing users to interact with the API of any Lotus Node on the Filecoin storage network. One such node is `api.chain.love`, which is a node hosted by the Outercore Engineering team. 6 | 7 | ## Features 8 | 9 | - Make storage deals with miners 10 | - Query storage ask prices 11 | - Construct and sign deal proposals 12 | - Send deal proposals over the network to miners 13 | - Send data to miners via data transfer and graphsync 14 | - Check status of a deal with a miner 15 | - Make retrieval deals with miners 16 | - Query retrieval ask prices 17 | - Run retrieval protocol, get data 18 | - Data transfer management 19 | - Ability to query data transfer (in or out) status 20 | - Market funds management 21 | - Check and add to market escrow balance 22 | - Local message signing capabilities and wallet management 23 | 24 | ## Roadmap 25 | 26 | - [ ] Cleanup and organization of functions 27 | - [ ] Godocs on main methods 28 | - [ ] Direct mempool integration (to avoid relying on a lotus gateway node) 29 | - [ ] Cleanup of dependency tree 30 | - [ ] Remove dependency on filecoin-ffi 31 | - [ ] Remove dependency on go-fil-markets 32 | - [ ] Remove dependency on main lotus repo 33 | - [ ] Good usage examples 34 | - [ ] Sample application using filclient 35 | - [ ] Integration testing 36 | 37 | ## License 38 | 39 | MIT 40 | -------------------------------------------------------------------------------- /apiprov.go: -------------------------------------------------------------------------------- 1 | package filclient 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/filecoin-project/go-address" 7 | "github.com/filecoin-project/go-state-types/crypto" 8 | "github.com/filecoin-project/lotus/api" 9 | "github.com/filecoin-project/lotus/chain/types" 10 | "github.com/filecoin-project/lotus/chain/wallet" 11 | ) 12 | 13 | type paychApiProvider struct { 14 | api.Gateway 15 | wallet *wallet.LocalWallet 16 | mp *MsgPusher 17 | } 18 | 19 | func (a *paychApiProvider) MpoolPushMessage(ctx context.Context, msg *types.Message, maxFee *api.MessageSendSpec) (*types.SignedMessage, error) { 20 | return a.mp.MpoolPushMessage(ctx, msg, maxFee) 21 | } 22 | 23 | func (a *paychApiProvider) WalletHas(ctx context.Context, addr address.Address) (bool, error) { 24 | return a.wallet.WalletHas(ctx, addr) 25 | } 26 | 27 | func (a *paychApiProvider) WalletSign(ctx context.Context, addr address.Address, data []byte) (*crypto.Signature, error) { 28 | return a.wallet.WalletSign(ctx, addr, data, api.MsgMeta{Type: api.MTUnknown}) 29 | } 30 | -------------------------------------------------------------------------------- /channelid.go: -------------------------------------------------------------------------------- 1 | package filclient 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "strings" 7 | 8 | datatransfer "github.com/filecoin-project/go-data-transfer" 9 | "github.com/libp2p/go-libp2p/core/peer" 10 | ) 11 | 12 | func ChannelIDFromString(id string) (*datatransfer.ChannelID, error) { 13 | if id == "" { 14 | return nil, fmt.Errorf("cannot parse empty string as channel id") 15 | } 16 | 17 | parts := strings.Split(id, "-") 18 | if len(parts) != 3 { 19 | return nil, fmt.Errorf("cannot parse channel id '%s': expected format 'initiator-responder-transferid'", id) 20 | } 21 | 22 | initiator, err := peer.Decode(parts[0]) 23 | if err != nil { 24 | return nil, fmt.Errorf("parsing initiator peer id '%s' in channel id '%s'", parts[0], id) 25 | } 26 | 27 | responder, err := peer.Decode(parts[1]) 28 | if err != nil { 29 | return nil, fmt.Errorf("parsing responder peer id '%s' in channel id '%s'", parts[1], id) 30 | } 31 | 32 | xferid, err := strconv.ParseUint(parts[2], 10, 64) 33 | if err != nil { 34 | return nil, fmt.Errorf("parsing transfer id '%s' in channel id '%s'", parts[2], id) 35 | } 36 | 37 | return &datatransfer.ChannelID{ 38 | Initiator: initiator, 39 | Responder: responder, 40 | ID: datatransfer.TransferID(xferid), 41 | }, nil 42 | } 43 | -------------------------------------------------------------------------------- /errors.go: -------------------------------------------------------------------------------- 1 | package filclient 2 | 3 | import "fmt" 4 | 5 | type ErrorCode int 6 | 7 | const ( 8 | ErrUnknown ErrorCode = iota 9 | 10 | // Failed to connect to a miner. 11 | ErrMinerConnectionFailed 12 | 13 | // There was an issue related to the Lotus API. 14 | ErrLotusError 15 | ) 16 | 17 | type Error struct { 18 | Code ErrorCode 19 | Inner error 20 | } 21 | 22 | func (code ErrorCode) String() string { 23 | switch code { 24 | case ErrUnknown: 25 | return "unknown" 26 | case ErrMinerConnectionFailed: 27 | return "miner connection failed" 28 | case ErrLotusError: 29 | return "lotus error" 30 | default: 31 | return "(invalid error code)" 32 | } 33 | } 34 | 35 | func (err *Error) Error() string { 36 | return fmt.Sprintf("%s: %s", err.Code, err.Inner) 37 | } 38 | 39 | func (err *Error) Unwrap() error { 40 | return err.Inner 41 | } 42 | 43 | func NewError(code ErrorCode, err error) *Error { 44 | return &Error{ 45 | Code: code, 46 | Inner: err, 47 | } 48 | } 49 | 50 | func NewErrUnknown(err error) error { 51 | return NewError(ErrUnknown, err) 52 | } 53 | 54 | func NewErrMinerConnectionFailed(err error) error { 55 | return NewError(ErrMinerConnectionFailed, err) 56 | } 57 | 58 | func NewErrLotusError(err error) error { 59 | return NewError(ErrLotusError, err) 60 | } 61 | -------------------------------------------------------------------------------- /filc/Makefile: -------------------------------------------------------------------------------- 1 | BINS:=filc 2 | 3 | filc: *.go 4 | go build 5 | 6 | .PHONY: clean 7 | clean: 8 | rm -f $(BINS) 9 | 10 | all: filc 11 | -------------------------------------------------------------------------------- /filc/README.md: -------------------------------------------------------------------------------- 1 | # filc 2 | 3 | filc is a simple command line interface for `filclient`. It's great for basic storage/retrieval deals, checking wallet balance, and testing. 4 | 5 | ## Setup 6 | 7 | ### Install 8 | Using filc requires some system dependencies 9 | 10 | Ubuntu/Debian: 11 | ``` 12 | sudo apt install ocl-icd-opencl-dev build-essential jq pkg-config libhwloc-dev wget -y && sudo apt upgrade -y 13 | ``` 14 | 15 | **Go** 16 | 17 | To build filc, you need a working installation of Go 1.16.4 or higher: 18 | 19 | ``` 20 | wget -c https://golang.org/dl/go1.16.4.linux-amd64.tar.gz -O - | sudo tar -xz -C /usr/local 21 | ``` 22 | 23 | >You’ll need to add /usr/local/go/bin to your path. For most Linux distributions you can run something like: 24 | >echo 'export PATH=$PATH:/usr/local/go/bin' >> ~/.bashrc && source ~/.bashrc 25 | > 26 | >See the official Golang installation instructions if you get stuck. 27 | 28 | **Build and install** 29 | 30 | Once all the dependencies are installed, you can build filc in the `filc` directory: 31 | 32 | 1. Clone the repository 33 | ``` 34 | git clone https://github.com/application-research/filclient.git 35 | cd filclient 36 | ``` 37 | 38 | 2. Build 39 | ``` 40 | make all filc 41 | ``` 42 | 43 | 3. Run 44 | ``` 45 | ./filc/filc help 46 | ``` 47 | 48 | ### Lotus Connection 49 | filc currently needs a connection to a synced Lotus node to function. By default, filc will attempt to connect to a node hosted at `localhost`. If you don't have a self-hosted Lotus node, an alternative address can be specified by setting the environment variable `FULLNODE_API_INFO` (you'll probably want `FULLNODE_API_INFO=wss://api.chain.love`). 50 | 51 | ### Wallet Setup 52 | Currently, filc will automatically generate a wallet address for you on first run. If you already have a wallet you'd like to use, you can grab the corresponding file starting with `O5` from your existing wallet folder (e.g. `~/.lotus/keystore/`) and place it into `~/.filc/wallet/`. 53 | 54 | ### Debugging 55 | 56 | For more verbose logging, set GOLOG_LOG_LEVEL with the modules and levels you want to see. Example - `GOLOG_LOG_LEVEL="filclient=debug"`. Run `filc print-loggers` to view a list of all available configurable loggers. 57 | -------------------------------------------------------------------------------- /filc/cmds.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "math/rand" 7 | "net/url" 8 | "os" 9 | "strings" 10 | "time" 11 | 12 | "github.com/application-research/filclient" 13 | "github.com/application-research/filclient/retrievehelper" 14 | "github.com/filecoin-project/boost/transport/httptransport" 15 | "github.com/filecoin-project/go-address" 16 | cborutil "github.com/filecoin-project/go-cbor-util" 17 | datatransfer "github.com/filecoin-project/go-data-transfer" 18 | "github.com/filecoin-project/go-fil-markets/storagemarket/network" 19 | "github.com/filecoin-project/go-state-types/big" 20 | "github.com/filecoin-project/lotus/chain/types" 21 | lcli "github.com/filecoin-project/lotus/cli" 22 | "github.com/google/uuid" 23 | "github.com/ipfs/go-blockservice" 24 | "github.com/ipfs/go-cid" 25 | chunker "github.com/ipfs/go-ipfs-chunker" 26 | offline "github.com/ipfs/go-ipfs-exchange-offline" 27 | files "github.com/ipfs/go-ipfs-files" 28 | logging "github.com/ipfs/go-log/v2" 29 | "github.com/ipfs/go-merkledag" 30 | unixfile "github.com/ipfs/go-unixfs/file" 31 | "github.com/ipfs/go-unixfs/importer" 32 | "github.com/ipld/go-car" 33 | "github.com/ipld/go-ipld-prime" 34 | cidlink "github.com/ipld/go-ipld-prime/linking/cid" 35 | basicnode "github.com/ipld/go-ipld-prime/node/basic" 36 | "github.com/ipld/go-ipld-prime/traversal" 37 | "github.com/ipld/go-ipld-prime/traversal/selector" 38 | "github.com/ipld/go-ipld-prime/traversal/selector/builder" 39 | textselector "github.com/ipld/go-ipld-selector-text-lite" 40 | dht "github.com/libp2p/go-libp2p-kad-dht" 41 | "github.com/libp2p/go-libp2p/core/host" 42 | inet "github.com/libp2p/go-libp2p/core/network" 43 | "github.com/libp2p/go-libp2p/core/peer" 44 | "github.com/multiformats/go-multiaddr" 45 | "github.com/urfave/cli/v2" 46 | "golang.org/x/xerrors" 47 | ) 48 | 49 | var printLoggersCmd = &cli.Command{ 50 | Name: "print-loggers", 51 | Usage: "Display loggers present in the program to help configure log levels", 52 | Action: func(cctx *cli.Context) error { 53 | loggers := logging.GetSubsystems() 54 | 55 | for _, logger := range loggers { 56 | fmt.Printf("%s\n", logger) 57 | } 58 | 59 | return nil 60 | }, 61 | } 62 | 63 | func tpr(s string, args ...interface{}) { 64 | fmt.Printf("[%s] "+s+"\n", append([]interface{}{time.Now().Format("15:04:05")}, args...)...) 65 | } 66 | 67 | var makeDealCmd = &cli.Command{ 68 | Name: "deal", 69 | Usage: "Make a storage deal with a miner", 70 | ArgsUsage: "", 71 | Flags: []cli.Flag{ 72 | flagMinerRequired, 73 | flagVerified, 74 | &cli.StringFlag{ 75 | Name: "announce", 76 | Usage: "the public multi-address from which to download the data (for deals with protocol v120)", 77 | }, 78 | }, 79 | Action: func(cctx *cli.Context) error { 80 | if !cctx.Args().Present() { 81 | return fmt.Errorf("please specify file to make deal for") 82 | } 83 | 84 | ddir := ddir(cctx) 85 | 86 | miner, err := parseMiner(cctx) 87 | if err != nil { 88 | return err 89 | } 90 | 91 | nd, err := setup(cctx.Context, ddir) 92 | if err != nil { 93 | return err 94 | } 95 | 96 | fc, closer, err := clientFromNode(cctx, nd, ddir) 97 | if err != nil { 98 | return err 99 | } 100 | defer closer() 101 | 102 | fi, err := os.Open(cctx.Args().First()) 103 | if err != nil { 104 | return err 105 | } 106 | 107 | bserv := blockservice.New(nd.Blockstore, nil) 108 | dserv := merkledag.NewDAGService(bserv) 109 | 110 | tpr("importing file...") 111 | spl := chunker.DefaultSplitter(fi) 112 | 113 | obj, err := importer.BuildDagFromReader(dserv, spl) 114 | if err != nil { 115 | return err 116 | } 117 | 118 | tpr("File CID: %s", obj.Cid()) 119 | 120 | tpr("getting ask from storage provider %s...", miner) 121 | ask, err := fc.GetAsk(cctx.Context, miner) 122 | if err != nil { 123 | return fmt.Errorf("getting ask from storage provider %s: %w", miner, err) 124 | } 125 | 126 | verified := parseVerified(cctx) 127 | removeUnsealed := parseRemoveUnsealed(cctx) 128 | 129 | price := ask.Ask.Ask.Price 130 | if verified { 131 | price = ask.Ask.Ask.VerifiedPrice 132 | tpr("storage provider ask for verified deals: %d", price) 133 | } else { 134 | tpr("storage provider ask: %d", price) 135 | } 136 | 137 | minPieceSize := ask.Ask.Ask.MinPieceSize 138 | proposal, err := fc.MakeDeal(cctx.Context, miner, obj.Cid(), price, minPieceSize, 2880*365, verified, removeUnsealed) 139 | if err != nil { 140 | return err 141 | } 142 | 143 | propnd, err := cborutil.AsIpld(proposal.DealProposal) 144 | if err != nil { 145 | return xerrors.Errorf("failed to compute deal proposal ipld node: %w", err) 146 | } 147 | 148 | tpr("proposal cid: %s", propnd.Cid()) 149 | 150 | if err := saveDealProposal(ddir, propnd.Cid(), proposal.DealProposal); err != nil { 151 | return err 152 | } 153 | 154 | proto, err := fc.DealProtocolForMiner(cctx.Context, miner) 155 | if err != nil { 156 | return err 157 | } 158 | 159 | tpr("storage provider supports deal protocol %s", proto) 160 | 161 | switch { 162 | case proto == filclient.DealProtocolv110: 163 | return makev110Deal(cctx, fc, miner, proposal, propnd.Cid(), obj.Cid()) 164 | case proto == filclient.DealProtocolv120: 165 | return makev120Deal(cctx, fc, nd.Host, miner, proposal, propnd.Cid()) 166 | default: 167 | return fmt.Errorf("unrecognized deal protocol %s", proto) 168 | } 169 | }, 170 | } 171 | 172 | func makev110Deal(cctx *cli.Context, fc *filclient.FilClient, miner address.Address, proposal *network.Proposal, propCid cid.Cid, dataCid cid.Cid) error { 173 | ctx := cctx.Context 174 | 175 | // Send the deal proposal 176 | _, err := fc.SendProposalV110(ctx, *proposal, propCid) 177 | if err != nil { 178 | return err 179 | } 180 | 181 | tpr("miner accepted the deal!") 182 | 183 | // Start the push transfer 184 | tpr("starting data transfer... %s", propCid) 185 | chanid, err := fc.StartDataTransfer(ctx, miner, propCid, dataCid) 186 | if err != nil { 187 | return err 188 | } 189 | 190 | // Periodically check the transfer status and output a log 191 | var lastStatus datatransfer.Status 192 | for { 193 | status, err := fc.TransferStatus(ctx, chanid) 194 | if err != nil { 195 | return err 196 | } 197 | 198 | statusChanged := status.Status != lastStatus 199 | logstr, err := logStatus(status, statusChanged) 200 | if err != nil { 201 | return err 202 | } 203 | if logstr != "" { 204 | tpr(logstr) 205 | } 206 | if status.Status == datatransfer.Completed { 207 | tpr("transfer completed, miner: %s, propcid: %s", miner, propCid) 208 | return nil 209 | } 210 | lastStatus = status.Status 211 | 212 | time.Sleep(time.Millisecond * 100) 213 | } 214 | return nil 215 | } 216 | 217 | func makev120Deal(cctx *cli.Context, fc *filclient.FilClient, h host.Host, miner address.Address, netprop *network.Proposal, propCid cid.Cid) error { 218 | var announceAddr multiaddr.Multiaddr 219 | tpr("filc host addr: %s", h.Addrs()) 220 | tpr("filc host peer: %s", h.ID()) 221 | announce := cctx.String("announce") 222 | if announce == "" { 223 | return fmt.Errorf("must specify announce address to make deals over deal v1.2.0 protocol %s", filclient.DealProtocolv120) 224 | } 225 | 226 | announceStr := announce + "/p2p/" + h.ID().String() 227 | announceAddr, err := multiaddr.NewMultiaddr(announceStr) 228 | if err != nil { 229 | return fmt.Errorf("parsing announce address '%s': %w", announceStr, err) 230 | } 231 | tpr("filc announce address: %s", announceAddr.String()) 232 | 233 | dbid := uint(rand.Uint32()) 234 | dealUUID := uuid.New() 235 | pullComplete := make(chan error) 236 | var lastStatus datatransfer.Status 237 | 238 | // Subscribe to pull transfer updates. 239 | unsubPullEvts, err := fc.Libp2pTransferMgr.Subscribe(func(evtdbid uint, st filclient.ChannelState) { 240 | if dbid != evtdbid { 241 | return 242 | } 243 | 244 | statusChanged := st.Status != lastStatus 245 | logstr, err := logStatus(&st, statusChanged) 246 | if err != nil { 247 | pullComplete <- err 248 | return 249 | } 250 | 251 | if logstr != "" { 252 | tpr(logstr) 253 | } 254 | 255 | if st.Status == datatransfer.Completed { 256 | tpr("transfer completed, miner: %s, propcid: %s", miner, propCid) 257 | pullComplete <- nil 258 | } 259 | 260 | lastStatus = st.Status 261 | }) 262 | if err != nil { 263 | return err 264 | } 265 | defer unsubPullEvts() 266 | 267 | // Keep the connection alive 268 | ctx, cancel := context.WithCancel(cctx.Context) 269 | defer cancel() 270 | go keepConnection(ctx, fc, h, miner, tpr) 271 | 272 | // In deal protocol v120 the transfer will be initiated by the 273 | // storage provider (a pull transfer) so we need to prepare for 274 | // the data request 275 | tpr("sending v1.2.0 deal proposal with dbid %d, deal uuid %s", dbid, dealUUID.String()) 276 | 277 | // Create an auth token to be used in the request 278 | authToken, err := httptransport.GenerateAuthToken() 279 | if err != nil { 280 | return xerrors.Errorf("generating auth token for deal: %w", err) 281 | } 282 | 283 | // Add an auth token for the data to the auth DB 284 | rootCid := netprop.Piece.Root 285 | size := netprop.Piece.RawBlockSize 286 | err = fc.Libp2pTransferMgr.PrepareForDataRequest(ctx, dbid, authToken, propCid, rootCid, size) 287 | if err != nil { 288 | return xerrors.Errorf("preparing for data request: %w", err) 289 | } 290 | 291 | // Send the deal proposal to the storage provider 292 | _, err = fc.SendProposalV120(ctx, dbid, *netprop, dealUUID, announceAddr, authToken) 293 | if err != nil { 294 | // Clean up auth token 295 | fc.Libp2pTransferMgr.CleanupPreparedRequest(ctx, dbid, authToken) //nolint:errcheck 296 | return err 297 | } 298 | 299 | tpr("miner accepted the deal!") 300 | 301 | // Wait for the transfer to complete (while outputting logs) 302 | select { 303 | case <-cctx.Context.Done(): 304 | return cctx.Context.Err() 305 | case err = <-pullComplete: 306 | } 307 | return err 308 | } 309 | 310 | func logStatus(status *filclient.ChannelState, changed bool) (string, error) { 311 | switch status.Status { 312 | case datatransfer.Failed: 313 | return "", fmt.Errorf("data transfer failed: %s", status.Message) 314 | case datatransfer.Cancelled: 315 | return "", fmt.Errorf("transfer cancelled: %s", status.Message) 316 | case datatransfer.Failing: 317 | return fmt.Sprintf("data transfer failing... %s", status.Message), nil 318 | // I guess we just wait until its failed all the way? 319 | case datatransfer.Requested: 320 | if changed { 321 | return "data transfer requested", nil 322 | } 323 | //fmt.Println("transfer is requested, hasnt started yet") 324 | // probably okay 325 | case datatransfer.TransferFinished, datatransfer.Finalizing, datatransfer.Completing: 326 | if changed { 327 | return "current state: " + status.StatusStr, nil 328 | } 329 | case datatransfer.Completed: 330 | return "transfer complete!", nil 331 | case datatransfer.Ongoing: 332 | return fmt.Sprintf("transfer progress: %d", status.Sent), nil 333 | default: 334 | return fmt.Sprintf("Unexpected data transfer state: %d (msg = %s)", status.Status, status.Message), nil 335 | } 336 | return "", nil 337 | } 338 | 339 | // keepConnection watches the connection to the miner, reconnecting if it goes 340 | // down 341 | func keepConnection(ctx context.Context, fc *filclient.FilClient, host host.Host, maddr address.Address, tpr func(s string, args ...interface{})) { 342 | pid, err := fc.ConnectToMiner(ctx, maddr) 343 | if err != nil { 344 | tpr("Unable to make initial connection to storage provider %s: %s", maddr, err) 345 | return 346 | } 347 | 348 | tpr("Watching connection to storage provider %s with peer ID %s", maddr, pid) 349 | cw := connWatcher{ 350 | pid: pid, 351 | disconnected: make(chan struct{}, 1), 352 | reconnect: func() { 353 | tpr("Connection to storage provider %s disconnected. Reconnecting...", maddr) 354 | _, err := fc.ConnectToMiner(ctx, maddr) 355 | if err == nil { 356 | tpr("Reconnected to storage provider %s. Waiting for storage provider to restart transfer...", maddr) 357 | } else { 358 | tpr("Failed to reconnect to storage provider %s: %s", maddr, err) 359 | } 360 | }, 361 | } 362 | host.Network().Notify(&cw) 363 | } 364 | 365 | type connWatcher struct { 366 | pid peer.ID 367 | reconnect func() 368 | disconnected chan struct{} 369 | } 370 | 371 | // Disconnected is called when a connection breaks 372 | func (c *connWatcher) Disconnected(n inet.Network, conn inet.Conn) { 373 | if conn.RemotePeer() != c.pid { 374 | return 375 | } 376 | 377 | select { 378 | case c.disconnected <- struct{}{}: 379 | go c.processDisconnect() 380 | default: 381 | } 382 | } 383 | 384 | func (c *connWatcher) processDisconnect() { 385 | // Sleep for a few seconds to prevent flapping 386 | time.Sleep(5 * time.Second) 387 | 388 | select { 389 | case <-c.disconnected: 390 | c.reconnect() 391 | // Do one more reconnect in case a disconnect happened while we were 392 | // reconnecting, 393 | go c.processDisconnect() 394 | default: 395 | } 396 | } 397 | 398 | func (c *connWatcher) Listen(n inet.Network, m multiaddr.Multiaddr) {} 399 | func (c *connWatcher) ListenClose(n inet.Network, m multiaddr.Multiaddr) {} 400 | func (c *connWatcher) Connected(n inet.Network, conn inet.Conn) {} 401 | func (c *connWatcher) OpenedStream(n inet.Network, stream inet.Stream) {} 402 | func (c *connWatcher) ClosedStream(n inet.Network, stream inet.Stream) {} 403 | 404 | var dealStatusCmd = &cli.Command{ 405 | Name: "deal-status", 406 | Usage: "Get on-chain deal status", 407 | ArgsUsage: "", 408 | Flags: []cli.Flag{ 409 | flagMinerRequired, 410 | flagDealUUID, 411 | }, 412 | Action: func(cctx *cli.Context) error { 413 | if !cctx.Args().Present() { 414 | return fmt.Errorf("proposal CID must be specified") 415 | } 416 | 417 | miner, err := parseMiner(cctx) 418 | if err != nil { 419 | return fmt.Errorf("invalid miner address: %w", err) 420 | } 421 | 422 | dealUUID, err := parseDealUUID(cctx) 423 | if err != nil { 424 | return fmt.Errorf("invalid deal UUID: %w", err) 425 | } 426 | 427 | cid, err := cid.Decode(cctx.Args().First()) 428 | if err != nil { 429 | return fmt.Errorf("invalid proposal CID: %w", err) 430 | } 431 | 432 | nd, err := setup(cctx.Context, ddir(cctx)) 433 | if err != nil { 434 | return fmt.Errorf("could not set up node: %w", err) 435 | } 436 | 437 | fc, closer, err := clientFromNode(cctx, nd, ddir(cctx)) 438 | if err != nil { 439 | return fmt.Errorf("could not initialize filclient: %w", err) 440 | } 441 | defer closer() 442 | 443 | var dealUUIDPtr *uuid.UUID 444 | if dealUUID != uuid.Nil { 445 | dealUUIDPtr = &dealUUID 446 | } 447 | 448 | dealStatus, err := fc.DealStatus(cctx.Context, miner, cid, dealUUIDPtr) 449 | if err != nil { 450 | return fmt.Errorf("could not get deal state from provider: %w", err) 451 | } 452 | 453 | printDealStatus(dealStatus) 454 | 455 | return nil 456 | }, 457 | } 458 | 459 | var infoCmd = &cli.Command{ 460 | Name: "info", 461 | Usage: "Display wallet information", 462 | ArgsUsage: " ", 463 | Action: func(cctx *cli.Context) error { 464 | ddir := ddir(cctx) 465 | 466 | nd, err := setup(cctx.Context, ddir) 467 | if err != nil { 468 | return err 469 | } 470 | 471 | api, closer, err := lcli.GetGatewayAPI(cctx) 472 | if err != nil { 473 | return err 474 | } 475 | defer closer() 476 | 477 | addr, err := nd.Wallet.GetDefault() 478 | if err != nil { 479 | return err 480 | } 481 | 482 | balance := big.NewInt(0) 483 | verifiedBalance := big.NewInt(0) 484 | 485 | act, err := api.StateGetActor(cctx.Context, addr, types.EmptyTSK) 486 | if err != nil { 487 | fmt.Println("NOTE - Actor not found on chain") 488 | } else { 489 | balance = act.Balance 490 | 491 | v, err := api.StateVerifiedClientStatus(cctx.Context, addr, types.EmptyTSK) 492 | if err != nil { 493 | return err 494 | } 495 | 496 | verifiedBalance = *v 497 | } 498 | 499 | fmt.Printf("Default client address: %v\n", addr) 500 | fmt.Printf("Balance: %v\n", types.FIL(balance)) 501 | fmt.Printf("Verified Balance: %v\n", types.FIL(verifiedBalance)) 502 | 503 | return nil 504 | }, 505 | } 506 | 507 | var getAskCmd = &cli.Command{ 508 | Name: "get-ask", 509 | Usage: "Query storage deal ask for a miner", 510 | ArgsUsage: "", 511 | Action: func(cctx *cli.Context) error { 512 | if !cctx.Args().Present() { 513 | return fmt.Errorf("please specify miner to query ask of") 514 | } 515 | 516 | ddir := ddir(cctx) 517 | 518 | miner, err := address.NewFromString(cctx.Args().First()) 519 | if err != nil { 520 | return err 521 | } 522 | 523 | fc, closer, err := getClient(cctx, ddir) 524 | if err != nil { 525 | return err 526 | } 527 | defer closer() 528 | 529 | ask, err := fc.GetAsk(cctx.Context, miner) 530 | if err != nil { 531 | return fmt.Errorf("failed to get ask: %s", err) 532 | } 533 | 534 | printAskResponse(ask.Ask.Ask) 535 | 536 | return nil 537 | }, 538 | } 539 | 540 | var listDealsCmd = &cli.Command{ 541 | Name: "list", 542 | Usage: "List local storage deal history", 543 | ArgsUsage: " ", 544 | Action: func(cctx *cli.Context) error { 545 | ddir := ddir(cctx) 546 | 547 | deals, err := listDeals(ddir) 548 | if err != nil { 549 | return err 550 | } 551 | 552 | for _, dcid := range deals { 553 | fmt.Println(dcid) 554 | } 555 | 556 | return nil 557 | }, 558 | } 559 | 560 | var retrieveFileCmd = &cli.Command{ 561 | Name: "retrieve", 562 | Usage: "Retrieve a file by CID from a miner", 563 | Description: "Retrieve a file by CID from a miner. If desired, multiple miners can be specified as fallbacks in case of a failure (comma-separated, no spaces).", 564 | ArgsUsage: "", 565 | Flags: []cli.Flag{ 566 | flagMiners, 567 | flagOutput, 568 | flagNetwork, 569 | flagDmPathSel, 570 | flagCar, 571 | }, 572 | Action: func(cctx *cli.Context) error { 573 | 574 | // Parse command input 575 | 576 | cidStr := cctx.Args().First() 577 | if cidStr == "" { 578 | return fmt.Errorf("please specify a CID to retrieve") 579 | } 580 | 581 | dmSelText := textselector.Expression(cctx.String(flagDmPathSel.Name)) 582 | 583 | miners, err := parseMiners(cctx) 584 | if err != nil { 585 | return err 586 | } 587 | 588 | output, err := parseOutput(cctx) 589 | if err != nil { 590 | return err 591 | } 592 | if output == "" { 593 | output = cidStr 594 | if dmSelText != "" { 595 | output += "_" + url.QueryEscape(string(dmSelText)) 596 | } 597 | } 598 | 599 | network := strings.ToLower(strings.TrimSpace(cctx.String("network"))) 600 | 601 | c, err := cid.Decode(cidStr) 602 | if err != nil { 603 | return err 604 | } 605 | 606 | // Get subselector node 607 | 608 | var selNode ipld.Node 609 | if dmSelText != "" { 610 | ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) 611 | 612 | selspec, err := textselector.SelectorSpecFromPath( 613 | dmSelText, 614 | true, 615 | 616 | // URGH - this is a direct copy from https://github.com/filecoin-project/go-fil-markets/blob/v1.12.0/shared/selectors.go#L10-L16 617 | // Unable to use it because we need the SelectorSpec, and markets exposes just a reified node 618 | ssb.ExploreRecursive( 619 | selector.RecursionLimitNone(), 620 | ssb.ExploreAll(ssb.ExploreRecursiveEdge()), 621 | ), 622 | ) 623 | if err != nil { 624 | return xerrors.Errorf("failed to parse text-selector '%s': %w", dmSelText, err) 625 | } 626 | 627 | selNode = selspec.Node() 628 | } 629 | 630 | // Set up node and filclient 631 | 632 | ddir := ddir(cctx) 633 | 634 | node, err := setup(cctx.Context, ddir) 635 | if err != nil { 636 | return err 637 | } 638 | 639 | fc, closer, err := clientFromNode(cctx, node, ddir) 640 | if err != nil { 641 | return err 642 | } 643 | defer closer() 644 | 645 | // Collect retrieval candidates and config. If one or more miners are 646 | // provided, use those with the requested cid as the root cid as the 647 | // candidate list. Otherwise, we can use the auto retrieve API endpoint 648 | // to automatically find some candidates to retrieve from. 649 | 650 | var candidates []FILRetrievalCandidate 651 | if len(miners) > 0 { 652 | for _, miner := range miners { 653 | candidates = append(candidates, FILRetrievalCandidate{ 654 | Miner: miner, 655 | RootCid: c, 656 | }) 657 | } 658 | } else { 659 | endpoint := "https://api.estuary.tech/retrieval-candidates" // TODO: don't hard code 660 | candidates_, err := node.GetRetrievalCandidates(endpoint, c) 661 | if err != nil { 662 | return fmt.Errorf("failed to get retrieval candidates: %w", err) 663 | } 664 | 665 | candidates = candidates_ 666 | } 667 | 668 | // Do the retrieval 669 | 670 | var networks []RetrievalAttempt 671 | 672 | if network == NetworkIPFS || network == NetworkAuto { 673 | if selNode != nil && !selNode.IsNull() { 674 | // Selector nodes are not compatible with IPFS 675 | if network == NetworkIPFS { 676 | log.Fatal("IPFS is not compatible with selector node") 677 | } else { 678 | log.Info("A selector node has been specified, skipping IPFS") 679 | } 680 | } else { 681 | networks = append(networks, &IPFSRetrievalAttempt{ 682 | Cid: c, 683 | }) 684 | } 685 | } 686 | 687 | if network == NetworkFIL || network == NetworkAuto { 688 | networks = append(networks, &FILRetrievalAttempt{ 689 | FilClient: fc, 690 | Cid: c, 691 | Candidates: candidates, 692 | SelNode: selNode, 693 | }) 694 | } 695 | 696 | if len(networks) == 0 { 697 | log.Fatalf("Unknown --network value \"%s\"", network) 698 | } 699 | 700 | stats, err := node.RetrieveFromBestCandidate(cctx.Context, networks) 701 | if err != nil { 702 | return err 703 | } 704 | 705 | printRetrievalStats(stats) 706 | 707 | // Save the output 708 | 709 | dservOffline := merkledag.NewDAGService(blockservice.New(node.Blockstore, offline.Exchange(node.Blockstore))) 710 | 711 | // if we used a selector - need to find the sub-root the user actually wanted to retrieve 712 | if dmSelText != "" { 713 | var subRootFound bool 714 | 715 | // no err check - we just compiled this before starting, but now we do not wrap a `*` 716 | selspec, _ := textselector.SelectorSpecFromPath(dmSelText, true, nil) //nolint:errcheck 717 | if err := retrievehelper.TraverseDag( 718 | cctx.Context, 719 | dservOffline, 720 | c, 721 | selspec.Node(), 722 | func(p traversal.Progress, n ipld.Node, r traversal.VisitReason) error { 723 | if r == traversal.VisitReason_SelectionMatch { 724 | 725 | if p.LastBlock.Path.String() != p.Path.String() { 726 | return xerrors.Errorf("unsupported selection path '%s' does not correspond to a node boundary (a.k.a. CID link)", p.Path.String()) 727 | } 728 | 729 | cidLnk, castOK := p.LastBlock.Link.(cidlink.Link) 730 | if !castOK { 731 | return xerrors.Errorf("cidlink cast unexpectedly failed on '%s'", p.LastBlock.Link.String()) 732 | } 733 | 734 | c = cidLnk.Cid 735 | subRootFound = true 736 | } 737 | return nil 738 | }, 739 | ); err != nil { 740 | return xerrors.Errorf("error while locating partial retrieval sub-root: %w", err) 741 | } 742 | 743 | if !subRootFound { 744 | return xerrors.Errorf("path selection '%s' does not match a node within %s", dmSelText, c) 745 | } 746 | } 747 | 748 | dnode, err := dservOffline.Get(cctx.Context, c) 749 | if err != nil { 750 | return err 751 | } 752 | 753 | if cctx.Bool(flagCar.Name) { 754 | // Write file as car file 755 | file, err := os.Create(output + ".car") 756 | if err != nil { 757 | return err 758 | } 759 | car.WriteCar(cctx.Context, dservOffline, []cid.Cid{c}, file) 760 | 761 | fmt.Println("Saved .car output to", output+".car") 762 | } else { 763 | // Otherwise write file as UnixFS File 764 | ufsFile, err := unixfile.NewUnixfsFile(cctx.Context, dservOffline, dnode) 765 | if err != nil { 766 | return err 767 | } 768 | 769 | if err := files.WriteTo(ufsFile, output); err != nil { 770 | return err 771 | } 772 | 773 | fmt.Println("Saved output to", output) 774 | } 775 | 776 | return nil 777 | }, 778 | } 779 | 780 | var queryRetrievalCmd = &cli.Command{ 781 | Name: "query-retrieval", 782 | Usage: "Query retrieval information for a CID", 783 | ArgsUsage: "", 784 | Flags: []cli.Flag{ 785 | flagMiner, 786 | }, 787 | Action: func(cctx *cli.Context) error { 788 | 789 | cidStr := cctx.Args().First() 790 | if cidStr == "" { 791 | return fmt.Errorf("please specify a CID to query retrieval of") 792 | } 793 | 794 | miner, err := parseMiner(cctx) 795 | if err != nil { 796 | return err 797 | } 798 | 799 | cid, err := cid.Decode(cidStr) 800 | if err != nil { 801 | return err 802 | } 803 | 804 | ddir := ddir(cctx) 805 | 806 | nd, err := setup(cctx.Context, ddir) 807 | if err != nil { 808 | return err 809 | } 810 | 811 | dht, err := dht.New(cctx.Context, nd.Host, dht.Mode(dht.ModeClient)) 812 | if err != nil { 813 | return err 814 | } 815 | 816 | providers, err := dht.FindProviders(cctx.Context, cid) 817 | if err != nil { 818 | return err 819 | } 820 | 821 | availableOnIPFS := len(providers) != 0 822 | 823 | if miner != address.Undef { 824 | fc, closer, err := clientFromNode(cctx, nd, ddir) 825 | if err != nil { 826 | return err 827 | } 828 | defer closer() 829 | 830 | query, err := fc.RetrievalQuery(cctx.Context, miner, cid) 831 | if err != nil { 832 | return err 833 | } 834 | 835 | printQueryResponse(query, availableOnIPFS) 836 | } else { 837 | fmt.Println("No miner specified") 838 | if availableOnIPFS { 839 | fmt.Println("Available on IPFS") 840 | } 841 | } 842 | 843 | return nil 844 | }, 845 | } 846 | 847 | var clearBlockstoreCmd = &cli.Command{ 848 | Name: "clear-blockstore", 849 | Usage: "Delete all retrieved file data in the blockstore", 850 | ArgsUsage: " ", 851 | Action: func(cctx *cli.Context) error { 852 | ddir := ddir(cctx) 853 | 854 | fmt.Println("clearing blockstore...") 855 | 856 | if err := os.RemoveAll(blockstorePath(ddir)); err != nil { 857 | return err 858 | } 859 | 860 | fmt.Println("done") 861 | 862 | return nil 863 | }, 864 | } 865 | -------------------------------------------------------------------------------- /filc/disk.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | crand "crypto/rand" 6 | "encoding/json" 7 | "fmt" 8 | "io/ioutil" 9 | "os" 10 | "path/filepath" 11 | 12 | "github.com/application-research/filclient" 13 | "github.com/application-research/filclient/keystore" 14 | "github.com/filecoin-project/go-state-types/builtin/v9/market" 15 | "github.com/filecoin-project/lotus/chain/types" 16 | "github.com/filecoin-project/lotus/chain/wallet" 17 | lcli "github.com/filecoin-project/lotus/cli" 18 | "github.com/ipfs/go-bitswap" 19 | bsnet "github.com/ipfs/go-bitswap/network" 20 | "github.com/ipfs/go-cid" 21 | "github.com/ipfs/go-datastore" 22 | flatfs "github.com/ipfs/go-ds-flatfs" 23 | levelds "github.com/ipfs/go-ds-leveldb" 24 | blockstore "github.com/ipfs/go-ipfs-blockstore" 25 | "github.com/libp2p/go-libp2p" 26 | dht "github.com/libp2p/go-libp2p-kad-dht" 27 | "github.com/libp2p/go-libp2p/core/crypto" 28 | "github.com/libp2p/go-libp2p/core/host" 29 | "github.com/libp2p/go-libp2p/core/metrics" 30 | "github.com/urfave/cli/v2" 31 | ) 32 | 33 | type dealData struct { 34 | Proposal *market.ClientDealProposal 35 | } 36 | 37 | func dealsPath(baseDir string) string { 38 | return filepath.Join(baseDir, "deals") 39 | } 40 | 41 | func keyPath(baseDir string) string { 42 | return filepath.Join(baseDir, "libp2p.key") 43 | } 44 | 45 | func blockstorePath(baseDir string) string { 46 | return filepath.Join(baseDir, "blockstore") 47 | } 48 | 49 | func datastorePath(baseDir string) string { 50 | return filepath.Join(baseDir, "datastore") 51 | } 52 | 53 | func walletPath(baseDir string) string { 54 | return filepath.Join(baseDir, "wallet") 55 | } 56 | 57 | func saveDealProposal(dataDir string, propcid cid.Cid, proposal *market.ClientDealProposal) error { 58 | dealsPath := dealsPath(dataDir) 59 | 60 | if err := os.MkdirAll(dealsPath, 0755); err != nil { 61 | return err 62 | } 63 | 64 | data := &dealData{ 65 | Proposal: proposal, 66 | } 67 | 68 | fi, err := os.Create(filepath.Join(dealsPath, propcid.String())) 69 | if err != nil { 70 | return err 71 | } 72 | defer fi.Close() 73 | 74 | if err := json.NewEncoder(fi).Encode(data); err != nil { 75 | return err 76 | } 77 | 78 | return nil 79 | } 80 | 81 | func listDeals(dataDir string) ([]cid.Cid, error) { 82 | elems, err := ioutil.ReadDir(dealsPath(dataDir)) 83 | if err != nil { 84 | return nil, err 85 | } 86 | 87 | var out []cid.Cid 88 | for _, e := range elems { 89 | fmt.Println(e.Name()) 90 | c, err := cid.Decode(e.Name()) 91 | if err == nil { 92 | out = append(out, c) 93 | } 94 | } 95 | return out, nil 96 | } 97 | 98 | func clientFromNode(cctx *cli.Context, nd *Node, dir string) (*filclient.FilClient, func(), error) { 99 | api, closer, err := lcli.GetGatewayAPI(cctx) 100 | if err != nil { 101 | return nil, nil, err 102 | } 103 | 104 | addr, err := nd.Wallet.GetDefault() 105 | if err != nil { 106 | return nil, nil, err 107 | } 108 | 109 | fc, err := filclient.NewClient(nd.Host, api, nd.Wallet, addr, nd.Blockstore, nd.Datastore, dir) 110 | if err != nil { 111 | return nil, nil, err 112 | } 113 | 114 | return fc, closer, nil 115 | } 116 | 117 | func getClient(cctx *cli.Context, dir string) (*filclient.FilClient, func(), error) { 118 | nd, err := setup(context.Background(), dir) 119 | if err != nil { 120 | return nil, nil, err 121 | } 122 | 123 | return clientFromNode(cctx, nd, dir) 124 | } 125 | 126 | type Node struct { 127 | Host host.Host 128 | 129 | Datastore datastore.Batching 130 | DHT *dht.IpfsDHT 131 | Blockstore blockstore.Blockstore 132 | Bitswap *bitswap.Bitswap 133 | 134 | Wallet *wallet.LocalWallet 135 | } 136 | 137 | func setup(ctx context.Context, cfgdir string) (*Node, error) { 138 | peerkey, err := loadOrInitPeerKey(keyPath(cfgdir)) 139 | if err != nil { 140 | return nil, err 141 | } 142 | 143 | bwc := metrics.NewBandwidthCounter() 144 | 145 | h, err := libp2p.New( 146 | //libp2p.ConnectionManager(connmgr.NewConnManager(500, 800, time.Minute)), 147 | libp2p.ListenAddrStrings("/ip4/0.0.0.0/tcp/6755"), 148 | libp2p.Identity(peerkey), 149 | libp2p.BandwidthReporter(bwc), 150 | ) 151 | if err != nil { 152 | return nil, err 153 | } 154 | 155 | bstoreDatastore, err := flatfs.CreateOrOpen(blockstorePath(cfgdir), flatfs.NextToLast(3), false) 156 | bstore := blockstore.NewBlockstoreNoPrefix(bstoreDatastore) 157 | if err != nil { 158 | return nil, fmt.Errorf("blockstore could not be opened (it may be incompatible after an update - try running %s subcommand to delete the blockstore and try again): %v", clearBlockstoreCmd.Name, err) 159 | } 160 | 161 | ds, err := levelds.NewDatastore(datastorePath(cfgdir), nil) 162 | if err != nil { 163 | return nil, err 164 | } 165 | 166 | dht, err := dht.New( 167 | ctx, 168 | h, 169 | dht.Mode(dht.ModeClient), 170 | dht.QueryFilter(dht.PublicQueryFilter), 171 | dht.RoutingTableFilter(dht.PublicRoutingTableFilter), 172 | dht.BootstrapPeersFunc(dht.GetDefaultBootstrapPeerAddrInfos), 173 | dht.Datastore(ds), 174 | dht.RoutingTablePeerDiversityFilter(dht.NewRTPeerDiversityFilter(h, 2, 3)), 175 | ) 176 | if err != nil { 177 | return nil, err 178 | } 179 | 180 | bsnet := bsnet.NewFromIpfsHost(h, dht) 181 | bswap := bitswap.New(ctx, bsnet, bstore) 182 | 183 | wallet, err := setupWallet(walletPath(cfgdir)) 184 | if err != nil { 185 | return nil, err 186 | } 187 | 188 | return &Node{ 189 | Host: h, 190 | Blockstore: bstore, 191 | DHT: dht, 192 | Datastore: ds, 193 | Bitswap: bswap, 194 | Wallet: wallet, 195 | }, nil 196 | } 197 | 198 | func loadOrInitPeerKey(kf string) (crypto.PrivKey, error) { 199 | data, err := ioutil.ReadFile(kf) 200 | if err != nil { 201 | if !os.IsNotExist(err) { 202 | return nil, err 203 | } 204 | 205 | k, _, err := crypto.GenerateEd25519Key(crand.Reader) 206 | if err != nil { 207 | return nil, err 208 | } 209 | 210 | data, err := crypto.MarshalPrivateKey(k) 211 | if err != nil { 212 | return nil, err 213 | } 214 | 215 | if err := ioutil.WriteFile(kf, data, 0600); err != nil { 216 | return nil, err 217 | } 218 | 219 | return k, nil 220 | } 221 | return crypto.UnmarshalPrivateKey(data) 222 | } 223 | 224 | func setupWallet(dir string) (*wallet.LocalWallet, error) { 225 | kstore, err := keystore.OpenOrInitKeystore(dir) 226 | if err != nil { 227 | return nil, err 228 | } 229 | 230 | wallet, err := wallet.NewWallet(kstore) 231 | if err != nil { 232 | return nil, err 233 | } 234 | 235 | addrs, err := wallet.WalletList(context.TODO()) 236 | if err != nil { 237 | return nil, err 238 | } 239 | 240 | if len(addrs) == 0 { 241 | _, err := wallet.WalletNew(context.TODO(), types.KTBLS) 242 | if err != nil { 243 | return nil, err 244 | } 245 | } 246 | 247 | return wallet, nil 248 | } 249 | -------------------------------------------------------------------------------- /filc/flags.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "github.com/urfave/cli/v2" 4 | 5 | var flagMiner = &cli.StringFlag{ 6 | Name: "miner", 7 | Aliases: []string{"m"}, 8 | } 9 | 10 | var flagMinerRequired = &cli.StringFlag{ 11 | Name: flagMiner.Name, 12 | Aliases: flagMiner.Aliases, 13 | Required: true, 14 | } 15 | 16 | var flagMiners = &cli.StringSliceFlag{ 17 | Name: "miners", 18 | Aliases: []string{"miner", "m"}, 19 | } 20 | 21 | var flagMinersRequired = &cli.StringSliceFlag{ 22 | Name: flagMiners.Name, 23 | Aliases: flagMiners.Aliases, 24 | Required: true, 25 | } 26 | 27 | var flagVerified = &cli.BoolFlag{ 28 | Name: "verified", 29 | } 30 | 31 | var removeUnsealed = &cli.BoolFlag{ 32 | Name: "remove-unsealed", 33 | } 34 | 35 | var flagOutput = &cli.StringFlag{ 36 | Name: "output", 37 | Aliases: []string{"o"}, 38 | } 39 | 40 | var flagNetwork = &cli.StringFlag{ 41 | Name: "network", 42 | Aliases: []string{"n"}, 43 | Usage: "which network to retrieve from [fil|ipfs|auto]", 44 | DefaultText: NetworkAuto, 45 | Value: NetworkAuto, 46 | } 47 | 48 | var flagCar = &cli.BoolFlag{ 49 | Name: "car", 50 | } 51 | 52 | var flagDealUUID = &cli.StringFlag{ 53 | Name: "deal-uuid", 54 | } 55 | 56 | const ( 57 | NetworkFIL = "fil" 58 | NetworkIPFS = "ipfs" 59 | NetworkAuto = "auto" 60 | ) 61 | 62 | var flagDmPathSel = &cli.StringFlag{ 63 | Name: "datamodel-path-selector", 64 | Usage: "a rudimentary (DM-level-only) text-path selector, allowing for sub-selection within a deal", 65 | } 66 | -------------------------------------------------------------------------------- /filc/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | logging "github.com/ipfs/go-log/v2" 8 | "github.com/mitchellh/go-homedir" 9 | cli "github.com/urfave/cli/v2" 10 | "go.uber.org/zap/zapcore" 11 | ) 12 | 13 | func init() { 14 | if os.Getenv("FULLNODE_API_INFO") == "" { 15 | os.Setenv("FULLNODE_API_INFO", "wss://api.chain.love") 16 | } 17 | } 18 | 19 | var log = logging.Logger("filc") 20 | 21 | func main() { 22 | logging.SetPrimaryCore(zapcore.NewCore(zapcore.NewConsoleEncoder(zapcore.EncoderConfig{ 23 | MessageKey: "message", 24 | TimeKey: "time", 25 | LevelKey: "level", 26 | 27 | EncodeLevel: zapcore.CapitalColorLevelEncoder, 28 | EncodeTime: zapcore.TimeEncoderOfLayout("15:04:05"), 29 | 30 | ConsoleSeparator: " ", 31 | }), os.Stdout, zapcore.DebugLevel)) 32 | 33 | logging.SetLogLevel("filc", "info") 34 | 35 | defer log.Sync() 36 | 37 | app := cli.NewApp() 38 | 39 | app.Commands = []*cli.Command{ 40 | printLoggersCmd, 41 | makeDealCmd, 42 | dealStatusCmd, 43 | getAskCmd, 44 | infoCmd, 45 | listDealsCmd, 46 | retrieveFileCmd, 47 | queryRetrievalCmd, 48 | clearBlockstoreCmd, 49 | } 50 | app.Flags = []cli.Flag{ 51 | &cli.StringFlag{ 52 | Name: "repo", 53 | Value: "~/.lotus", 54 | }, 55 | } 56 | 57 | // Store config dir in metadata 58 | ddir, err := homedir.Expand("~/.filc") 59 | if err != nil { 60 | fmt.Println("could not set config dir: ", err) 61 | } 62 | app.Metadata = map[string]interface{}{ 63 | "ddir": ddir, 64 | } 65 | 66 | // ...and make sure the directory exists 67 | if err := os.MkdirAll(ddir, 0755); err != nil { 68 | fmt.Println("could not create config directory: ", err) 69 | os.Exit(1) 70 | } 71 | 72 | if err := app.Run(os.Args); err != nil { 73 | fmt.Println(err) 74 | os.Exit(1) 75 | } 76 | } 77 | 78 | // Get config directory from CLI metadata. 79 | func ddir(cctx *cli.Context) string { 80 | mDdir := cctx.App.Metadata["ddir"] 81 | switch ddir := mDdir.(type) { 82 | case string: 83 | return ddir 84 | default: 85 | panic("ddir should be present in CLI metadata") 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /filc/parsing.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io/fs" 6 | "strings" 7 | 8 | "github.com/filecoin-project/go-address" 9 | "github.com/google/uuid" 10 | "github.com/urfave/cli/v2" 11 | ) 12 | 13 | // Read a single miner from the CLI, returning address.Undef if none is 14 | // provided. 15 | func parseMiner(cctx *cli.Context) (address.Address, error) { 16 | minerStringRaw := cctx.String(flagMiner.Name) 17 | 18 | miner, err := address.NewFromString(minerStringRaw) 19 | if err != nil { 20 | return address.Undef, fmt.Errorf("failed to parse miner: %s: %w", minerStringRaw, err) 21 | } 22 | 23 | return miner, nil 24 | } 25 | 26 | // Read a comma-separated or multi flag list of miners from the CLI. 27 | func parseMiners(cctx *cli.Context) ([]address.Address, error) { 28 | // Each minerStringsRaw element may contain multiple comma-separated values 29 | minerStringsRaw := cctx.StringSlice(flagMiners.Name) 30 | 31 | // Split any comma-separated minerStringsRaw elements 32 | var minerStrings []string 33 | for _, raw := range minerStringsRaw { 34 | minerStrings = append(minerStrings, strings.Split(raw, ",")...) 35 | } 36 | 37 | var miners []address.Address 38 | for _, ms := range minerStrings { 39 | 40 | miner, err := address.NewFromString(ms) 41 | if err != nil { 42 | return nil, fmt.Errorf("failed to parse miner %s: %w", ms, err) 43 | } 44 | 45 | miners = append(miners, miner) 46 | } 47 | 48 | return miners, nil 49 | } 50 | 51 | // Get whether to use a verified deal or not. 52 | func parseVerified(cctx *cli.Context) bool { 53 | return cctx.Bool(flagVerified.Name) 54 | } 55 | 56 | // Get whether to ask SP to remove unsealed copy. 57 | func parseRemoveUnsealed(cctx *cli.Context) bool { 58 | return cctx.Bool(removeUnsealed.Name) 59 | } 60 | 61 | // Get the destination file to write the output to, erroring if not a valid 62 | // path. This early error check is important because you don't want to do a 63 | // bunch of work, only to end up crashing when you try to write the file. 64 | func parseOutput(cctx *cli.Context) (string, error) { 65 | path := cctx.String(flagOutput.Name) 66 | 67 | if path != "" && !fs.ValidPath(path) { 68 | return "", fmt.Errorf("invalid output location '%s'", path) 69 | } 70 | 71 | return path, nil 72 | } 73 | 74 | func parseDealUUID(cctx *cli.Context) (uuid.UUID, error) { 75 | dealUUIDStr := cctx.String(flagDealUUID.Name) 76 | 77 | if dealUUIDStr == "" { 78 | return uuid.Nil, nil 79 | } 80 | 81 | dealUUID, err := uuid.Parse(dealUUIDStr) 82 | if err != nil { 83 | return uuid.Nil, fmt.Errorf("failed to parse deal UUID '%s'", dealUUIDStr) 84 | } 85 | 86 | return dealUUID, nil 87 | } 88 | -------------------------------------------------------------------------------- /filc/printing.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/dustin/go-humanize" 7 | "github.com/filecoin-project/go-fil-markets/retrievalmarket" 8 | "github.com/filecoin-project/go-fil-markets/storagemarket" 9 | "github.com/filecoin-project/go-state-types/big" 10 | "github.com/filecoin-project/lotus/chain/types" 11 | ) 12 | 13 | func printAskResponse(ask *storagemarket.StorageAsk) { 14 | fmt.Printf(`ASK RESPONSE 15 | ----- 16 | Miner: %v 17 | Price (Unverified): %v (%v) 18 | Price (Verified): %v (%v) 19 | Min Piece Size: %v 20 | Max Piece Size: %v 21 | `, 22 | ask.Miner, 23 | ask.Price, types.FIL(ask.Price), 24 | ask.VerifiedPrice, types.FIL(ask.VerifiedPrice), 25 | ask.MinPieceSize, 26 | ask.MaxPieceSize, 27 | ) 28 | } 29 | 30 | func printDealStatus(state *storagemarket.ProviderDealState) { 31 | fmt.Printf(`DEAL STATUS 32 | ----- 33 | Deal State: %s 34 | Proposal CID: %s 35 | Add Funds CID: %s 36 | Publish CID: %s 37 | Deal ID: %d 38 | Fast Retrieval: %t 39 | `, 40 | storagemarket.DealStates[state.State], 41 | state.ProposalCid, 42 | state.AddFundsCid, 43 | state.PublishCid, 44 | state.DealID, 45 | state.FastRetrieval, 46 | ) 47 | 48 | stateProposalLabel, err := state.Proposal.Label.ToString() 49 | if err != nil { 50 | fmt.Printf("Message: %s\n", state.Message) 51 | stateProposalLabel = "" 52 | } 53 | if state.Proposal != nil { 54 | fmt.Printf(`Proposal: 55 | Piece CID: %s 56 | Piece Size: %d (%s) 57 | Verified Deal: %t 58 | Client: %s 59 | Provider: %s 60 | Label: %s 61 | Start Epoch: %d 62 | End Epoch: %d 63 | Storage Price Per Epoch: %d (%s) 64 | Provider Collateral: %d (%s) 65 | Client Collateral: %d (%d) 66 | `, 67 | state.Proposal.PieceCID, 68 | state.Proposal.PieceSize, humanize.IBytes(uint64(state.Proposal.PieceSize)), 69 | state.Proposal.VerifiedDeal, 70 | state.Proposal.Client, 71 | state.Proposal.Provider, 72 | stateProposalLabel, 73 | state.Proposal.StartEpoch, 74 | state.Proposal.EndEpoch, 75 | state.Proposal.StoragePricePerEpoch, types.FIL(state.Proposal.StoragePricePerEpoch), 76 | state.Proposal.ProviderCollateral, types.FIL(state.Proposal.ProviderCollateral), 77 | state.Proposal.ClientCollateral, types.FIL(state.Proposal.ClientCollateral), 78 | ) 79 | } 80 | 81 | if state.Message != "" { 82 | fmt.Printf("Message: %s\n", state.Message) 83 | } 84 | } 85 | 86 | func printRetrievalStats(stats RetrievalStats) { 87 | switch stats := stats.(type) { 88 | case *FILRetrievalStats: 89 | fmt.Printf(`RETRIEVAL STATS (FIL) 90 | ----- 91 | Size: %v (%v) 92 | Duration: %v 93 | Average Speed: %v (%v/s) 94 | Ask Price: %v (%v) 95 | Total Payment: %v (%v) 96 | Num Payments: %v 97 | Peer: %v 98 | `, 99 | stats.Size, humanize.IBytes(stats.Size), 100 | stats.Duration, 101 | stats.AverageSpeed, humanize.IBytes(stats.AverageSpeed), 102 | stats.AskPrice, types.FIL(stats.AskPrice), 103 | stats.TotalPayment, types.FIL(stats.TotalPayment), 104 | stats.NumPayments, 105 | stats.Peer, 106 | ) 107 | case *IPFSRetrievalStats: 108 | fmt.Printf(`RETRIEVAL STATS (IPFS) 109 | ----- 110 | Size: %v (%v) 111 | Duration: %v 112 | Average Speed: %v 113 | `, 114 | stats.ByteSize, humanize.IBytes(stats.ByteSize), 115 | stats.Duration, 116 | stats.GetAverageBytesPerSecond(), 117 | ) 118 | } 119 | } 120 | 121 | func printQueryResponse(query *retrievalmarket.QueryResponse, availableOnIPFS bool) { 122 | var status string 123 | switch query.Status { 124 | case retrievalmarket.QueryResponseAvailable: 125 | status = "Available" 126 | case retrievalmarket.QueryResponseUnavailable: 127 | status = "Unavailable" 128 | case retrievalmarket.QueryResponseError: 129 | status = "Error" 130 | default: 131 | status = fmt.Sprintf("Unrecognized Status (%d)", query.Status) 132 | } 133 | 134 | var pieceCIDFound string 135 | switch query.PieceCIDFound { 136 | case retrievalmarket.QueryItemAvailable: 137 | pieceCIDFound = "Available" 138 | case retrievalmarket.QueryItemUnavailable: 139 | pieceCIDFound = "Unavailable" 140 | case retrievalmarket.QueryItemUnknown: 141 | pieceCIDFound = "Unknown" 142 | default: 143 | pieceCIDFound = fmt.Sprintf("Unrecognized (%d)", query.PieceCIDFound) 144 | } 145 | 146 | total := big.Add(query.UnsealPrice, big.Mul(big.NewIntUnsigned(query.Size), query.MinPricePerByte)) 147 | fmt.Printf(`QUERY RESPONSE 148 | ----- 149 | Status: %v 150 | Piece CID Found: %v 151 | Size: %v (%v) 152 | Unseal Price: %v (%v) 153 | Min Price Per Byte: %v (%v) 154 | Total Retrieval Price: %v (%v) 155 | Payment Address: %v 156 | Max Payment Interval: %v (%v) 157 | Max Payment Interval Increase: %v (%v) 158 | `, 159 | status, 160 | pieceCIDFound, 161 | query.Size, humanize.IBytes(query.Size), 162 | query.UnsealPrice, types.FIL(query.UnsealPrice), 163 | query.MinPricePerByte, types.FIL(query.MinPricePerByte), 164 | total, types.FIL(total), 165 | query.PaymentAddress, 166 | query.MaxPaymentInterval, humanize.IBytes(query.MaxPaymentInterval), 167 | query.MaxPaymentIntervalIncrease, humanize.IBytes(query.MaxPaymentIntervalIncrease), 168 | ) 169 | 170 | if query.Message != "" { 171 | fmt.Printf("Message: %v\n", query.Message) 172 | } 173 | 174 | if availableOnIPFS { 175 | fmt.Printf("-----\nAvaiable on IPFS") 176 | } 177 | } 178 | -------------------------------------------------------------------------------- /filc/retrieval.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "net/url" 9 | "os" 10 | "path" 11 | "sort" 12 | "strings" 13 | "sync" 14 | "time" 15 | 16 | "github.com/application-research/filclient" 17 | "github.com/application-research/filclient/retrievehelper" 18 | "github.com/dustin/go-humanize" 19 | "github.com/filecoin-project/go-address" 20 | "github.com/filecoin-project/go-fil-markets/retrievalmarket" 21 | "github.com/filecoin-project/go-state-types/big" 22 | "github.com/filecoin-project/lotus/chain/types" 23 | "github.com/ipfs/go-blockservice" 24 | "github.com/ipfs/go-cid" 25 | ipldformat "github.com/ipfs/go-ipld-format" 26 | "github.com/ipfs/go-merkledag" 27 | "github.com/ipld/go-ipld-prime" 28 | "golang.org/x/term" 29 | "golang.org/x/xerrors" 30 | ) 31 | 32 | // A retrieval attempt is a configuration for performing a specific retrieval 33 | // over a specific network 34 | type RetrievalAttempt interface { 35 | Retrieve(context.Context, *Node) (RetrievalStats, error) 36 | } 37 | 38 | type IPFSRetrievalAttempt struct { 39 | Cid cid.Cid 40 | } 41 | 42 | func (attempt *IPFSRetrievalAttempt) Retrieve(ctx context.Context, node *Node) (RetrievalStats, error) { 43 | ctx, cancel := context.WithCancel(ctx) 44 | defer cancel() 45 | 46 | log.Info("Searching IPFS for CID...") 47 | 48 | providers := node.DHT.FindProvidersAsync(ctx, attempt.Cid, 0) 49 | 50 | // Ready will be true if we connected to at least one provider, false if no 51 | // miners successfully connected 52 | ready := make(chan bool, 1) 53 | go func() { 54 | for { 55 | select { 56 | case provider, ok := <-providers: 57 | if !ok { 58 | ready <- false 59 | return 60 | } 61 | 62 | // If no addresses are listed for the provider, we should just 63 | // skip it 64 | if len(provider.Addrs) == 0 { 65 | log.Debugf("Skipping IPFS provider with no addresses %s", provider.ID) 66 | continue 67 | } 68 | 69 | log.Infof("Connected to IPFS provider %s", provider.ID) 70 | ready <- true 71 | case <-ctx.Done(): 72 | return 73 | } 74 | } 75 | }() 76 | 77 | select { 78 | // TODO: also add connection timeout 79 | case <-ctx.Done(): 80 | return nil, ctx.Err() 81 | case ready := <-ready: 82 | if !ready { 83 | return nil, fmt.Errorf("couldn't find CID") 84 | } 85 | } 86 | 87 | // If we were able to connect to at least one of the providers, go ahead 88 | // with the retrieval 89 | 90 | var progressLk sync.Mutex 91 | var bytesRetrieved uint64 = 0 92 | startTime := time.Now() 93 | 94 | log.Info("Starting retrieval") 95 | 96 | bserv := blockservice.New(node.Blockstore, node.Bitswap) 97 | dserv := merkledag.NewDAGService(bserv) 98 | //dsess := dserv.Session(ctx) 99 | 100 | cset := cid.NewSet() 101 | if err := merkledag.Walk(ctx, func(ctx context.Context, c cid.Cid) ([]*ipldformat.Link, error) { 102 | node, err := dserv.Get(ctx, c) 103 | if err != nil { 104 | return nil, err 105 | } 106 | 107 | // Only count leaf nodes toward the total size 108 | if len(node.Links()) == 0 { 109 | progressLk.Lock() 110 | nodeSize, err := node.Size() 111 | if err != nil { 112 | nodeSize = 0 113 | } 114 | bytesRetrieved += nodeSize 115 | printProgress(bytesRetrieved) 116 | progressLk.Unlock() 117 | } 118 | 119 | if c.Type() == cid.Raw { 120 | return nil, nil 121 | } 122 | 123 | return node.Links(), nil 124 | }, attempt.Cid, cset.Visit, merkledag.Concurrent()); err != nil { 125 | return nil, err 126 | } 127 | 128 | log.Info("IPFS retrieval succeeded") 129 | 130 | return &IPFSRetrievalStats{ 131 | ByteSize: bytesRetrieved, 132 | Duration: time.Since(startTime), 133 | }, nil 134 | } 135 | 136 | type FILRetrievalAttempt struct { 137 | FilClient *filclient.FilClient 138 | Cid cid.Cid 139 | Candidates []FILRetrievalCandidate 140 | SelNode ipld.Node 141 | 142 | // Disable sorting of candidates based on preferability 143 | NoSort bool 144 | } 145 | 146 | func (attempt *FILRetrievalAttempt) Retrieve(ctx context.Context, node *Node) (RetrievalStats, error) { 147 | // If no miners are provided, there's nothing else we can do 148 | if len(attempt.Candidates) == 0 { 149 | log.Info("No miners were provided, will not attempt FIL retrieval") 150 | return nil, xerrors.Errorf("retrieval failed: no miners were provided") 151 | } 152 | 153 | // If IPFS retrieval was unavailable, do a full FIL retrieval. Start with 154 | // querying all the candidates for sorting. 155 | 156 | log.Info("Querying FIL retrieval candidates...") 157 | 158 | type CandidateQuery struct { 159 | Candidate FILRetrievalCandidate 160 | Response *retrievalmarket.QueryResponse 161 | } 162 | checked := 0 163 | var queries []CandidateQuery 164 | var queriesLk sync.Mutex 165 | 166 | var wg sync.WaitGroup 167 | wg.Add(len(attempt.Candidates)) 168 | 169 | for _, candidate := range attempt.Candidates { 170 | 171 | // Copy into loop, cursed go 172 | candidate := candidate 173 | 174 | go func() { 175 | defer wg.Done() 176 | 177 | query, err := attempt.FilClient.RetrievalQuery(ctx, candidate.Miner, candidate.RootCid) 178 | if err != nil { 179 | log.Debugf("Retrieval query for miner %s failed: %v", candidate.Miner, err) 180 | return 181 | } 182 | 183 | queriesLk.Lock() 184 | queries = append(queries, CandidateQuery{Candidate: candidate, Response: query}) 185 | checked++ 186 | fmt.Fprintf(os.Stderr, "%v/%v\r", checked, len(attempt.Candidates)) 187 | queriesLk.Unlock() 188 | }() 189 | } 190 | 191 | wg.Wait() 192 | 193 | log.Infof("Got back %v retrieval query results of a total of %v candidates", len(queries), len(attempt.Candidates)) 194 | 195 | if len(queries) == 0 { 196 | return nil, xerrors.Errorf("retrieval failed: queries failed for all miners") 197 | } 198 | 199 | // After we got the query results, sort them with respect to the candidate 200 | // selection config as long as noSort isn't requested (TODO - more options) 201 | 202 | if !attempt.NoSort { 203 | sort.Slice(queries, func(i, j int) bool { 204 | a := queries[i].Response 205 | b := queries[i].Response 206 | 207 | // Always prefer unsealed to sealed, no matter what 208 | if a.UnsealPrice.IsZero() && !b.UnsealPrice.IsZero() { 209 | return true 210 | } 211 | 212 | // Select lower price, or continue if equal 213 | aTotalPrice := totalCost(a) 214 | bTotalPrice := totalCost(b) 215 | if !aTotalPrice.Equals(bTotalPrice) { 216 | return aTotalPrice.LessThan(bTotalPrice) 217 | } 218 | 219 | // Select smaller size, or continue if equal 220 | if a.Size != b.Size { 221 | return a.Size < b.Size 222 | } 223 | 224 | return false 225 | }) 226 | } 227 | 228 | // Now attempt retrievals in serial from first to last, until one works. 229 | // stats will get set if a retrieval succeeds - if no retrievals work, it 230 | // will still be nil after the loop finishes 231 | var stats *FILRetrievalStats = nil 232 | for _, query := range queries { 233 | log.Infof("Attempting FIL retrieval with miner %s from root CID %s (%s)", query.Candidate.Miner, query.Candidate.RootCid, types.FIL(totalCost(query.Response))) 234 | 235 | if attempt.SelNode != nil && !attempt.SelNode.IsNull() { 236 | log.Infof("Using selector %s", attempt.SelNode) 237 | } 238 | 239 | proposal, err := retrievehelper.RetrievalProposalForAsk(query.Response, query.Candidate.RootCid, attempt.SelNode) 240 | if err != nil { 241 | log.Debugf("Failed to create retrieval proposal with candidate miner %s: %v", query.Candidate.Miner, err) 242 | continue 243 | } 244 | 245 | var bytesReceived uint64 246 | stats_, err := attempt.FilClient.RetrieveContentWithProgressCallback( 247 | ctx, 248 | query.Candidate.Miner, 249 | proposal, 250 | func(bytesReceived_ uint64) { 251 | bytesReceived = bytesReceived_ 252 | printProgress(bytesReceived) 253 | }, 254 | ) 255 | if err != nil { 256 | log.Errorf("Failed to retrieve content with candidate miner %s: %v", query.Candidate.Miner, err) 257 | continue 258 | } 259 | 260 | stats = &FILRetrievalStats{RetrievalStats: *stats_} 261 | break 262 | } 263 | 264 | if stats == nil { 265 | return nil, xerrors.New("retrieval failed for all miners") 266 | } 267 | 268 | log.Info("FIL retrieval succeeded") 269 | 270 | return stats, nil 271 | } 272 | 273 | type FILRetrievalCandidate struct { 274 | Miner address.Address 275 | RootCid cid.Cid 276 | DealID uint 277 | } 278 | 279 | func (node *Node) GetRetrievalCandidates(endpoint string, c cid.Cid) ([]FILRetrievalCandidate, error) { 280 | 281 | endpointURL, err := url.Parse(endpoint) 282 | if err != nil { 283 | return nil, xerrors.Errorf("endpoint %s is not a valid url", endpoint) 284 | } 285 | endpointURL.Path = path.Join(endpointURL.Path, c.String()) 286 | 287 | resp, err := http.Get(endpointURL.String()) 288 | if err != nil { 289 | return nil, err 290 | } 291 | defer resp.Body.Close() 292 | if resp.StatusCode != http.StatusOK { 293 | return nil, fmt.Errorf("http request to endpoint %s got status %v", endpointURL, resp.StatusCode) 294 | } 295 | 296 | var res []FILRetrievalCandidate 297 | 298 | if err := json.NewDecoder(resp.Body).Decode(&res); err != nil { 299 | return nil, xerrors.Errorf("could not unmarshal http response for cid %s", c) 300 | } 301 | 302 | return res, nil 303 | } 304 | 305 | type RetrievalStats interface { 306 | GetByteSize() uint64 307 | GetDuration() time.Duration 308 | GetAverageBytesPerSecond() uint64 309 | } 310 | 311 | type FILRetrievalStats struct { 312 | filclient.RetrievalStats 313 | } 314 | 315 | func (stats *FILRetrievalStats) GetByteSize() uint64 { 316 | return stats.Size 317 | } 318 | 319 | func (stats *FILRetrievalStats) GetDuration() time.Duration { 320 | return stats.Duration 321 | } 322 | 323 | func (stats *FILRetrievalStats) GetAverageBytesPerSecond() uint64 { 324 | return stats.AverageSpeed 325 | } 326 | 327 | type IPFSRetrievalStats struct { 328 | ByteSize uint64 329 | Duration time.Duration 330 | } 331 | 332 | func (stats *IPFSRetrievalStats) GetByteSize() uint64 { 333 | return stats.ByteSize 334 | } 335 | 336 | func (stats *IPFSRetrievalStats) GetDuration() time.Duration { 337 | return stats.Duration 338 | } 339 | 340 | func (stats *IPFSRetrievalStats) GetAverageBytesPerSecond() uint64 { 341 | return uint64(float64(stats.ByteSize) / stats.Duration.Seconds()) 342 | } 343 | 344 | // Takes a list of network configs to attempt to retrieve from, in order. Valid 345 | // structs for the interface: IPFSRetrievalConfig, FILRetrievalConfig 346 | func (node *Node) RetrieveFromBestCandidate( 347 | ctx context.Context, 348 | attempts []RetrievalAttempt, 349 | ) (RetrievalStats, error) { 350 | for _, attempt := range attempts { 351 | stats, err := attempt.Retrieve(ctx, node) 352 | if err == nil { 353 | return stats, nil 354 | } 355 | } 356 | 357 | return nil, fmt.Errorf("all retrieval attempts failed") 358 | } 359 | 360 | func totalCost(qres *retrievalmarket.QueryResponse) big.Int { 361 | return big.Add(big.Mul(qres.MinPricePerByte, big.NewIntUnsigned(qres.Size)), qres.UnsealPrice) 362 | } 363 | 364 | func printProgress(bytesReceived uint64) { 365 | str := fmt.Sprintf("%v (%v)", bytesReceived, humanize.IBytes(bytesReceived)) 366 | 367 | termWidth, _, err := term.GetSize(int(os.Stdin.Fd())) 368 | strLen := len(str) 369 | if err == nil { 370 | 371 | if strLen < termWidth { 372 | // If the string is shorter than the terminal width, pad right side 373 | // with spaces to remove old text 374 | str = strings.Join([]string{str, strings.Repeat(" ", termWidth-strLen)}, "") 375 | } else if strLen > termWidth { 376 | // If the string doesn't fit in the terminal, cut it down to a size 377 | // that fits 378 | str = str[:termWidth] 379 | } 380 | } 381 | 382 | fmt.Fprintf(os.Stderr, "%s\r", str) 383 | } 384 | -------------------------------------------------------------------------------- /filclient_test.go: -------------------------------------------------------------------------------- 1 | package filclient 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "math/rand" 7 | "os" 8 | "path/filepath" 9 | "sync" 10 | "testing" 11 | "time" 12 | 13 | cborutil "github.com/filecoin-project/go-cbor-util" 14 | datatransfer "github.com/filecoin-project/go-data-transfer" 15 | "github.com/filecoin-project/go-jsonrpc" 16 | "github.com/filecoin-project/lotus/api" 17 | lotusactors "github.com/filecoin-project/lotus/chain/actors" 18 | lotustypes "github.com/filecoin-project/lotus/chain/types" 19 | "github.com/filecoin-project/lotus/chain/wallet" 20 | lcli "github.com/filecoin-project/lotus/cli" 21 | "github.com/filecoin-project/lotus/itests/kit" 22 | lotusrepo "github.com/filecoin-project/lotus/node/repo" 23 | filbuiltin "github.com/filecoin-project/specs-actors/v6/actors/builtin" 24 | filminer "github.com/filecoin-project/specs-actors/v6/actors/builtin/miner" 25 | "github.com/ipfs/go-blockservice" 26 | "github.com/ipfs/go-datastore" 27 | flatfs "github.com/ipfs/go-ds-flatfs" 28 | leveldb "github.com/ipfs/go-ds-leveldb" 29 | blockstore "github.com/ipfs/go-ipfs-blockstore" 30 | chunk "github.com/ipfs/go-ipfs-chunker" 31 | "github.com/ipfs/go-merkledag" 32 | "github.com/ipfs/go-unixfs/importer" 33 | "github.com/libp2p/go-libp2p/core/peer" 34 | "github.com/multiformats/go-multiaddr" 35 | "github.com/stretchr/testify/require" 36 | "github.com/urfave/cli/v2" 37 | "golang.org/x/xerrors" 38 | ) 39 | 40 | type DummyDataGen struct { 41 | TargetByteLen uint64 42 | progress uint64 43 | } 44 | 45 | func NewDummyDataGen(targetByteLen uint64) *DummyDataGen { 46 | return &DummyDataGen{ 47 | TargetByteLen: targetByteLen, 48 | progress: 0, 49 | } 50 | } 51 | 52 | func (reader *DummyDataGen) Read(p []byte) (int, error) { 53 | i := 0 54 | for ; i < len(p); i += 1 { 55 | // End if we read enough bytes 56 | if reader.progress >= reader.TargetByteLen { 57 | return i, io.EOF 58 | } 59 | 60 | // Otherwise write the next byte 61 | p[i] = byte(rand.Uint32() % 0xff) 62 | 63 | reader.progress += 1 64 | } 65 | 66 | return i, nil 67 | } 68 | 69 | // func TestRetrieval(t *testing.T) { 70 | // app := cli.NewApp() 71 | // app.Action = func(cctx *cli.Context) error { 72 | // client, miner, ensemble, fc, closer := initEnsemble(t, cctx) 73 | // defer closer() 74 | 75 | // // Create dummy deal on miner 76 | // res, file := client.CreateImportFile(cctx.Context, 1, 256<<20) 77 | // pieceInfo, err := client.ClientDealPieceCID(cctx.Context, res.Root) 78 | // require.NoError(t, err) 79 | // dh := kit.NewDealHarness(t, client, miner, miner) 80 | // dp := dh.DefaultStartDealParams() 81 | // dp.DealStartEpoch = abi.ChainEpoch(4 << 10) 82 | // dp.Data = &storagemarket.DataRef{ 83 | // TransferType: storagemarket.TTManual, 84 | // Root: res.Root, 85 | // PieceCid: &pieceInfo.PieceCID, 86 | // PieceSize: pieceInfo.PieceSize.Unpadded(), 87 | // } 88 | // proposalCid := dh.StartDeal(cctx.Context, dp) 89 | // require.Eventually(t, func() bool { 90 | // cd, _ := client.ClientGetDealInfo(cctx.Context, *proposalCid) 91 | // return cd.State == storagemarket.StorageDealCheckForAcceptance 92 | // }, 30*time.Second, 1*time.Second) 93 | 94 | // carFileDir := t.TempDir() 95 | // carFilePath := filepath.Join(carFileDir, "out.car") 96 | // require.NoError(t, client.ClientGenCar(cctx.Context, api.FileRef{Path: file}, carFilePath)) 97 | // require.NoError(t, miner.DealsImportData(cctx.Context, *proposalCid, carFilePath)) 98 | 99 | // query, err := fc.RetrievalQuery(cctx.Context, miner.ActorAddr, res.Root) 100 | // err := fc.RetrieveContent(cctx.Context, miner.ActorAddr, &retrievalmarket.DealProposal{ 101 | // PayloadCID: res.Root, 102 | // ID: query., 103 | // }) 104 | 105 | // return nil 106 | // } 107 | // if err := app.Run([]string{""}); err != nil { 108 | // t.Fatalf("App failed: %v", err) 109 | // } 110 | // } 111 | 112 | func TestStorage(t *testing.T) { 113 | app := cli.NewApp() 114 | app.Action = func(cctx *cli.Context) error { 115 | _, miner, _, fc, closer := initEnsemble(t, cctx) 116 | defer closer() 117 | 118 | ctx := cctx.Context 119 | 120 | bserv := blockservice.New(fc.blockstore, nil) 121 | dserv := merkledag.NewDAGService(bserv) 122 | 123 | spl := chunk.DefaultSplitter(NewDummyDataGen(128 << 20)) 124 | 125 | obj, err := importer.BuildDagFromReader(dserv, spl) 126 | if err != nil { 127 | t.Fatalf("Could not build test data DAG: %v", err) 128 | } 129 | 130 | version, err := fc.GetMinerVersion(cctx.Context, miner.ActorAddr) 131 | require.NoError(t, err) 132 | fmt.Printf("Miner Version: %s\n", version) 133 | 134 | addr, err := miner.ActorAddress(ctx) 135 | require.NoError(t, err) 136 | 137 | fmt.Printf("Testing storage deal for miner %s\n", addr) 138 | 139 | ask, err := fc.GetAsk(ctx, addr) 140 | require.NoError(t, err) 141 | 142 | _, err = fc.LockMarketFunds( 143 | ctx, 144 | lotustypes.FIL(lotustypes.NewInt(1000000000000000)), // FIXME - no idea what's reasonable 145 | ) 146 | require.NoError(t, err) 147 | 148 | proposal, err := fc.MakeDeal(ctx, addr, obj.Cid(), ask.Ask.Ask.Price, 0, 2880*365, false, false) 149 | require.NoError(t, err) 150 | 151 | fmt.Printf("Sending proposal\n") 152 | 153 | propnd, err := cborutil.AsIpld(proposal.DealProposal) 154 | if err != nil { 155 | return xerrors.Errorf("failed to compute deal proposal ipld node: %w", err) 156 | } 157 | 158 | propCid := propnd.Cid() 159 | 160 | _, err = fc.SendProposalV110(ctx, *proposal, propCid) 161 | require.NoError(t, err) 162 | 163 | var chanid *datatransfer.ChannelID 164 | var chanidLk sync.Mutex 165 | res := make(chan error, 1) 166 | 167 | finish := func(err error) { 168 | select { 169 | case res <- err: 170 | default: 171 | } 172 | } 173 | 174 | unsubscribe := fc.SubscribeToDataTransferEvents(func(event datatransfer.Event, state datatransfer.ChannelState) { 175 | chanidLk.Lock() 176 | chanidCopy := *chanid 177 | chanidLk.Unlock() 178 | 179 | // Skip messages not related to this channel 180 | if state.ChannelID() != chanidCopy { 181 | return 182 | } 183 | 184 | switch event.Code { 185 | case datatransfer.CleanupComplete: // FIXME previously this was waiting for a code that would never come - not sure if CleanupComplete is right here.... 186 | finish(nil) 187 | case datatransfer.Error: 188 | finish(fmt.Errorf("data transfer failed")) 189 | default: 190 | fmt.Printf("Other event code \"%s\" (%v)", datatransfer.Events[event.Code], event.Code) 191 | } 192 | }) 193 | defer unsubscribe() 194 | 195 | fmt.Printf("Starting data transfer\n") 196 | 197 | chanidLk.Lock() 198 | chanid, err = fc.StartDataTransfer(ctx, miner.ActorAddr, propCid, obj.Cid()) 199 | chanidLk.Unlock() 200 | require.NoError(t, err) 201 | 202 | select { 203 | case err := <-res: 204 | if err != nil { 205 | t.Fatalf("Data transfer error: %v", err) 206 | } 207 | case <-cctx.Done(): 208 | } 209 | 210 | if err := fc.dataTransfer.CloseDataTransferChannel(ctx, *chanid); err != nil { 211 | t.Fatalf("Failed to close data transfer channel") 212 | } 213 | 214 | fmt.Printf("Data transfer finished\n") 215 | 216 | // TODO: bad position for testing retrieval query to peer 217 | query, err := fc.RetrievalQueryToPeer(ctx, peer.AddrInfo{ID: miner.Libp2p.PeerID, Addrs: []multiaddr.Multiaddr{miner.ListenAddr}}, obj.Cid()) 218 | require.NoError(t, err) 219 | 220 | fmt.Printf("query: %#v\n", query) 221 | 222 | return nil 223 | } 224 | if err := app.Run([]string{""}); err != nil { 225 | t.Fatalf("App failed: %v", err) 226 | } 227 | } 228 | 229 | // -- Setup functions 230 | 231 | // Create and set up an ensemble with linked filclient 232 | func initEnsemble(t *testing.T, cctx *cli.Context) (*kit.TestFullNode, *kit.TestMiner, *kit.Ensemble, *FilClient, func()) { 233 | 234 | fmt.Printf("Initializing test network...\n") 235 | 236 | kit.QuietMiningLogs() 237 | client, miner, ensemble := kit.EnsembleMinimal(t, 238 | kit.ThroughRPC(), // so filclient can talk to it 239 | kit.MockProofs(), // we don't care about proper sealing/proofs 240 | kit.SectorSize(512<<20), // 512MiB sectors 241 | kit.GenesisNetworkVersion(15), 242 | kit.DisableLibp2p(), 243 | ) 244 | ensemble.InterconnectAll().BeginMining(50 * time.Millisecond) 245 | 246 | // set the *optional* on-chain multiaddr 247 | // the mind boggles: there is no API call for that - got to assemble your own msg 248 | { 249 | minfo, err := miner.FullNode.StateMinerInfo(cctx.Context, miner.ActorAddr, lotustypes.EmptyTSK) 250 | require.NoError(t, err) 251 | 252 | maddrNop2p, _ := multiaddr.SplitFunc(miner.ListenAddr, func(c multiaddr.Component) bool { 253 | return c.Protocol().Code == multiaddr.P_P2P 254 | }) 255 | 256 | params, aerr := lotusactors.SerializeParams(&filminer.ChangeMultiaddrsParams{NewMultiaddrs: [][]byte{maddrNop2p.Bytes()}}) 257 | require.NoError(t, aerr) 258 | 259 | _, err = miner.FullNode.MpoolPushMessage(cctx.Context, &lotustypes.Message{ 260 | To: miner.ActorAddr, 261 | From: minfo.Worker, 262 | Value: lotustypes.NewInt(0), 263 | Method: filbuiltin.MethodsMiner.ChangeMultiaddrs, 264 | Params: params, 265 | }, nil) 266 | require.NoError(t, err) 267 | } 268 | 269 | fmt.Printf("Test client fullnode running on %s\n", client.ListenAddr) 270 | os.Setenv("FULLNODE_API_INFO", client.ListenAddr.String()) 271 | 272 | client.WaitTillChain(cctx.Context, kit.BlockMinedBy(miner.ActorAddr)) 273 | 274 | // FilClient initialization 275 | fmt.Printf("Initializing filclient...\n") 276 | 277 | // give filc the pre-funded wallet from the client 278 | ki, err := client.WalletExport(cctx.Context, client.DefaultKey.Address) 279 | require.NoError(t, err) 280 | lr, err := lotusrepo.NewMemory(nil).Lock(lotusrepo.Wallet) 281 | require.NoError(t, err) 282 | ks, err := lr.KeyStore() 283 | require.NoError(t, err) 284 | wallet, err := wallet.NewWallet(ks) 285 | require.NoError(t, err) 286 | _, err = wallet.WalletImport(cctx.Context, ki) 287 | require.NoError(t, err) 288 | 289 | h, err := ensemble.Mocknet().GenPeer() 290 | if err != nil { 291 | t.Fatalf("Could not gen p2p peer: %v", err) 292 | } 293 | ensemble.Mocknet().LinkAll() 294 | api, closer := initAPI(t, cctx) 295 | bs := initBlockstore(t) 296 | ds := initDatastore(t) 297 | fc, err := NewClient(h, api, wallet, client.DefaultKey.Address, bs, ds, t.TempDir()) 298 | if err != nil { 299 | t.Fatalf("Could not initialize FilClient: %v", err) 300 | } 301 | 302 | time.Sleep(time.Millisecond * 500) 303 | 304 | return client, miner, ensemble, fc, closer 305 | } 306 | 307 | func initAPI(t *testing.T, cctx *cli.Context) (api.Gateway, jsonrpc.ClientCloser) { 308 | api, closer, err := lcli.GetGatewayAPI(cctx) 309 | if err != nil { 310 | t.Fatalf("Could not initialize Lotus API gateway: %v", err) 311 | } 312 | 313 | return api, closer 314 | } 315 | 316 | func initBlockstore(t *testing.T) blockstore.Blockstore { 317 | parseShardFunc, err := flatfs.ParseShardFunc("/repo/flatfs/shard/v1/next-to-last/3") 318 | if err != nil { 319 | t.Fatalf("Blockstore parse shard func failed: %v", err) 320 | } 321 | 322 | ds, err := flatfs.CreateOrOpen(filepath.Join(t.TempDir(), "blockstore"), parseShardFunc, false) 323 | if err != nil { 324 | t.Fatalf("Could not initialize blockstore: %v", err) 325 | } 326 | 327 | bs := blockstore.NewBlockstoreNoPrefix(ds) 328 | 329 | return bs 330 | } 331 | 332 | func initDatastore(t *testing.T) datastore.Batching { 333 | ds, err := leveldb.NewDatastore(filepath.Join(t.TempDir(), "datastore"), nil) 334 | if err != nil { 335 | t.Fatalf("Could not initialize datastore: %v", err) 336 | } 337 | 338 | return ds 339 | } 340 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/application-research/filclient 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/dustin/go-humanize v1.0.1 7 | github.com/filecoin-project/boost v1.5.1-rc5 8 | github.com/filecoin-project/go-address v1.1.0 9 | github.com/filecoin-project/go-cbor-util v0.0.1 10 | github.com/filecoin-project/go-commp-utils v0.1.4 11 | github.com/filecoin-project/go-data-transfer v1.15.3 12 | github.com/filecoin-project/go-fil-commcid v0.1.0 13 | github.com/filecoin-project/go-fil-commp-hashhash v0.2.0 14 | github.com/filecoin-project/go-fil-markets v1.26.0 15 | github.com/filecoin-project/go-jsonrpc v0.2.1 16 | github.com/filecoin-project/go-state-types v0.9.9 17 | github.com/filecoin-project/lotus v1.18.3-0.20230110150616-2995a530dcc7 18 | github.com/filecoin-project/specs-actors/v6 v6.0.2 19 | github.com/google/uuid v1.3.0 20 | github.com/ipfs/go-bitswap v0.12.0 21 | github.com/ipfs/go-blockservice v0.5.0 22 | github.com/ipfs/go-cid v0.3.2 23 | github.com/ipfs/go-datastore v0.6.0 24 | github.com/ipfs/go-ds-flatfs v0.5.1 25 | github.com/ipfs/go-ds-leveldb v0.5.0 26 | github.com/ipfs/go-graphsync v0.13.2 27 | github.com/ipfs/go-ipfs-blockstore v1.2.0 28 | github.com/ipfs/go-ipfs-chunker v0.0.5 29 | github.com/ipfs/go-ipfs-exchange-offline v0.3.0 30 | github.com/ipfs/go-ipfs-files v0.3.0 31 | github.com/ipfs/go-ipld-format v0.4.0 32 | github.com/ipfs/go-log/v2 v2.5.1 33 | github.com/ipfs/go-merkledag v0.10.0 34 | github.com/ipfs/go-unixfs v0.4.2 35 | github.com/ipld/go-car v0.6.0 36 | github.com/ipld/go-codec-dagpb v1.6.0 37 | github.com/ipld/go-ipld-prime v0.20.0 38 | github.com/ipld/go-ipld-selector-text-lite v0.0.1 39 | github.com/libp2p/go-libp2p v0.23.4 40 | github.com/libp2p/go-libp2p-kad-dht v0.20.0 41 | github.com/mitchellh/go-homedir v1.1.0 42 | github.com/multiformats/go-multiaddr v0.8.0 43 | github.com/stretchr/testify v1.8.1 44 | github.com/urfave/cli/v2 v2.24.4 45 | github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc 46 | go.opentelemetry.io/otel v1.13.0 47 | go.opentelemetry.io/otel/trace v1.13.0 48 | go.uber.org/zap v1.24.0 49 | golang.org/x/term v0.5.0 50 | golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 51 | ) 52 | 53 | require ( 54 | contrib.go.opencensus.io/exporter/prometheus v0.4.2 // indirect 55 | github.com/BurntSushi/toml v1.2.1 // indirect 56 | github.com/DataDog/zstd v1.5.2 // indirect 57 | github.com/GeertJohan/go.incremental v1.0.0 // indirect 58 | github.com/GeertJohan/go.rice v1.0.3 // indirect 59 | github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee // indirect 60 | github.com/Kubuxu/imtui v0.0.0-20210401140320-41663d68d0fa // indirect 61 | github.com/StackExchange/wmi v1.2.1 // indirect 62 | github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect 63 | github.com/akavel/rsrc v0.8.0 // indirect 64 | github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect 65 | github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect 66 | github.com/benbjohnson/clock v1.3.0 // indirect 67 | github.com/beorn7/perks v1.0.1 // indirect 68 | github.com/bep/debounce v1.2.1 // indirect 69 | github.com/buger/goterm v1.0.3 // indirect 70 | github.com/cespare/xxhash v1.1.0 // indirect 71 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 72 | github.com/chzyer/readline v1.5.0 // indirect 73 | github.com/cilium/ebpf v0.7.0 // indirect 74 | github.com/containerd/cgroups v1.0.4 // indirect 75 | github.com/coreos/go-systemd/v22 v22.5.0 // indirect 76 | github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect 77 | github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect 78 | github.com/cskr/pubsub v1.0.2 // indirect 79 | github.com/daaku/go.zipexe v1.0.2 // indirect 80 | github.com/davecgh/go-spew v1.1.1 // indirect 81 | github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect 82 | github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect 83 | github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e // indirect 84 | github.com/dgraph-io/badger/v2 v2.2007.3 // indirect 85 | github.com/dgraph-io/ristretto v0.1.0 // indirect 86 | github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect 87 | github.com/docker/go-units v0.5.0 // indirect 88 | github.com/drand/drand v1.3.0 // indirect 89 | github.com/drand/kyber v1.1.7 // indirect 90 | github.com/drand/kyber-bls12381 v0.2.1 // indirect 91 | github.com/elastic/go-sysinfo v1.7.0 // indirect 92 | github.com/elastic/go-windows v1.0.0 // indirect 93 | github.com/elastic/gosigar v0.14.2 // indirect 94 | github.com/emirpasic/gods v1.18.1 // indirect 95 | github.com/fatih/color v1.13.0 // indirect 96 | github.com/filecoin-project/dagstore v0.6.0 // indirect 97 | github.com/filecoin-project/filecoin-ffi v0.30.4-0.20200910194244-f640612a1a1f // indirect 98 | github.com/filecoin-project/go-amt-ipld/v2 v2.1.0 // indirect 99 | github.com/filecoin-project/go-amt-ipld/v3 v3.1.0 // indirect 100 | github.com/filecoin-project/go-amt-ipld/v4 v4.0.0 // indirect 101 | github.com/filecoin-project/go-bitfield v0.2.4 // indirect 102 | github.com/filecoin-project/go-commp-utils/nonffi v0.0.0-20220905160352-62059082a837 // indirect 103 | github.com/filecoin-project/go-crypto v0.0.1 // indirect 104 | github.com/filecoin-project/go-ds-versioning v0.1.2 // indirect 105 | github.com/filecoin-project/go-hamt-ipld v0.1.5 // indirect 106 | github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 // indirect 107 | github.com/filecoin-project/go-hamt-ipld/v3 v3.1.0 // indirect 108 | github.com/filecoin-project/go-legs v0.4.9 // indirect 109 | github.com/filecoin-project/go-padreader v0.0.1 // indirect 110 | github.com/filecoin-project/go-paramfetch v0.0.4 // indirect 111 | github.com/filecoin-project/go-statemachine v1.0.3 // indirect 112 | github.com/filecoin-project/go-statestore v0.2.0 // indirect 113 | github.com/filecoin-project/go-storedcounter v0.1.0 // indirect 114 | github.com/filecoin-project/pubsub v1.0.0 // indirect 115 | github.com/filecoin-project/specs-actors v0.9.15 // indirect 116 | github.com/filecoin-project/specs-actors/v2 v2.3.6 // indirect 117 | github.com/filecoin-project/specs-actors/v3 v3.1.2 // indirect 118 | github.com/filecoin-project/specs-actors/v4 v4.0.2 // indirect 119 | github.com/filecoin-project/specs-actors/v5 v5.0.6 // indirect 120 | github.com/filecoin-project/specs-actors/v7 v7.0.1 // indirect 121 | github.com/filecoin-project/specs-actors/v8 v8.0.1 // indirect 122 | github.com/filecoin-project/specs-storage v0.4.1 // indirect 123 | github.com/flynn/noise v1.0.0 // indirect 124 | github.com/francoispqt/gojay v1.2.13 // indirect 125 | github.com/fsnotify/fsnotify v1.6.0 // indirect 126 | github.com/gbrlsnchs/jwt/v3 v3.0.1 // indirect 127 | github.com/gdamore/encoding v1.0.0 // indirect 128 | github.com/gdamore/tcell/v2 v2.2.0 // indirect 129 | github.com/go-kit/kit v0.12.0 // indirect 130 | github.com/go-kit/log v0.2.1 // indirect 131 | github.com/go-logfmt/logfmt v0.5.1 // indirect 132 | github.com/go-logr/logr v1.2.3 // indirect 133 | github.com/go-logr/stdr v1.2.2 // indirect 134 | github.com/go-ole/go-ole v1.2.5 // indirect 135 | github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect 136 | github.com/godbus/dbus/v5 v5.1.0 // indirect 137 | github.com/gogo/googleapis v1.4.1 // indirect 138 | github.com/gogo/protobuf v1.3.2 // indirect 139 | github.com/golang/glog v1.0.0 // indirect 140 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 141 | github.com/golang/mock v1.6.0 // indirect 142 | github.com/golang/protobuf v1.5.2 // indirect 143 | github.com/golang/snappy v0.0.4 // indirect 144 | github.com/google/gopacket v1.1.19 // indirect 145 | github.com/gorilla/mux v1.8.0 // indirect 146 | github.com/gorilla/websocket v1.5.0 // indirect 147 | github.com/graph-gophers/graphql-go v1.2.0 // indirect 148 | github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026 // indirect 149 | github.com/hannahhoward/cbor-gen-for v0.0.0-20200817222906-ea96cece81f1 // indirect 150 | github.com/hannahhoward/go-pubsub v1.0.0 // indirect 151 | github.com/hashicorp/errwrap v1.1.0 // indirect 152 | github.com/hashicorp/go-multierror v1.1.1 // indirect 153 | github.com/hashicorp/golang-lru v0.5.4 // indirect 154 | github.com/huin/goupnp v1.0.3 // indirect 155 | github.com/icza/backscanner v0.0.0-20210726202459-ac2ffc679f94 // indirect 156 | github.com/ipfs/bbloom v0.0.4 // indirect 157 | github.com/ipfs/go-bitfield v1.0.0 // indirect 158 | github.com/ipfs/go-block-format v0.1.1 // indirect 159 | github.com/ipfs/go-cidutil v0.1.0 // indirect 160 | github.com/ipfs/go-ds-badger2 v0.1.2 // indirect 161 | github.com/ipfs/go-ds-measure v0.2.0 // indirect 162 | github.com/ipfs/go-filestore v1.2.0 // indirect 163 | github.com/ipfs/go-fs-lock v0.0.7 // indirect 164 | github.com/ipfs/go-ipfs-blocksutil v0.0.1 // indirect 165 | github.com/ipfs/go-ipfs-cmds v0.8.2 // indirect 166 | github.com/ipfs/go-ipfs-delay v0.0.1 // indirect 167 | github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect 168 | github.com/ipfs/go-ipfs-exchange-interface v0.2.0 // indirect 169 | github.com/ipfs/go-ipfs-http-client v0.4.0 // indirect 170 | github.com/ipfs/go-ipfs-posinfo v0.0.1 // indirect 171 | github.com/ipfs/go-ipfs-pq v0.0.2 // indirect 172 | github.com/ipfs/go-ipfs-routing v0.3.0 // indirect 173 | github.com/ipfs/go-ipfs-util v0.0.2 // indirect 174 | github.com/ipfs/go-ipld-cbor v0.0.6 // indirect 175 | github.com/ipfs/go-ipld-legacy v0.1.1 // indirect 176 | github.com/ipfs/go-ipns v0.3.0 // indirect 177 | github.com/ipfs/go-libipfs v0.4.0 // indirect 178 | github.com/ipfs/go-log v1.0.5 // indirect 179 | github.com/ipfs/go-metrics-interface v0.0.1 // indirect 180 | github.com/ipfs/go-path v0.3.0 // indirect 181 | github.com/ipfs/go-peertaskqueue v0.8.0 // indirect 182 | github.com/ipfs/go-unixfsnode v1.5.1 // indirect 183 | github.com/ipfs/go-verifcid v0.0.2 // indirect 184 | github.com/ipfs/interface-go-ipfs-core v0.7.0 // indirect 185 | github.com/ipld/go-car/v2 v2.5.1 // indirect 186 | github.com/ipld/go-ipld-adl-hamt v0.0.0-20230103232215-ec18ad32db9b // indirect 187 | github.com/ipni/index-provider v0.10.1 // indirect 188 | github.com/ipni/storetheindex v0.5.3-0.20221203123030-16745cb63f15 // indirect 189 | github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 // indirect 190 | github.com/jackpal/go-nat-pmp v1.0.2 // indirect 191 | github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c // indirect 192 | github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect 193 | github.com/jbenet/goprocess v0.1.4 // indirect 194 | github.com/jessevdk/go-flags v1.4.0 // indirect 195 | github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect 196 | github.com/jpillora/backoff v1.0.0 // indirect 197 | github.com/kelseyhightower/envconfig v1.4.0 // indirect 198 | github.com/kilic/bls12-381 v0.0.0-20200820230200-6b2c19996391 // indirect 199 | github.com/klauspost/compress v1.15.15 // indirect 200 | github.com/klauspost/cpuid/v2 v2.2.4 // indirect 201 | github.com/koron/go-ssdp v0.0.3 // indirect 202 | github.com/libp2p/go-buffer-pool v0.1.0 // indirect 203 | github.com/libp2p/go-cidranger v1.1.0 // indirect 204 | github.com/libp2p/go-flow-metrics v0.1.0 // indirect 205 | github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect 206 | github.com/libp2p/go-libp2p-connmgr v0.4.0 // indirect 207 | github.com/libp2p/go-libp2p-core v0.20.1 // indirect 208 | github.com/libp2p/go-libp2p-gostream v0.5.0 // indirect 209 | github.com/libp2p/go-libp2p-http v0.4.0 // indirect 210 | github.com/libp2p/go-libp2p-kbucket v0.5.0 // indirect 211 | github.com/libp2p/go-libp2p-noise v0.5.0 // indirect 212 | github.com/libp2p/go-libp2p-peerstore v0.8.0 // indirect 213 | github.com/libp2p/go-libp2p-pubsub v0.8.3 // indirect 214 | github.com/libp2p/go-libp2p-record v0.2.0 // indirect 215 | github.com/libp2p/go-libp2p-routing-helpers v0.6.0 // indirect 216 | github.com/libp2p/go-libp2p-tls v0.5.0 // indirect 217 | github.com/libp2p/go-maddr-filter v0.1.0 // indirect 218 | github.com/libp2p/go-msgio v0.3.0 // indirect 219 | github.com/libp2p/go-nat v0.1.0 // indirect 220 | github.com/libp2p/go-netroute v0.2.1 // indirect 221 | github.com/libp2p/go-openssl v0.1.0 // indirect 222 | github.com/libp2p/go-reuseport v0.2.0 // indirect 223 | github.com/libp2p/go-yamux/v4 v4.0.0 // indirect 224 | github.com/lucas-clemente/quic-go v0.29.1 // indirect 225 | github.com/lucasb-eyer/go-colorful v1.0.3 // indirect 226 | github.com/magefile/mage v1.9.0 // indirect 227 | github.com/marten-seemann/qtls-go1-18 v0.1.4 // indirect 228 | github.com/marten-seemann/qtls-go1-19 v0.1.2 // indirect 229 | github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect 230 | github.com/mattn/go-colorable v0.1.13 // indirect 231 | github.com/mattn/go-isatty v0.0.17 // indirect 232 | github.com/mattn/go-pointer v0.0.1 // indirect 233 | github.com/mattn/go-runewidth v0.0.13 // indirect 234 | github.com/mattn/go-sqlite3 v1.14.10 // indirect 235 | github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect 236 | github.com/miekg/dns v1.1.50 // indirect 237 | github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect 238 | github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect 239 | github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect 240 | github.com/minio/sha256-simd v1.0.1-0.20230130105256-d9c3aea9e949 // indirect 241 | github.com/mr-tron/base58 v1.2.0 // indirect 242 | github.com/multiformats/go-base32 v0.1.0 // indirect 243 | github.com/multiformats/go-base36 v0.2.0 // indirect 244 | github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect 245 | github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect 246 | github.com/multiformats/go-multibase v0.1.1 // indirect 247 | github.com/multiformats/go-multicodec v0.8.0 // indirect 248 | github.com/multiformats/go-multihash v0.2.1 // indirect 249 | github.com/multiformats/go-multistream v0.3.3 // indirect 250 | github.com/multiformats/go-varint v0.0.7 // indirect 251 | github.com/nikkolasg/hexjson v0.0.0-20181101101858-78e39397e00c // indirect 252 | github.com/nkovacs/streamquote v1.0.0 // indirect 253 | github.com/nxadm/tail v1.4.8 // indirect 254 | github.com/onsi/ginkgo v1.16.5 // indirect 255 | github.com/onsi/gomega v1.24.1 // indirect 256 | github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect 257 | github.com/opentracing/opentracing-go v1.2.0 // indirect 258 | github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect 259 | github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect 260 | github.com/pkg/errors v0.9.1 // indirect 261 | github.com/pmezard/go-difflib v1.0.0 // indirect 262 | github.com/polydawn/refmt v0.89.0 // indirect 263 | github.com/prometheus/client_golang v1.14.0 // indirect 264 | github.com/prometheus/client_model v0.3.0 // indirect 265 | github.com/prometheus/common v0.39.0 // indirect 266 | github.com/prometheus/procfs v0.9.0 // indirect 267 | github.com/prometheus/statsd_exporter v0.23.0 // indirect 268 | github.com/raulk/clock v1.1.0 // indirect 269 | github.com/raulk/go-watchdog v1.3.0 // indirect 270 | github.com/rivo/uniseg v0.2.0 // indirect 271 | github.com/rs/cors v1.7.0 // indirect 272 | github.com/russross/blackfriday/v2 v2.1.0 // indirect 273 | github.com/shirou/gopsutil v2.18.12+incompatible // indirect 274 | github.com/sirupsen/logrus v1.9.0 // indirect 275 | github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect 276 | github.com/spaolacci/murmur3 v1.1.0 // indirect 277 | github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect 278 | github.com/twmb/murmur3 v1.1.6 // indirect 279 | github.com/valyala/bytebufferpool v1.0.0 // indirect 280 | github.com/valyala/fasttemplate v1.0.1 // indirect 281 | github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba // indirect 282 | github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect 283 | github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect 284 | github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect 285 | github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect 286 | github.com/whyrusleeping/ledger-filecoin-go v0.9.1-0.20201010031517-c3dcc1bddce4 // indirect 287 | github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 // indirect 288 | github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect 289 | github.com/zondax/hid v0.9.1 // indirect 290 | github.com/zondax/ledger-go v0.12.1 // indirect 291 | go.opencensus.io v0.24.0 // indirect 292 | go.uber.org/atomic v1.10.0 // indirect 293 | go.uber.org/dig v1.16.1 // indirect 294 | go.uber.org/fx v1.19.1 // indirect 295 | go.uber.org/multierr v1.9.0 // indirect 296 | go4.org v0.0.0-20200411211856-f5505b9728dd // indirect 297 | golang.org/x/crypto v0.5.0 // indirect 298 | golang.org/x/exp v0.0.0-20230124142953-7f5a42a36c7e // indirect 299 | golang.org/x/mod v0.7.0 // indirect 300 | golang.org/x/net v0.5.0 // indirect 301 | golang.org/x/sync v0.1.0 // indirect 302 | golang.org/x/sys v0.6.0 // indirect 303 | golang.org/x/text v0.6.0 // indirect 304 | golang.org/x/time v0.3.0 // indirect 305 | golang.org/x/tools v0.5.0 // indirect 306 | google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect 307 | google.golang.org/grpc v1.47.0 // indirect 308 | google.golang.org/protobuf v1.28.1 // indirect 309 | gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect 310 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect 311 | gopkg.in/yaml.v2 v2.4.0 // indirect 312 | gopkg.in/yaml.v3 v3.0.1 // indirect 313 | howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect 314 | lukechampine.com/blake3 v1.1.7 // indirect 315 | ) 316 | 317 | replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi 318 | -------------------------------------------------------------------------------- /keystore/keystore.go: -------------------------------------------------------------------------------- 1 | package keystore 2 | 3 | import ( 4 | "encoding/json" 5 | "io/ioutil" 6 | "os" 7 | "path/filepath" 8 | 9 | "github.com/filecoin-project/lotus/chain/types" 10 | "github.com/whyrusleeping/base32" 11 | "golang.org/x/xerrors" 12 | ) 13 | 14 | type DiskKeyStore struct { 15 | path string 16 | } 17 | 18 | func OpenOrInitKeystore(p string) (*DiskKeyStore, error) { 19 | if _, err := os.Stat(p); err == nil { 20 | return &DiskKeyStore{p}, nil 21 | } else if !os.IsNotExist(err) { 22 | return nil, err 23 | } 24 | 25 | if err := os.Mkdir(p, 0700); err != nil { 26 | return nil, err 27 | } 28 | 29 | return &DiskKeyStore{p}, nil 30 | } 31 | 32 | var kstrPermissionMsg = "permissions of key: '%s' are too relaxed, " + 33 | "required: 0600, got: %#o" 34 | 35 | // List lists all the keys stored in the KeyStore 36 | func (fsr *DiskKeyStore) List() ([]string, error) { 37 | 38 | dir, err := os.Open(fsr.path) 39 | if err != nil { 40 | return nil, xerrors.Errorf("opening dir to list keystore: %w", err) 41 | } 42 | defer dir.Close() //nolint:errcheck 43 | files, err := dir.Readdir(-1) 44 | if err != nil { 45 | return nil, xerrors.Errorf("reading keystore dir: %w", err) 46 | } 47 | keys := make([]string, 0, len(files)) 48 | for _, f := range files { 49 | if f.Mode()&0077 != 0 { 50 | return nil, xerrors.Errorf(kstrPermissionMsg, f.Name(), f.Mode()) 51 | } 52 | name, err := base32.RawStdEncoding.DecodeString(f.Name()) 53 | if err != nil { 54 | return nil, xerrors.Errorf("decoding key: '%s': %w", f.Name(), err) 55 | } 56 | keys = append(keys, string(name)) 57 | } 58 | return keys, nil 59 | } 60 | 61 | // Get gets a key out of keystore and returns types.KeyInfo coresponding to named key 62 | func (fsr *DiskKeyStore) Get(name string) (types.KeyInfo, error) { 63 | 64 | encName := base32.RawStdEncoding.EncodeToString([]byte(name)) 65 | keyPath := filepath.Join(fsr.path, encName) 66 | 67 | fstat, err := os.Stat(keyPath) 68 | if os.IsNotExist(err) { 69 | return types.KeyInfo{}, xerrors.Errorf("opening key '%s': %w", name, types.ErrKeyInfoNotFound) 70 | } else if err != nil { 71 | return types.KeyInfo{}, xerrors.Errorf("opening key '%s': %w", name, err) 72 | } 73 | 74 | if fstat.Mode()&0077 != 0 { 75 | return types.KeyInfo{}, xerrors.Errorf(kstrPermissionMsg, name, fstat.Mode()) 76 | } 77 | 78 | file, err := os.Open(keyPath) 79 | if err != nil { 80 | return types.KeyInfo{}, xerrors.Errorf("opening key '%s': %w", name, err) 81 | } 82 | defer file.Close() //nolint: errcheck // read only op 83 | 84 | data, err := ioutil.ReadAll(file) 85 | if err != nil { 86 | return types.KeyInfo{}, xerrors.Errorf("reading key '%s': %w", name, err) 87 | } 88 | 89 | var res types.KeyInfo 90 | err = json.Unmarshal(data, &res) 91 | if err != nil { 92 | return types.KeyInfo{}, xerrors.Errorf("decoding key '%s': %w", name, err) 93 | } 94 | 95 | return res, nil 96 | } 97 | 98 | // Put saves key info under given name 99 | func (fsr *DiskKeyStore) Put(name string, info types.KeyInfo) error { 100 | 101 | encName := base32.RawStdEncoding.EncodeToString([]byte(name)) 102 | keyPath := filepath.Join(fsr.path, encName) 103 | 104 | _, err := os.Stat(keyPath) 105 | if err == nil { 106 | return xerrors.Errorf("checking key before put '%s': %w", name, types.ErrKeyExists) 107 | } else if !os.IsNotExist(err) { 108 | return xerrors.Errorf("checking key before put '%s': %w", name, err) 109 | } 110 | 111 | keyData, err := json.Marshal(info) 112 | if err != nil { 113 | return xerrors.Errorf("encoding key '%s': %w", name, err) 114 | } 115 | 116 | err = ioutil.WriteFile(keyPath, keyData, 0600) 117 | if err != nil { 118 | return xerrors.Errorf("writing key '%s': %w", name, err) 119 | } 120 | return nil 121 | } 122 | 123 | func (fsr *DiskKeyStore) Delete(name string) error { 124 | 125 | encName := base32.RawStdEncoding.EncodeToString([]byte(name)) 126 | keyPath := filepath.Join(fsr.path, encName) 127 | 128 | _, err := os.Stat(keyPath) 129 | if os.IsNotExist(err) { 130 | return xerrors.Errorf("checking key before delete '%s': %w", name, types.ErrKeyInfoNotFound) 131 | } else if err != nil { 132 | return xerrors.Errorf("checking key before delete '%s': %w", name, err) 133 | } 134 | 135 | err = os.Remove(keyPath) 136 | if err != nil { 137 | return xerrors.Errorf("deleting key '%s': %w", name, err) 138 | } 139 | return nil 140 | } 141 | -------------------------------------------------------------------------------- /libp2ptransfermgr.go: -------------------------------------------------------------------------------- 1 | package filclient 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "strconv" 8 | "sync" 9 | "time" 10 | 11 | "github.com/filecoin-project/boost/transport/httptransport" 12 | boosttypes "github.com/filecoin-project/boost/transport/types" 13 | datatransfer "github.com/filecoin-project/go-data-transfer" 14 | "github.com/ipfs/go-cid" 15 | "github.com/ipfs/go-datastore" 16 | "github.com/ipfs/go-datastore/namespace" 17 | "github.com/ipfs/go-datastore/query" 18 | "github.com/libp2p/go-libp2p/core/peer" 19 | "golang.org/x/xerrors" 20 | ) 21 | 22 | // libp2pCarServer is mocked out by the tests 23 | type libp2pCarServer interface { 24 | ID() peer.ID 25 | Start(ctx context.Context) error 26 | Get(id string) (*httptransport.Libp2pTransfer, error) 27 | Subscribe(f httptransport.EventListenerFn) httptransport.UnsubFn 28 | CancelTransfer(ctx context.Context, id string) (*boosttypes.TransferState, error) 29 | Matching(f httptransport.MatchFn) ([]*httptransport.Libp2pTransfer, error) 30 | } 31 | 32 | // libp2pTransferManager watches events for a libp2p data transfer 33 | type libp2pTransferManager struct { 34 | dtServer libp2pCarServer 35 | authDB *httptransport.AuthTokenDB 36 | dtds datastore.Batching 37 | opts libp2pTransferManagerOpts 38 | ctx context.Context 39 | cancelCtx context.CancelFunc 40 | ticker *time.Ticker 41 | 42 | listenerLk sync.Mutex 43 | listener func(uint, ChannelState) 44 | unsub func() 45 | } 46 | 47 | type libp2pTransferManagerOpts struct { 48 | xferTimeout time.Duration 49 | authCheckPeriod time.Duration 50 | } 51 | 52 | func newLibp2pTransferManager(dtServer libp2pCarServer, ds datastore.Batching, authDB *httptransport.AuthTokenDB, opts libp2pTransferManagerOpts) *libp2pTransferManager { 53 | ds = namespace.Wrap(ds, datastore.NewKey("/data-transfers")) 54 | return &libp2pTransferManager{ 55 | dtServer: dtServer, 56 | dtds: ds, 57 | authDB: authDB, 58 | opts: opts, 59 | } 60 | } 61 | 62 | func (m *libp2pTransferManager) Stop() { 63 | m.ticker.Stop() 64 | m.cancelCtx() 65 | m.unsub() 66 | } 67 | 68 | func (m *libp2pTransferManager) Start(ctx context.Context) error { 69 | m.ctx, m.cancelCtx = context.WithCancel(ctx) 70 | 71 | // subscribe to notifications from the data transfer server 72 | m.unsub = m.subscribe() 73 | 74 | // Every tick, check transfers to see if any have expired 75 | m.ticker = time.NewTicker(m.opts.authCheckPeriod) 76 | go func() { 77 | for range m.ticker.C { 78 | m.checkTransferExpiry(ctx) 79 | } 80 | }() 81 | 82 | return m.dtServer.Start(ctx) 83 | } 84 | 85 | func (m *libp2pTransferManager) checkTransferExpiry(ctx context.Context) { 86 | // Delete expired auth tokens from the auth DB 87 | expired, err := m.authDB.DeleteExpired(ctx, time.Now().Add(-m.opts.xferTimeout)) 88 | if err != nil { 89 | log.Errorw("deleting expired tokens from auth DB", "err", err) 90 | return 91 | } 92 | 93 | // For each expired auth token 94 | for _, val := range expired { 95 | // Get the active transfer associated with the auth token 96 | activeXfer, err := m.dtServer.Get(val.ID) 97 | if err != nil && !xerrors.Is(err, httptransport.ErrTransferNotFound) { 98 | log.Errorw("getting transfer", "id", val.ID, "err", err) 99 | continue 100 | } 101 | 102 | // Note that there may not be an active transfer (it may have not 103 | // started, errored out or completed) 104 | if activeXfer != nil { 105 | // Cancel the transfer 106 | _, err := m.dtServer.CancelTransfer(ctx, val.ID) 107 | if err != nil && !xerrors.Is(err, httptransport.ErrTransferNotFound) { 108 | log.Errorw("canceling transfer", "id", val.ID, "err", err) 109 | continue 110 | } 111 | } 112 | 113 | // Check if the transfer already completed 114 | completedXfer, err := m.getCompletedTransfer(val.ID) 115 | if err != nil && !xerrors.Is(err, datastore.ErrNotFound) { 116 | log.Errorw("getting completed transfer", "id", val.ID, "err", err) 117 | continue 118 | } 119 | // If the transfer had already completed, nothing more to do 120 | if completedXfer != nil && completedXfer.Status == boosttypes.TransferStatusCompleted { 121 | continue 122 | } 123 | 124 | // The transfer didn't start, or was canceled or errored out before 125 | // completing, so fire a transfer error event 126 | dbid, err := strconv.ParseUint(val.ID, 10, 64) 127 | if err != nil { 128 | log.Errorw("parsing dbid in libp2p transfer manager event", "id", val.ID, "err", err) 129 | continue 130 | } 131 | 132 | st := boosttypes.TransferState{ 133 | ID: val.ID, 134 | LocalAddr: m.dtServer.ID().String(), 135 | } 136 | 137 | if activeXfer != nil { 138 | st = activeXfer.State() 139 | } 140 | st.Status = boosttypes.TransferStatusFailed 141 | if st.Message == "" { 142 | st.Message = fmt.Sprintf("timed out waiting %s for transfer to complete", m.opts.xferTimeout) 143 | } 144 | 145 | if completedXfer == nil { 146 | // Save the transfer status in persistent storage 147 | err = m.saveCompletedTransfer(val.ID, st) 148 | if err != nil { 149 | log.Errorf("saving completed transfer: %s", err) 150 | continue 151 | } 152 | } 153 | 154 | // Fire transfer error event 155 | m.listenerLk.Lock() 156 | if m.listener != nil { 157 | m.listener(uint(dbid), m.toDTState(st)) 158 | } 159 | m.listenerLk.Unlock() 160 | } 161 | } 162 | 163 | // PrepareForDataRequest prepares to receive a data transfer request with the 164 | // given auth token 165 | func (m *libp2pTransferManager) PrepareForDataRequest(ctx context.Context, id uint, authToken string, proposalCid cid.Cid, payloadCid cid.Cid, size uint64) error { 166 | err := m.authDB.Put(ctx, authToken, httptransport.AuthValue{ 167 | ID: fmt.Sprintf("%d", id), 168 | ProposalCid: proposalCid, 169 | PayloadCid: payloadCid, 170 | Size: size, 171 | }) 172 | if err != nil { 173 | return fmt.Errorf("adding new auth token: %w", err) 174 | } 175 | return nil 176 | } 177 | 178 | // CleanupPreparedRequest is called to remove the auth token for a request 179 | // when the request is no longer expected (eg because the provider rejected 180 | // the deal proposal) 181 | func (m *libp2pTransferManager) CleanupPreparedRequest(ctx context.Context, dbid uint, authToken string) error { 182 | // Delete the auth token for the request 183 | delerr := m.authDB.Delete(ctx, authToken) 184 | 185 | // Cancel any related transfer 186 | dbidstr := fmt.Sprintf("%d", dbid) 187 | _, cancelerr := m.dtServer.CancelTransfer(ctx, dbidstr) 188 | if cancelerr != nil && xerrors.Is(cancelerr, httptransport.ErrTransferNotFound) { 189 | // Ignore transfer not found error 190 | cancelerr = nil 191 | } 192 | 193 | if delerr != nil { 194 | return delerr 195 | } 196 | return cancelerr 197 | } 198 | 199 | // Subscribe to state change events from the libp2p server 200 | func (m *libp2pTransferManager) Subscribe(listener func(dbid uint, st ChannelState)) (func(), error) { 201 | m.listenerLk.Lock() 202 | defer m.listenerLk.Unlock() 203 | 204 | if m.listener != nil { 205 | // Only need one for the current use case, we can add more if needed later 206 | return nil, fmt.Errorf("only one listener allowed") 207 | } 208 | 209 | m.listener = listener 210 | 211 | unsub := func() { 212 | m.listenerLk.Lock() 213 | defer m.listenerLk.Unlock() 214 | 215 | m.listener = nil 216 | } 217 | return unsub, nil 218 | } 219 | 220 | // Convert from libp2p server events to data transfer events 221 | func (m *libp2pTransferManager) subscribe() (unsub func()) { 222 | return m.dtServer.Subscribe(func(dbidstr string, st boosttypes.TransferState) { 223 | dbid, err := strconv.ParseUint(dbidstr, 10, 64) 224 | if err != nil { 225 | log.Errorf("cannot parse dbid '%s' in libp2p transfer manager event: %s", dbidstr, err) 226 | return 227 | } 228 | if st.Status == boosttypes.TransferStatusFailed { 229 | // If the transfer fails, don't fire the failure event yet. 230 | // Wait until the transfer timeout (so that the data receiver has 231 | // a chance to make another request) 232 | log.Infow("libp2p data transfer error", "dbid", dbid, "remote", st.RemoteAddr, "message", st.Message) 233 | return 234 | } 235 | if st.Status == boosttypes.TransferStatusCompleted { 236 | if err := m.saveCompletedTransfer(dbidstr, st); err != nil { 237 | log.Errorf("saving completed transfer: %s", err) 238 | } 239 | } 240 | 241 | m.listenerLk.Lock() 242 | defer m.listenerLk.Unlock() 243 | if m.listener != nil { 244 | m.listener(uint(dbid), m.toDTState(st)) 245 | } 246 | }) 247 | } 248 | 249 | // All returns all active and completed transfers 250 | func (m *libp2pTransferManager) All() (map[string]ChannelState, error) { 251 | // Get all active transfers 252 | all := make(map[string]ChannelState) 253 | _, err := m.dtServer.Matching(func(xfer *httptransport.Libp2pTransfer) (bool, error) { 254 | all[xfer.ID] = m.toDTState(xfer.State()) 255 | return true, nil 256 | }) 257 | if err != nil { 258 | return nil, err 259 | } 260 | 261 | // Get all persisted (completed) transfers 262 | err = m.forEachCompletedTransfer(func(id string, st boosttypes.TransferState) bool { 263 | all[id] = m.toDTState(st) 264 | return true 265 | }) 266 | return all, err 267 | } 268 | 269 | func (m *libp2pTransferManager) byId(id string) (*ChannelState, error) { 270 | // Check active transfers 271 | xfer, err := m.dtServer.Get(id) 272 | if err == nil { 273 | st := m.toDTState(xfer.State()) 274 | return &st, nil 275 | } 276 | if !xerrors.Is(err, httptransport.ErrTransferNotFound) { 277 | return nil, err 278 | } 279 | 280 | // Check completed transfers 281 | boostSt, err := m.getCompletedTransfer(id) 282 | if err != nil { 283 | return nil, err 284 | } 285 | st := m.toDTState(*boostSt) 286 | return &st, nil 287 | } 288 | 289 | // byRemoteAddrAndPayloadCid returns the transfer with the matching remote 290 | // address and payload cid 291 | func (m *libp2pTransferManager) byRemoteAddrAndPayloadCid(remoteAddr string, payloadCid cid.Cid) (*ChannelState, error) { 292 | // Check active transfers 293 | matches, err := m.dtServer.Matching(func(xfer *httptransport.Libp2pTransfer) (bool, error) { 294 | match := xfer.RemoteAddr == remoteAddr && xfer.PayloadCid == payloadCid 295 | return match, nil 296 | }) 297 | if err != nil { 298 | return nil, err 299 | } 300 | 301 | if len(matches) > 0 { 302 | st := m.toDTState(matches[0].State()) 303 | return &st, nil 304 | } 305 | 306 | // Check completed transfers 307 | var chanst *ChannelState 308 | err = m.forEachCompletedTransfer(func(id string, st boosttypes.TransferState) bool { 309 | if st.RemoteAddr == remoteAddr && st.PayloadCid == payloadCid { 310 | dtst := m.toDTState(st) 311 | chanst = &dtst 312 | return true 313 | } 314 | return false 315 | }) 316 | return chanst, err 317 | } 318 | 319 | func (m *libp2pTransferManager) toDTState(st boosttypes.TransferState) ChannelState { 320 | status := datatransfer.Ongoing 321 | switch st.Status { 322 | case boosttypes.TransferStatusStarted: 323 | status = datatransfer.Requested 324 | case boosttypes.TransferStatusCompleted: 325 | status = datatransfer.Completed 326 | case boosttypes.TransferStatusFailed: 327 | status = datatransfer.Failed 328 | } 329 | 330 | // The datatransfer channel ID is no longer used, but fill in 331 | // its fields anyway so that the parts of the UI that depend on it 332 | // still work 333 | xferid, _ := strconv.ParseUint(st.ID, 10, 64) //nolint:errcheck 334 | remoteAddr := st.RemoteAddr 335 | if remoteAddr == "" { 336 | // If the remote peer never tried to connect to us we don't have 337 | // a remote peer ID. However datatransfer.ChannelID requires a 338 | // remote peer ID. So just fill it in with the local peer ID so 339 | // that it doesn't fail JSON parsing when sent in an RPC. 340 | remoteAddr = st.LocalAddr 341 | } 342 | chid := datatransfer.ChannelID{ 343 | Initiator: parsePeerID(st.LocalAddr), 344 | Responder: parsePeerID(remoteAddr), 345 | ID: datatransfer.TransferID(xferid), 346 | } 347 | 348 | return ChannelState{ 349 | SelfPeer: parsePeerID(st.LocalAddr), 350 | RemotePeer: parsePeerID(remoteAddr), 351 | Status: status, 352 | StatusStr: datatransfer.Statuses[status], 353 | Sent: st.Sent, 354 | Received: st.Received, 355 | Message: st.Message, 356 | BaseCid: st.PayloadCid.String(), 357 | ChannelID: chid, 358 | TransferID: st.ID, 359 | TransferType: BoostTransfer, 360 | } 361 | } 362 | 363 | func (m *libp2pTransferManager) getCompletedTransfer(id string) (*boosttypes.TransferState, error) { 364 | data, err := m.dtds.Get(m.ctx, datastore.NewKey(id)) 365 | if err != nil { 366 | if xerrors.Is(err, datastore.ErrNotFound) { 367 | return nil, fmt.Errorf("getting transfer status for id '%s': %w", id, err) 368 | } 369 | return nil, fmt.Errorf("getting transfer status for id '%s' from datastore: %w", id, err) 370 | } 371 | 372 | var st boosttypes.TransferState 373 | err = json.Unmarshal(data, &st) 374 | if err != nil { 375 | return nil, fmt.Errorf("unmarshaling transfer status: %w", err) 376 | } 377 | 378 | return &st, nil 379 | } 380 | 381 | func (m *libp2pTransferManager) saveCompletedTransfer(id string, st boosttypes.TransferState) error { 382 | data, err := json.Marshal(st) 383 | if err != nil { 384 | return fmt.Errorf("json marshalling transfer status: %w", err) 385 | } 386 | 387 | err = m.dtds.Put(m.ctx, datastore.NewKey(id), data) 388 | if err != nil { 389 | return fmt.Errorf("storing transfer status to datastore: %w", err) 390 | } 391 | 392 | return nil 393 | } 394 | 395 | func (m *libp2pTransferManager) forEachCompletedTransfer(cb func(string, boosttypes.TransferState) bool) error { 396 | qres, err := m.dtds.Query(m.ctx, query.Query{}) 397 | if err != nil { 398 | return xerrors.Errorf("query error: %w", err) 399 | } 400 | defer qres.Close() //nolint:errcheck 401 | 402 | for r := range qres.Next() { 403 | var st boosttypes.TransferState 404 | err = json.Unmarshal(r.Value, &st) 405 | if err != nil { 406 | return fmt.Errorf("unmarshaling json from datastore: %w", err) 407 | } 408 | 409 | ok := cb(removeLeadingSlash(r.Key), st) 410 | if !ok { 411 | return nil 412 | } 413 | } 414 | 415 | return nil 416 | } 417 | 418 | func removeLeadingSlash(key string) string { 419 | if key[0] == '/' { 420 | return key[1:] 421 | } 422 | return key 423 | } 424 | 425 | func parsePeerID(pidstr string) peer.ID { 426 | pid, err := peer.Decode(pidstr) 427 | if err != nil { 428 | log.Warnf("couldnt decode pid '%s': %s", pidstr, err) 429 | return "" 430 | } 431 | return pid 432 | } 433 | -------------------------------------------------------------------------------- /libp2ptransfermgr_test.go: -------------------------------------------------------------------------------- 1 | package filclient 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | "time" 8 | 9 | "github.com/filecoin-project/boost/transport/httptransport" 10 | boosttypes "github.com/filecoin-project/boost/transport/types" 11 | datatransfer "github.com/filecoin-project/go-data-transfer" 12 | "github.com/ipfs/go-cid" 13 | "github.com/ipfs/go-datastore" 14 | "github.com/ipfs/go-datastore/sync" 15 | "github.com/libp2p/go-libp2p/core/peer" 16 | "github.com/stretchr/testify/require" 17 | ) 18 | 19 | func TestLibp2pTransferManager(t *testing.T) { 20 | runTest := func(t *testing.T, completeBeforeExpiry bool) { 21 | opts := libp2pTransferManagerOpts{ 22 | xferTimeout: 100 * time.Millisecond, 23 | authCheckPeriod: 200 * time.Millisecond, 24 | } 25 | ctx, cancel := context.WithTimeout(context.Background(), opts.authCheckPeriod*2) 26 | defer cancel() 27 | 28 | ds := sync.MutexWrap(datastore.NewMapDatastore()) 29 | server := &mockLibp2pCarServer{} 30 | authDB := httptransport.NewAuthTokenDB(ds) 31 | 32 | tm := newLibp2pTransferManager(server, ds, authDB, opts) 33 | 34 | type evt struct { 35 | dbid uint 36 | st ChannelState 37 | } 38 | evts := make(chan evt, 1024) 39 | unsub, err := tm.Subscribe(func(dbid uint, st ChannelState) { 40 | evts <- evt{dbid: dbid, st: st} 41 | }) 42 | require.NoError(t, err) 43 | defer unsub() 44 | 45 | err = tm.Start(ctx) 46 | require.NoError(t, err) 47 | 48 | authToken, err := httptransport.GenerateAuthToken() 49 | require.NoError(t, err) 50 | proposalCid, err := cid.Parse("bafkqaaa") 51 | require.NoError(t, err) 52 | payloadCid, err := cid.Parse("bafkqaab") 53 | require.NoError(t, err) 54 | dbid := uint(1) 55 | size := uint64(1024) 56 | err = tm.PrepareForDataRequest(ctx, dbid, authToken, proposalCid, payloadCid, size) 57 | require.NoError(t, err) 58 | 59 | // Expect an "Ongoing" event to be passed through to the subscriber 60 | dbidstr := fmt.Sprintf("%d", dbid) 61 | st := boosttypes.TransferState{ 62 | Status: boosttypes.TransferStatusOngoing, 63 | } 64 | server.simulateEvent(dbidstr, st) 65 | require.Len(t, evts, 1) 66 | e := <-evts 67 | require.Equal(t, datatransfer.Ongoing, e.st.Status) 68 | 69 | if completeBeforeExpiry { 70 | // Simulate a "Completed" event 71 | st := boosttypes.TransferState{ 72 | Status: boosttypes.TransferStatusCompleted, 73 | } 74 | server.simulateEvent(dbidstr, st) 75 | require.Len(t, evts, 1) 76 | e := <-evts 77 | require.Equal(t, datatransfer.Completed, e.st.Status) 78 | } 79 | 80 | // Simulate a Failed event 81 | st = boosttypes.TransferState{ 82 | Status: boosttypes.TransferStatusFailed, 83 | } 84 | server.simulateEvent(dbidstr, st) 85 | require.Len(t, evts, 0) 86 | 87 | // Wait until the auth token expires, and the auth token check runs 88 | select { 89 | case <-ctx.Done(): 90 | // The server fired a Complete event so we expected the manager not 91 | // to fire a Failed event 92 | if completeBeforeExpiry { 93 | return 94 | } 95 | 96 | // The server did not fire a Complete event, so we were expecting 97 | // the manager to fire a Failed event 98 | require.Fail(t, "timed out waiting for Failed event") 99 | case e = <-evts: 100 | // The server fired a Complete event, so we did not expect the 101 | // manager to fire a Failed event 102 | if completeBeforeExpiry { 103 | require.Fail(t, "unexpected event %s", e.st.StatusStr) 104 | return 105 | } 106 | 107 | // The server did not fire a Complete event, so we expect the 108 | // manager to fire a Failed event 109 | require.Equal(t, datatransfer.Failed, e.st.Status) 110 | } 111 | } 112 | 113 | t.Run("pass if complete event before token expiry", func(t *testing.T) { 114 | runTest(t, true) 115 | }) 116 | 117 | t.Run("fail if no complete event", func(t *testing.T) { 118 | runTest(t, false) 119 | }) 120 | } 121 | 122 | type mockLibp2pCarServer struct { 123 | listener httptransport.EventListenerFn 124 | } 125 | 126 | func (m *mockLibp2pCarServer) simulateEvent(dbidstr string, st boosttypes.TransferState) { 127 | m.listener(dbidstr, st) 128 | } 129 | 130 | func (m *mockLibp2pCarServer) Subscribe(listener httptransport.EventListenerFn) httptransport.UnsubFn { 131 | m.listener = listener 132 | return func() {} 133 | } 134 | 135 | func (m *mockLibp2pCarServer) ID() peer.ID { 136 | return "pid" 137 | } 138 | 139 | func (m *mockLibp2pCarServer) Start(ctx context.Context) error { 140 | return nil 141 | } 142 | 143 | func (m *mockLibp2pCarServer) Get(id string) (*httptransport.Libp2pTransfer, error) { 144 | return nil, nil 145 | } 146 | 147 | func (m *mockLibp2pCarServer) CancelTransfer(ctx context.Context, id string) (*boosttypes.TransferState, error) { 148 | return nil, nil 149 | } 150 | 151 | func (m *mockLibp2pCarServer) Matching(f httptransport.MatchFn) ([]*httptransport.Libp2pTransfer, error) { 152 | return nil, nil 153 | } 154 | 155 | var _ libp2pCarServer = (*mockLibp2pCarServer)(nil) 156 | -------------------------------------------------------------------------------- /msgpusher.go: -------------------------------------------------------------------------------- 1 | package filclient 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | 8 | "github.com/filecoin-project/go-address" 9 | "github.com/filecoin-project/go-state-types/abi" 10 | "github.com/filecoin-project/go-state-types/big" 11 | "github.com/filecoin-project/lotus/api" 12 | "github.com/filecoin-project/lotus/chain/types" 13 | "github.com/filecoin-project/lotus/chain/wallet" 14 | ) 15 | 16 | // a simple nonce tracking message pusher that assumes it is the only thing with access to the given key 17 | type MsgPusher struct { 18 | gapi api.Gateway 19 | w *wallet.LocalWallet 20 | 21 | nlk sync.Mutex 22 | nonces map[address.Address]uint64 23 | } 24 | 25 | func NewMsgPusher(gapi api.Gateway, w *wallet.LocalWallet) *MsgPusher { 26 | return &MsgPusher{ 27 | gapi: gapi, 28 | w: w, 29 | nonces: make(map[address.Address]uint64), 30 | } 31 | } 32 | 33 | func (mp *MsgPusher) MpoolPushMessage(ctx context.Context, msg *types.Message, maxFee *api.MessageSendSpec) (*types.SignedMessage, error) { 34 | mp.nlk.Lock() 35 | defer mp.nlk.Unlock() 36 | 37 | kaddr, err := mp.gapi.StateAccountKey(ctx, msg.From, types.EmptyTSK) 38 | if err != nil { 39 | return nil, err 40 | } 41 | 42 | n, ok := mp.nonces[kaddr] 43 | if !ok { 44 | act, err := mp.gapi.StateGetActor(ctx, kaddr, types.EmptyTSK) 45 | if err != nil { 46 | return nil, err 47 | } 48 | 49 | n = act.Nonce 50 | mp.nonces[kaddr] = n 51 | } 52 | 53 | msg.Nonce = n 54 | 55 | estim, err := mp.gapi.GasEstimateMessageGas(ctx, msg, &api.MessageSendSpec{}, types.EmptyTSK) 56 | if err != nil { 57 | return nil, fmt.Errorf("failed to estimate gas: %w", err) 58 | } 59 | 60 | estim.GasFeeCap = abi.NewTokenAmount(4000000000) 61 | estim.GasPremium = big.Mul(estim.GasPremium, big.NewInt(2)) 62 | 63 | sig, err := mp.w.WalletSign(ctx, kaddr, estim.Cid().Bytes(), api.MsgMeta{Type: api.MTChainMsg}) 64 | if err != nil { 65 | return nil, err 66 | } 67 | 68 | smsg := &types.SignedMessage{ 69 | Message: *estim, 70 | Signature: *sig, 71 | } 72 | 73 | _, err = mp.gapi.MpoolPush(ctx, smsg) 74 | if err != nil { 75 | return nil, err 76 | } 77 | 78 | mp.nonces[kaddr]++ 79 | 80 | return smsg, nil 81 | } 82 | -------------------------------------------------------------------------------- /rep/event.go: -------------------------------------------------------------------------------- 1 | package rep 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/filecoin-project/go-address" 7 | "github.com/filecoin-project/go-fil-markets/retrievalmarket" 8 | "github.com/filecoin-project/go-state-types/big" 9 | "github.com/ipfs/go-cid" 10 | "github.com/libp2p/go-libp2p/core/peer" 11 | ) 12 | 13 | type Phase string 14 | 15 | const ( 16 | // QueryPhase involves a connect, query-ask, success|failure 17 | QueryPhase Phase = "query" 18 | // RetrievalPhase involves the full data retrieval: connect, proposed, accepted, first-byte-received, success|failure 19 | RetrievalPhase Phase = "retrieval" 20 | ) 21 | 22 | type Code string 23 | 24 | const ( 25 | ConnectedCode Code = "connected" 26 | QueryAskedCode Code = "query-asked" 27 | ProposedCode Code = "proposed" 28 | AcceptedCode Code = "accepted" 29 | FirstByteCode Code = "first-byte-received" 30 | FailureCode Code = "failure" 31 | SuccessCode Code = "success" 32 | ) 33 | 34 | type RetrievalEvent interface { 35 | // Code returns the type of event this is 36 | Code() Code 37 | // Phase returns what phase of a retrieval this even occurred on 38 | Phase() Phase 39 | // PayloadCid returns the CID being requested 40 | PayloadCid() cid.Cid 41 | // StorageProviderId returns the peer ID of the storage provider if this 42 | // retrieval was requested via peer ID 43 | StorageProviderId() peer.ID 44 | // StorageProviderAddr returns the peer address of the storage provider if 45 | // this retrieval was requested via peer address 46 | StorageProviderAddr() address.Address 47 | } 48 | 49 | var ( 50 | _ RetrievalEvent = RetrievalEventConnect{} 51 | _ RetrievalEvent = RetrievalEventQueryAsk{} 52 | _ RetrievalEvent = RetrievalEventProposed{} 53 | _ RetrievalEvent = RetrievalEventAccepted{} 54 | _ RetrievalEvent = RetrievalEventFirstByte{} 55 | _ RetrievalEvent = RetrievalEventFailure{} 56 | _ RetrievalEvent = RetrievalEventSuccess{} 57 | ) 58 | 59 | type RetrievalEventConnect struct { 60 | phase Phase 61 | payloadCid cid.Cid 62 | storageProviderId peer.ID 63 | storageProviderAddr address.Address 64 | } 65 | 66 | func NewRetrievalEventConnect(phase Phase, payloadCid cid.Cid, storageProviderId peer.ID, storageProviderAddr address.Address) RetrievalEventConnect { 67 | return RetrievalEventConnect{phase, payloadCid, storageProviderId, storageProviderAddr} 68 | } 69 | 70 | type RetrievalEventQueryAsk struct { 71 | phase Phase 72 | payloadCid cid.Cid 73 | storageProviderId peer.ID 74 | storageProviderAddr address.Address 75 | queryResponse retrievalmarket.QueryResponse 76 | } 77 | 78 | func NewRetrievalEventQueryAsk(phase Phase, payloadCid cid.Cid, storageProviderId peer.ID, storageProviderAddr address.Address, queryResponse retrievalmarket.QueryResponse) RetrievalEventQueryAsk { 79 | return RetrievalEventQueryAsk{phase, payloadCid, storageProviderId, storageProviderAddr, queryResponse} 80 | } 81 | 82 | type RetrievalEventProposed struct { 83 | phase Phase 84 | payloadCid cid.Cid 85 | storageProviderId peer.ID 86 | storageProviderAddr address.Address 87 | } 88 | 89 | func NewRetrievalEventProposed(phase Phase, payloadCid cid.Cid, storageProviderId peer.ID, storageProviderAddr address.Address) RetrievalEventProposed { 90 | return RetrievalEventProposed{phase, payloadCid, storageProviderId, storageProviderAddr} 91 | } 92 | 93 | type RetrievalEventAccepted struct { 94 | phase Phase 95 | payloadCid cid.Cid 96 | storageProviderId peer.ID 97 | storageProviderAddr address.Address 98 | } 99 | 100 | func NewRetrievalEventAccepted(phase Phase, payloadCid cid.Cid, storageProviderId peer.ID, storageProviderAddr address.Address) RetrievalEventAccepted { 101 | return RetrievalEventAccepted{phase, payloadCid, storageProviderId, storageProviderAddr} 102 | } 103 | 104 | type RetrievalEventFirstByte struct { 105 | phase Phase 106 | payloadCid cid.Cid 107 | storageProviderId peer.ID 108 | storageProviderAddr address.Address 109 | } 110 | 111 | func NewRetrievalEventFirstByte(phase Phase, payloadCid cid.Cid, storageProviderId peer.ID, storageProviderAddr address.Address) RetrievalEventFirstByte { 112 | return RetrievalEventFirstByte{phase, payloadCid, storageProviderId, storageProviderAddr} 113 | } 114 | 115 | type RetrievalEventFailure struct { 116 | phase Phase 117 | payloadCid cid.Cid 118 | storageProviderId peer.ID 119 | storageProviderAddr address.Address 120 | errorMessage string 121 | } 122 | 123 | func NewRetrievalEventFailure(phase Phase, payloadCid cid.Cid, storageProviderId peer.ID, storageProviderAddr address.Address, errorMessage string) RetrievalEventFailure { 124 | return RetrievalEventFailure{phase, payloadCid, storageProviderId, storageProviderAddr, errorMessage} 125 | } 126 | 127 | type RetrievalEventSuccess struct { 128 | phase Phase 129 | payloadCid cid.Cid 130 | storageProviderId peer.ID 131 | storageProviderAddr address.Address 132 | receivedSize uint64 133 | receivedCids int64 134 | duration time.Duration 135 | totalPayment big.Int 136 | } 137 | 138 | func NewRetrievalEventSuccess(phase Phase, payloadCid cid.Cid, storageProviderId peer.ID, storageProviderAddr address.Address, receivedSize uint64, receivedCids int64, duration time.Duration, totalPayment big.Int) RetrievalEventSuccess { 139 | return RetrievalEventSuccess{phase, payloadCid, storageProviderId, storageProviderAddr, receivedSize, receivedCids, duration, totalPayment} 140 | } 141 | 142 | func (r RetrievalEventConnect) Code() Code { return ConnectedCode } 143 | func (r RetrievalEventConnect) Phase() Phase { return r.phase } 144 | func (r RetrievalEventConnect) PayloadCid() cid.Cid { return r.payloadCid } 145 | func (r RetrievalEventConnect) StorageProviderId() peer.ID { return r.storageProviderId } 146 | func (r RetrievalEventConnect) StorageProviderAddr() address.Address { return r.storageProviderAddr } 147 | func (r RetrievalEventQueryAsk) Code() Code { return QueryAskedCode } 148 | func (r RetrievalEventQueryAsk) Phase() Phase { return r.phase } 149 | func (r RetrievalEventQueryAsk) PayloadCid() cid.Cid { return r.payloadCid } 150 | func (r RetrievalEventQueryAsk) StorageProviderId() peer.ID { return r.storageProviderId } 151 | func (r RetrievalEventQueryAsk) StorageProviderAddr() address.Address { return r.storageProviderAddr } 152 | 153 | // QueryResponse returns the response from a storage provider to a query-ask 154 | func (r RetrievalEventQueryAsk) QueryResponse() retrievalmarket.QueryResponse { return r.queryResponse } 155 | func (r RetrievalEventProposed) Code() Code { return ProposedCode } 156 | func (r RetrievalEventProposed) Phase() Phase { return r.phase } 157 | func (r RetrievalEventProposed) PayloadCid() cid.Cid { return r.payloadCid } 158 | func (r RetrievalEventProposed) StorageProviderId() peer.ID { return r.storageProviderId } 159 | func (r RetrievalEventProposed) StorageProviderAddr() address.Address { return r.storageProviderAddr } 160 | func (r RetrievalEventAccepted) Code() Code { return AcceptedCode } 161 | func (r RetrievalEventAccepted) Phase() Phase { return r.phase } 162 | func (r RetrievalEventAccepted) PayloadCid() cid.Cid { return r.payloadCid } 163 | func (r RetrievalEventAccepted) StorageProviderId() peer.ID { return r.storageProviderId } 164 | func (r RetrievalEventAccepted) StorageProviderAddr() address.Address { return r.storageProviderAddr } 165 | func (r RetrievalEventFirstByte) Code() Code { return FirstByteCode } 166 | func (r RetrievalEventFirstByte) Phase() Phase { return r.phase } 167 | func (r RetrievalEventFirstByte) PayloadCid() cid.Cid { return r.payloadCid } 168 | func (r RetrievalEventFirstByte) StorageProviderId() peer.ID { return r.storageProviderId } 169 | func (r RetrievalEventFirstByte) StorageProviderAddr() address.Address { return r.storageProviderAddr } 170 | func (r RetrievalEventFailure) Code() Code { return FailureCode } 171 | func (r RetrievalEventFailure) Phase() Phase { return r.phase } 172 | func (r RetrievalEventFailure) PayloadCid() cid.Cid { return r.payloadCid } 173 | func (r RetrievalEventFailure) StorageProviderId() peer.ID { return r.storageProviderId } 174 | func (r RetrievalEventFailure) StorageProviderAddr() address.Address { return r.storageProviderAddr } 175 | 176 | // ErrorMessage returns a string form of the error that caused the retrieval 177 | // failure 178 | func (r RetrievalEventFailure) ErrorMessage() string { return r.errorMessage } 179 | func (r RetrievalEventSuccess) Code() Code { return SuccessCode } 180 | func (r RetrievalEventSuccess) Phase() Phase { return r.phase } 181 | func (r RetrievalEventSuccess) PayloadCid() cid.Cid { return r.payloadCid } 182 | func (r RetrievalEventSuccess) StorageProviderId() peer.ID { return r.storageProviderId } 183 | func (r RetrievalEventSuccess) StorageProviderAddr() address.Address { return r.storageProviderAddr } 184 | func (r RetrievalEventSuccess) Duration() time.Duration { return r.duration } 185 | func (r RetrievalEventSuccess) TotalPayment() big.Int { return r.totalPayment } 186 | 187 | // ReceivedSize returns the number of bytes received 188 | func (r RetrievalEventSuccess) ReceivedSize() uint64 { return r.receivedSize } 189 | 190 | // ReceivedCids returns the number of (non-unique) CIDs received so far - note 191 | // that a block can exist in more than one place in the DAG so this may not 192 | // equal the total number of blocks transferred 193 | func (r RetrievalEventSuccess) ReceivedCids() int64 { return r.receivedCids } 194 | -------------------------------------------------------------------------------- /rep/publisher.go: -------------------------------------------------------------------------------- 1 | package rep 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | ) 7 | 8 | type RetrievalSubscriber interface { 9 | OnRetrievalEvent(RetrievalEvent) 10 | } 11 | 12 | type RetrievalEventPublisher struct { 13 | ctx context.Context 14 | // Lock for the subscribers list 15 | subscribersLk sync.RWMutex 16 | // The list of subscribers 17 | subscribers map[int]RetrievalSubscriber 18 | idx int 19 | events chan RetrievalEvent 20 | } 21 | 22 | // A return function unsubscribing a subscribed Subscriber via the Subscribe() function 23 | type UnsubscribeFn func() 24 | 25 | func New(ctx context.Context) *RetrievalEventPublisher { 26 | ep := &RetrievalEventPublisher{ 27 | ctx: ctx, 28 | subscribers: make(map[int]RetrievalSubscriber, 0), 29 | events: make(chan RetrievalEvent, 16), 30 | } 31 | go ep.loop() 32 | 33 | return ep 34 | } 35 | 36 | func (ep *RetrievalEventPublisher) loop() { 37 | for { 38 | select { 39 | case <-ep.ctx.Done(): 40 | return 41 | case event := <-ep.events: 42 | ep.subscribersLk.RLock() 43 | subscribers := make([]RetrievalSubscriber, 0, len(ep.subscribers)) 44 | for _, subscriber := range ep.subscribers { 45 | subscribers = append(subscribers, subscriber) 46 | } 47 | ep.subscribersLk.RUnlock() 48 | 49 | for _, subscriber := range subscribers { 50 | subscriber.OnRetrievalEvent(event) 51 | } 52 | } 53 | } 54 | } 55 | 56 | func (ep *RetrievalEventPublisher) Subscribe(subscriber RetrievalSubscriber) UnsubscribeFn { 57 | // Lock writes on the subscribers list 58 | ep.subscribersLk.Lock() 59 | defer ep.subscribersLk.Unlock() 60 | 61 | // increment the index so we can assign a unique one to this subscriber so 62 | // our unregister function works 63 | idx := ep.idx 64 | ep.idx++ 65 | ep.subscribers[idx] = subscriber 66 | 67 | // return unregister function 68 | return func() { 69 | ep.subscribersLk.Lock() 70 | defer ep.subscribersLk.Unlock() 71 | delete(ep.subscribers, idx) 72 | } 73 | } 74 | 75 | func (ep *RetrievalEventPublisher) Publish(event RetrievalEvent) { 76 | select { 77 | case <-ep.ctx.Done(): 78 | case ep.events <- event: 79 | } 80 | } 81 | 82 | // Returns the number of retrieval event subscribers 83 | func (ep *RetrievalEventPublisher) SubscriberCount() int { 84 | ep.subscribersLk.RLock() 85 | defer ep.subscribersLk.RUnlock() 86 | return len(ep.subscribers) 87 | } 88 | -------------------------------------------------------------------------------- /rep/publisher_test.go: -------------------------------------------------------------------------------- 1 | package rep_test 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | 8 | "github.com/application-research/filclient/rep" 9 | "github.com/filecoin-project/go-address" 10 | "github.com/filecoin-project/go-state-types/abi" 11 | "github.com/ipfs/go-cid" 12 | "github.com/libp2p/go-libp2p/core/peer" 13 | "github.com/stretchr/testify/require" 14 | ) 15 | 16 | var testCid1 = mustCid("bafybeihrqe2hmfauph5yfbd6ucv7njqpiy4tvbewlvhzjl4bhnyiu6h7pm") 17 | 18 | type sub struct { 19 | ping chan rep.RetrievalEvent 20 | } 21 | 22 | func (us *sub) OnRetrievalEvent(evt rep.RetrievalEvent) { 23 | if us.ping != nil { 24 | us.ping <- evt 25 | } 26 | } 27 | 28 | func TestSubscribeAndUnsubscribe(t *testing.T) { 29 | pub := rep.New(context.Background()) 30 | sub := &sub{} 31 | require.Equal(t, 0, pub.SubscriberCount(), "has no subscribers") 32 | 33 | unsub := pub.Subscribe(sub) 34 | unsub2 := pub.Subscribe(sub) 35 | 36 | require.Equal(t, 2, pub.SubscriberCount(), "registered both subscribers") 37 | unsub() 38 | require.Equal(t, 1, pub.SubscriberCount(), "unregistered first subscriber") 39 | unsub2() 40 | require.Equal(t, 0, pub.SubscriberCount(), "unregistered second subscriber") 41 | } 42 | 43 | func TestEventing(t *testing.T) { 44 | ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) 45 | defer cancel() 46 | 47 | pub := rep.New(ctx) 48 | sub := &sub{ping: make(chan rep.RetrievalEvent)} 49 | 50 | pub.Subscribe(sub) 51 | pid := peer.NewPeerRecord().PeerID 52 | pub.Publish(rep.NewRetrievalEventConnect(rep.QueryPhase, testCid1, pid, address.Undef)) 53 | pub.Publish(rep.NewRetrievalEventSuccess(rep.RetrievalPhase, testCid1, "", address.TestAddress, 101, 202, time.Millisecond*303, abi.NewTokenAmount(404))) 54 | 55 | evt := <-sub.ping 56 | require.Equal(t, rep.QueryPhase, evt.Phase()) 57 | require.Equal(t, testCid1, evt.PayloadCid()) 58 | require.Equal(t, pid, evt.StorageProviderId()) 59 | require.Equal(t, address.Undef, evt.StorageProviderAddr()) 60 | require.Equal(t, rep.ConnectedCode, evt.Code()) 61 | _, ok := evt.(rep.RetrievalEventConnect) 62 | require.True(t, ok) 63 | 64 | evt = <-sub.ping 65 | require.Equal(t, rep.RetrievalPhase, evt.Phase()) 66 | require.Equal(t, testCid1, evt.PayloadCid()) 67 | require.Equal(t, peer.ID(""), evt.StorageProviderId()) 68 | require.Equal(t, address.TestAddress, evt.StorageProviderAddr()) 69 | require.Equal(t, rep.SuccessCode, evt.Code()) 70 | res, ok := evt.(rep.RetrievalEventSuccess) 71 | require.True(t, ok) 72 | require.Equal(t, uint64(101), res.ReceivedSize()) 73 | require.Equal(t, int64(202), res.ReceivedCids()) 74 | require.Equal(t, time.Millisecond*303, res.Duration()) 75 | require.Equal(t, abi.NewTokenAmount(404), res.TotalPayment()) 76 | } 77 | 78 | func mustCid(cstr string) cid.Cid { 79 | c, err := cid.Decode(cstr) 80 | if err != nil { 81 | panic(err) 82 | } 83 | return c 84 | } 85 | -------------------------------------------------------------------------------- /retrievehelper/params.go: -------------------------------------------------------------------------------- 1 | package retrievehelper 2 | 3 | import ( 4 | "github.com/filecoin-project/go-fil-markets/retrievalmarket" 5 | "github.com/filecoin-project/go-fil-markets/shared" 6 | "github.com/ipfs/go-cid" 7 | "github.com/ipld/go-ipld-prime" 8 | selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" 9 | ) 10 | 11 | var dealIdGen = shared.NewTimeCounter() 12 | 13 | func RetrievalProposalForAsk(ask *retrievalmarket.QueryResponse, c cid.Cid, optionalSelector ipld.Node) (*retrievalmarket.DealProposal, error) { 14 | 15 | if optionalSelector == nil { 16 | optionalSelector = selectorparse.CommonSelector_ExploreAllRecursively 17 | } 18 | 19 | params, err := retrievalmarket.NewParamsV1( 20 | ask.MinPricePerByte, 21 | ask.MaxPaymentInterval, 22 | ask.MaxPaymentIntervalIncrease, 23 | optionalSelector, 24 | nil, 25 | ask.UnsealPrice, 26 | ) 27 | if err != nil { 28 | return nil, err 29 | } 30 | return &retrievalmarket.DealProposal{ 31 | PayloadCID: c, 32 | ID: retrievalmarket.DealID(dealIdGen.Next()), 33 | Params: params, 34 | }, nil 35 | } 36 | -------------------------------------------------------------------------------- /retrievehelper/selector.go: -------------------------------------------------------------------------------- 1 | package retrievehelper 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "io" 8 | 9 | // must be imported to init() raw-codec support 10 | _ "github.com/ipld/go-ipld-prime/codec/raw" 11 | 12 | "github.com/ipfs/go-cid" 13 | mdagipld "github.com/ipfs/go-ipld-format" 14 | dagpb "github.com/ipld/go-codec-dagpb" 15 | "github.com/ipld/go-ipld-prime" 16 | cidlink "github.com/ipld/go-ipld-prime/linking/cid" 17 | basicnode "github.com/ipld/go-ipld-prime/node/basic" 18 | "github.com/ipld/go-ipld-prime/traversal" 19 | "github.com/ipld/go-ipld-prime/traversal/selector" 20 | selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" 21 | ) 22 | 23 | func TraverseDag( 24 | ctx context.Context, 25 | ds mdagipld.DAGService, 26 | startFrom cid.Cid, 27 | optionalSelector ipld.Node, 28 | visitCallback traversal.AdvVisitFn, 29 | ) error { 30 | 31 | if optionalSelector == nil { 32 | optionalSelector = selectorparse.CommonSelector_MatchAllRecursively 33 | } 34 | 35 | parsedSelector, err := selector.ParseSelector(optionalSelector) 36 | if err != nil { 37 | return err 38 | } 39 | 40 | // not sure what this is for TBH: we also provide ctx in &traversal.Config{} 41 | linkContext := ipld.LinkContext{Ctx: ctx} 42 | 43 | // this is what allows us to understand dagpb 44 | nodePrototypeChooser := dagpb.AddSupportToChooser( 45 | func(ipld.Link, ipld.LinkContext) (ipld.NodePrototype, error) { 46 | return basicnode.Prototype.Any, nil 47 | }, 48 | ) 49 | 50 | // this is how we implement GETs 51 | linkSystem := cidlink.DefaultLinkSystem() 52 | linkSystem.StorageReadOpener = func(lctx ipld.LinkContext, lnk ipld.Link) (io.Reader, error) { 53 | cl, isCid := lnk.(cidlink.Link) 54 | if !isCid { 55 | return nil, fmt.Errorf("unexpected link type %#v", lnk) 56 | } 57 | 58 | node, err := ds.Get(lctx.Ctx, cl.Cid) 59 | if err != nil { 60 | return nil, err 61 | } 62 | 63 | return bytes.NewBuffer(node.RawData()), nil 64 | } 65 | 66 | // this is how we pull the start node out of the DS 67 | startLink := cidlink.Link{Cid: startFrom} 68 | startNodePrototype, err := nodePrototypeChooser(startLink, linkContext) 69 | if err != nil { 70 | return err 71 | } 72 | startNode, err := linkSystem.Load( 73 | linkContext, 74 | startLink, 75 | startNodePrototype, 76 | ) 77 | if err != nil { 78 | return err 79 | } 80 | 81 | // this is the actual execution, invoking the supplied callback 82 | return traversal.Progress{ 83 | Cfg: &traversal.Config{ 84 | Ctx: ctx, 85 | LinkSystem: linkSystem, 86 | LinkTargetNodePrototypeChooser: nodePrototypeChooser, 87 | }, 88 | }.WalkAdv(startNode, parsedSelector, visitCallback) 89 | } 90 | --------------------------------------------------------------------------------