├── .bingo ├── .gitignore ├── README.md ├── Variables.mk ├── buf.mod ├── go.mod ├── gomplate.mod ├── govvv.mod ├── gox.mod ├── protoc-gen-buf-check-breaking.mod ├── protoc-gen-buf-check-lint.mod └── variables.env ├── .github └── workflows │ ├── build.yml │ ├── publish-js-libs.yml │ ├── release.yml │ └── test.yml ├── .gitignore ├── CODE_OF_CONDUCT.md ├── LICENSE ├── Makefile ├── README.md ├── access.go ├── api ├── apitest │ ├── apitest.go │ └── docker-compose.yml ├── cast │ └── cast.go ├── client │ ├── client.go │ ├── client_test.go │ └── testdata │ │ ├── file1.jpg │ │ └── file2.jpg ├── common │ └── common.go ├── pb │ └── buckets │ │ ├── buckets.pb.go │ │ ├── buckets.proto │ │ └── javascript │ │ ├── package-lock.json │ │ └── package.json └── service.go ├── buckets.go ├── buf.yaml ├── buildinfo └── buildinfo.go ├── buildtools └── install_protoc.bash ├── cmd ├── buck │ ├── cli │ │ ├── add.go │ │ ├── cli.go │ │ ├── init.go │ │ ├── pinning.go │ │ ├── pull.go │ │ ├── push.go │ │ ├── roles.go │ │ ├── util.go │ │ └── watch.go │ └── main.go ├── buckd │ ├── Dockerfile │ ├── Dockerfile.dev │ ├── docker-compose-dev.yml │ ├── docker-compose.yml │ └── main.go ├── cmd.go ├── config.go ├── output.go └── watch.go ├── collection ├── bucket.go ├── collection.go └── mergemap │ ├── LICENSE │ └── mergemap.go ├── create.go ├── dag ├── crypto.go ├── dag.go └── pinning.go ├── dist └── install.tmpl ├── dns └── dns.go ├── gateway ├── Makefile ├── assets.go ├── bucketfs.go ├── buckets.go ├── gateway.go ├── ipfs.go ├── pinning.go ├── pinning_test.go ├── public │ ├── css │ │ ├── all.min.css │ │ └── style.css │ ├── html │ │ ├── 404.gohtml │ │ ├── confirm.gohtml │ │ ├── consent.gohtml │ │ ├── error.gohtml │ │ ├── index.gohtml │ │ └── unixfs.gohtml │ ├── img │ │ ├── android-chrome-192x192.png │ │ ├── android-chrome-512x512.png │ │ ├── apple-touch-icon.png │ │ ├── favicon-16x16.png │ │ ├── favicon-32x32.png │ │ ├── favicon.ico │ │ ├── hex.svg │ │ └── site.webmanifest │ └── webfonts │ │ ├── fa-brands-400.eot │ │ ├── fa-brands-400.svg │ │ ├── fa-brands-400.ttf │ │ ├── fa-brands-400.woff │ │ ├── fa-brands-400.woff2 │ │ ├── fa-regular-400.eot │ │ ├── fa-regular-400.svg │ │ ├── fa-regular-400.ttf │ │ ├── fa-regular-400.woff │ │ ├── fa-regular-400.woff2 │ │ ├── fa-solid-900.eot │ │ ├── fa-solid-900.svg │ │ ├── fa-solid-900.ttf │ │ ├── fa-solid-900.woff │ │ └── fa-solid-900.woff2 ├── push.go ├── push_test.go └── threads.go ├── go.mod ├── go.sum ├── info.go ├── ipns ├── ipns.go └── store │ ├── store.go │ └── store_test.go ├── list.go ├── local ├── access.go ├── add.go ├── bucket.go ├── bucket_test.go ├── buckets.go ├── buckets_test.go ├── crypto.go ├── diff.go ├── list.go ├── options.go ├── pull.go ├── push.go ├── repo.go ├── repo_test.go ├── testdata │ ├── a │ │ ├── bar.txt │ │ ├── foo.txt │ │ └── one │ │ │ ├── baz.txt │ │ │ ├── buz.txt │ │ │ └── two │ │ │ ├── boo.txt │ │ │ └── fuz.txt │ ├── b │ │ ├── foo.txt │ │ └── one │ │ │ ├── baz.txt │ │ │ ├── muz.txt │ │ │ ├── three │ │ │ └── far.txt │ │ │ └── two │ │ │ └── fuz.txt │ └── c │ │ ├── one.jpg │ │ └── two.jpg └── watch.go ├── move.go ├── options.go ├── path.go ├── pinning ├── openapi │ ├── .openapi-generator-ignore │ ├── .openapi-generator │ │ ├── FILES │ │ └── VERSION │ ├── api │ │ └── openapi.yaml │ └── go │ │ ├── README.md │ │ ├── model_failure.go │ │ ├── model_failure_error.go │ │ ├── model_pin.go │ │ ├── model_pin_results.go │ │ ├── model_pin_status.go │ │ ├── model_query.go │ │ ├── model_status.go │ │ └── model_text_matching_strategy.go ├── queue │ ├── queue.go │ └── queue_test.go └── service.go ├── pull.go ├── push.go ├── remove.go ├── scripts ├── gen_js_protos.bash ├── protoc_gen_plugin.bash └── publish_js_protos.bash └── set.go /.bingo/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Ignore everything 3 | * 4 | 5 | # But not these files: 6 | !.gitignore 7 | !*.mod 8 | !README.md 9 | !Variables.mk 10 | !variables.env 11 | 12 | *tmp.mod 13 | -------------------------------------------------------------------------------- /.bingo/README.md: -------------------------------------------------------------------------------- 1 | # Project Development Dependencies. 2 | 3 | This is directory which stores Go modules with pinned buildable package that is used within this repository, managed by https://github.com/bwplotka/bingo. 4 | 5 | * Run `bingo get` to install all tools having each own module file in this directory. 6 | * Run `bingo get ` to install that have own module file in this directory. 7 | * For Makefile: Make sure to put `include .bingo/Variables.mk` in your Makefile, then use $() variable where is the .bingo/.mod. 8 | * For shell: Run `source .bingo/variables.env` to source all environment variable for each tool 9 | * See https://github.com/bwplotka/bingo or -h on how to add, remove or change binaries dependencies. 10 | 11 | ## Requirements 12 | 13 | * Go 1.14+ 14 | -------------------------------------------------------------------------------- /.bingo/Variables.mk: -------------------------------------------------------------------------------- 1 | # Auto generated binary variables helper managed by https://github.com/bwplotka/bingo v0.2.3. DO NOT EDIT. 2 | # All tools are designed to be build inside $GOBIN. 3 | GOPATH ?= $(shell go env GOPATH) 4 | GOBIN ?= $(firstword $(subst :, ,${GOPATH}))/bin 5 | GO ?= $(shell which go) 6 | 7 | # Bellow generated variables ensure that every time a tool under each variable is invoked, the correct version 8 | # will be used; reinstalling only if needed. 9 | # For example for buf variable: 10 | # 11 | # In your main Makefile (for non array binaries): 12 | # 13 | #include .bingo/Variables.mk # Assuming -dir was set to .bingo . 14 | # 15 | #command: $(BUF) 16 | # @echo "Running buf" 17 | # @$(BUF) 18 | # 19 | BUF := $(GOBIN)/buf-v0.20.5 20 | $(BUF): .bingo/buf.mod 21 | @# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies. 22 | @echo "(re)installing $(GOBIN)/buf-v0.20.5" 23 | @cd .bingo && $(GO) build -mod=mod -modfile=buf.mod -o=$(GOBIN)/buf-v0.20.5 "github.com/bufbuild/buf/cmd/buf" 24 | 25 | GOMPLATE := $(GOBIN)/gomplate-v3.8.0 26 | $(GOMPLATE): .bingo/gomplate.mod 27 | @# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies. 28 | @echo "(re)installing $(GOBIN)/gomplate-v3.8.0" 29 | @cd .bingo && $(GO) build -mod=mod -modfile=gomplate.mod -o=$(GOBIN)/gomplate-v3.8.0 "github.com/hairyhenderson/gomplate/v3/cmd/gomplate" 30 | 31 | GOVVV := $(GOBIN)/govvv-v0.3.0 32 | $(GOVVV): .bingo/govvv.mod 33 | @# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies. 34 | @echo "(re)installing $(GOBIN)/govvv-v0.3.0" 35 | @cd .bingo && $(GO) build -mod=mod -modfile=govvv.mod -o=$(GOBIN)/govvv-v0.3.0 "github.com/ahmetb/govvv" 36 | 37 | GOX := $(GOBIN)/gox-v1.0.1 38 | $(GOX): .bingo/gox.mod 39 | @# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies. 40 | @echo "(re)installing $(GOBIN)/gox-v1.0.1" 41 | @cd .bingo && $(GO) build -mod=mod -modfile=gox.mod -o=$(GOBIN)/gox-v1.0.1 "github.com/mitchellh/gox" 42 | 43 | PROTOC_GEN_BUF_CHECK_BREAKING := $(GOBIN)/protoc-gen-buf-check-breaking-v0.20.5 44 | $(PROTOC_GEN_BUF_CHECK_BREAKING): .bingo/protoc-gen-buf-check-breaking.mod 45 | @# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies. 46 | @echo "(re)installing $(GOBIN)/protoc-gen-buf-check-breaking-v0.20.5" 47 | @cd .bingo && $(GO) build -mod=mod -modfile=protoc-gen-buf-check-breaking.mod -o=$(GOBIN)/protoc-gen-buf-check-breaking-v0.20.5 "github.com/bufbuild/buf/cmd/protoc-gen-buf-check-breaking" 48 | 49 | PROTOC_GEN_BUF_CHECK_LINT := $(GOBIN)/protoc-gen-buf-check-lint-v0.20.5 50 | $(PROTOC_GEN_BUF_CHECK_LINT): .bingo/protoc-gen-buf-check-lint.mod 51 | @# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies. 52 | @echo "(re)installing $(GOBIN)/protoc-gen-buf-check-lint-v0.20.5" 53 | @cd .bingo && $(GO) build -mod=mod -modfile=protoc-gen-buf-check-lint.mod -o=$(GOBIN)/protoc-gen-buf-check-lint-v0.20.5 "github.com/bufbuild/buf/cmd/protoc-gen-buf-check-lint" 54 | 55 | -------------------------------------------------------------------------------- /.bingo/buf.mod: -------------------------------------------------------------------------------- 1 | module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT 2 | 3 | go 1.14 4 | 5 | require github.com/bufbuild/buf v0.20.5 // cmd/buf 6 | -------------------------------------------------------------------------------- /.bingo/go.mod: -------------------------------------------------------------------------------- 1 | module _ // Fake go.mod auto-created by 'bingo' for go -moddir compatibility with non-Go projects. Commit this file, together with other .mod files. 2 | 3 | go 1.14 4 | -------------------------------------------------------------------------------- /.bingo/gomplate.mod: -------------------------------------------------------------------------------- 1 | module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT 2 | 3 | go 1.14 4 | 5 | require github.com/hairyhenderson/gomplate/v3 v3.8.0 // cmd/gomplate 6 | -------------------------------------------------------------------------------- /.bingo/govvv.mod: -------------------------------------------------------------------------------- 1 | module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT 2 | 3 | go 1.14 4 | 5 | require github.com/ahmetb/govvv v0.3.0 6 | -------------------------------------------------------------------------------- /.bingo/gox.mod: -------------------------------------------------------------------------------- 1 | module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT 2 | 3 | go 1.14 4 | 5 | require github.com/mitchellh/gox v1.0.1 6 | -------------------------------------------------------------------------------- /.bingo/protoc-gen-buf-check-breaking.mod: -------------------------------------------------------------------------------- 1 | module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT 2 | 3 | go 1.14 4 | 5 | require github.com/bufbuild/buf v0.20.5 // cmd/protoc-gen-buf-check-breaking 6 | -------------------------------------------------------------------------------- /.bingo/protoc-gen-buf-check-lint.mod: -------------------------------------------------------------------------------- 1 | module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT 2 | 3 | go 1.14 4 | 5 | require github.com/bufbuild/buf v0.20.5 // cmd/protoc-gen-buf-check-lint 6 | -------------------------------------------------------------------------------- /.bingo/variables.env: -------------------------------------------------------------------------------- 1 | # Auto generated binary variables helper managed by https://github.com/bwplotka/bingo v0.2.3. DO NOT EDIT. 2 | # All tools are designed to be build inside $GOBIN. 3 | # Those variables will work only until 'bingo get' was invoked, or if tools were installed via Makefile's Variables.mk. 4 | local gobin=$(go env GOBIN) 5 | 6 | if [ -z "$gobin" ]; then 7 | gobin="$(go env GOPATH)/bin" 8 | fi 9 | 10 | 11 | BUF="${gobin}/buf-v0.20.5" 12 | 13 | GOMPLATE="${gobin}/gomplate-v3.8.0" 14 | 15 | GOVVV="${gobin}/govvv-v0.3.0" 16 | 17 | GOX="${gobin}/gox-v1.0.1" 18 | 19 | PROTOC_GEN_BUF_CHECK_BREAKING="${gobin}/protoc-gen-buf-check-breaking-v0.20.5" 20 | 21 | PROTOC_GEN_BUF_CHECK_LINT="${gobin}/protoc-gen-buf-check-lint-v0.20.5" 22 | 23 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | on: 3 | push: 4 | branches: 5 | - master 6 | pull_request: 7 | branches: 8 | - master 9 | jobs: 10 | buf: 11 | name: Buf 12 | runs-on: ubuntu-latest 13 | steps: 14 | - name: checkout 15 | uses: actions/checkout@v1 16 | with: 17 | ref: master 18 | - name: checkout-master 19 | run: git checkout master 20 | - name: checkout 21 | uses: actions/checkout@v1 22 | - name: make local 23 | run: make buf-local 24 | buck: 25 | name: Buck CLI 26 | runs-on: ubuntu-latest 27 | container: golang:1.16.0-buster 28 | steps: 29 | - name: checkout 30 | uses: actions/checkout@v1 31 | - name: build 32 | run: make build-buck 33 | buckd: 34 | name: Buck Daemon 35 | runs-on: ubuntu-latest 36 | container: golang:1.16.0-buster 37 | steps: 38 | - name: checkout 39 | uses: actions/checkout@v1 40 | - name: build 41 | run: make build-buckd 42 | -------------------------------------------------------------------------------- /.github/workflows/publish-js-libs.yml: -------------------------------------------------------------------------------- 1 | name: Publish JS Libs 2 | on: 3 | release: 4 | types: [published] 5 | jobs: 6 | publish-js-libs: 7 | name: Publish JS libs 8 | runs-on: ubuntu-latest 9 | steps: 10 | - name: Check out code 11 | uses: actions/checkout@v1 12 | - name: Install build tools 13 | run: | 14 | sudo apt-get update 15 | sudo apt-get install -y build-essential 16 | - name: Set up Go 17 | uses: actions/setup-go@v1 18 | with: 19 | go-version: 1.16 20 | - name: Setup env 21 | env: 22 | ACTIONS_ALLOW_UNSECURE_COMMANDS: true 23 | run: | 24 | echo "::set-env name=GOPATH::$(go env GOPATH)" 25 | echo "::add-path::$(go env GOPATH)/bin" 26 | - name: Set up Node 27 | uses: actions/setup-node@v1 28 | with: 29 | node-version: "14.x" 30 | registry-url: "https://registry.npmjs.org" 31 | - name: Generate JS libs 32 | run: | 33 | make js-protos 34 | - name: Publish JS libs 35 | run: | 36 | ./scripts/publish_js_protos.bash -v ${{ github.event.release.tag_name }} -t ${{ secrets.NPM_AUTH_TOKEN }} -p ${{ github.event.release.prerelease }} 37 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | on: 3 | release: 4 | types: [created] 5 | jobs: 6 | release-platform-builds: 7 | name: Release Builds 8 | runs-on: ubuntu-latest 9 | container: golang:1.16.0-buster 10 | steps: 11 | - name: Check out code 12 | uses: actions/checkout@v1 13 | - name: Cache dependencies 14 | id: cache-dependencies 15 | uses: actions/cache@v1 16 | with: 17 | path: ~/go/pkg/mod 18 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 19 | restore-keys: | 20 | ${{ runner.os }}-go- 21 | - name: Get dependencies 22 | if: steps.cache-dependencies.outputs.cache-hit != 'true' 23 | run: | 24 | export PATH=${PATH}:`go env GOPATH`/bin 25 | go get -v -t -d ./... 26 | - name: Build release artifacts 27 | run: | 28 | BIN_VERSION=${{ github.event.release.tag_name }} make build-releases 29 | echo $(ls ./build/dist/) 30 | - name: Upload multiple assets to release 31 | uses: AButler/upload-release-assets@v2.0 32 | with: 33 | files: "build/dist/*" 34 | repo-token: ${{ secrets.GITHUB_TOKEN }} 35 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | on: 3 | push: 4 | branches: 5 | - master 6 | pull_request: 7 | branches: 8 | - master 9 | jobs: 10 | test: 11 | name: Test 12 | runs-on: [self-hosted, buckets1] 13 | container: golang:1.16.0-buster 14 | services: 15 | threads: 16 | image: textile/go-threads:534a6d0 17 | env: 18 | THREADS_APIADDR: /ip4/0.0.0.0/tcp/5000 19 | THREADS_APIPROXYADDR: /ip4/0.0.0.0/tcp/5050 20 | ipfs: 21 | image: ipfs/go-ipfs:v0.8.0 22 | env: 23 | IPFS_PROFILE: test 24 | steps: 25 | - name: checkout 26 | uses: actions/checkout@v1 27 | - name: test 28 | env: 29 | SKIP_SERVICES: true 30 | THREADS_API_ADDR: threads:5000 31 | IPFS_API_MULTIADDR: /dns4/ipfs/tcp/5001 32 | run: make test 33 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | vendor/ 16 | 17 | # JS 18 | node_modules/ 19 | 20 | # IDEs 21 | .idea/ 22 | .vscode/ 23 | 24 | # Project 25 | .env 26 | buildtools/protoc 27 | buildtools/protoc-gen-go 28 | build/ 29 | **/javascript/api 30 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to make participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies within all project spaces, and it also applies when 49 | an individual is representing the project or its community in public spaces. 50 | Examples of representing a project or community include using an official 51 | project e-mail address, posting via an official social media account, or acting 52 | as an appointed representative at an online or offline event. Representation of 53 | a project may be further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at contact@textile.io. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | 75 | For answers to common questions about this code of conduct, see 76 | https://www.contributor-covenant.org/faq 77 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018-2021 textile.io 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | include .bingo/Variables.mk 2 | 3 | .DEFAULT_GOAL=build 4 | 5 | BIN_BUILD_FLAGS?=CGO_ENABLED=0 6 | BIN_VERSION?="git" 7 | GOVVV_FLAGS=$(shell $(GOVVV) -flags -version $(BIN_VERSION) -pkg $(shell go list ./buildinfo)) 8 | 9 | build: $(GOVVV) 10 | $(BIN_BUILD_FLAGS) go build -ldflags="${GOVVV_FLAGS}" ./... 11 | .PHONY: build 12 | 13 | build-buck: $(GOVVV) 14 | $(BIN_BUILD_FLAGS) go build -ldflags="${GOVVV_FLAGS}" ./cmd/buck 15 | .PHONY: build-buck 16 | 17 | build-buckd: $(GOVVV) 18 | $(BIN_BUILD_FLAGS) go build -ldflags="${GOVVV_FLAGS}" ./cmd/buckd 19 | .PHONY: build-buckd 20 | 21 | install: $(GOVVV) 22 | $(BIN_BUILD_FLAGS) go install -ldflags="${GOVVV_FLAGS}" ./... 23 | .PHONY: install 24 | 25 | install-buck: $(GOVVV) 26 | $(BIN_BUILD_FLAGS) go install -ldflags="${GOVVV_FLAGS}" ./cmd/buck 27 | .PHONY: install-buck 28 | 29 | install-buckd: $(GOVVV) 30 | $(BIN_BUILD_FLAGS) go install -ldflags="${GOVVV_FLAGS}" ./cmd/buckd 31 | .PHONY: install-buckd 32 | 33 | define gen_release_files 34 | $(GOX) -osarch=$(3) -output="build/$(2)/$(2)_${BIN_VERSION}_{{.OS}}-{{.Arch}}/$(2)" -ldflags="${GOVVV_FLAGS}" $(1) 35 | mkdir -p build/dist; \ 36 | cd build/$(2); \ 37 | for release in *; do \ 38 | cp ../../LICENSE ../../README.md $${release}/; \ 39 | if [ $${release} != *"windows"* ]; then \ 40 | BIN_FILE=$(2) $(GOMPLATE) -f ../../dist/install.tmpl -o "$${release}/install"; \ 41 | tar -czvf ../dist/$${release}.tar.gz $${release}; \ 42 | else \ 43 | zip -r ../dist/$${release}.zip $${release}; \ 44 | fi; \ 45 | done 46 | endef 47 | 48 | build-buck-release: $(GOX) $(GOVVV) $(GOMPLATE) 49 | $(call gen_release_files,./cmd/buck,buck,"linux/amd64 linux/386 linux/arm darwin/amd64 windows/amd64") 50 | .PHONY: build-buck-release 51 | 52 | build-buckd-release: $(GOX) $(GOVVV) $(GOMPLATE) 53 | $(call gen_release_files,./cmd/buckd,buckd,"linux/amd64 linux/386 linux/arm darwin/amd64 windows/amd64") 54 | .PHONY: build-buckd-release 55 | 56 | build-releases: build-buck-release build-buckd-release 57 | .PHONY: build-releases 58 | 59 | buck-up: 60 | docker-compose -f cmd/buckd/docker-compose-dev.yml up --build 61 | 62 | buck-stop: 63 | docker-compose -f cmd/buckd/docker-compose-dev.yml stop 64 | 65 | buck-clean: 66 | docker-compose -f cmd/buckd/docker-compose-dev.yml down -v --remove-orphans 67 | 68 | test: 69 | go test -race -timeout 30m ./... 70 | .PHONY: test 71 | 72 | clean-protos: 73 | find . -type f -name '*.pb.go' -delete 74 | find . -type f -name '*pb_test.go' -delete 75 | .PHONY: clean-protos 76 | 77 | clean-js-protos: 78 | find . -type f -name '*pb.js' ! -path "*/node_modules/*" -delete 79 | find . -type f -name '*pb.d.ts' ! -path "*/node_modules/*" -delete 80 | find . -type f -name '*pb_service.js' ! -path "*/node_modules/*" -delete 81 | find . -type f -name '*pb_service.d.ts' ! -path "*/node_modules/*" -delete 82 | .PHONY: clean-js-protos 83 | 84 | install-protoc: 85 | cd buildtools && ./install_protoc.bash 86 | 87 | PROTOCGENGO=$(shell pwd)/buildtools/protoc-gen-go 88 | protos: install-protoc clean-protos 89 | PATH=$(PROTOCGENGO):$(PATH) ./scripts/protoc_gen_plugin.bash \ 90 | --proto_path=. \ 91 | --plugin_name=go \ 92 | --plugin_out=. \ 93 | --plugin_opt=plugins=grpc,paths=source_relative 94 | .PHONY: protos 95 | 96 | js-protos: install-protoc clean-js-protos 97 | ./scripts/gen_js_protos.bash 98 | 99 | # local is what we run when testing locally. 100 | # This does breaking change detection against our local git repository. 101 | .PHONY: buf-local 102 | buf-local: $(BUF) 103 | $(BUF) check lint 104 | # $(BUF) check breaking --against-input '.git#branch=master' 105 | 106 | # https is what we run when testing in most CI providers. 107 | # This does breaking change detection against our remote HTTPS git repository. 108 | .PHONY: buf-https 109 | buf-https: $(BUF) 110 | $(BUF) check lint 111 | # $(BUF) check breaking --against-input "$(HTTPS_GIT)#branch=master" 112 | 113 | # ssh is what we run when testing in CI providers that provide ssh public key authentication. 114 | # This does breaking change detection against our remote HTTPS ssh repository. 115 | # This is especially useful for private repositories. 116 | .PHONY: buf-ssh 117 | buf-ssh: $(BUF) 118 | $(BUF) check lint 119 | # $(BUF) check breaking --against-input "$(SSH_GIT)#branch=master" 120 | -------------------------------------------------------------------------------- /api/apitest/apitest.go: -------------------------------------------------------------------------------- 1 | package apitest 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "os" 8 | "os/exec" 9 | "path" 10 | "runtime" 11 | "testing" 12 | "time" 13 | 14 | httpapi "github.com/ipfs/go-ipfs-http-client" 15 | logging "github.com/ipfs/go-log/v2" 16 | ma "github.com/multiformats/go-multiaddr" 17 | "github.com/phayes/freeport" 18 | "github.com/stretchr/testify/require" 19 | "github.com/textileio/go-buckets" 20 | "github.com/textileio/go-buckets/api/common" 21 | "github.com/textileio/go-buckets/ipns" 22 | dbc "github.com/textileio/go-threads/api/client" 23 | "github.com/textileio/go-threads/core/did" 24 | tdb "github.com/textileio/go-threads/db" 25 | nc "github.com/textileio/go-threads/net/api/client" 26 | tutil "github.com/textileio/go-threads/util" 27 | ) 28 | 29 | func NewService(t *testing.T) (listenAddr string, host did.DID) { 30 | err := tutil.SetLogLevels(map[string]logging.LogLevel{ 31 | "buckets": logging.LevelDebug, 32 | "buckets/api": logging.LevelDebug, 33 | "buckets/ipns": logging.LevelDebug, 34 | "buckets/dns": logging.LevelDebug, 35 | }) 36 | require.NoError(t, err) 37 | 38 | threadsAddr := GetThreadsApiAddr() 39 | net, err := nc.NewClient(threadsAddr, common.GetClientRPCOpts(threadsAddr)...) 40 | require.NoError(t, err) 41 | 42 | // @todo: Use service description to build client 43 | doc, err := net.GetServices(context.Background()) 44 | require.NoError(t, err) 45 | 46 | db, err := dbc.NewClient(threadsAddr, common.GetClientRPCOpts(threadsAddr)...) 47 | require.NoError(t, err) 48 | ipfs, err := httpapi.NewApi(GetIPFSApiMultiAddr()) 49 | require.NoError(t, err) 50 | ipnsms := tdb.NewTxMapDatastore() 51 | ipnsm, err := ipns.NewManager(ipnsms, ipfs) 52 | require.NoError(t, err) 53 | lib, err := buckets.NewBuckets(net, db, ipfs, ipnsm, nil) 54 | require.NoError(t, err) 55 | 56 | listenPort, err := freeport.GetFreePort() 57 | require.NoError(t, err) 58 | listenAddr = fmt.Sprintf("127.0.0.1:%d", listenPort) 59 | server, proxy, err := common.GetServerAndProxy(lib, listenAddr, "127.0.0.1:0") 60 | require.NoError(t, err) 61 | 62 | t.Cleanup(func() { 63 | require.NoError(t, proxy.Close()) 64 | server.Stop() 65 | require.NoError(t, lib.Close()) 66 | require.NoError(t, ipnsm.Close()) 67 | require.NoError(t, ipnsms.Close()) 68 | require.NoError(t, db.Close()) 69 | require.NoError(t, net.Close()) 70 | }) 71 | 72 | return listenAddr, doc.ID 73 | } 74 | 75 | // GetThreadsApiAddr returns env value or default. 76 | func GetThreadsApiAddr() string { 77 | env := os.Getenv("THREADS_API_ADDR") 78 | if env != "" { 79 | return env 80 | } 81 | return "127.0.0.1:4002" 82 | } 83 | 84 | // GetIPFSApiMultiAddr returns env value or default. 85 | func GetIPFSApiMultiAddr() ma.Multiaddr { 86 | env := os.Getenv("IPFS_API_MULTIADDR") 87 | if env != "" { 88 | return tutil.MustParseAddr(env) 89 | } 90 | return tutil.MustParseAddr("/ip4/127.0.0.1/tcp/5012") 91 | } 92 | 93 | // StartServices starts an ipfs and threads node for tests. 94 | func StartServices() (cleanup func()) { 95 | _, currentFilePath, _, _ := runtime.Caller(0) 96 | dirpath := path.Dir(currentFilePath) 97 | 98 | makeDown := func() { 99 | cmd := exec.Command( 100 | "docker-compose", 101 | "-f", 102 | fmt.Sprintf("%s/docker-compose.yml", dirpath), 103 | "down", 104 | "-v", 105 | "--remove-orphans", 106 | ) 107 | cmd.Stdout = os.Stdout 108 | cmd.Stderr = os.Stderr 109 | if err := cmd.Run(); err != nil { 110 | log.Fatalf("docker-compose down: %s", err) 111 | } 112 | } 113 | makeDown() 114 | 115 | cmd := exec.Command( 116 | "docker-compose", 117 | "-f", 118 | fmt.Sprintf("%s/docker-compose.yml", dirpath), 119 | "build", 120 | ) 121 | cmd.Stdout = os.Stdout 122 | cmd.Stderr = os.Stderr 123 | if err := cmd.Run(); err != nil { 124 | log.Fatalf("docker-compose build: %s", err) 125 | } 126 | cmd = exec.Command( 127 | "docker-compose", 128 | "-f", 129 | fmt.Sprintf("%s/docker-compose.yml", dirpath), 130 | "up", 131 | "-V", 132 | ) 133 | //cmd.Stdout = os.Stdout 134 | //cmd.Stderr = os.Stderr 135 | if err := cmd.Start(); err != nil { 136 | log.Fatalf("running docker-compose: %s", err) 137 | } 138 | 139 | limit := 20 140 | retries := 0 141 | var err error 142 | for retries < limit { 143 | err = checkServices() 144 | if err == nil { 145 | break 146 | } 147 | time.Sleep(time.Second) 148 | retries++ 149 | } 150 | if retries == limit { 151 | makeDown() 152 | if err != nil { 153 | log.Fatalf("connecting to services: %s", err) 154 | } 155 | log.Fatalf("max retries exhausted connecting to services") 156 | } 157 | return makeDown 158 | } 159 | 160 | func checkServices() error { 161 | ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) 162 | defer cancel() 163 | 164 | // Check Threads 165 | threadsAddr := GetThreadsApiAddr() 166 | tc, err := nc.NewClient(threadsAddr, common.GetClientRPCOpts(threadsAddr)...) 167 | if err != nil { 168 | return err 169 | } 170 | if _, err := tc.GetServices(ctx); err != nil { 171 | return err 172 | } 173 | 174 | // Check IPFS 175 | ic, err := httpapi.NewApi(GetIPFSApiMultiAddr()) 176 | if err != nil { 177 | return err 178 | } 179 | if _, err = ic.Key().Self(ctx); err != nil { 180 | return err 181 | } 182 | return nil 183 | } 184 | -------------------------------------------------------------------------------- /api/apitest/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | threads: 4 | image: textile/go-threads:f604018-m1 5 | environment: 6 | - THREADS_APIADDR=/ip4/0.0.0.0/tcp/5000 7 | - THREADS_APIPROXYADDR=/ip4/0.0.0.0/tcp/5050 8 | ports: 9 | - "4007:4006" 10 | - "4007:4006/udp" 11 | - "127.0.0.1:4002:5000" 12 | - "127.0.0.1:4052:5050" 13 | ipfs: 14 | image: textile/go-ipfs:v0.8.0-m1 15 | environment: 16 | - IPFS_PROFILE=test 17 | ports: 18 | - "127.0.0.1:5012:5001" -------------------------------------------------------------------------------- /api/cast/cast.go: -------------------------------------------------------------------------------- 1 | package cast 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "github.com/textileio/go-buckets" 7 | pb "github.com/textileio/go-buckets/api/pb/buckets" 8 | "github.com/textileio/go-buckets/collection" 9 | "github.com/textileio/go-threads/core/did" 10 | "github.com/textileio/go-threads/core/thread" 11 | ) 12 | 13 | func BucketToPb(bucket *buckets.Bucket) *pb.Bucket { 14 | pmd := make(map[string]*pb.Metadata) 15 | for p, md := range bucket.Metadata { 16 | pmd[p] = MetadataToPb(md) 17 | } 18 | return &pb.Bucket{ 19 | Thread: bucket.Thread.String(), 20 | Key: bucket.Key, 21 | Owner: string(bucket.Owner), 22 | Name: bucket.Name, 23 | Version: int32(bucket.Version), 24 | LinkKey: bucket.LinkKey, 25 | Path: bucket.Path, 26 | Metadata: pmd, 27 | CreatedAt: bucket.CreatedAt, 28 | UpdatedAt: bucket.UpdatedAt, 29 | } 30 | } 31 | 32 | func BucketFromPb(bucket *pb.Bucket) (buckets.Bucket, error) { 33 | name := "unnamed" 34 | if bucket.Name != "" { 35 | name = bucket.Name 36 | } 37 | id, err := thread.Decode(bucket.Thread) 38 | if err != nil { 39 | return buckets.Bucket{}, err 40 | } 41 | md := make(map[string]collection.Metadata) 42 | for p, m := range bucket.Metadata { 43 | md[p] = MetadataFromPb(m) 44 | } 45 | return buckets.Bucket{ 46 | Thread: id, 47 | Bucket: collection.Bucket{ 48 | Key: bucket.Key, 49 | Owner: did.DID(bucket.Owner), 50 | Name: name, 51 | Version: int(bucket.Version), 52 | LinkKey: bucket.LinkKey, 53 | Path: bucket.Path, 54 | Metadata: md, 55 | CreatedAt: bucket.CreatedAt, 56 | UpdatedAt: bucket.UpdatedAt, 57 | }, 58 | }, nil 59 | } 60 | 61 | func MetadataToPb(md collection.Metadata) *pb.Metadata { 62 | var info []byte 63 | if md.Info != nil { 64 | info, _ = json.Marshal(md.Info) 65 | } 66 | return &pb.Metadata{ 67 | Key: md.Key, 68 | Roles: RolesToPb(md.Roles), 69 | UpdatedAt: md.UpdatedAt, 70 | Info: info, 71 | } 72 | } 73 | 74 | func MetadataFromPb(md *pb.Metadata) collection.Metadata { 75 | var info map[string]interface{} 76 | if md.Info != nil { 77 | _ = json.Unmarshal(md.Info, &info) 78 | } 79 | return collection.Metadata{ 80 | Key: md.Key, 81 | Roles: RolesFromPb(md.Roles), 82 | UpdatedAt: md.UpdatedAt, 83 | Info: info, 84 | } 85 | } 86 | 87 | func ItemToPb(item *buckets.PathItem) *pb.PathItem { 88 | items := make([]*pb.PathItem, len(item.Items)) 89 | for j, i := range item.Items { 90 | items[j] = ItemToPb(&i) 91 | } 92 | return &pb.PathItem{ 93 | Cid: item.Cid, 94 | Name: item.Name, 95 | Path: item.Path, 96 | Size: item.Size, 97 | IsDir: item.IsDir, 98 | Items: items, 99 | ItemsCount: item.ItemsCount, 100 | Metadata: MetadataToPb(item.Metadata), 101 | } 102 | } 103 | 104 | func RolesToPb(roles map[did.DID]collection.Role) map[string]pb.PathAccessRole { 105 | proles := make(map[string]pb.PathAccessRole) 106 | for k, r := range roles { 107 | var pr pb.PathAccessRole 108 | switch r { 109 | case collection.ReaderRole: 110 | pr = pb.PathAccessRole_PATH_ACCESS_ROLE_READER 111 | case collection.WriterRole: 112 | pr = pb.PathAccessRole_PATH_ACCESS_ROLE_WRITER 113 | case collection.AdminRole: 114 | pr = pb.PathAccessRole_PATH_ACCESS_ROLE_ADMIN 115 | default: 116 | pr = pb.PathAccessRole_PATH_ACCESS_ROLE_UNSPECIFIED 117 | } 118 | proles[string(k)] = pr 119 | } 120 | return proles 121 | } 122 | 123 | func RolesFromPb(roles map[string]pb.PathAccessRole) map[did.DID]collection.Role { 124 | croles := make(map[did.DID]collection.Role) 125 | for k, pr := range roles { 126 | var r collection.Role 127 | switch pr { 128 | case pb.PathAccessRole_PATH_ACCESS_ROLE_READER: 129 | r = collection.ReaderRole 130 | case pb.PathAccessRole_PATH_ACCESS_ROLE_WRITER: 131 | r = collection.WriterRole 132 | case pb.PathAccessRole_PATH_ACCESS_ROLE_ADMIN: 133 | r = collection.AdminRole 134 | default: 135 | r = collection.NoneRole 136 | } 137 | croles[did.DID(k)] = r 138 | } 139 | return croles 140 | } 141 | 142 | func InfoToPb(info map[string]interface{}) ([]byte, error) { 143 | return json.Marshal(info) 144 | } 145 | 146 | func InfoFromPb(info []byte) (map[string]interface{}, error) { 147 | var pinfo map[string]interface{} 148 | if err := json.Unmarshal(info, &pinfo); err != nil { 149 | return nil, err 150 | } 151 | return pinfo, nil 152 | } 153 | 154 | func LinksToPb(links buckets.Links) *pb.Links { 155 | return &pb.Links{ 156 | Url: links.URL, 157 | Www: links.WWW, 158 | Ipns: links.IPNS, 159 | Bps: links.BPS, 160 | } 161 | } 162 | 163 | func LinksFromPb(links *pb.Links) buckets.Links { 164 | return buckets.Links{ 165 | URL: links.Url, 166 | WWW: links.Www, 167 | IPNS: links.Ipns, 168 | BPS: links.Bps, 169 | } 170 | } 171 | -------------------------------------------------------------------------------- /api/client/testdata/file1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/textileio/go-buckets/89def8d01abaafa36b18eaef8c2b78c871cfff8c/api/client/testdata/file1.jpg -------------------------------------------------------------------------------- /api/client/testdata/file2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/textileio/go-buckets/89def8d01abaafa36b18eaef8c2b78c871cfff8c/api/client/testdata/file2.jpg -------------------------------------------------------------------------------- /api/common/common.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "crypto/tls" 5 | "errors" 6 | gnet "net" 7 | "net/http" 8 | "strings" 9 | 10 | "github.com/improbable-eng/grpc-web/go/grpcweb" 11 | logging "github.com/ipfs/go-log/v2" 12 | "github.com/textileio/go-buckets" 13 | "github.com/textileio/go-buckets/api" 14 | pb "github.com/textileio/go-buckets/api/pb/buckets" 15 | "github.com/textileio/go-threads/core/did" 16 | "google.golang.org/grpc" 17 | "google.golang.org/grpc/credentials" 18 | ) 19 | 20 | var log = logging.Logger("buckets/api") 21 | 22 | func GetClientRPCOpts(target string) (opts []grpc.DialOption) { 23 | creds := did.RPCCredentials{} 24 | if strings.Contains(target, "443") { 25 | tcreds := credentials.NewTLS(&tls.Config{}) 26 | opts = append(opts, grpc.WithTransportCredentials(tcreds)) 27 | creds.Secure = true 28 | } else { 29 | opts = append(opts, grpc.WithInsecure()) 30 | } 31 | opts = append(opts, grpc.WithPerRPCCredentials(creds)) 32 | return opts 33 | } 34 | 35 | func GetServerAndProxy(lib *buckets.Buckets, listenAddr, listenAddrProxy string) (*grpc.Server, *http.Server, error) { 36 | server := grpc.NewServer() 37 | listener, err := gnet.Listen("tcp", listenAddr) 38 | if err != nil { 39 | return nil, nil, err 40 | } 41 | go func() { 42 | pb.RegisterAPIServiceServer(server, api.NewService(lib)) 43 | if err := server.Serve(listener); err != nil && !errors.Is(err, grpc.ErrServerStopped) { 44 | log.Errorf("server error: %v", err) 45 | } 46 | }() 47 | webrpc := grpcweb.WrapServer( 48 | server, 49 | grpcweb.WithOriginFunc(func(origin string) bool { 50 | return true 51 | }), 52 | grpcweb.WithAllowedRequestHeaders([]string{"Origin"}), 53 | grpcweb.WithWebsockets(true), 54 | grpcweb.WithWebsocketOriginFunc(func(req *http.Request) bool { 55 | return true 56 | })) 57 | proxy := &http.Server{ 58 | Addr: listenAddrProxy, 59 | } 60 | proxy.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 61 | if webrpc.IsGrpcWebRequest(r) || 62 | webrpc.IsAcceptableGrpcCorsRequest(r) || 63 | webrpc.IsGrpcWebSocketRequest(r) { 64 | webrpc.ServeHTTP(w, r) 65 | } 66 | }) 67 | go func() { 68 | if err := proxy.ListenAndServe(); err != nil && err != http.ErrServerClosed { 69 | log.Errorf("proxy error: %v", err) 70 | } 71 | }() 72 | return server, proxy, nil 73 | } 74 | -------------------------------------------------------------------------------- /api/pb/buckets/javascript/package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@textile/buckets-grpc", 3 | "version": "0.0.0", 4 | "lockfileVersion": 1, 5 | "requires": true, 6 | "dependencies": { 7 | "@improbable-eng/grpc-web": { 8 | "version": "0.14.0", 9 | "resolved": "https://registry.npmjs.org/@improbable-eng/grpc-web/-/grpc-web-0.14.0.tgz", 10 | "integrity": "sha512-ag1PTMWpBZKGi6GrEcZ4lkU5Qag23Xjo10BmnK9qyx4TMmSVcWmQ3rECirfQzm2uogrM9n1M6xfOpFsJP62ivA==", 11 | "requires": { 12 | "browser-headers": "^0.4.1" 13 | } 14 | }, 15 | "@types/google-protobuf": { 16 | "version": "3.7.4", 17 | "resolved": "https://registry.npmjs.org/@types/google-protobuf/-/google-protobuf-3.7.4.tgz", 18 | "integrity": "sha512-6PjMFKl13cgB4kRdYtvyjKl8VVa0PXS2IdVxHhQ8GEKbxBkyJtSbaIeK1eZGjDKN7dvUh4vkOvU9FMwYNv4GQQ==" 19 | }, 20 | "browser-headers": { 21 | "version": "0.4.1", 22 | "resolved": "https://registry.npmjs.org/browser-headers/-/browser-headers-0.4.1.tgz", 23 | "integrity": "sha512-CA9hsySZVo9371qEHjHZtYxV2cFtVj5Wj/ZHi8ooEsrtm4vOnl9Y9HmyYWk9q+05d7K3rdoAE0j3MVEFVvtQtg==" 24 | }, 25 | "google-protobuf": { 26 | "version": "3.15.3", 27 | "resolved": "https://registry.npmjs.org/google-protobuf/-/google-protobuf-3.15.3.tgz", 28 | "integrity": "sha512-+q6w/pTPS8hmdeHe+OWO7PRKOkqtPM4+dxUUOtC9lLgiaLUd7FQq+0EkTt9UmEHf2KMigkbV1fIxSr73t/JG/A==" 29 | }, 30 | "ts-protoc-gen": { 31 | "version": "0.14.0", 32 | "resolved": "https://registry.npmjs.org/ts-protoc-gen/-/ts-protoc-gen-0.14.0.tgz", 33 | "integrity": "sha512-2z6w2HioMCMVNcgNHBcEvudmQfzrn+3BjAlz+xgYZ9L0o8n8UG8WUiTJcbXHFiEg2SU8IltwH2pm1otLoMSKwg==", 34 | "dev": true, 35 | "requires": { 36 | "google-protobuf": "^3.6.1" 37 | } 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /api/pb/buckets/javascript/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@textile/buckets-grpc", 3 | "version": "0.0.0", 4 | "description": "A client for interacting with the Buckets gRPC API.", 5 | "repository": { 6 | "type": "git", 7 | "url": "git+https://github.com/textileio/go-buckets.git" 8 | }, 9 | "author": "Textile", 10 | "license": "MIT", 11 | "files": [ 12 | "api/pb/buckets/buckets_pb.js", 13 | "api/pb/buckets/buckets_pb_service.js", 14 | "api/pb/buckets/buckets_pb.d.ts", 15 | "api/pb/buckets/buckets_pb_service.d.ts" 16 | ], 17 | "dependencies": { 18 | "@improbable-eng/grpc-web": "^0.14.0", 19 | "@types/google-protobuf": "^3.7.4", 20 | "google-protobuf": "^3.15.3" 21 | }, 22 | "devDependencies": { 23 | "ts-protoc-gen": "^0.14.0" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /buf.yaml: -------------------------------------------------------------------------------- 1 | build: 2 | roots: 3 | - . 4 | excludes: 5 | - api/pb/buckets/javascript 6 | lint: 7 | use: 8 | - DEFAULT 9 | except: 10 | - PACKAGE_VERSION_SUFFIX 11 | breaking: 12 | use: 13 | - FILE 14 | -------------------------------------------------------------------------------- /buildinfo/buildinfo.go: -------------------------------------------------------------------------------- 1 | package buildinfo 2 | 3 | import "fmt" 4 | 5 | var ( 6 | // GitCommit is set by govvv at build time. 7 | GitCommit = "" 8 | // GitBranch is set by govvv at build time. 9 | GitBranch = "" 10 | // GitState is set by govvv at build time. 11 | GitState = "" 12 | // GitSummary is set by govvv at build time. 13 | GitSummary = "" 14 | // BuildDate is set by govvv at build time. 15 | BuildDate = "" 16 | // Version is set by govvv at build time. 17 | Version = "git" 18 | ) 19 | 20 | // Summary prints a summary of all build info. 21 | func Summary() string { 22 | return fmt.Sprintf( 23 | "\tversion:\t%s\n\tbuild date:\t%s\n\tgit summary:\t%s\n\tgit branch:\t%s\n\tgit commit:\t%s\n\tgit state:\t%s", 24 | Version, 25 | BuildDate, 26 | GitSummary, 27 | GitBranch, 28 | GitCommit, 29 | GitState, 30 | ) 31 | } 32 | -------------------------------------------------------------------------------- /buildtools/install_protoc.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eo pipefail 3 | 4 | if [ ! -d ./protoc ]; then 5 | OS=$(uname) 6 | if [ $OS = "Darwin" ]; then 7 | OS="osx" 8 | fi 9 | VERSION=3.13.0 10 | ZIPNAME=protoc-$VERSION-$OS-x86_64 11 | DOWNLOADLINK=https://github.com/protocolbuffers/protobuf/releases/download/v$VERSION/$ZIPNAME.zip 12 | curl -LO $DOWNLOADLINK 13 | unzip $ZIPNAME.zip -d protoc 14 | rm $ZIPNAME.zip 15 | fi 16 | 17 | if [ ! -d ./protoc-gen-go ]; then 18 | git clone --single-branch --depth 1 --branch "v1.4.3" https://github.com/golang/protobuf.git 19 | cd protobuf 20 | go build -o ../protoc-gen-go/protoc-gen-go ./protoc-gen-go 21 | cd .. 22 | rm -rf protobuf 23 | fi 24 | -------------------------------------------------------------------------------- /cmd/buck/cli/add.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/ipfs/go-cid" 7 | "github.com/manifoldco/promptui" 8 | "github.com/spf13/cobra" 9 | "github.com/textileio/go-buckets/cmd" 10 | "github.com/textileio/go-buckets/local" 11 | ) 12 | 13 | var addCmd = &cobra.Command{ 14 | Use: "add [cid] [path]", 15 | Short: "Adds a UnixFs DAG locally at path", 16 | Long: `Adds a UnixFs DAG locally at path, merging with existing content.`, 17 | Args: cobra.ExactArgs(2), 18 | Run: func(c *cobra.Command, args []string) { 19 | yes, err := c.Flags().GetBool("yes") 20 | cmd.ErrCheck(err) 21 | target, err := cid.Decode(args[0]) 22 | cmd.ErrCheck(err) 23 | conf, err := bucks.NewConfigFromCmd(c, ".") 24 | cmd.ErrCheck(err) 25 | ctx, cancel := context.WithTimeout(context.Background(), cmd.PushTimeout) 26 | defer cancel() 27 | buck, err := bucks.GetLocalBucket(ctx, conf) 28 | cmd.ErrCheck(err) 29 | events := make(chan local.Event) 30 | defer close(events) 31 | go handleEvents(events) 32 | err = buck.AddRemoteCid( 33 | ctx, 34 | target, 35 | args[1], 36 | local.WithSelectMerge(getSelectMergeStrategy(yes)), 37 | local.WithAddEvents(events), 38 | ) 39 | cmd.ErrCheck(err) 40 | cmd.Success("Merged %s with %s", target, args[1]) 41 | }, 42 | } 43 | 44 | func getSelectMergeStrategy(auto bool) local.SelectMergeFunc { 45 | return func(desc string, isDir bool) (s local.MergeStrategy, err error) { 46 | if isDir { 47 | if auto { 48 | return local.Merge, nil 49 | } 50 | prompt := promptui.Select{ 51 | Label: desc, 52 | Items: []local.MergeStrategy{local.Skip, local.Merge, local.Replace}, 53 | } 54 | _, res, err := prompt.Run() 55 | if err != nil { 56 | return s, err 57 | } 58 | return local.MergeStrategy(res), nil 59 | } else { 60 | if auto { 61 | return local.Replace, nil 62 | } 63 | prompt := promptui.Prompt{ 64 | Label: desc, 65 | IsConfirm: true, 66 | } 67 | if _, err := prompt.Run(); err != nil { 68 | return local.Skip, nil 69 | } 70 | return local.Replace, nil 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /cmd/buck/cli/init.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | 8 | cid "github.com/ipfs/go-cid" 9 | "github.com/manifoldco/promptui" 10 | "github.com/spf13/cobra" 11 | "github.com/textileio/go-buckets/cmd" 12 | "github.com/textileio/go-buckets/local" 13 | ) 14 | 15 | var initCmd = &cobra.Command{ 16 | Use: "init", 17 | Short: "Initialize a new or existing bucket", 18 | Long: `Initializes a new or existing bucket. 19 | 20 | A .textile config directory and a seed file will be created in the current working directory. 21 | Existing configs will not be overwritten. 22 | 23 | Use the '--existing' flag to interactively select an existing remote bucket. 24 | Use the '--cid' flag to initialize from an existing UnixFS DAG. 25 | Use the '--unfreeze' flag to retrieve '--cid' from known or imported deals. 26 | 27 | By default, if the remote bucket exists, remote objects are pulled and merged with local changes. 28 | Use the '--soft' flag to accept all local changes, including deletions. 29 | Use the '--hard' flag to discard all local changes. 30 | `, 31 | Args: cobra.ExactArgs(0), 32 | Run: func(c *cobra.Command, args []string) { 33 | conf, err := bucks.NewConfigFromCmd(c, ".") 34 | cmd.ErrCheck(err) 35 | 36 | quiet, err := c.Flags().GetBool("quiet") 37 | cmd.ErrCheck(err) 38 | 39 | existing := conf.Thread.Defined() && conf.Key != "" 40 | chooseExisting, err := c.Flags().GetBool("existing") 41 | cmd.ErrCheck(err) 42 | if existing && chooseExisting { 43 | chooseExisting = false // Nothing left to choose 44 | } 45 | 46 | var strategy local.InitStrategy 47 | soft, err := c.Flags().GetBool("soft") 48 | cmd.ErrCheck(err) 49 | hard, err := c.Flags().GetBool("hard") 50 | cmd.ErrCheck(err) 51 | if soft && hard { 52 | cmd.Fatal(errors.New("--soft and --hard cannot by used together")) 53 | } 54 | if soft { 55 | strategy = local.Soft 56 | } else if hard { 57 | strategy = local.Hard 58 | } else { 59 | strategy = local.Hybrid 60 | } 61 | 62 | var xcid cid.Cid 63 | xcids, err := c.Flags().GetString("cid") 64 | cmd.ErrCheck(err) 65 | if xcids != "" { 66 | xcid, err = cid.Decode(xcids) 67 | cmd.ErrCheck(err) 68 | } 69 | if (existing || chooseExisting) && xcid.Defined() { 70 | cmd.Fatal(errors.New("--cid cannot be used with an existing bucket")) 71 | } 72 | 73 | var name string 74 | var private bool 75 | if !existing && !chooseExisting { 76 | if c.Flags().Changed("name") { 77 | name, err = c.Flags().GetString("name") 78 | cmd.ErrCheck(err) 79 | } else { 80 | namep := promptui.Prompt{ 81 | Label: "Enter a name for your new bucket (optional)", 82 | } 83 | name, err = namep.Run() 84 | if err != nil { 85 | cmd.End("") 86 | } 87 | } 88 | if c.Flags().Changed("private") { 89 | private, err = c.Flags().GetBool("private") 90 | cmd.ErrCheck(err) 91 | } else { 92 | privp := promptui.Prompt{ 93 | Label: "Encrypt bucket contents", 94 | IsConfirm: true, 95 | } 96 | if _, err = privp.Run(); err == nil { 97 | private = true 98 | } 99 | } 100 | } 101 | 102 | if chooseExisting { 103 | ctx, cancel := context.WithTimeout(context.Background(), cmd.Timeout) 104 | defer cancel() 105 | list, err := bucks.RemoteBuckets(ctx, conf.Thread, conf.Identity) 106 | cmd.ErrCheck(err) 107 | if len(list) == 0 { 108 | cmd.Fatal(fmt.Errorf("no existing buckets found")) 109 | } 110 | prompt := promptui.Select{ 111 | Label: "Which existing bucket do you want to init from?", 112 | Items: list, 113 | Templates: &promptui.SelectTemplates{ 114 | Active: fmt.Sprintf(`{{ "%s" | cyan }} {{ .Name | bold }} {{ .Key | faint | bold }}`, 115 | promptui.IconSelect), 116 | Inactive: `{{ .Name | faint }} {{ .Key | faint | bold }}`, 117 | Selected: aurora.Sprintf(aurora.BrightBlack("> Selected bucket {{ .Name | white | bold }}")), 118 | }, 119 | } 120 | index, _, err := prompt.Run() 121 | if err != nil { 122 | cmd.End("") 123 | } 124 | selected := list[index] 125 | name = selected.Name 126 | conf.Thread = selected.Thread 127 | conf.Key = selected.Key 128 | existing = true 129 | } 130 | 131 | if !conf.Thread.Defined() { 132 | //ctx, cancel := context.WithTimeout(bucks.Context(context.Background()), cmd.Timeout) 133 | //defer cancel() 134 | //selected := bucks.Clients().SelectThread( 135 | // ctx, 136 | // "Buckets are written to a threadDB. Select or create a new one", 137 | // aurora.Sprintf(aurora.BrightBlack("> Selected threadDB {{ .Label | white | bold }}")), 138 | // true) 139 | //if selected.Label == "Create new" { 140 | // if selected.Name == "" { 141 | // prompt := promptui.Prompt{ 142 | // Label: "Enter a name for your new threadDB (optional)", 143 | // } 144 | // selected.Name, err = prompt.Run() 145 | // if err != nil { 146 | // cmd.End("") 147 | // } 148 | // } 149 | // ctx = common.NewThreadNameContext(ctx, selected.Name) 150 | // conf.Thread = thread.NewIDV1(thread.Raw, 32) 151 | // err = bucks.Clients().Threads.NewDB(ctx, conf.Thread, db.WithNewManagedName(selected.Name)) 152 | // cmd.ErrCheck(err) 153 | //} else { 154 | // conf.Thread = selected.ID 155 | //} 156 | } 157 | 158 | ctx, cancel := context.WithTimeout(context.Background(), cmd.Timeout) 159 | defer cancel() 160 | 161 | var events chan local.Event 162 | if !quiet { 163 | events = make(chan local.Event) 164 | defer close(events) 165 | go handleEvents(events) 166 | } 167 | buck, err := bucks.NewBucket( 168 | ctx, 169 | conf, 170 | local.WithName(name), 171 | local.WithPrivate(private), 172 | local.WithCid(xcid), 173 | local.WithStrategy(strategy), 174 | local.WithInitEvents(events)) 175 | cmd.ErrCheck(err) 176 | 177 | links, err := buck.RemoteLinks(ctx, "") 178 | cmd.ErrCheck(err) 179 | printLinks(links, DefaultFormat) 180 | 181 | var msg string 182 | if !existing { 183 | msg = "Initialized %s as a new empty bucket" 184 | if xcid.Defined() { 185 | msg = "Initialized %s as a new bootstrapped bucket" 186 | } 187 | } else { 188 | msg = "Initialized %s from an existing bucket" 189 | } 190 | 191 | bp, err := buck.Path() 192 | cmd.ErrCheck(err) 193 | cmd.Success(msg, aurora.White(bp).Bold()) 194 | }, 195 | } 196 | -------------------------------------------------------------------------------- /cmd/buck/cli/pinning.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "io" 8 | "os" 9 | "os/exec" 10 | "runtime" 11 | "time" 12 | 13 | "github.com/ipfs/go-cid" 14 | "github.com/spf13/cobra" 15 | "github.com/textileio/go-buckets/cmd" 16 | ) 17 | 18 | var pinningCmd = &cobra.Command{ 19 | Use: "pinning", 20 | Short: "Manage IPFS pinning service", 21 | Long: `Enable/disable bucket as an IPFS Pinning Service.`, 22 | Args: cobra.ExactArgs(0), 23 | } 24 | 25 | var pinningEnableCmd = &cobra.Command{ 26 | Use: "enable", 27 | Short: "Enable bucket as an IPFS Pinning Service", 28 | Long: `Enables the bucket as an IPFS Pinning Service for a locally running IPFS node.`, 29 | Args: cobra.ExactArgs(0), 30 | Run: func(c *cobra.Command, args []string) { 31 | conf, err := bucks.NewConfigFromCmd(c, ".") 32 | cmd.ErrCheck(err) 33 | ctx, cancel := context.WithTimeout(context.Background(), cmd.Timeout) 34 | defer cancel() 35 | buck, err := bucks.GetLocalBucket(ctx, conf) 36 | cmd.ErrCheck(err) 37 | 38 | token, err := buck.GetIdentityToken(time.Hour * 24 * 30) 39 | cmd.ErrCheck(err) 40 | 41 | links, err := buck.RemoteLinks(ctx, "") 42 | cmd.ErrCheck(err) 43 | 44 | doExec( 45 | fmt.Sprintf("ipfs pin remote service add %s %s %s", buck.Key(), links.BPS, string(token)), 46 | nil) 47 | 48 | cmd.Success("Enabled bucket as pinning service: %s", aurora.White(buck.Key()).Bold()) 49 | }, 50 | } 51 | 52 | var pinningDisableCmd = &cobra.Command{ 53 | Use: "disable", 54 | Short: "Disable bucket as an IPFS Pinning Service", 55 | Long: `Disables the bucket as an IPFS Pinning Service for a locally running IPFS node.`, 56 | Args: cobra.ExactArgs(0), 57 | Run: func(c *cobra.Command, args []string) { 58 | conf, err := bucks.NewConfigFromCmd(c, ".") 59 | cmd.ErrCheck(err) 60 | ctx, cancel := context.WithTimeout(context.Background(), cmd.Timeout) 61 | defer cancel() 62 | buck, err := bucks.GetLocalBucket(ctx, conf) 63 | cmd.ErrCheck(err) 64 | 65 | doExec(fmt.Sprintf("ipfs pin remote service rm %s", buck.Key()), nil) 66 | 67 | cmd.Success("Disabled bucket as pinning service") 68 | }, 69 | } 70 | 71 | var pinningAddCmd = &cobra.Command{ 72 | Use: "add [path]", 73 | Short: "Add path to local IPFS node and pin to bucket", 74 | Long: `Adds the path to the locally running IPFS node and pins the resulting CID to the bucket`, 75 | Args: cobra.ExactArgs(1), 76 | Run: func(c *cobra.Command, args []string) { 77 | name, err := c.Flags().GetString("name") 78 | cmd.ErrCheck(err) 79 | 80 | conf, err := bucks.NewConfigFromCmd(c, ".") 81 | cmd.ErrCheck(err) 82 | ctx, cancel := context.WithTimeout(context.Background(), cmd.Timeout) 83 | defer cancel() 84 | buck, err := bucks.GetLocalBucket(ctx, conf) 85 | cmd.ErrCheck(err) 86 | 87 | var buf bytes.Buffer 88 | doExec(fmt.Sprintf("ipfs add -Qr --cid-version=1 %s", args[0]), &buf) 89 | 90 | doExec( 91 | fmt.Sprintf("ipfs pin remote add --background --service=%s --name=%s %s", 92 | buck.Key(), 93 | name, 94 | buf.String(), 95 | ), 96 | os.Stdout, 97 | ) 98 | }, 99 | } 100 | 101 | var pinningRmCmd = &cobra.Command{ 102 | Use: "rm [name|CID]", 103 | Short: "Remove pin by name or CID from bucket", 104 | Long: "Removes a pin by name or CID from the bucket.", 105 | Args: cobra.ExactArgs(1), 106 | Run: func(c *cobra.Command, args []string) { 107 | conf, err := bucks.NewConfigFromCmd(c, ".") 108 | cmd.ErrCheck(err) 109 | ctx, cancel := context.WithTimeout(context.Background(), cmd.Timeout) 110 | defer cancel() 111 | buck, err := bucks.GetLocalBucket(ctx, conf) 112 | cmd.ErrCheck(err) 113 | 114 | var match string 115 | if _, err := cid.Decode(args[0]); err == nil { 116 | match = fmt.Sprintf("--cid=%s", args[0]) 117 | } else { 118 | match = fmt.Sprintf("--name=%s", args[0]) 119 | } 120 | 121 | doExec( 122 | fmt.Sprintf( 123 | "ipfs pin remote rm --service=%s --status=queued,pinning,pinned,failed %s", 124 | buck.Key(), 125 | match, 126 | ), 127 | os.Stdout, 128 | ) 129 | }, 130 | } 131 | 132 | var pinningLsCmd = &cobra.Command{ 133 | Use: "ls", 134 | Short: "Disable bucket as an IPFS Pinning Service", 135 | Long: `Disables the bucket as an IPFS Pinning Service for a locally running IPFS node.`, 136 | Args: cobra.ExactArgs(0), 137 | Run: func(c *cobra.Command, args []string) { 138 | conf, err := bucks.NewConfigFromCmd(c, ".") 139 | cmd.ErrCheck(err) 140 | ctx, cancel := context.WithTimeout(context.Background(), cmd.Timeout) 141 | defer cancel() 142 | buck, err := bucks.GetLocalBucket(ctx, conf) 143 | cmd.ErrCheck(err) 144 | 145 | doExec( 146 | fmt.Sprintf("ipfs pin remote ls --service=%s --status=queued,pinning,pinned,failed", buck.Key()), 147 | os.Stdout) 148 | }, 149 | } 150 | 151 | func doExec(c string, out io.Writer) { 152 | var com *exec.Cmd 153 | if runtime.GOOS == "windows" { 154 | com = exec.Command("cmd", "/C", c) 155 | } else { 156 | com = exec.Command("bash", "-c", c) 157 | } 158 | if out != nil { 159 | com.Stdout = out 160 | } 161 | err := com.Run() 162 | cmd.ErrCheck(err) 163 | } 164 | -------------------------------------------------------------------------------- /cmd/buck/cli/pull.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | 7 | "github.com/spf13/cobra" 8 | "github.com/textileio/go-buckets/cmd" 9 | "github.com/textileio/go-buckets/local" 10 | ) 11 | 12 | var pullCmd = &cobra.Command{ 13 | Use: "pull", 14 | Short: "Pull bucket object changes", 15 | Long: `Pulls paths that have been added to and paths that have been removed or differ from the remote bucket root. 16 | 17 | Use the '--hard' flag to discard all local changes. 18 | Use the '--force' flag to pull all remote objects, even if they already exist locally. 19 | `, 20 | Args: cobra.ExactArgs(0), 21 | Run: func(c *cobra.Command, args []string) { 22 | force, err := c.Flags().GetBool("force") 23 | cmd.ErrCheck(err) 24 | hard, err := c.Flags().GetBool("hard") 25 | cmd.ErrCheck(err) 26 | yes, err := c.Flags().GetBool("yes") 27 | cmd.ErrCheck(err) 28 | quiet, err := c.Flags().GetBool("quiet") 29 | cmd.ErrCheck(err) 30 | conf, err := bucks.NewConfigFromCmd(c, ".") 31 | cmd.ErrCheck(err) 32 | ctx, cancel := context.WithTimeout(context.Background(), cmd.PullTimeout) 33 | defer cancel() 34 | buck, err := bucks.GetLocalBucket(ctx, conf) 35 | cmd.ErrCheck(err) 36 | var events chan local.Event 37 | if !quiet { 38 | events = make(chan local.Event) 39 | defer close(events) 40 | go handleEvents(events) 41 | } 42 | roots, err := buck.PullRemote( 43 | ctx, 44 | local.WithConfirm(getConfirm("Discard %d local changes", yes)), 45 | local.WithForce(force), 46 | local.WithHard(hard), 47 | local.WithEvents(events)) 48 | if errors.Is(err, local.ErrAborted) { 49 | cmd.End("") 50 | } else if errors.Is(err, local.ErrUpToDate) { 51 | cmd.End("Everything up-to-date") 52 | } else if err != nil { 53 | cmd.Fatal(err) 54 | } 55 | cmd.Message("%s", aurora.White(roots.Remote).Bold()) 56 | }, 57 | } 58 | -------------------------------------------------------------------------------- /cmd/buck/cli/push.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "strings" 7 | 8 | "github.com/spf13/cobra" 9 | "github.com/textileio/go-buckets" 10 | "github.com/textileio/go-buckets/cmd" 11 | "github.com/textileio/go-buckets/local" 12 | ) 13 | 14 | const nonFastForwardMsg = "the root of your bucket is behind (try `%s` before pushing again)" 15 | 16 | var pushCmd = &cobra.Command{ 17 | Use: "push", 18 | Short: "Push bucket object changes", 19 | Long: `Pushes paths that have been added to and paths that have been removed or differ from the local bucket root. 20 | 21 | Use the '--force' flag to allow a non-fast-forward update. 22 | `, 23 | Args: cobra.ExactArgs(0), 24 | Run: func(c *cobra.Command, args []string) { 25 | force, err := c.Flags().GetBool("force") 26 | cmd.ErrCheck(err) 27 | yes, err := c.Flags().GetBool("yes") 28 | cmd.ErrCheck(err) 29 | quiet, err := c.Flags().GetBool("quiet") 30 | cmd.ErrCheck(err) 31 | conf, err := bucks.NewConfigFromCmd(c, ".") 32 | cmd.ErrCheck(err) 33 | ctx, cancel := context.WithTimeout(context.Background(), cmd.PushTimeout) 34 | defer cancel() 35 | buck, err := bucks.GetLocalBucket(ctx, conf) 36 | cmd.ErrCheck(err) 37 | 38 | var events chan local.Event 39 | if !quiet { 40 | events = make(chan local.Event) 41 | defer close(events) 42 | go handleEvents(events) 43 | } 44 | roots, err := buck.PushLocal( 45 | ctx, 46 | local.WithConfirm(getConfirm("Push %d changes", yes)), 47 | local.WithForce(force), 48 | local.WithEvents(events), 49 | ) 50 | if errors.Is(err, local.ErrAborted) { 51 | cmd.End("") 52 | } else if errors.Is(err, local.ErrUpToDate) { 53 | cmd.End("Everything up-to-date") 54 | } else if err != nil && strings.Contains(err.Error(), buckets.ErrNonFastForward.Error()) { 55 | cmd.Fatal(errors.New(nonFastForwardMsg), aurora.Cyan("buck pull")) 56 | } else if err != nil { 57 | cmd.Fatal(err) 58 | } 59 | cmd.Message("%s", aurora.White(roots.Remote).Bold()) 60 | }, 61 | } 62 | -------------------------------------------------------------------------------- /cmd/buck/cli/roles.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/manifoldco/promptui" 8 | "github.com/spf13/cobra" 9 | "github.com/textileio/go-buckets/cmd" 10 | "github.com/textileio/go-buckets/collection" 11 | "github.com/textileio/go-threads/core/did" 12 | ) 13 | 14 | var rolesCmd = &cobra.Command{ 15 | Use: "roles", 16 | Aliases: []string{ 17 | "role", 18 | }, 19 | Short: "Object access role management", 20 | Long: `Manages remote bucket object access roles.`, 21 | Args: cobra.ExactArgs(0), 22 | } 23 | 24 | var rolesGrantCmd = &cobra.Command{ 25 | Use: "grant [identity] [path]", 26 | Short: "Grant remote object access roles", 27 | Long: `Grants remote object access roles to an identity. 28 | 29 | Identity must be a multibase encoded public key. A "*" value will set the default access role for an object. 30 | 31 | Access roles: 32 | "none": Revokes all access. 33 | "reader": Grants read-only access. 34 | "writer": Grants read and write access. 35 | "admin": Grants read, write, delete and role editing access. 36 | `, 37 | Args: cobra.RangeArgs(1, 2), 38 | Run: func(c *cobra.Command, args []string) { 39 | roleStr, err := c.Flags().GetString("role") 40 | cmd.ErrCheck(err) 41 | conf, err := bucks.NewConfigFromCmd(c, ".") 42 | cmd.ErrCheck(err) 43 | ctx, cancel := context.WithTimeout(context.Background(), cmd.Timeout) 44 | defer cancel() 45 | buck, err := bucks.GetLocalBucket(ctx, conf) 46 | cmd.ErrCheck(err) 47 | if roleStr == "" { 48 | roles := []string{"None", "Reader", "Writer", "Admin"} 49 | prompt := promptui.Select{ 50 | Label: "Select a role", 51 | Items: roles, 52 | Templates: &promptui.SelectTemplates{ 53 | Active: fmt.Sprintf(`{{ "%s" | cyan }} {{ . | bold }}`, promptui.IconSelect), 54 | Inactive: `{{ . | faint }}`, 55 | Selected: aurora.Sprintf(aurora.BrightBlack("> Selected role {{ . | white | bold }}")), 56 | }, 57 | } 58 | index, _, err := prompt.Run() 59 | if err != nil { 60 | cmd.End("") 61 | } 62 | roleStr = roles[index] 63 | } 64 | role, err := collection.NewRoleFromString(roleStr) 65 | if err != nil { 66 | cmd.Err(fmt.Errorf("access role must be one of: none, reader, writer, or admin")) 67 | } 68 | var pth string 69 | if len(args) > 1 { 70 | pth = args[1] 71 | } 72 | res, err := buck.PushPathAccessRoles(ctx, pth, map[did.DID]collection.Role{ 73 | did.DID(args[0]): role, 74 | }) 75 | cmd.ErrCheck(err) 76 | var data [][]string 77 | if len(res) > 0 { 78 | for i, r := range res { 79 | data = append(data, []string{string(i), r.String()}) 80 | } 81 | } 82 | if len(data) > 0 { 83 | cmd.RenderTable([]string{"identity", "role"}, data) 84 | } 85 | cmd.Success("Updated access roles for path %s", aurora.White(pth).Bold()) 86 | }, 87 | } 88 | 89 | var rolesLsCmd = &cobra.Command{ 90 | Use: "ls [path]", 91 | Aliases: []string{ 92 | "list", 93 | }, 94 | Short: "List top-level or nested bucket object access roles", 95 | Long: `Lists top-level or nested bucket object access roles.`, 96 | Args: cobra.MaximumNArgs(1), 97 | Run: func(c *cobra.Command, args []string) { 98 | conf, err := bucks.NewConfigFromCmd(c, ".") 99 | cmd.ErrCheck(err) 100 | ctx, cancel := context.WithTimeout(context.Background(), cmd.Timeout) 101 | defer cancel() 102 | buck, err := bucks.GetLocalBucket(ctx, conf) 103 | cmd.ErrCheck(err) 104 | var pth string 105 | if len(args) > 0 { 106 | pth = args[0] 107 | } 108 | res, err := buck.PullPathAccessRoles(ctx, pth) 109 | cmd.ErrCheck(err) 110 | var data [][]string 111 | if len(res) > 0 { 112 | for i, r := range res { 113 | data = append(data, []string{string(i), r.String()}) 114 | } 115 | } 116 | if len(data) > 0 { 117 | cmd.RenderTable([]string{"identity", "role"}, data) 118 | } 119 | cmd.Message("Found %d access roles", aurora.White(len(data)).Bold()) 120 | }, 121 | } 122 | -------------------------------------------------------------------------------- /cmd/buck/cli/util.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "runtime" 7 | 8 | pb "github.com/cheggaaa/pb/v3" 9 | "github.com/manifoldco/promptui" 10 | "github.com/textileio/go-buckets/cmd" 11 | "github.com/textileio/go-buckets/local" 12 | ) 13 | 14 | func getConfirm(label string, auto bool) local.ConfirmDiffFunc { 15 | return func(diff []local.Change) bool { 16 | if auto { 17 | return true 18 | } 19 | for _, c := range diff { 20 | cf := local.ChangeColor(c.Type) 21 | cmd.Message("%s %s", cf(local.ChangeType(c.Type)), cf(c.Rel)) 22 | } 23 | prompt := promptui.Prompt{ 24 | Label: fmt.Sprintf(label, len(diff)), 25 | IsConfirm: true, 26 | } 27 | if _, err := prompt.Run(); err != nil { 28 | return false 29 | } 30 | return true 31 | } 32 | } 33 | 34 | func handleEvents(events chan local.Event) { 35 | var bar *pb.ProgressBar 36 | if runtime.GOOS != "windows" { 37 | bar = pb.New(0) 38 | bar.Set(pb.Bytes, true) 39 | tmp := `{{string . "prefix"}}{{counters . }} {{bar . "[" "=" ">" "-" "]"}} {{percent . }} {{etime . }}{{string . "suffix"}}` 40 | bar.SetTemplate(pb.ProgressBarTemplate(tmp)) 41 | } 42 | 43 | clear := func() { 44 | if bar != nil { 45 | _, _ = fmt.Fprintf(os.Stderr, "\033[2K\r") 46 | } 47 | } 48 | 49 | for e := range events { 50 | switch e.Type { 51 | case local.EventProgress: 52 | if bar == nil { 53 | continue 54 | } 55 | bar.SetTotal(e.Size) 56 | bar.SetCurrent(e.Complete) 57 | if !bar.IsStarted() { 58 | bar.Start() 59 | } 60 | bar.Write() 61 | case local.EventFileComplete: 62 | clear() 63 | _, _ = fmt.Fprintf(os.Stdout, 64 | "+ %s %s %s\n", 65 | e.Cid, 66 | e.Path, 67 | formatBytes(e.Size, false), 68 | ) 69 | if bar != nil && bar.IsStarted() { 70 | bar.Write() 71 | } 72 | case local.EventFileRemoved: 73 | clear() 74 | _, _ = fmt.Fprintf(os.Stdout, "- %s\n", e.Path) 75 | if bar != nil && bar.IsStarted() { 76 | bar.Write() 77 | } 78 | } 79 | } 80 | } 81 | 82 | // Copied from https://github.com/cheggaaa/pb/blob/master/v3/util.go 83 | const ( 84 | _KiB = 1024 85 | _MiB = 1048576 86 | _GiB = 1073741824 87 | _TiB = 1099511627776 88 | 89 | _kB = 1e3 90 | _MB = 1e6 91 | _GB = 1e9 92 | _TB = 1e12 93 | ) 94 | 95 | // Copied from https://github.com/cheggaaa/pb/blob/master/v3/util.go 96 | func formatBytes(i int64, useSIPrefix bool) (result string) { 97 | if !useSIPrefix { 98 | switch { 99 | case i >= _TiB: 100 | result = fmt.Sprintf("%.02f TiB", float64(i)/_TiB) 101 | case i >= _GiB: 102 | result = fmt.Sprintf("%.02f GiB", float64(i)/_GiB) 103 | case i >= _MiB: 104 | result = fmt.Sprintf("%.02f MiB", float64(i)/_MiB) 105 | case i >= _KiB: 106 | result = fmt.Sprintf("%.02f KiB", float64(i)/_KiB) 107 | default: 108 | result = fmt.Sprintf("%d B", i) 109 | } 110 | } else { 111 | switch { 112 | case i >= _TB: 113 | result = fmt.Sprintf("%.02f TB", float64(i)/_TB) 114 | case i >= _GB: 115 | result = fmt.Sprintf("%.02f GB", float64(i)/_GB) 116 | case i >= _MB: 117 | result = fmt.Sprintf("%.02f MB", float64(i)/_MB) 118 | case i >= _kB: 119 | result = fmt.Sprintf("%.02f kB", float64(i)/_kB) 120 | default: 121 | result = fmt.Sprintf("%d B", i) 122 | } 123 | } 124 | return 125 | } 126 | -------------------------------------------------------------------------------- /cmd/buck/cli/watch.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/textileio/go-buckets/local" 8 | ) 9 | 10 | //var watchCmd = &cobra.Command{ 11 | // Use: "watch", 12 | // Short: "Watch auto-pushes local changes to the remote", 13 | // Long: `Watch auto-pushes local changes to the remote.`, 14 | // Args: cobra.ExactArgs(0), 15 | // Run: func(c *cobra.Command, args []string) { 16 | // conf, err := bucks.NewConfigFromCmd(c, ".") 17 | // cmd.ErrCheck(err) 18 | // ctx, cancel := context.WithCancel(context.Background()) 19 | // defer cancel() 20 | // buck, err := bucks.GetLocalBucket(ctx, conf) 21 | // cmd.ErrCheck(err) 22 | // bp, err := buck.Path() 23 | // cmd.ErrCheck(err) 24 | // events := make(chan local.Event) 25 | // defer close(events) 26 | // go handleWatchEvents(events) 27 | // state, err := buck.Watch(ctx, local.WithWatchEvents(events), local.WithOffline(true)) 28 | // cmd.ErrCheck(err) 29 | // for s := range state { 30 | // switch s.State { 31 | // case cmd.Online: 32 | // cmd.Success("Watching %s for changes...", aurora.White(bp).Bold()) 33 | // case cmd.Offline: 34 | // if s.Aborted { 35 | // cmd.Fatal(s.Err) 36 | // } else { 37 | // cmd.Message("Not connected. Trying to connect...") 38 | // } 39 | // } 40 | // } 41 | // }, 42 | //} 43 | 44 | func handleWatchEvents(events chan local.Event) { 45 | for e := range events { 46 | switch e.Type { 47 | case local.EventFileComplete: 48 | _, _ = fmt.Fprintf(os.Stdout, 49 | "+ %s %s %s\n", 50 | e.Cid, 51 | e.Path, 52 | formatBytes(e.Size, false), 53 | ) 54 | case local.EventFileRemoved: 55 | _, _ = fmt.Fprintf(os.Stdout, "- %s\n", e.Path) 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /cmd/buck/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/spf13/cobra" 5 | bc "github.com/textileio/go-buckets/api/client" 6 | "github.com/textileio/go-buckets/api/common" 7 | "github.com/textileio/go-buckets/cmd" 8 | buck "github.com/textileio/go-buckets/cmd/buck/cli" 9 | "github.com/textileio/go-buckets/local" 10 | ) 11 | 12 | const defaultTarget = "127.0.0.1:5000" 13 | 14 | var client *bc.Client 15 | 16 | func init() { 17 | buck.Init(rootCmd) 18 | 19 | rootCmd.PersistentFlags().String("api", defaultTarget, "API target") 20 | } 21 | 22 | func main() { 23 | cmd.ErrCheck(rootCmd.Execute()) 24 | } 25 | 26 | var rootCmd = &cobra.Command{ 27 | Use: buck.Name, 28 | Short: "Bucket Client", 29 | Long: `The Bucket Client. 30 | 31 | Manages files and folders in an object storage bucket.`, 32 | PersistentPreRun: func(c *cobra.Command, args []string) { 33 | config := local.DefaultConfConfig() 34 | target := cmd.GetFlagOrEnvValue(c, "api", config.EnvPrefix) 35 | 36 | var err error 37 | client, err = bc.NewClient(target, common.GetClientRPCOpts(target)...) 38 | cmd.ErrCheck(err) 39 | buck.SetBucks(local.NewBuckets(client, config)) 40 | }, 41 | PersistentPostRun: func(c *cobra.Command, args []string) { 42 | cmd.ErrCheck(client.Close()) 43 | }, 44 | Args: cobra.ExactArgs(0), 45 | } 46 | -------------------------------------------------------------------------------- /cmd/buckd/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.16.0-buster 2 | MAINTAINER Textile 3 | 4 | # This is (in large part) copied (with love) from 5 | # https://hub.docker.com/r/ipfs/go-ipfs/dockerfile 6 | 7 | # Install deps 8 | RUN apt-get update && apt-get install -y \ 9 | libssl-dev \ 10 | ca-certificates 11 | 12 | ENV SRC_DIR /go-buckets 13 | 14 | # Download packages first so they can be cached. 15 | COPY go.mod go.sum $SRC_DIR/ 16 | RUN cd $SRC_DIR \ 17 | && CGO_ENABLED=0 go mod download 18 | 19 | COPY . $SRC_DIR 20 | 21 | # Build the thing. 22 | RUN cd $SRC_DIR \ 23 | && BIN_BUILD_FLAGS="CGO_ENABLED=0 GOOS=linux" make build-buckd 24 | 25 | # Get su-exec, a very minimal tool for dropping privileges, 26 | # and tini, a very minimal init daemon for containers 27 | ENV SUEXEC_VERSION v0.2 28 | ENV TINI_VERSION v0.19.0 29 | RUN set -eux; \ 30 | dpkgArch="$(dpkg --print-architecture)"; \ 31 | case "${dpkgArch##*-}" in \ 32 | "amd64" | "armhf" | "arm64") tiniArch="tini-static-$dpkgArch" ;;\ 33 | *) echo >&2 "unsupported architecture: ${dpkgArch}"; exit 1 ;; \ 34 | esac; \ 35 | cd /tmp \ 36 | && git clone https://github.com/ncopa/su-exec.git \ 37 | && cd su-exec \ 38 | && git checkout -q $SUEXEC_VERSION \ 39 | && make su-exec-static \ 40 | && cd /tmp \ 41 | && wget -q -O tini https://github.com/krallin/tini/releases/download/$TINI_VERSION/$tiniArch \ 42 | && chmod +x tini 43 | 44 | # Now comes the actual target image, which aims to be as small as possible. 45 | FROM busybox:1.31.1-glibc 46 | LABEL maintainer="Textile " 47 | 48 | # Get the binary, entrypoint script, and TLS CAs from the build container. 49 | ENV SRC_DIR /go-buckets 50 | COPY --from=0 $SRC_DIR/buckd /usr/local/bin/buckd 51 | COPY --from=0 /tmp/su-exec/su-exec-static /sbin/su-exec 52 | COPY --from=0 /tmp/tini /sbin/tini 53 | COPY --from=0 /etc/ssl/certs /etc/ssl/certs 54 | 55 | # This shared lib (part of glibc) doesn't seem to be included with busybox. 56 | COPY --from=0 /lib/*-linux-gnu*/libdl.so.2 /lib/ 57 | 58 | # Copy over SSL libraries. 59 | COPY --from=0 /usr/lib/*-linux-gnu*/libssl.so* /usr/lib/ 60 | COPY --from=0 /usr/lib/*-linux-gnu*/libcrypto.so* /usr/lib/ 61 | 62 | # addrApi; can be exposed to the public. 63 | EXPOSE 5000 64 | # addrApiProxy; can be exposed to the public. 65 | EXPOSE 5050 66 | # addrGateway; can be exposed to the public. 67 | EXPOSE 8000 68 | 69 | # Create the repo directory. 70 | ENV BUCKETS_PATH /data/buckets 71 | RUN mkdir -p $BUCKETS_PATH \ 72 | && adduser -D -h $BUCKETS_PATH -u 1000 -G users buckets \ 73 | && chown buckets:users $BUCKETS_PATH 74 | 75 | # Switch to a non-privileged user. 76 | USER buckets 77 | 78 | # Expose the repo as a volume. 79 | # Important this happens after the USER directive so permission are correct. 80 | VOLUME $BUCKETS_PATH 81 | 82 | ENTRYPOINT ["/sbin/tini", "--", "buckd"] 83 | 84 | CMD ["--datastoreBadgerRepo=/data/buckets"] 85 | -------------------------------------------------------------------------------- /cmd/buckd/Dockerfile.dev: -------------------------------------------------------------------------------- 1 | FROM golang:1.16.0-buster 2 | 3 | RUN apt-get update 4 | 5 | RUN go get github.com/go-delve/delve/cmd/dlv 6 | 7 | ENV SRC_DIR /go-buckets 8 | 9 | COPY go.mod go.sum $SRC_DIR/ 10 | RUN cd $SRC_DIR \ 11 | && CGO_ENABLED=0 go mod download 12 | 13 | COPY . $SRC_DIR 14 | 15 | RUN --mount=type=cache,target=/root/.cache/go-build cd $SRC_DIR \ 16 | && CGO_ENABLED=0 GOOS=linux go build -gcflags "all=-N -l" -o buckd cmd/buckd/main.go 17 | 18 | FROM debian:buster 19 | LABEL maintainer="Textile " 20 | 21 | ENV SRC_DIR /go-buckets 22 | COPY --from=0 /go/bin/dlv /usr/local/bin/dlv 23 | COPY --from=0 $SRC_DIR/buckd /usr/local/bin/buckd 24 | 25 | EXPOSE 5000 26 | EXPOSE 5050 27 | EXPOSE 8000 28 | EXPOSE 40000 29 | 30 | ENV BUCKETS_PATH /data/buckets 31 | RUN adduser --home $BUCKETS_PATH --disabled-login --gecos "" --ingroup users buckets 32 | 33 | USER buckets 34 | 35 | VOLUME $BUCKETS_PATH 36 | 37 | ENTRYPOINT ["dlv", "--listen=0.0.0.0:40000", "--headless=true", "--accept-multiclient", "--continue", "--api-version=2", "exec", "/usr/local/bin/buckd"] 38 | 39 | CMD ["--", "--datastoreBadgerRepo=/data/buckets"] 40 | -------------------------------------------------------------------------------- /cmd/buckd/docker-compose-dev.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | buckets: 4 | build: 5 | context: ../../ 6 | dockerfile: ./cmd/buckd/Dockerfile.dev 7 | volumes: 8 | - buckets_data:/data/buckets 9 | platform: linux/amd64 10 | environment: 11 | - BUCK_LOG_DEBUG=true 12 | - BUCK_ADDR_API=0.0.0.0:5000 13 | - BUCK_ADDR_API_PROXY=0.0.0.0:5050 14 | - BUCK_ADDR_GATEWAY=0.0.0.0:8000 15 | - BUCK_DATASTORE_TYPE=badger 16 | - BUCK_GATEWAY_URL 17 | - BUCK_GATEWAY_SUBDOMAINS 18 | - BUCK_GATEWAY_WWW_DOMAIN 19 | - BUCK_THREADS_ADDR=threads:5000 20 | - BUCK_THREADS_GATEWAY_URL=http://127.0.0.1:7000 21 | - BUCK_IPFS_MULTIADDR=/dns4/ipfs/tcp/5001 22 | - BUCK_IPNS_REPUBLISH_SCHEDULE 23 | - BUCK_IPNS_REPUBLISH_CONCURRENCY 24 | - BUCK_CLOUDFLARE_DNS_ZONE_ID 25 | - BUCK_CLOUDFLARE_DNS_TOKEN 26 | ports: 27 | - "127.0.0.1:5000:5000" 28 | - "127.0.0.1:5050:5050" 29 | - "127.0.0.1:8000:8000" 30 | - "127.0.0.1:40000:40000" 31 | security_opt: 32 | - "seccomp:unconfined" 33 | cap_add: 34 | - SYS_PTRACE 35 | depends_on: 36 | - threads 37 | - ipfs 38 | restart: unless-stopped 39 | threads: 40 | image: textile/threads:sha-43a1673 41 | volumes: 42 | - threads_data:/data/threads 43 | platform: linux/amd64 44 | environment: 45 | - THREADS_DEBUG=true 46 | - THREADS_APIADDR=/ip4/0.0.0.0/tcp/5000 47 | - THREADS_APIPROXYADDR=/ip4/0.0.0.0/tcp/5050 48 | - THREADS_GATEWAYADDR=0.0.0.0:8000 49 | ports: 50 | - "4066:4006" 51 | - "4066:4006/udp" 52 | - "127.0.0.1:4050:5050" 53 | - "127.0.0.1:7000:8000" 54 | restart: unless-stopped 55 | ipfs: 56 | image: textile/go-ipfs:sha-ce693d7 57 | volumes: 58 | - ipfs_data:/data/ipfs 59 | platform: linux/amd64 60 | environment: 61 | - IPFS_PROFILE=test 62 | ports: 63 | - "4011:4001" 64 | - "4011:4001/udp" 65 | - "127.0.0.1:8081:8080" 66 | restart: unless-stopped 67 | volumes: 68 | buckets_data: 69 | threads_data: 70 | ipfs_data: 71 | -------------------------------------------------------------------------------- /cmd/buckd/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | buckets: 4 | image: textile/buckets:sha-7db2e12 5 | volumes: 6 | - buckets_data:/data/buckets 7 | platform: linux/amd64 8 | environment: 9 | - BUCK_LOG_DEBUG 10 | - BUCK_ADDR_API=0.0.0.0:5000 11 | - BUCK_ADDR_API_PROXY=0.0.0.0:5050 12 | - BUCK_ADDR_GATEWAY=0.0.0.0:8000 13 | - BUCK_DATASTORE_TYPE 14 | - BUCK_DATASTORE_MONGO_URI 15 | - BUCK_DATASTORE_MONGO_NAME 16 | - BUCK_GATEWAY_URL 17 | - BUCK_GATEWAY_SUBDOMAINS 18 | - BUCK_GATEWAY_WWW_DOMAIN 19 | - BUCK_THREADS_ADDR=threads:5000 20 | - BUCK_THREADS_GATEWAY_URL=http://127.0.0.1:7000 21 | - BUCK_IPFS_MULTIADDR=/dns4/ipfs/tcp/5001 22 | - BUCK_IPNS_REPUBLISH_SCHEDULE 23 | - BUCK_IPNS_REPUBLISH_CONCURRENCY 24 | - BUCK_CLOUDFLARE_DNS_ZONE_ID 25 | - BUCK_CLOUDFLARE_DNS_TOKEN 26 | ports: 27 | - "5000:5000" 28 | - "5050:5050" 29 | - "8000:8000" 30 | depends_on: 31 | - threads 32 | - ipfs 33 | restart: unless-stopped 34 | threads: 35 | image: textile/threads:sha-43a1673 36 | volumes: 37 | - threads_data:/data/threads 38 | platform: linux/amd64 39 | environment: 40 | - THREADS_APIADDR=/ip4/0.0.0.0/tcp/5000 41 | - THREADS_APIPROXYADDR=/ip4/0.0.0.0/tcp/5050 42 | - THREADS_GATEWAYADDR=0.0.0.0:8000 43 | ports: 44 | - "4066:4006" 45 | - "4066:4006/udp" 46 | - "4050:5050" 47 | - "7000:8000" 48 | restart: unless-stopped 49 | ipfs: 50 | image: textile/go-ipfs:sha-ce693d7 51 | volumes: 52 | - ipfs_data:/data/ipfs 53 | platform: linux/amd64 54 | ports: 55 | - "4011:4001" 56 | - "4011:4001/udp" 57 | - "8081:8080" 58 | restart: unless-stopped 59 | volumes: 60 | buckets_data: 61 | threads_data: 62 | ipfs_data: 63 | -------------------------------------------------------------------------------- /cmd/cmd.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/manifoldco/promptui" 7 | "github.com/spf13/viper" 8 | ) 9 | 10 | var ( 11 | // Timeout is the default timeout used for most commands. 12 | Timeout = time.Minute * 10 13 | // PushTimeout is the command timeout used when pushing bucket changes. 14 | PushTimeout = time.Hour * 24 15 | // PullTimeout is the command timeout used when pulling bucket changes. 16 | PullTimeout = time.Hour * 24 17 | 18 | // Bold is a styler used to make the output text bold. 19 | Bold = promptui.Styler(promptui.FGBold) 20 | ) 21 | 22 | // Flag describes a command flag. 23 | type Flag struct { 24 | Key string 25 | DefValue interface{} 26 | } 27 | 28 | // Config describes a command config params and file info. 29 | type Config struct { 30 | Viper *viper.Viper 31 | File string 32 | Dir string 33 | Name string 34 | Flags map[string]Flag 35 | EnvPre string 36 | Global bool 37 | } 38 | 39 | // ConfConfig is used to generate new messages configs. 40 | type ConfConfig struct { 41 | Dir string // Config directory base name 42 | Name string // Name of the mailbox config file 43 | Type string // Type is the type of config file (yaml/json) 44 | EnvPrefix string // A prefix that will be expected on env vars 45 | } 46 | 47 | // NewConfig uses values from ConfConfig to contruct a new config. 48 | func (cc ConfConfig) NewConfig(pth string, flags map[string]Flag, global bool) (c *Config, fileExists bool, err error) { 49 | v := viper.New() 50 | v.SetConfigType(cc.Type) 51 | c = &Config{ 52 | Viper: v, 53 | Dir: cc.Dir, 54 | Name: cc.Name, 55 | Flags: flags, 56 | EnvPre: cc.EnvPrefix, 57 | Global: global, 58 | } 59 | fileExists = FindConfigFile(c, pth) 60 | return c, fileExists, nil 61 | } 62 | -------------------------------------------------------------------------------- /cmd/config.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "strings" 8 | 9 | "github.com/mitchellh/go-homedir" 10 | ma "github.com/multiformats/go-multiaddr" 11 | "github.com/spf13/cobra" 12 | "github.com/spf13/viper" 13 | ) 14 | 15 | // InitConfigCmd adds a config generator command to the root command. 16 | // The command will write the config file to dir. 17 | func InitConfigCmd(rootCmd *cobra.Command, v *viper.Viper, dir string) { 18 | configCmd := &cobra.Command{ 19 | Use: "config", 20 | Short: "Config utils", 21 | Long: `Config file utilities.`, 22 | } 23 | rootCmd.AddCommand(configCmd) 24 | createCmd := &cobra.Command{ 25 | Use: "create", 26 | Short: "Create config", 27 | Long: `Create a config file.`, 28 | Run: func(c *cobra.Command, args []string) { 29 | WriteConfig(c, v, dir) 30 | }, 31 | } 32 | configCmd.AddCommand(createCmd) 33 | createCmd.Flags().String( 34 | "dir", 35 | "", 36 | "Directory to write config (default ${HOME}/"+dir+")") 37 | } 38 | 39 | const maxSearchHeight = 50 40 | 41 | // InitConfig returns a function that can be used to search for and load a config file. 42 | func InitConfig(conf *Config) func() { 43 | return func() { 44 | FindConfigFile(conf, ".") 45 | } 46 | } 47 | 48 | // FindConfigFile searches up the path for a config file. 49 | // True is returned is a config file was found and successfully loaded. 50 | func FindConfigFile(conf *Config, pth string) bool { 51 | found := false 52 | h := 1 53 | for h <= maxSearchHeight && !found { 54 | found = initConfig(conf.Viper, conf.File, pth, conf.Dir, conf.Name, conf.EnvPre, conf.Global) 55 | npth := filepath.Dir(pth) 56 | if npth == string(os.PathSeparator) && pth == string(os.PathSeparator) { 57 | return found 58 | } 59 | pth = npth 60 | h++ 61 | } 62 | return found 63 | } 64 | 65 | func initConfig(v *viper.Viper, file, pre, cdir, name, envPre string, global bool) bool { 66 | if file != "" { 67 | v.SetConfigFile(file) 68 | } else { 69 | v.AddConfigPath(filepath.Join(pre, cdir)) // local config takes priority 70 | if global { 71 | home, err := homedir.Dir() 72 | if err != nil { 73 | panic(err) 74 | } 75 | v.AddConfigPath(filepath.Join(home, cdir)) 76 | } 77 | v.SetConfigName(name) 78 | } 79 | 80 | v.SetEnvPrefix(envPre) 81 | v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) 82 | v.AutomaticEnv() 83 | if err := v.ReadInConfig(); err != nil && strings.Contains(err.Error(), "Not Found") { 84 | return false 85 | } 86 | return true 87 | } 88 | 89 | // WriteConfig writes the viper config based on the command. 90 | func WriteConfig(c *cobra.Command, v *viper.Viper, name string) { 91 | var dir string 92 | if !c.Flag("dir").Changed { 93 | home, err := homedir.Dir() 94 | if err != nil { 95 | Fatal(err) 96 | } 97 | dir = filepath.Join(home, name) 98 | if err = os.MkdirAll(dir, os.ModePerm); err != nil { 99 | Fatal(err) 100 | } 101 | } else { 102 | dir = c.Flag("dir").Value.String() 103 | } 104 | 105 | filename := filepath.Join(dir, "config.yml") 106 | if _, err := os.Stat(filename); err == nil { 107 | Fatal(fmt.Errorf("%s already exists", filename)) 108 | } 109 | if err := v.WriteConfigAs(filename); err != nil { 110 | Fatal(err) 111 | } 112 | } 113 | 114 | // WriteConfigToHome writes config to the home directory. 115 | func WriteConfigToHome(config *Config) { 116 | home, err := homedir.Dir() 117 | ErrCheck(err) 118 | dir := filepath.Join(home, config.Dir) 119 | err = os.MkdirAll(dir, os.ModePerm) 120 | ErrCheck(err) 121 | filename := filepath.Join(dir, config.Name+".yml") 122 | err = config.Viper.WriteConfigAs(filename) 123 | ErrCheck(err) 124 | } 125 | 126 | // BindFlags binds the flags to the viper config values. 127 | func BindFlags(v *viper.Viper, root *cobra.Command, flags map[string]Flag) error { 128 | for n, f := range flags { 129 | if err := v.BindPFlag(f.Key, root.PersistentFlags().Lookup(n)); err != nil { 130 | return err 131 | } 132 | v.SetDefault(f.Key, f.DefValue) 133 | } 134 | return nil 135 | } 136 | 137 | // GetFlagOrEnvValue first load a value for the key from the command flags. 138 | // If no value was found, the value for the corresponding env variable is returned. 139 | func GetFlagOrEnvValue(c *cobra.Command, k, envPre string) (v string) { 140 | changed := c.Flags().Changed(k) 141 | v, err := c.Flags().GetString(k) 142 | if err == nil && changed { 143 | return 144 | } 145 | env := os.Getenv(fmt.Sprintf("%s_%s", envPre, strings.ToUpper(k))) 146 | if env != "" { 147 | return env 148 | } 149 | return v 150 | } 151 | 152 | // ExpandConfigVars evaluates the viper config file's expressions. 153 | func ExpandConfigVars(v *viper.Viper, flags map[string]Flag) { 154 | for _, f := range flags { 155 | if f.Key != "" { 156 | if str, ok := v.Get(f.Key).(string); ok { 157 | v.Set(f.Key, os.ExpandEnv(str)) 158 | } 159 | } 160 | } 161 | } 162 | 163 | // AddrFromStr returns a multiaddress from the string. 164 | func AddrFromStr(str string) ma.Multiaddr { 165 | addr, err := ma.NewMultiaddr(str) 166 | if err != nil { 167 | Fatal(err) 168 | } 169 | return addr 170 | } 171 | -------------------------------------------------------------------------------- /cmd/output.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "os" 7 | "os/signal" 8 | "path/filepath" 9 | "runtime" 10 | "strings" 11 | 12 | logging "github.com/ipfs/go-log/v2" 13 | aurora2 "github.com/logrusorgru/aurora" 14 | "github.com/olekukonko/tablewriter" 15 | "google.golang.org/grpc/status" 16 | ) 17 | 18 | var aurora = aurora2.NewAurora(runtime.GOOS != "windows") 19 | 20 | func Message(format string, args ...interface{}) { 21 | if format == "" { 22 | return 23 | } 24 | fmt.Println(aurora.Sprintf(aurora.BrightBlack("> "+format), args...)) 25 | } 26 | 27 | func Success(format string, args ...interface{}) { 28 | fmt.Println(aurora.Sprintf(aurora.Cyan("> Success! %s"), 29 | aurora.Sprintf(aurora.BrightBlack(format), args...))) 30 | } 31 | 32 | func Warn(format string, args ...interface{}) { 33 | if format == "" { 34 | return 35 | } 36 | fmt.Println(aurora.Sprintf(aurora.Yellow("> Warning! %s"), 37 | aurora.Sprintf(aurora.BrightBlack(format), args...))) 38 | } 39 | 40 | func Err(err error, args ...interface{}) { 41 | var msg string 42 | stat, ok := status.FromError(err) 43 | if ok { 44 | msg = stat.Message() 45 | } else { 46 | msg = err.Error() 47 | } 48 | words := strings.SplitN(msg, " ", 2) 49 | words[0] = strings.Title(words[0]) 50 | msg = strings.Join(words, " ") 51 | 52 | fmt.Println(aurora.Sprintf(aurora.Red("> Error! %s"), 53 | aurora.Sprintf(aurora.BrightBlack(msg), args...))) 54 | } 55 | 56 | func End(format string, args ...interface{}) { 57 | Message(format, args...) 58 | os.Exit(0) 59 | } 60 | 61 | func Fatal(err error, args ...interface{}) { 62 | Err(err, args...) 63 | os.Exit(1) 64 | } 65 | 66 | func ErrCheck(err error, args ...interface{}) { 67 | if err != nil { 68 | Fatal(err, args...) 69 | } 70 | } 71 | 72 | func LogErr(err error, args ...interface{}) { 73 | if err != nil { 74 | Err(err, args...) 75 | } 76 | } 77 | 78 | func RenderJSON(data interface{}) { 79 | bytes, err := json.MarshalIndent(data, "", " ") 80 | ErrCheck(err) 81 | fmt.Println(string(bytes)) 82 | } 83 | 84 | func RenderTable(header []string, data [][]string) { 85 | fmt.Println() 86 | table := tablewriter.NewWriter(os.Stdout) 87 | table.SetHeader(header) 88 | table.SetAutoWrapText(false) 89 | table.SetAutoFormatHeaders(true) 90 | table.SetHeaderAlignment(tablewriter.ALIGN_LEFT) 91 | table.SetAlignment(tablewriter.ALIGN_LEFT) 92 | table.SetCenterSeparator("") 93 | table.SetColumnSeparator("") 94 | table.SetRowSeparator("") 95 | table.SetHeaderLine(false) 96 | table.SetBorder(false) 97 | table.SetTablePadding("\t") 98 | table.SetNoWhiteSpace(false) 99 | headersColors := make([]tablewriter.Colors, len(header)) 100 | for i := range headersColors { 101 | headersColors[i] = tablewriter.Colors{tablewriter.FgHiBlackColor} 102 | } 103 | table.SetHeaderColor(headersColors...) 104 | table.AppendBulk(data) 105 | table.Render() 106 | fmt.Println() 107 | } 108 | 109 | func HandleInterrupt(stop func()) { 110 | quit := make(chan os.Signal) 111 | signal.Notify(quit, os.Interrupt) 112 | <-quit 113 | fmt.Println("Gracefully stopping... (press Ctrl+C again to force)") 114 | stop() 115 | os.Exit(1) 116 | } 117 | 118 | func SetupDefaultLoggingConfig(file string) error { 119 | if file != "" { 120 | if err := os.MkdirAll(filepath.Dir(file), os.ModePerm); err != nil { 121 | return err 122 | } 123 | } 124 | c := logging.Config{ 125 | Format: logging.ColorizedOutput, 126 | Stderr: true, 127 | File: file, 128 | Level: logging.LevelError, 129 | } 130 | logging.SetupLogging(c) 131 | return nil 132 | } 133 | -------------------------------------------------------------------------------- /cmd/watch.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "strings" 6 | "time" 7 | 8 | backoff "github.com/cenkalti/backoff/v4" 9 | "google.golang.org/grpc/codes" 10 | "google.golang.org/grpc/status" 11 | ) 12 | 13 | // WatchState is used to inform Watch callers about the connection state. 14 | type WatchState struct { 15 | // State of the watch connection (online/offline). 16 | State ConnectionState 17 | // Err returned by the watch operation. 18 | Err error 19 | // Aborted indicates whether or not the associated error aborted the watch. 20 | // (Connectivity related errors do not abort the watch.) 21 | Aborted bool 22 | } 23 | 24 | // ConnectionState indicates an online/offline state. 25 | type ConnectionState int 26 | 27 | const ( 28 | // Offline indicates the remote is currently not reachable. 29 | Offline ConnectionState = iota 30 | // Online indicates a connection with the remote has been established. 31 | Online 32 | ) 33 | 34 | func (cs ConnectionState) String() string { 35 | switch cs { 36 | case Online: 37 | return "online" 38 | case Offline: 39 | return "offline" 40 | default: 41 | return "unknown state" 42 | } 43 | } 44 | 45 | // WatchFunc is a function wrapper for a function used by Watch. 46 | type WatchFunc func(context.Context) (<-chan WatchState, error) 47 | 48 | // Watch calls watchFunc until it returns an error. 49 | // Normally, watchFunc would block while doing work that can fail, 50 | // e.g., the local network goes offline. 51 | // If watchFunc return an error, it will be called 52 | // again at the given interval so long as the returned error is non-fatal. 53 | // Returns a channel of watch connectivity states. 54 | // Cancel context to stop watching. 55 | func Watch(ctx context.Context, watchFunc WatchFunc, reconnectInterval time.Duration) (<-chan WatchState, error) { 56 | bc := backoff.NewConstantBackOff(reconnectInterval) 57 | outerState := make(chan WatchState) 58 | go func() { 59 | defer close(outerState) 60 | err := backoff.Retry(func() error { 61 | state, err := watchFunc(ctx) 62 | if err != nil { 63 | outerState <- WatchState{Err: err, Aborted: true} 64 | return nil // Stop retrying 65 | } 66 | for s := range state { 67 | outerState <- s 68 | if s.Err != nil { 69 | if s.Aborted { 70 | return nil // Stop retrying 71 | } else { 72 | return s.Err // Connection error, keep trying 73 | } 74 | } 75 | } 76 | return nil 77 | }, bc) 78 | if err != nil { 79 | outerState <- WatchState{Err: err, Aborted: true} 80 | } 81 | }() 82 | return outerState, nil 83 | } 84 | 85 | // IsConnectionError returns true if the error is related to a dropped connection. 86 | func IsConnectionError(err error) bool { 87 | return status.Code(err) == codes.Unavailable || strings.Contains(err.Error(), "RST_STREAM") 88 | } 89 | -------------------------------------------------------------------------------- /collection/mergemap/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013, Peter Bourgon, SoundCloud Ltd. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | Redistributions in binary form must reproduce the above copyright notice, this 11 | list of conditions and the following disclaimer in the documentation and/or 12 | other materials provided with the distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 15 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 16 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 18 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 20 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 21 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 22 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 23 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | -------------------------------------------------------------------------------- /collection/mergemap/mergemap.go: -------------------------------------------------------------------------------- 1 | package mergemap 2 | 3 | import "reflect" 4 | 5 | var MaxDepth = 32 6 | 7 | // Merge recursively merges the src and dst maps. Key conflicts are resolved by 8 | // preferring src, or recursively descending, if both src and dst are maps. 9 | func Merge(dst, src map[string]interface{}) map[string]interface{} { 10 | return merge(dst, src, 0) 11 | } 12 | 13 | func merge(dst, src map[string]interface{}, depth int) map[string]interface{} { 14 | if depth > MaxDepth { 15 | return dst 16 | } 17 | for key, srcVal := range src { 18 | if dstVal, ok := dst[key]; ok { 19 | srcMap, srcMapOk := mapify(srcVal) 20 | dstMap, dstMapOk := mapify(dstVal) 21 | if srcMapOk && dstMapOk { 22 | srcVal = merge(dstMap, srcMap, depth+1) 23 | } 24 | } 25 | dst[key] = srcVal 26 | } 27 | return dst 28 | } 29 | 30 | func mapify(i interface{}) (map[string]interface{}, bool) { 31 | value := reflect.ValueOf(i) 32 | if value.Kind() == reflect.Map { 33 | m := map[string]interface{}{} 34 | for _, k := range value.MapKeys() { 35 | m[k.String()] = value.MapIndex(k).Interface() 36 | } 37 | return m, true 38 | } 39 | return map[string]interface{}{}, false 40 | } 41 | -------------------------------------------------------------------------------- /create.go: -------------------------------------------------------------------------------- 1 | package buckets 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/ipfs/interface-go-ipfs-core/path" 9 | "github.com/textileio/dcrypto" 10 | "github.com/textileio/go-buckets/collection" 11 | "github.com/textileio/go-buckets/dag" 12 | "github.com/textileio/go-threads/core/did" 13 | core "github.com/textileio/go-threads/core/thread" 14 | "github.com/textileio/go-threads/db" 15 | ) 16 | 17 | // Create a new bucket using identity. 18 | // See CreateOption for more details. 19 | func (b *Buckets) Create( 20 | ctx context.Context, 21 | identity did.Token, 22 | opts ...CreateOption, 23 | ) (*Bucket, *Seed, int64, error) { 24 | args := &CreateOptions{} 25 | for _, opt := range opts { 26 | opt(args) 27 | } 28 | 29 | if args.Thread.Defined() { 30 | if err := args.Thread.Validate(); err != nil { 31 | return nil, nil, 0, fmt.Errorf("invalid thread id: %v", err) 32 | } 33 | } else { 34 | args.Thread = core.NewRandomIDV1() 35 | if err := b.db.NewDB( 36 | ctx, 37 | args.Thread, 38 | db.WithNewManagedName(args.Name), 39 | db.WithNewManagedToken(identity), 40 | ); err != nil { 41 | return nil, nil, 0, fmt.Errorf("creating new thread: %v", err) 42 | } 43 | } 44 | 45 | _, owner, err := b.net.ValidateIdentity(ctx, identity) 46 | if err != nil { 47 | return nil, nil, 0, fmt.Errorf("validating identity: %v", err) 48 | } 49 | 50 | // Create bucket keys if private 51 | var linkKey, fileKey []byte 52 | if args.Private { 53 | var err error 54 | linkKey, err = dcrypto.NewKey() 55 | if err != nil { 56 | return nil, nil, 0, err 57 | } 58 | fileKey, err = dcrypto.NewKey() 59 | if err != nil { 60 | return nil, nil, 0, err 61 | } 62 | } 63 | 64 | // Make a random seed, which ensures a bucket's uniqueness 65 | seed, err := dag.MakeBucketSeed(fileKey) 66 | if err != nil { 67 | return nil, nil, 0, fmt.Errorf("making bucket seed: %v", err) 68 | } 69 | 70 | // Create the bucket directory 71 | var pth path.Resolved 72 | if args.Cid.Defined() { 73 | ctx, pth, err = dag.CreateBucketPathWithCid( 74 | ctx, 75 | b.ipfs, 76 | "", 77 | args.Cid, 78 | linkKey, 79 | fileKey, 80 | seed, 81 | ) 82 | if err != nil { 83 | return nil, nil, 0, fmt.Errorf("creating bucket with cid: %v", err) 84 | } 85 | } else { 86 | ctx, pth, err = dag.CreateBucketPath(ctx, b.ipfs, seed, linkKey) 87 | if err != nil { 88 | return nil, nil, 0, fmt.Errorf("creating bucket: %v", err) 89 | } 90 | } 91 | 92 | // Create top-level metadata 93 | now := time.Now() 94 | md := map[string]collection.Metadata{ 95 | "": collection.NewDefaultMetadata(owner, fileKey, now), 96 | collection.SeedName: { 97 | Roles: make(map[did.DID]collection.Role), 98 | UpdatedAt: now.UnixNano(), 99 | }, 100 | } 101 | 102 | // Create a new IPNS key 103 | key, err := b.ipns.CreateKey(ctx, args.Thread) 104 | if err != nil { 105 | return nil, nil, 0, fmt.Errorf("creating IPNS key: %v", err) 106 | } 107 | 108 | // Create the bucket using the IPNS key as instance ID 109 | instance, err := b.c.New( 110 | ctx, 111 | args.Thread, 112 | key, 113 | owner, 114 | pth, 115 | now, 116 | md, 117 | identity, 118 | collection.WithBucketName(args.Name), 119 | collection.WithBucketKey(linkKey), 120 | ) 121 | if err != nil { 122 | return nil, nil, 0, err 123 | } 124 | 125 | seedInfo := &Seed{ 126 | Cid: seed.Cid(), 127 | } 128 | if instance.IsPrivate() { 129 | fileKey, err := instance.GetFileEncryptionKeyForPath("") 130 | if err != nil { 131 | return nil, nil, 0, err 132 | } 133 | data, err := dag.DecryptData(seed.RawData(), fileKey) 134 | if err != nil { 135 | return nil, nil, 0, err 136 | } 137 | seedInfo.Data = data 138 | } else { 139 | seedInfo.Data = seed.RawData() 140 | } 141 | 142 | // Publish the new bucket's address to the name system 143 | go b.ipns.Publish(pth, instance.Key) 144 | 145 | log.Debugf("created %s", key) 146 | return instanceToBucket(args.Thread, instance), seedInfo, dag.GetPinnedBytes(ctx), nil 147 | } 148 | -------------------------------------------------------------------------------- /dag/pinning.go: -------------------------------------------------------------------------------- 1 | package dag 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | 8 | ipld "github.com/ipfs/go-ipld-format" 9 | iface "github.com/ipfs/interface-go-ipfs-core" 10 | "github.com/ipfs/interface-go-ipfs-core/path" 11 | ) 12 | 13 | const ( 14 | // pinNotRecursiveMsg is used to match an IPFS "recursively pinned already" error. 15 | pinNotRecursiveMsg = "'from' cid was not recursively pinned already" 16 | ) 17 | 18 | var ( 19 | // ErrStorageQuotaExhausted indicates the requested operation exceeds the storage allowance. 20 | ErrStorageQuotaExhausted = errors.New("storage quota exhausted") 21 | ) 22 | 23 | type ctxKey string 24 | 25 | // BucketOwner provides owner context to the bucket service. 26 | type BucketOwner struct { 27 | StorageUsed int64 28 | StorageAvailable int64 29 | StorageDelta int64 30 | } 31 | 32 | // NewOwnerContext returns a new bucket owner context. 33 | func NewOwnerContext(ctx context.Context, owner *BucketOwner) context.Context { 34 | return context.WithValue(ctx, ctxKey("bucketOwner"), owner) 35 | } 36 | 37 | // OwnerFromContext returns a bucket owner from the context if available. 38 | func OwnerFromContext(ctx context.Context) (*BucketOwner, bool) { 39 | owner, ok := ctx.Value(ctxKey("bucketOwner")).(*BucketOwner) 40 | return owner, ok 41 | } 42 | 43 | // AddPinnedBytes adds the provided delta to a running total for context. 44 | func AddPinnedBytes(ctx context.Context, delta int64) context.Context { 45 | total, _ := ctx.Value(ctxKey("pinnedBytes")).(int64) 46 | ctx = context.WithValue(ctx, ctxKey("pinnedBytes"), total+delta) 47 | owner, ok := OwnerFromContext(ctx) 48 | if ok { 49 | owner.StorageUsed += delta 50 | owner.StorageAvailable -= delta 51 | owner.StorageDelta += delta 52 | ctx = NewOwnerContext(ctx, owner) 53 | } 54 | return ctx 55 | } 56 | 57 | // GetPinnedBytes returns the total pinned bytes for context. 58 | func GetPinnedBytes(ctx context.Context) int64 { 59 | pinned, _ := ctx.Value(ctxKey("pinnedBytes")).(int64) 60 | return pinned 61 | } 62 | 63 | // PinBlocks pins blocks, accounting for sum bytes pinned for context. 64 | func PinBlocks(ctx context.Context, ipfs iface.CoreAPI, nodes []ipld.Node) (context.Context, error) { 65 | var totalAddedSize int64 66 | for _, n := range nodes { 67 | s, err := n.Stat() 68 | if err != nil { 69 | return ctx, fmt.Errorf("getting size of node: %v", err) 70 | } 71 | totalAddedSize += int64(s.CumulativeSize) 72 | } 73 | 74 | // Check context owner's storage allowance 75 | owner, ok := OwnerFromContext(ctx) 76 | if ok && totalAddedSize > owner.StorageAvailable { 77 | return ctx, ErrStorageQuotaExhausted 78 | } 79 | 80 | if err := ipfs.Dag().Pinning().AddMany(ctx, nodes); err != nil { 81 | return ctx, fmt.Errorf("pinning set of nodes: %v", err) 82 | } 83 | return AddPinnedBytes(ctx, totalAddedSize), nil 84 | } 85 | 86 | // AddAndPinNodes adds and pins nodes, accounting for sum bytes pinned for context. 87 | func AddAndPinNodes(ctx context.Context, ipfs iface.CoreAPI, nodes []ipld.Node) (context.Context, error) { 88 | if err := ipfs.Dag().AddMany(ctx, nodes); err != nil { 89 | return ctx, err 90 | } 91 | return PinBlocks(ctx, ipfs, nodes) 92 | } 93 | 94 | // UpdateOrAddPin moves the pin at from to to. 95 | // If from is nil, a new pin as placed at to. 96 | func UpdateOrAddPin(ctx context.Context, ipfs iface.CoreAPI, from, to path.Path) (context.Context, error) { 97 | toSize, err := GetPathSize(ctx, ipfs, to) 98 | if err != nil { 99 | return ctx, fmt.Errorf("getting size of destination dag: %v", err) 100 | } 101 | 102 | fromSize, err := GetPathSize(ctx, ipfs, from) 103 | if err != nil { 104 | return ctx, fmt.Errorf("getting size of current dag: %v", err) 105 | } 106 | deltaSize := -fromSize + toSize 107 | 108 | // Check context owner's storage allowance 109 | owner, ok := OwnerFromContext(ctx) 110 | if ok && deltaSize > owner.StorageAvailable { 111 | return ctx, ErrStorageQuotaExhausted 112 | } 113 | 114 | if from == nil { 115 | if err := ipfs.Pin().Add(ctx, to); err != nil { 116 | return ctx, err 117 | } 118 | } else { 119 | if err := ipfs.Pin().Update(ctx, from, to); err != nil { 120 | if err.Error() == pinNotRecursiveMsg { 121 | return ctx, ipfs.Pin().Add(ctx, to) 122 | } 123 | return ctx, err 124 | } 125 | } 126 | return AddPinnedBytes(ctx, deltaSize), nil 127 | } 128 | 129 | // UnpinPath unpins path and accounts for sum bytes pinned for context. 130 | func UnpinPath(ctx context.Context, ipfs iface.CoreAPI, path path.Path) (context.Context, error) { 131 | if err := ipfs.Pin().Rm(ctx, path); err != nil { 132 | return ctx, err 133 | } 134 | size, err := GetPathSize(ctx, ipfs, path) 135 | if err != nil { 136 | return ctx, fmt.Errorf("getting size of removed node: %v", err) 137 | } 138 | return AddPinnedBytes(ctx, -size), nil 139 | } 140 | 141 | // UnpinBranch walks a the node at path, decrypting (if needed) and unpinning all nodes 142 | func UnpinBranch(ctx context.Context, ipfs iface.CoreAPI, p path.Resolved, key []byte) (context.Context, error) { 143 | n, _, err := ResolveNodeAtPath(ctx, ipfs, p, key) 144 | if err != nil { 145 | return ctx, err 146 | } 147 | for _, l := range n.Links() { 148 | if l.Name == "" { 149 | continue // Data nodes will never be pinned directly 150 | } 151 | lp := path.IpfsPath(l.Cid) 152 | ctx, err = UnpinPath(ctx, ipfs, lp) 153 | if err != nil { 154 | return ctx, err 155 | } 156 | ctx, err = UnpinBranch(ctx, ipfs, lp, key) 157 | if err != nil { 158 | return ctx, err 159 | } 160 | } 161 | return ctx, nil 162 | } 163 | 164 | // UnpinNodeAndBranch unpins a node and its entire branch, accounting for sum bytes pinned for context. 165 | func UnpinNodeAndBranch( 166 | ctx context.Context, 167 | ipfs iface.CoreAPI, 168 | pth path.Resolved, 169 | key []byte, 170 | ) (context.Context, error) { 171 | ctx, err := UnpinBranch(ctx, ipfs, pth, key) 172 | if err != nil { 173 | return ctx, err 174 | } 175 | return UnpinPath(ctx, ipfs, pth) 176 | } 177 | -------------------------------------------------------------------------------- /dist/install.tmpl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -Eeuo pipefail 3 | 4 | # From https://github.com/ipfs/go-ipfs/blob/ccef991a194beaedf009b1d0702b1150db3da0a6/cmd/ipfs/dist/install.sh 5 | # 6 | # Installation script for textile. It tries to move $bin in one of the 7 | # directories stored in $binpaths. 8 | 9 | INSTALL_DIR="$(dirname "$0")" 10 | 11 | file="$INSTALL_DIR/{{ .Env.BIN_FILE }}" 12 | binpaths="/usr/local/bin /usr/bin" 13 | 14 | # This variable contains a nonzero length string in case the script fails 15 | # because of missing write permissions. 16 | is_write_perm_missing="" 17 | 18 | for binpath in $binpaths; do 19 | if mv "$file" "$binpath/$file" 2>/dev/null; then 20 | echo "Moved $file to $binpath" 21 | exit 0 22 | else 23 | if test -d "$binpath" && ! test -w "$binpath"; then 24 | is_write_perm_missing=1 25 | fi 26 | fi 27 | done 28 | 29 | echo "We cannot install $file in one of the directories $binpaths" 30 | 31 | if test -n "$is_write_perm_missing"; then 32 | echo "It seems that we do not have the necessary write permissions." 33 | echo "Perhaps try running this script as a privileged user:" 34 | echo 35 | echo " sudo $0" 36 | echo 37 | fi 38 | 39 | exit 1 -------------------------------------------------------------------------------- /dns/dns.go: -------------------------------------------------------------------------------- 1 | package dns 2 | 3 | import ( 4 | "fmt" 5 | 6 | cf "github.com/cloudflare/cloudflare-go" 7 | logging "github.com/ipfs/go-log/v2" 8 | ) 9 | 10 | var log = logging.Logger("buckets/dns") 11 | 12 | const IPFSGateway = "cloudflare-ipfs.com" 13 | 14 | // Manager wraps a CloudflareClient client. 15 | type Manager struct { 16 | Domain string 17 | 18 | api *cf.API 19 | zoneID string 20 | } 21 | 22 | // NewManager return a cloudflare-backed dns updating client. 23 | func NewManager(domain string, zoneID string, token string) (*Manager, error) { 24 | api, err := cf.NewWithAPIToken(token) 25 | if err != nil { 26 | return nil, err 27 | } 28 | return &Manager{ 29 | Domain: domain, 30 | api: api, 31 | zoneID: zoneID, 32 | }, nil 33 | } 34 | 35 | // NewCNAME enters a new dns record for a CNAME. 36 | func (m *Manager) NewCNAME(name string, target string) (*cf.DNSRecord, error) { 37 | res, err := m.api.CreateDNSRecord(m.zoneID, cf.DNSRecord{ 38 | Type: "CNAME", 39 | Name: name, 40 | Content: target, 41 | Proxied: false, 42 | }) 43 | if err != nil { 44 | return nil, err 45 | } 46 | log.Debugf("created CNAME record %s -> %s", name, target) 47 | return &res.Result, nil 48 | } 49 | 50 | // NewTXT enters a new dns record for a TXT. 51 | func (m *Manager) NewTXT(name string, content string) (*cf.DNSRecord, error) { 52 | res, err := m.api.CreateDNSRecord(m.zoneID, cf.DNSRecord{ 53 | Type: "TXT", 54 | Name: name, 55 | Content: content, 56 | }) 57 | if err != nil { 58 | return nil, err 59 | } 60 | log.Debugf("created TXT record %s -> %s", name, content) 61 | return &res.Result, nil 62 | } 63 | 64 | // NewDNSLink enters a two dns records to enable DNS link. 65 | func (m *Manager) NewDNSLink(subdomain string, hash string) ([]*cf.DNSRecord, error) { 66 | cname, err := m.NewCNAME(subdomain, IPFSGateway) 67 | if err != nil { 68 | return nil, err 69 | } 70 | 71 | name := CreateDNSLinkName(subdomain) 72 | content := CreateDNSLinkContent(hash) 73 | txt, err := m.NewTXT(name, content) 74 | if err != nil { 75 | // Cleanup the orphaned cname record 76 | _ = m.DeleteRecord(cname.ID) 77 | return nil, err 78 | } 79 | 80 | log.Debugf("created DNSLink record %s -> %s", subdomain, hash) 81 | return []*cf.DNSRecord{cname, txt}, nil 82 | } 83 | 84 | // UpdateRecord updates an existing record. 85 | func (m *Manager) UpdateRecord(id, rtype, name, content string) error { 86 | if err := m.api.UpdateDNSRecord(m.zoneID, id, cf.DNSRecord{ 87 | Type: rtype, 88 | Name: name, 89 | Content: content, 90 | }); err != nil { 91 | return err 92 | } 93 | log.Debugf("updated record %s -> %s", name, content) 94 | return nil 95 | } 96 | 97 | // DeleteRecord removes a record by ID from dns. 98 | func (m *Manager) DeleteRecord(id string) error { 99 | if err := m.api.DeleteDNSRecord(m.zoneID, id); err != nil { 100 | return err 101 | } 102 | log.Debugf("deleted record %s", id) 103 | return nil 104 | } 105 | 106 | // CreateDNSLinkName converts a subdomain into the Name format for dnslink TXT entries. 107 | func CreateDNSLinkName(subdomain string) string { 108 | return fmt.Sprintf("_dnslink.%s", subdomain) 109 | } 110 | 111 | // CreateDNSLinkContent converts a hash into the Content format for dnslink TXT entries. 112 | func CreateDNSLinkContent(hash string) string { 113 | return fmt.Sprintf("dnslink=/ipfs/%s", hash) 114 | } 115 | -------------------------------------------------------------------------------- /gateway/Makefile: -------------------------------------------------------------------------------- 1 | ASSET_DIRS = $(shell find ./public/ -type d) 2 | ASSET_FILES = $(shell find ./public/ -type f -name '*') 3 | 4 | assets.go: ./public/ $(ASSET_DIRS) $(ASSET_FILES) 5 | go-assets-builder . -p gateway -o assets.go -------------------------------------------------------------------------------- /gateway/bucketfs.go: -------------------------------------------------------------------------------- 1 | package gateway 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "net/http" 8 | gopath "path" 9 | "strings" 10 | 11 | "github.com/gin-gonic/gin" 12 | "github.com/gin-gonic/gin/render" 13 | "github.com/textileio/go-assets" 14 | "github.com/textileio/go-buckets" 15 | "github.com/textileio/go-buckets/ipns" 16 | "github.com/textileio/go-threads/core/did" 17 | core "github.com/textileio/go-threads/core/thread" 18 | ) 19 | 20 | type fileSystem struct { 21 | *assets.FileSystem 22 | } 23 | 24 | func (f *fileSystem) Exists(prefix, path string) bool { 25 | pth := strings.TrimPrefix(path, prefix) 26 | if pth == "/" { 27 | return false 28 | } 29 | _, ok := f.Files[pth] 30 | return ok 31 | } 32 | 33 | type serveBucketFS interface { 34 | GetThread(key string) (core.ID, error) 35 | Exists(ctx context.Context, thread core.ID, bucket, pth string, token did.Token) (bool, string) 36 | Write(c *gin.Context, ctx context.Context, thread core.ID, bucket, pth string, token did.Token) error 37 | ValidHost() string 38 | } 39 | 40 | type bucketFS struct { 41 | lib *buckets.Buckets 42 | ipns *ipns.Manager 43 | domain string 44 | } 45 | 46 | func serveBucket(fs serveBucketFS) gin.HandlerFunc { 47 | return func(c *gin.Context) { 48 | key, err := bucketFromHost(c.Request.Host, fs.ValidHost()) 49 | if err != nil { 50 | return 51 | } 52 | thread, err := fs.GetThread(key) 53 | if err != nil { 54 | return 55 | } 56 | token := did.Token(c.Query("token")) 57 | 58 | var content string 59 | ctx, cancel := context.WithTimeout(context.Background(), handlerTimeout) 60 | defer cancel() 61 | exists, target := fs.Exists(ctx, thread, key, c.Request.URL.Path, token) 62 | if exists { 63 | content = c.Request.URL.Path 64 | } else if len(target) != 0 { 65 | content = gopath.Join(c.Request.URL.Path, target) 66 | } 67 | if len(content) != 0 { 68 | if err := fs.Write(c, ctx, thread, key, content, token); err != nil { 69 | renderError(c, http.StatusInternalServerError, err) 70 | } else { 71 | c.Abort() 72 | } 73 | } 74 | } 75 | } 76 | 77 | func (f *bucketFS) GetThread(bkey string) (id core.ID, err error) { 78 | key, err := f.ipns.Store().GetByCid(bkey) 79 | if err != nil { 80 | return 81 | } 82 | return key.ThreadID, nil 83 | } 84 | 85 | func (f *bucketFS) Exists(ctx context.Context, thread core.ID, key, pth string, token did.Token) (ok bool, name string) { 86 | if key == "" || pth == "/" { 87 | return 88 | } 89 | rep, _, err := f.lib.ListPath(ctx, thread, key, token, pth) 90 | if err != nil { 91 | return 92 | } 93 | if rep.IsDir { 94 | for _, item := range rep.Items { 95 | if item.Name == "index.html" { 96 | return false, item.Name 97 | } 98 | } 99 | return 100 | } 101 | return true, "" 102 | } 103 | 104 | func (f *bucketFS) Write(c *gin.Context, ctx context.Context, thread core.ID, key, pth string, token did.Token) error { 105 | r, err := f.lib.PullPath(ctx, thread, key, token, pth) 106 | if err != nil { 107 | return fmt.Errorf("pulling path: %v", err) 108 | } 109 | defer r.Close() 110 | 111 | ct, mr, err := detectReaderOrPathContentType(r, pth) 112 | if err != nil { 113 | return fmt.Errorf("detecting content-type: %v", err) 114 | } 115 | c.Writer.Header().Set("Content-Type", ct) 116 | c.Render(200, render.Reader{ContentLength: -1, Reader: mr}) 117 | return nil 118 | } 119 | 120 | func (f *bucketFS) ValidHost() string { 121 | return f.domain 122 | } 123 | 124 | func (g *Gateway) renderWWWBucket(c *gin.Context, key string) { 125 | ipnskey, err := g.ipns.Store().GetByCid(key) 126 | if err != nil { 127 | render404(c) 128 | return 129 | } 130 | ctx, cancel := context.WithTimeout(context.Background(), handlerTimeout) 131 | defer cancel() 132 | token := did.Token(c.Query("token")) 133 | rep, _, err := g.lib.ListPath(ctx, ipnskey.ThreadID, key, token, "") 134 | if err != nil { 135 | render404(c) 136 | return 137 | } 138 | for _, item := range rep.Items { 139 | if item.Name == "index.html" { 140 | r, err := g.lib.PullPath(ctx, ipnskey.ThreadID, key, token, item.Name) 141 | if err != nil { 142 | render404(c) 143 | return 144 | } 145 | 146 | ct, mr, err := detectReaderOrPathContentType(r, item.Name) 147 | if err != nil { 148 | renderError(c, http.StatusInternalServerError, fmt.Errorf("detecting content-type: %v", err)) 149 | return 150 | } 151 | c.Writer.Header().Set("Content-Type", ct) 152 | c.Render(200, render.Reader{ContentLength: -1, Reader: mr}) 153 | r.Close() 154 | } 155 | } 156 | renderError(c, http.StatusNotFound, errors.New("an index.html file was not found in this bucket")) 157 | } 158 | 159 | func bucketFromHost(host, valid string) (key string, err error) { 160 | parts := strings.SplitN(host, ".", 2) 161 | hostport := parts[len(parts)-1] 162 | hostparts := strings.SplitN(hostport, ":", 2) 163 | if hostparts[0] != valid || valid == "" { 164 | err = errors.New("invalid bucket host") 165 | return 166 | } 167 | return parts[0], nil 168 | } 169 | -------------------------------------------------------------------------------- /gateway/buckets.go: -------------------------------------------------------------------------------- 1 | package gateway 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | gopath "path" 8 | "strconv" 9 | "strings" 10 | "time" 11 | 12 | "github.com/gin-gonic/gin" 13 | "github.com/gin-gonic/gin/render" 14 | "github.com/textileio/go-buckets/collection" 15 | "github.com/textileio/go-threads/core/did" 16 | core "github.com/textileio/go-threads/core/thread" 17 | ) 18 | 19 | func (g *Gateway) bucketHandler(c *gin.Context) { 20 | thread, err := g.getThread(c) 21 | if err != nil { 22 | renderError(c, http.StatusBadRequest, err) 23 | return 24 | } 25 | g.renderBucket(c, thread, c.Param("key"), c.Param("path")) 26 | } 27 | 28 | func (g *Gateway) renderBucket(c *gin.Context, thread core.ID, key, pth string) { 29 | pth = strings.TrimPrefix(pth, "/") 30 | token := did.Token(c.Query("token")) 31 | 32 | ctx, cancel := context.WithTimeout(context.Background(), handlerTimeout) 33 | defer cancel() 34 | rep, buck, err := g.lib.ListPath(ctx, thread, key, token, pth) 35 | if err != nil { 36 | render404(c) 37 | return 38 | } 39 | if !rep.IsDir { 40 | r, err := g.lib.PullPath(ctx, thread, buck.Key, token, pth) 41 | if err != nil { 42 | render404(c) 43 | return 44 | } 45 | defer r.Close() 46 | 47 | ct, mr, err := detectReaderOrPathContentType(r, pth) 48 | if err != nil { 49 | renderError(c, http.StatusInternalServerError, fmt.Errorf("detecting content-type: %v", err)) 50 | return 51 | } 52 | c.Writer.Header().Set("Content-Type", ct) 53 | c.Render(200, render.Reader{ContentLength: -1, Reader: mr}) 54 | } else { 55 | var base string 56 | if g.subdomains { 57 | base = collection.Name 58 | } else { 59 | base = gopath.Join("thread", thread.String(), collection.Name) 60 | } 61 | var links []link 62 | for _, item := range rep.Items { 63 | pth := gopath.Join(base, strings.Replace(item.Path, buck.Path, buck.Key, 1)) 64 | if token.Defined() { 65 | pth += "?token=" + string(token) 66 | } 67 | links = append(links, link{ 68 | Name: item.Name, 69 | Path: pth, 70 | Size: byteCountDecimal(item.Size), 71 | Links: strconv.Itoa(len(item.Items)), 72 | }) 73 | } 74 | var name string 75 | if len(buck.Name) != 0 { 76 | name = buck.Name 77 | } else { 78 | name = buck.Key 79 | } 80 | root := strings.Replace(rep.Path, buck.Path, name, 1) 81 | back := gopath.Dir(gopath.Join(base, strings.Replace(rep.Path, buck.Path, buck.Key, 1))) 82 | if token.Defined() { 83 | back += "?token=" + string(token) 84 | } 85 | c.HTML(http.StatusOK, "/public/html/unixfs.gohtml", gin.H{ 86 | "Title": "Index of /" + root, 87 | "Root": "/" + root, 88 | "Path": rep.Path, 89 | "Updated": time.Unix(0, buck.UpdatedAt).String(), 90 | "Back": back, 91 | "Links": links, 92 | }) 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /gateway/ipfs.go: -------------------------------------------------------------------------------- 1 | package gateway 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | gopath "path" 9 | "strings" 10 | 11 | "github.com/gin-gonic/gin" 12 | "github.com/gin-gonic/gin/render" 13 | files "github.com/ipfs/go-ipfs-files" 14 | iface "github.com/ipfs/interface-go-ipfs-core" 15 | "github.com/ipfs/interface-go-ipfs-core/path" 16 | "github.com/libp2p/go-libp2p-core/peer" 17 | ) 18 | 19 | func (g *Gateway) ipfsHandler(c *gin.Context) { 20 | base := fmt.Sprintf("ipfs/%s", c.Param("root")) 21 | pth := fmt.Sprintf("/%s%s", base, c.Param("path")) 22 | g.renderIPFSPath(c, base, pth) 23 | } 24 | 25 | func (g *Gateway) renderIPFSPath(c *gin.Context, base, pth string) { 26 | ctx, cancel := context.WithTimeout(context.Background(), handlerTimeout) 27 | defer cancel() 28 | pth = strings.TrimSuffix(pth, "/") 29 | f, err := g.openPath(ctx, path.New(pth)) 30 | if err != nil { 31 | if err == iface.ErrIsDir { 32 | var root, dir, back string 33 | parts := strings.Split(pth, "/") 34 | if len(parts) > 2 { 35 | root = strings.Join(parts[:3], "/") 36 | dir = strings.Join(parts[3:], "/") 37 | back = gopath.Dir(dir) 38 | } 39 | if dir == "" { 40 | back = "" 41 | } 42 | if !g.subdomains { 43 | dir = gopath.Join(base, dir) 44 | if back != "" { 45 | back = gopath.Join(base, back) 46 | } 47 | } 48 | lctx, lcancel := context.WithTimeout(context.Background(), handlerTimeout) 49 | defer lcancel() 50 | ilinks, err := g.ipfs.Object().Links(lctx, path.New(pth)) 51 | if err != nil { 52 | renderError(c, http.StatusNotFound, err) 53 | return 54 | } 55 | var links []link 56 | for _, l := range ilinks { 57 | links = append(links, link{ 58 | Name: l.Name, 59 | Path: gopath.Join(dir, l.Name), 60 | Size: byteCountDecimal(int64(l.Size)), 61 | }) 62 | } 63 | var index string 64 | if strings.HasPrefix(base, "ipns") { 65 | index = gopath.Join("/", base, dir) 66 | } else { 67 | index = gopath.Join(root, dir) 68 | } 69 | if !g.subdomains { 70 | dir = strings.TrimPrefix(strings.Replace(dir, base, "", 1), "/") 71 | } 72 | params := gin.H{ 73 | "Title": "Index of " + index, 74 | "Root": "/" + dir, 75 | "Path": pth, 76 | "Updated": "", 77 | "Back": strings.TrimPrefix(back, "/"), 78 | "Links": links, 79 | } 80 | c.HTML(http.StatusOK, "/public/html/unixfs.gohtml", params) 81 | return 82 | } 83 | 84 | renderError(c, http.StatusBadRequest, err) 85 | return 86 | } 87 | defer f.Close() 88 | 89 | ct, r, err := detectReaderOrPathContentType(f, pth) 90 | if err != nil { 91 | renderError(c, http.StatusInternalServerError, fmt.Errorf("detecting mime: %s", err)) 92 | return 93 | } 94 | 95 | c.Writer.Header().Set("Content-Type", ct) 96 | c.Render(200, render.Reader{ContentLength: -1, Reader: r}) 97 | } 98 | 99 | func (g *Gateway) openPath(ctx context.Context, pth path.Path) (io.ReadCloser, error) { 100 | f, err := g.ipfs.Unixfs().Get(ctx, pth) 101 | if err != nil { 102 | return nil, err 103 | } 104 | var file files.File 105 | switch f := f.(type) { 106 | case files.File: 107 | file = f 108 | case files.Directory: 109 | return nil, iface.ErrIsDir 110 | default: 111 | return nil, iface.ErrNotSupported 112 | } 113 | return file, nil 114 | } 115 | 116 | func (g *Gateway) ipnsHandler(c *gin.Context) { 117 | g.renderIPNSKey(c, c.Param("key"), c.Param("path")) 118 | } 119 | 120 | func (g *Gateway) renderIPNSKey(c *gin.Context, key, pth string) { 121 | // @todo: Lookup key and render from local content if exists 122 | ctx, cancel := context.WithTimeout(context.Background(), handlerTimeout) 123 | defer cancel() 124 | root, err := g.ipfs.Name().Resolve(ctx, key) 125 | if err != nil { 126 | renderError(c, http.StatusNotFound, err) 127 | return 128 | } 129 | base := fmt.Sprintf("ipns/%s", key) 130 | g.renderIPFSPath(c, base, gopath.Join(root.String(), pth)) 131 | } 132 | 133 | func (g *Gateway) p2pHandler(c *gin.Context) { 134 | g.renderP2PKey(c, c.Param("key")) 135 | } 136 | 137 | func (g *Gateway) renderP2PKey(c *gin.Context, key string) { 138 | pid, err := peer.Decode(key) 139 | if err != nil { 140 | renderError(c, http.StatusBadRequest, err) 141 | return 142 | } 143 | ctx, cancel := context.WithTimeout(context.Background(), handlerTimeout) 144 | defer cancel() 145 | info, err := g.ipfs.Dht().FindPeer(ctx, pid) 146 | if err != nil { 147 | renderError(c, http.StatusNotFound, err) 148 | return 149 | } 150 | c.JSON(http.StatusOK, info) 151 | } 152 | 153 | func (g *Gateway) ipldHandler(c *gin.Context) { 154 | pth := fmt.Sprintf("%s%s", c.Param("root"), strings.TrimSuffix(c.Param("path"), "/")) 155 | g.renderP2PKey(c, pth) 156 | } 157 | 158 | func (g *Gateway) renderIPLDPath(c *gin.Context, pth string) { 159 | ctx, cancel := context.WithTimeout(context.Background(), handlerTimeout) 160 | defer cancel() 161 | node, err := g.ipfs.Object().Get(ctx, path.New(pth)) 162 | if err != nil { 163 | renderError(c, http.StatusNotFound, err) 164 | return 165 | } 166 | c.JSON(http.StatusOK, node) 167 | } 168 | -------------------------------------------------------------------------------- /gateway/public/css/style.css: -------------------------------------------------------------------------------- 1 | html { 2 | box-sizing: border-box; 3 | margin: 0; 4 | padding: 0; 5 | height: 100%; 6 | } 7 | 8 | *, *:before, *:after { 9 | box-sizing: inherit; 10 | } 11 | 12 | body { 13 | margin: 0; 14 | padding: 2em; 15 | font-family: monospace, sans-serif; 16 | color: #666666; 17 | background-color: #222222; 18 | height: 100%; 19 | } 20 | 21 | .logo { 22 | position: absolute; 23 | } 24 | 25 | .title { 26 | line-height: 2em; 27 | font-size: 0.9em; 28 | margin-top: 5em; 29 | margin-bottom: 1em; 30 | } 31 | 32 | .title span.yellow { 33 | color: #FFCE00; 34 | } 35 | 36 | .updated { 37 | font-size: 0.8em; 38 | margin: 2em 0 1em; 39 | } 40 | 41 | a { 42 | color: #FFB6D5; 43 | text-decoration: none; 44 | } 45 | 46 | ul, li { 47 | list-style: none; 48 | width: 100%; 49 | } 50 | 51 | ul { 52 | margin: 0; 53 | padding: 0; 54 | } 55 | 56 | li { 57 | line-height: 2em; 58 | font-size: 0.9em; 59 | } 60 | 61 | li span.right { 62 | float: right; 63 | } 64 | 65 | li:nth-child(even) { 66 | background: rgba(255,182,213,0.05); 67 | } 68 | 69 | li:nth-child(odd) { 70 | background: none; 71 | } 72 | 73 | .aligner { 74 | display: flex; 75 | align-items: center; 76 | justify-content: center; 77 | flex-direction: column; 78 | height: 100%; 79 | } 80 | 81 | .aligner-item { 82 | max-width: 60%; 83 | } 84 | 85 | .aligner-item p { 86 | text-align: center; 87 | line-height: 1.5em; 88 | } 89 | 90 | .icon-big { 91 | font-size: 4em; 92 | } 93 | -------------------------------------------------------------------------------- /gateway/public/html/404.gohtml: -------------------------------------------------------------------------------- 1 | {{template "header" "404 Not Found"}} 2 |
3 |
4 | 5 |
6 |
7 |

Nothing to see here!

8 |
9 |
10 | {{template "footer"}} 11 | -------------------------------------------------------------------------------- /gateway/public/html/confirm.gohtml: -------------------------------------------------------------------------------- 1 | {{template "header" "Email Address Confirmed"}} 2 |
3 |
4 | 5 |
6 |
7 |

You have been correctly authenticated. You may now close this window!

8 |
9 |
10 | {{template "footer"}} 11 | -------------------------------------------------------------------------------- /gateway/public/html/consent.gohtml: -------------------------------------------------------------------------------- 1 | {{template "header" "Invite Accepted"}} 2 |
3 |
4 | 5 |
6 |
7 |

{{.Email}} is now a member of the {{.Org}} organization.

8 |

If you haven't already, download the Hub CLI and initialize a new account. Check out the docs for more. You may now close this window!

9 |
10 |
11 | {{template "footer"}} 12 | -------------------------------------------------------------------------------- /gateway/public/html/error.gohtml: -------------------------------------------------------------------------------- 1 | {{template "header" "Oops!"}} 2 |
3 |
4 | 5 |
6 |
7 |

{{.Code}} Error: {{.Error}}

8 |
9 |
10 | {{template "footer"}} 11 | -------------------------------------------------------------------------------- /gateway/public/html/index.gohtml: -------------------------------------------------------------------------------- 1 | {{define "header"}} 2 | 3 | 4 | 5 | 6 | {{.}} 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 18 | {{end}} 19 | 20 | {{define "footer"}} 21 | 22 | 23 | {{end}} 24 | -------------------------------------------------------------------------------- /gateway/public/html/unixfs.gohtml: -------------------------------------------------------------------------------- 1 | {{template "header" .Title}} 2 |
3 | {{ if ne .Root "" }}{{.Root}}: {{ end }}{{.Path}} 4 |
5 |
    6 | {{ if ne .Back "" }} 7 |
  • ..
    • 8 | {{ end }} 9 | {{range .Links}} 10 |
  • {{.Name}}{{.Size}}
    • 11 | {{end}} 12 |
13 | {{ if ne .Updated "" }} 14 |
Updated {{.Updated}}
15 | {{ end }} 16 | {{template "footer"}} 17 | -------------------------------------------------------------------------------- /gateway/public/img/android-chrome-192x192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/textileio/go-buckets/89def8d01abaafa36b18eaef8c2b78c871cfff8c/gateway/public/img/android-chrome-192x192.png -------------------------------------------------------------------------------- /gateway/public/img/android-chrome-512x512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/textileio/go-buckets/89def8d01abaafa36b18eaef8c2b78c871cfff8c/gateway/public/img/android-chrome-512x512.png -------------------------------------------------------------------------------- /gateway/public/img/apple-touch-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/textileio/go-buckets/89def8d01abaafa36b18eaef8c2b78c871cfff8c/gateway/public/img/apple-touch-icon.png -------------------------------------------------------------------------------- /gateway/public/img/favicon-16x16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/textileio/go-buckets/89def8d01abaafa36b18eaef8c2b78c871cfff8c/gateway/public/img/favicon-16x16.png -------------------------------------------------------------------------------- /gateway/public/img/favicon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/textileio/go-buckets/89def8d01abaafa36b18eaef8c2b78c871cfff8c/gateway/public/img/favicon-32x32.png -------------------------------------------------------------------------------- /gateway/public/img/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/textileio/go-buckets/89def8d01abaafa36b18eaef8c2b78c871cfff8c/gateway/public/img/favicon.ico -------------------------------------------------------------------------------- /gateway/public/img/site.webmanifest: -------------------------------------------------------------------------------- 1 | {"name":"","short_name":"","icons":[{"src":"/android-chrome-192x192.png","sizes":"192x192","type":"image/png"},{"src":"/android-chrome-512x512.png","sizes":"512x512","type":"image/png"}],"theme_color":"#ffffff","background_color":"#ffffff","display":"standalone"} -------------------------------------------------------------------------------- /gateway/public/webfonts/fa-brands-400.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/textileio/go-buckets/89def8d01abaafa36b18eaef8c2b78c871cfff8c/gateway/public/webfonts/fa-brands-400.eot -------------------------------------------------------------------------------- /gateway/public/webfonts/fa-brands-400.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/textileio/go-buckets/89def8d01abaafa36b18eaef8c2b78c871cfff8c/gateway/public/webfonts/fa-brands-400.ttf -------------------------------------------------------------------------------- /gateway/public/webfonts/fa-brands-400.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/textileio/go-buckets/89def8d01abaafa36b18eaef8c2b78c871cfff8c/gateway/public/webfonts/fa-brands-400.woff -------------------------------------------------------------------------------- /gateway/public/webfonts/fa-brands-400.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/textileio/go-buckets/89def8d01abaafa36b18eaef8c2b78c871cfff8c/gateway/public/webfonts/fa-brands-400.woff2 -------------------------------------------------------------------------------- /gateway/public/webfonts/fa-regular-400.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/textileio/go-buckets/89def8d01abaafa36b18eaef8c2b78c871cfff8c/gateway/public/webfonts/fa-regular-400.eot -------------------------------------------------------------------------------- /gateway/public/webfonts/fa-regular-400.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/textileio/go-buckets/89def8d01abaafa36b18eaef8c2b78c871cfff8c/gateway/public/webfonts/fa-regular-400.ttf -------------------------------------------------------------------------------- /gateway/public/webfonts/fa-regular-400.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/textileio/go-buckets/89def8d01abaafa36b18eaef8c2b78c871cfff8c/gateway/public/webfonts/fa-regular-400.woff -------------------------------------------------------------------------------- /gateway/public/webfonts/fa-regular-400.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/textileio/go-buckets/89def8d01abaafa36b18eaef8c2b78c871cfff8c/gateway/public/webfonts/fa-regular-400.woff2 -------------------------------------------------------------------------------- /gateway/public/webfonts/fa-solid-900.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/textileio/go-buckets/89def8d01abaafa36b18eaef8c2b78c871cfff8c/gateway/public/webfonts/fa-solid-900.eot -------------------------------------------------------------------------------- /gateway/public/webfonts/fa-solid-900.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/textileio/go-buckets/89def8d01abaafa36b18eaef8c2b78c871cfff8c/gateway/public/webfonts/fa-solid-900.ttf -------------------------------------------------------------------------------- /gateway/public/webfonts/fa-solid-900.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/textileio/go-buckets/89def8d01abaafa36b18eaef8c2b78c871cfff8c/gateway/public/webfonts/fa-solid-900.woff -------------------------------------------------------------------------------- /gateway/public/webfonts/fa-solid-900.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/textileio/go-buckets/89def8d01abaafa36b18eaef8c2b78c871cfff8c/gateway/public/webfonts/fa-solid-900.woff2 -------------------------------------------------------------------------------- /gateway/push.go: -------------------------------------------------------------------------------- 1 | package gateway 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "mime" 8 | "mime/multipart" 9 | "net/http" 10 | "time" 11 | 12 | "github.com/gin-gonic/gin" 13 | "github.com/ipfs/interface-go-ipfs-core/path" 14 | "github.com/textileio/go-buckets" 15 | "github.com/textileio/go-buckets/dag" 16 | core "github.com/textileio/go-threads/core/thread" 17 | ) 18 | 19 | // UploadTimeout is the max time taken to push files to a bucket. 20 | var UploadTimeout = time.Hour 21 | 22 | const chunkSize = 1024 * 32 23 | 24 | // PostError wraps errors as JSON. 25 | type PostError struct { 26 | Error string `json:"error"` 27 | } 28 | 29 | type chanErr struct { 30 | code int 31 | err error 32 | } 33 | 34 | // PushPathsResult wraps a single path result. 35 | type PushPathsResult struct { 36 | Path string `json:"path"` 37 | Cid string `json:"cid"` 38 | Size int64 `json:"size"` 39 | } 40 | 41 | // PushPathsResults wraps all path results. 42 | type PushPathsResults struct { 43 | Results []PushPathsResult `json:"results"` 44 | Pinned int64 `json:"pinned"` 45 | Bucket *buckets.Bucket `json:"bucket"` 46 | } 47 | 48 | // bucketPushPathsHandler handles bucket pushes over HTTP. 49 | // 50 | // For example: 51 | // > curl -H "Authorization: Bearer " \ 52 | // -F push=@ \ 53 | // -F push=@ \ 54 | // http://127.0.0.1:8000/b/ 55 | // 56 | // To ensure updates are fast-forward-only, use the root query param on the request URL: 57 | // ?root=/ipfs/ 58 | func (g *Gateway) bucketPushPathsHandler(c *gin.Context) { 59 | thread, err := g.getThread(c) 60 | if err != nil { 61 | c.AbortWithStatusJSON(http.StatusBadRequest, PostError{ 62 | Error: err.Error(), 63 | }) 64 | return 65 | } 66 | g.pushBucketPaths(c, thread, c.Param("key")) 67 | } 68 | 69 | func (g *Gateway) pushBucketPaths(c *gin.Context, thread core.ID, key string) { 70 | token, ok := getAuth(c) 71 | if !ok { 72 | c.AbortWithStatusJSON(http.StatusUnauthorized, PostError{ 73 | Error: fmt.Sprintf("authorization required"), 74 | }) 75 | return 76 | } 77 | var root path.Resolved 78 | if v, ok := c.GetQuery("root"); ok { 79 | var err error 80 | root, err = dag.NewResolvedPath(v) 81 | if err != nil { 82 | c.AbortWithStatusJSON(http.StatusBadRequest, PostError{ 83 | Error: fmt.Sprintf("parsing root param: %v", err), 84 | }) 85 | return 86 | } 87 | } 88 | 89 | _, params, err := mime.ParseMediaType(c.GetHeader("Content-Type")) 90 | if err != nil { 91 | c.AbortWithStatusJSON(http.StatusBadRequest, PostError{ 92 | Error: fmt.Sprintf("parsing content-type: %v", err), 93 | }) 94 | return 95 | } 96 | boundary, ok := params["boundary"] 97 | if !ok { 98 | c.AbortWithStatusJSON(http.StatusBadRequest, PostError{ 99 | Error: "invalid multipart boundary", 100 | }) 101 | return 102 | } 103 | 104 | ctx, cancel := context.WithTimeout(context.Background(), UploadTimeout) 105 | defer cancel() 106 | in, out, errs := g.lib.PushPaths(ctx, thread, key, token, root) 107 | if len(errs) != 0 { 108 | err := <-errs 109 | c.AbortWithStatusJSON(http.StatusBadRequest, PostError{ 110 | Error: fmt.Sprintf("starting push: %v", err), 111 | }) 112 | return 113 | } 114 | 115 | errCh := make(chan chanErr) 116 | go func() { 117 | defer close(in) 118 | mr := multipart.NewReader(c.Request.Body, boundary) 119 | buf := make([]byte, chunkSize) 120 | for { 121 | part, err := mr.NextPart() 122 | if err == io.EOF { 123 | return 124 | } else if err != nil { 125 | errCh <- chanErr{ 126 | code: http.StatusInternalServerError, 127 | err: fmt.Errorf("reading part: %v", err), 128 | } 129 | return 130 | } 131 | for { 132 | n, err := part.Read(buf) 133 | input := buckets.PushPathsInput{ 134 | Path: part.FileName(), 135 | } 136 | if n > 0 { 137 | input.Chunk = make([]byte, n) 138 | copy(input.Chunk, buf[:n]) 139 | in <- input 140 | } else if err == io.EOF { 141 | in <- input 142 | part.Close() 143 | break 144 | } else if err != nil { 145 | errCh <- chanErr{ 146 | code: http.StatusInternalServerError, 147 | err: fmt.Errorf("reading part: %v", err), 148 | } 149 | part.Close() 150 | return 151 | } 152 | } 153 | } 154 | }() 155 | 156 | results := PushPathsResults{} 157 | for { 158 | select { 159 | case res := <-out: 160 | results.Results = append(results.Results, PushPathsResult{ 161 | Path: res.Path, 162 | Cid: res.Cid.String(), 163 | Size: res.Size, 164 | }) 165 | results.Bucket = res.Bucket 166 | results.Pinned = res.Pinned 167 | case err := <-errs: 168 | if err != nil { 169 | c.AbortWithStatusJSON(http.StatusBadRequest, PostError{ 170 | Error: err.Error(), 171 | }) 172 | } else { 173 | c.JSON(http.StatusCreated, results) 174 | } 175 | return 176 | case err := <-errCh: 177 | c.AbortWithStatusJSON(err.code, PostError{ 178 | Error: err.err.Error(), 179 | }) 180 | return 181 | } 182 | } 183 | } 184 | -------------------------------------------------------------------------------- /gateway/push_test.go: -------------------------------------------------------------------------------- 1 | package gateway_test 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "crypto/rand" 7 | "fmt" 8 | "io" 9 | "io/ioutil" 10 | "mime/multipart" 11 | "net/http" 12 | "os" 13 | "testing" 14 | 15 | "github.com/stretchr/testify/assert" 16 | "github.com/stretchr/testify/require" 17 | "github.com/textileio/go-threads/core/did" 18 | ) 19 | 20 | func Test_PushBucketPaths(t *testing.T) { 21 | gw := newGateway(t) 22 | 23 | token := newIdentityToken(t) 24 | buck, _, _, err := gw.Buckets().Create(context.Background(), token) 25 | require.NoError(t, err) 26 | 27 | files := make(map[string]*os.File) 28 | files["file"] = getRandomFile(t, 1024) 29 | files["dir/file"] = getRandomFile(t, 1024) 30 | files["dir/dir/file"] = getRandomFile(t, 1024) 31 | 32 | url := fmt.Sprintf("%s/thread/%s/buckets/%s?root=%s", gw.Url(), buck.Thread, buck.Key, buck.Path) 33 | pushBucketPaths(t, url, token, files) 34 | 35 | item, _, err := gw.Buckets().ListPath(context.Background(), buck.Thread, buck.Key, token, "") 36 | require.NoError(t, err) 37 | assert.True(t, item.IsDir) 38 | assert.Len(t, item.Items, 3) // .textileseed, file, dir 39 | 40 | item, _, err = gw.Buckets().ListPath(context.Background(), buck.Thread, buck.Key, token, "dir") 41 | require.NoError(t, err) 42 | assert.True(t, item.IsDir) 43 | assert.Len(t, item.Items, 2) // file, dir 44 | 45 | item, _, err = gw.Buckets().ListPath(context.Background(), buck.Thread, buck.Key, token, "dir/dir") 46 | require.NoError(t, err) 47 | assert.True(t, item.IsDir) 48 | assert.Len(t, item.Items, 1) // file 49 | 50 | item, _, err = gw.Buckets().ListPath(context.Background(), buck.Thread, buck.Key, token, "file") 51 | require.NoError(t, err) 52 | assert.False(t, item.IsDir) 53 | assert.Equal(t, 1024, int(item.Size)) 54 | 55 | item, _, err = gw.Buckets().ListPath(context.Background(), buck.Thread, buck.Key, token, "dir/file") 56 | require.NoError(t, err) 57 | assert.False(t, item.IsDir) 58 | assert.Equal(t, 1024, int(item.Size)) 59 | 60 | item, _, err = gw.Buckets().ListPath(context.Background(), buck.Thread, buck.Key, token, "dir/dir/file") 61 | require.NoError(t, err) 62 | assert.False(t, item.IsDir) 63 | assert.Equal(t, 1024, int(item.Size)) 64 | } 65 | 66 | func getRandomFile(t *testing.T, size int64) *os.File { 67 | tmp, err := ioutil.TempFile("", "") 68 | require.NoError(t, err) 69 | _, err = io.CopyN(tmp, rand.Reader, size) 70 | require.NoError(t, err) 71 | _, err = tmp.Seek(0, 0) 72 | require.NoError(t, err) 73 | return tmp 74 | } 75 | 76 | func pushBucketPaths(t *testing.T, url string, token did.Token, files map[string]*os.File) { 77 | var b bytes.Buffer 78 | w := multipart.NewWriter(&b) 79 | for pth, f := range files { 80 | fw, err := w.CreateFormFile("push", pth) 81 | require.NoError(t, err) 82 | _, err = io.Copy(fw, f) 83 | require.NoError(t, err) 84 | } 85 | err := w.Close() 86 | require.NoError(t, err) 87 | 88 | req, err := http.NewRequest("POST", url, &b) 89 | require.NoError(t, err) 90 | 91 | req.Header.Set("Content-Type", w.FormDataContentType()) 92 | req.Header.Set("Authorization", "Bearer "+string(token)) 93 | 94 | c := &http.Client{} 95 | res, err := c.Do(req) 96 | require.NoError(t, err) 97 | assert.Equal(t, http.StatusCreated, res.StatusCode) 98 | } 99 | -------------------------------------------------------------------------------- /gateway/threads.go: -------------------------------------------------------------------------------- 1 | package gateway 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "net/http" 7 | gopath "path" 8 | 9 | "github.com/gin-gonic/gin" 10 | "github.com/textileio/go-buckets/collection" 11 | "github.com/textileio/go-threads/core/did" 12 | core "github.com/textileio/go-threads/core/thread" 13 | ) 14 | 15 | func (g *Gateway) threadHandler(c *gin.Context) { 16 | thread, err := core.Decode(c.Param("thread")) 17 | if err != nil { 18 | renderError(c, http.StatusBadRequest, errors.New("invalid thread ID")) 19 | return 20 | } 21 | g.renderThread(c, thread) 22 | } 23 | 24 | func (g *Gateway) renderThread(c *gin.Context, thread core.ID) { 25 | token := did.Token(c.Query("token")) 26 | 27 | ctx, cancel := context.WithTimeout(context.Background(), handlerTimeout) 28 | defer cancel() 29 | rep, err := g.lib.List(ctx, thread, token) 30 | if err != nil { 31 | render404(c) 32 | return 33 | } 34 | links := make([]link, len(rep)) 35 | for i, r := range rep { 36 | var name string 37 | if r.Name != "" { 38 | name = r.Name 39 | } else { 40 | name = r.Key 41 | } 42 | p := gopath.Join("thread", thread.String(), collection.Name, r.Key) 43 | if token.Defined() { 44 | p += "?token=" + string(token) 45 | } 46 | links[i] = link{ 47 | Name: name, 48 | Path: p, 49 | Size: "", 50 | Links: "", 51 | } 52 | } 53 | c.HTML(http.StatusOK, "/public/html/unixfs.gohtml", gin.H{ 54 | "Title": "Index of " + gopath.Join("/thread", thread.String(), collection.Name), 55 | "Root": "/", 56 | "Path": "", 57 | "Updated": "", 58 | "Back": "", 59 | "Links": links, 60 | }) 61 | } 62 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/textileio/go-buckets 2 | 3 | go 1.16 4 | 5 | require ( 6 | github.com/alecthomas/jsonschema v0.0.0-20191017121752-4bb6e3fae4f2 7 | github.com/aws/aws-sdk-go v1.32.11 // indirect 8 | github.com/cenkalti/backoff/v4 v4.0.2 9 | github.com/cheggaaa/pb/v3 v3.0.5 10 | github.com/cloudflare/cloudflare-go v0.11.6 11 | github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect 12 | github.com/fatih/color v1.9.0 // indirect 13 | github.com/gin-contrib/location v0.0.2 14 | github.com/gin-contrib/static v0.0.0-20191128031702-f81c604d8ac2 15 | github.com/gin-gonic/gin v1.6.3 16 | github.com/gogo/googleapis v1.4.0 // indirect 17 | github.com/golang/protobuf v1.4.3 18 | github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf // indirect 19 | github.com/google/go-cmp v0.5.4 // indirect 20 | github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f // indirect 21 | github.com/grpc-ecosystem/go-grpc-middleware v1.2.2 // indirect 22 | github.com/improbable-eng/grpc-web v0.13.0 23 | github.com/ipfs/go-blockservice v0.1.4 24 | github.com/ipfs/go-cid v0.0.7 25 | github.com/ipfs/go-datastore v0.4.5 26 | github.com/ipfs/go-ds-flatfs v0.4.4 27 | github.com/ipfs/go-ipfs-blockstore v1.0.3 28 | github.com/ipfs/go-ipfs-chunker v0.0.5 29 | github.com/ipfs/go-ipfs-ds-help v1.0.0 30 | github.com/ipfs/go-ipfs-exchange-offline v0.0.1 31 | github.com/ipfs/go-ipfs-files v0.0.8 32 | github.com/ipfs/go-ipfs-http-client v0.1.0 33 | github.com/ipfs/go-ipld-cbor v0.0.5 34 | github.com/ipfs/go-ipld-format v0.2.0 35 | github.com/ipfs/go-log/v2 v2.1.2-0.20200626104915-0016c0b4b3e4 36 | github.com/ipfs/go-merkledag v0.3.2 37 | github.com/ipfs/go-pinning-service-http-client v0.1.0 38 | github.com/ipfs/go-unixfs v0.2.4 39 | github.com/ipfs/interface-go-ipfs-core v0.4.0 40 | github.com/jbenet/go-is-domain v1.0.3 41 | github.com/json-iterator/go v1.1.10 // indirect 42 | github.com/libp2p/go-libp2p-core v0.7.0 43 | github.com/libp2p/go-libp2p-pubsub v0.4.1 // indirect 44 | github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 45 | github.com/lunixbochs/vtclean v1.0.0 // indirect 46 | github.com/manifoldco/promptui v0.7.0 47 | github.com/mattn/go-colorable v0.1.8 // indirect 48 | github.com/mattn/go-runewidth v0.0.10 // indirect 49 | github.com/mitchellh/go-homedir v1.1.0 50 | github.com/mitchellh/mapstructure v1.3.0 // indirect 51 | github.com/multiformats/go-multiaddr v0.3.1 52 | github.com/multiformats/go-multibase v0.0.3 53 | github.com/multiformats/go-multihash v0.0.14 54 | github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect 55 | github.com/oklog/ulid/v2 v2.0.2 56 | github.com/olekukonko/tablewriter v0.0.4 57 | github.com/onsi/ginkgo v1.14.0 // indirect 58 | github.com/pelletier/go-toml v1.7.0 // indirect 59 | github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 60 | github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a // indirect 61 | github.com/rivo/uniseg v0.2.0 // indirect 62 | github.com/rs/cors v1.7.0 63 | github.com/smartystreets/assertions v1.0.1 // indirect 64 | github.com/spf13/afero v1.2.2 // indirect 65 | github.com/spf13/cast v1.3.1 // indirect 66 | github.com/spf13/cobra v1.1.3 67 | github.com/spf13/jwalterweatherman v1.1.0 // indirect 68 | github.com/spf13/viper v1.7.1 69 | github.com/stretchr/testify v1.7.0 70 | github.com/textileio/dcrypto v0.0.1 71 | github.com/textileio/go-assets v0.0.0-20200430191519-b341e634e2b7 72 | github.com/textileio/go-datastore-extensions v1.0.1 73 | github.com/textileio/go-ds-badger3 v0.0.0-20210324034212-7b7fb3be3d1c 74 | github.com/textileio/go-ds-mongo v0.1.5-0.20201230201018-2b7fdca787a5 75 | github.com/textileio/go-threads v1.0.3-0.20210331042803-3a1b7e46e91f 76 | github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a // indirect 77 | github.com/whyrusleeping/cbor-gen v0.0.0-20210118024343-169e9d70c0c2 // indirect 78 | github.com/xdg/stringprep v1.0.0 // indirect 79 | go.mongodb.org/mongo-driver v1.4.1 // indirect 80 | go.opencensus.io v0.22.6 // indirect 81 | go.uber.org/multierr v1.6.0 // indirect 82 | golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad // indirect 83 | golang.org/x/exp v0.0.0-20200513190911-00229845015e // indirect 84 | golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect 85 | golang.org/x/net v0.0.0-20210119194325-5f4716e94777 // indirect 86 | golang.org/x/sync v0.0.0-20201207232520-09787c993a3a 87 | golang.org/x/sys v0.0.0-20210324051608-47abb6519492 // indirect 88 | golang.org/x/text v0.3.5 // indirect 89 | golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect 90 | google.golang.org/genproto v0.0.0-20200624020401-64a14ca9d1ad // indirect 91 | google.golang.org/grpc v1.35.0 92 | google.golang.org/protobuf v1.25.0 93 | gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect 94 | gopkg.in/ini.v1 v1.55.0 // indirect 95 | honnef.co/go/tools v0.0.1-2020.1.4 // indirect 96 | ) 97 | 98 | replace github.com/ipfs/go-pinning-service-http-client => github.com/textileio/go-pinning-service-http-client v0.1.1-0.20210328174252-bc12e73b9a56 99 | -------------------------------------------------------------------------------- /info.go: -------------------------------------------------------------------------------- 1 | package buckets 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/ipfs/interface-go-ipfs-core/path" 9 | "github.com/textileio/go-buckets/collection" 10 | "github.com/textileio/go-threads/core/did" 11 | core "github.com/textileio/go-threads/core/thread" 12 | ) 13 | 14 | // PushPathInfo pushes arbitrary info/metadata to a path, 15 | // allowing users to add application specific path metadata. 16 | func (b *Buckets) PushPathInfo( 17 | ctx context.Context, 18 | thread core.ID, 19 | key string, 20 | identity did.Token, 21 | root path.Resolved, 22 | pth string, 23 | info map[string]interface{}, 24 | ) (*Bucket, error) { 25 | txn, err := b.NewTxn(thread, key, identity) 26 | if err != nil { 27 | return nil, err 28 | } 29 | defer txn.Close() 30 | return txn.PushPathInfo(ctx, root, pth, info) 31 | } 32 | 33 | // PushPathInfo is Txn based PushPathInfo. 34 | func (t *Txn) PushPathInfo( 35 | ctx context.Context, 36 | root path.Resolved, 37 | pth string, 38 | info map[string]interface{}, 39 | ) (*Bucket, error) { 40 | pth, err := parsePath(pth) 41 | if err != nil { 42 | return nil, err 43 | } 44 | 45 | instance, err := t.b.c.GetSafe(ctx, t.thread, t.key, collection.WithIdentity(t.identity)) 46 | if err != nil { 47 | return nil, err 48 | } 49 | if root != nil && root.String() != instance.Path { 50 | return nil, ErrNonFastForward 51 | } 52 | 53 | md, mdPath, ok := instance.GetMetadataForPath(pth, false) 54 | if !ok { 55 | return nil, fmt.Errorf("could not resolve path: %s", pth) 56 | } 57 | var target collection.Metadata 58 | if mdPath != pth { // If the metadata is inherited from a parent, create a new entry 59 | target = collection.Metadata{ 60 | Info: make(map[string]interface{}), 61 | } 62 | } else { 63 | target = md 64 | } 65 | if target.Info == nil { 66 | target.Info = make(map[string]interface{}) 67 | } 68 | 69 | var changed bool 70 | for k, v := range info { 71 | if x, ok := target.Info[k]; ok && x == v { 72 | continue 73 | } 74 | target.Info[k] = v 75 | changed = true 76 | } 77 | if changed { 78 | instance.UpdatedAt = time.Now().UnixNano() 79 | target.UpdatedAt = instance.UpdatedAt 80 | instance.Metadata[pth] = target 81 | if err := t.b.c.Save(ctx, t.thread, instance, collection.WithIdentity(t.identity)); err != nil { 82 | return nil, err 83 | } 84 | } 85 | 86 | log.Debugf("pushed info for %s in %s", pth, t.key) 87 | return instanceToBucket(t.thread, instance), nil 88 | } 89 | 90 | // PullPathInfo pulls all info at a path. 91 | func (b *Buckets) PullPathInfo( 92 | ctx context.Context, 93 | thread core.ID, 94 | key string, 95 | identity did.Token, 96 | pth string, 97 | ) (map[string]interface{}, error) { 98 | if err := thread.Validate(); err != nil { 99 | return nil, fmt.Errorf("invalid thread id: %v", err) 100 | } 101 | pth, err := parsePath(pth) 102 | if err != nil { 103 | return nil, err 104 | } 105 | instance, err := b.c.GetSafe(ctx, thread, key, collection.WithIdentity(identity)) 106 | if err != nil { 107 | return nil, err 108 | } 109 | md, _, ok := instance.GetMetadataForPath(pth, false) 110 | if !ok { 111 | return nil, fmt.Errorf("could not resolve path: %s", pth) 112 | } 113 | if md.Info == nil { 114 | md.Info = make(map[string]interface{}) 115 | } 116 | 117 | log.Debugf("pulled info for %s in %s", pth, key) 118 | return md.Info, nil 119 | } 120 | -------------------------------------------------------------------------------- /ipns/ipns.go: -------------------------------------------------------------------------------- 1 | package ipns 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "sync" 7 | "time" 8 | 9 | ds "github.com/ipfs/go-datastore" 10 | logging "github.com/ipfs/go-log/v2" 11 | iface "github.com/ipfs/interface-go-ipfs-core" 12 | "github.com/ipfs/interface-go-ipfs-core/options" 13 | "github.com/ipfs/interface-go-ipfs-core/path" 14 | "github.com/libp2p/go-libp2p-core/peer" 15 | mbase "github.com/multiformats/go-multibase" 16 | s "github.com/textileio/go-buckets/ipns/store" 17 | "github.com/textileio/go-threads/core/thread" 18 | tutil "github.com/textileio/go-threads/util" 19 | ) 20 | 21 | var log = logging.Logger("buckets/ipns") 22 | 23 | const ( 24 | // publishTimeout 25 | publishTimeout = time.Minute * 2 26 | // maxCancelPublishTries is the number of time cancelling a publish is allowed to fail. 27 | maxCancelPublishTries = 10 28 | ) 29 | 30 | // Manager handles bucket name publishing to IPNS. 31 | type Manager struct { 32 | store *s.Store 33 | keyAPI iface.KeyAPI 34 | nameAPI iface.NameAPI 35 | 36 | keyLocks map[string]chan struct{} 37 | ctxsLock sync.Mutex 38 | ctxs map[string]context.CancelFunc 39 | sync.Mutex 40 | } 41 | 42 | // NewManager returns a new IPNS manager. 43 | func NewManager(store ds.TxnDatastore, ipfs iface.CoreAPI) (*Manager, error) { 44 | return &Manager{ 45 | store: s.NewStore(store), 46 | keyAPI: ipfs.Key(), 47 | nameAPI: ipfs.Name(), 48 | ctxs: make(map[string]context.CancelFunc), 49 | keyLocks: make(map[string]chan struct{}), 50 | }, nil 51 | } 52 | 53 | // Store returns the key store. 54 | func (m *Manager) Store() *s.Store { 55 | return m.store 56 | } 57 | 58 | // CreateKey generates and saves a new IPNS key. 59 | func (m *Manager) CreateKey(ctx context.Context, dbID thread.ID) (keyID string, err error) { 60 | key, err := m.keyAPI.Generate(ctx, tutil.MakeToken(), options.Key.Type(options.Ed25519Key)) 61 | if err != nil { 62 | return 63 | } 64 | keyID, err = peer.ToCid(key.ID()).StringOfBase(mbase.Base36) 65 | if err != nil { 66 | return 67 | } 68 | if err = m.store.Create(key.Name(), keyID, dbID); err != nil { 69 | return 70 | } 71 | return keyID, nil 72 | } 73 | 74 | // RemoveKey removes an IPNS key. 75 | func (m *Manager) RemoveKey(ctx context.Context, keyID string) error { 76 | key, err := m.store.GetByCid(keyID) 77 | if err != nil { 78 | return err 79 | } 80 | if _, err = m.keyAPI.Remove(ctx, key.Name); err != nil { 81 | return err 82 | } 83 | return m.store.Delete(key.Name) 84 | } 85 | 86 | // Publish publishes a path to IPNS with key ID. 87 | // Publishing can take up to a minute. Pending publishes are cancelled by consecutive 88 | // calls with the same key ID, which results in only the most recent publish succeeding. 89 | func (m *Manager) Publish(pth path.Path, keyID string) { 90 | ptl := m.getSemaphore(keyID) 91 | try := 0 92 | for { 93 | select { 94 | case ptl <- struct{}{}: 95 | pctx, cancel := context.WithTimeout(context.Background(), publishTimeout) 96 | m.ctxsLock.Lock() 97 | m.ctxs[keyID] = cancel 98 | m.ctxsLock.Unlock() 99 | if err := m.publishUnsafe(pctx, pth, keyID); err != nil { 100 | if !errors.Is(err, context.Canceled) { 101 | // Logging as a warning because this often fails with "context deadline exceeded", 102 | // even if the entry can be found on the network (not fully saturated). 103 | log.Warnf("error publishing path %s: %v", pth, err) 104 | } else { 105 | log.Debugf("publishing path %s was cancelled: %v", pth, err) 106 | } 107 | } 108 | cancel() 109 | m.ctxsLock.Lock() 110 | delete(m.ctxs, keyID) 111 | m.ctxsLock.Unlock() 112 | <-ptl 113 | return 114 | default: 115 | m.ctxsLock.Lock() 116 | cancel, ok := m.ctxs[keyID] 117 | m.ctxsLock.Unlock() 118 | if ok { 119 | cancel() 120 | } else { 121 | try++ 122 | if try > maxCancelPublishTries { 123 | log.Warnf("failed to publish path %s: max tries exceeded", pth) 124 | return 125 | } else { 126 | log.Debugf("failed to cancel publish (%v tries remaining)", maxCancelPublishTries-try) 127 | } 128 | } 129 | } 130 | } 131 | } 132 | 133 | // Close all pending publishes. 134 | func (m *Manager) Close() error { 135 | m.Lock() 136 | defer m.Unlock() 137 | m.ctxsLock.Lock() 138 | defer m.ctxsLock.Unlock() 139 | for _, cancel := range m.ctxs { 140 | cancel() 141 | } 142 | return nil 143 | } 144 | 145 | func (m *Manager) publishUnsafe(ctx context.Context, pth path.Path, keyID string) error { 146 | key, err := m.store.GetByCid(keyID) 147 | if err != nil { 148 | return err 149 | } 150 | entry, err := m.nameAPI.Publish(ctx, pth, options.Name.Key(key.Name)) 151 | if err != nil { 152 | return err 153 | } 154 | log.Debugf("published %s => %s", entry.Value(), entry.Name()) 155 | return nil 156 | } 157 | 158 | func (m *Manager) getSemaphore(key string) chan struct{} { 159 | var ptl chan struct{} 160 | var ok bool 161 | m.Lock() 162 | defer m.Unlock() 163 | if ptl, ok = m.keyLocks[key]; !ok { 164 | ptl = make(chan struct{}, 1) 165 | m.keyLocks[key] = ptl 166 | } 167 | return ptl 168 | } 169 | -------------------------------------------------------------------------------- /ipns/store/store.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "bytes" 5 | "encoding/gob" 6 | "time" 7 | 8 | ds "github.com/ipfs/go-datastore" 9 | "github.com/textileio/go-threads/core/thread" 10 | ) 11 | 12 | var ( 13 | dsPrefix = ds.NewKey("/ipns") 14 | dsCid = dsPrefix.ChildString("cid") 15 | ) 16 | 17 | type Key struct { 18 | Name string 19 | Cid string 20 | ThreadID thread.ID 21 | CreatedAt time.Time 22 | } 23 | 24 | type Store struct { 25 | store ds.TxnDatastore 26 | } 27 | 28 | func NewStore(store ds.TxnDatastore) *Store { 29 | return &Store{store: store} 30 | } 31 | 32 | func (s *Store) Create(name, cid string, threadID thread.ID) error { 33 | var buf bytes.Buffer 34 | if err := gob.NewEncoder(&buf).Encode(Key{ 35 | Name: name, 36 | Cid: cid, 37 | ThreadID: threadID, 38 | CreatedAt: time.Now(), 39 | }); err != nil { 40 | return err 41 | } 42 | 43 | txn, err := s.store.NewTransaction(false) 44 | if err != nil { 45 | return err 46 | } 47 | defer txn.Discard() 48 | 49 | // Add key value 50 | if err := txn.Put(dsPrefix.ChildString(name), buf.Bytes()); err != nil { 51 | return err 52 | } 53 | 54 | // Add "indexes" 55 | if err := txn.Put(dsCid.ChildString(cid), []byte(name)); err != nil { 56 | return err 57 | } 58 | 59 | return txn.Commit() 60 | } 61 | 62 | func (s *Store) Get(name string) (*Key, error) { 63 | val, err := s.store.Get(dsPrefix.ChildString(name)) 64 | if err != nil { 65 | return nil, err 66 | } 67 | return decode(val) 68 | } 69 | 70 | func decode(v []byte) (*Key, error) { 71 | var buf bytes.Buffer 72 | buf.Write(v) 73 | dec := gob.NewDecoder(&buf) 74 | var key Key 75 | if err := dec.Decode(&key); err != nil { 76 | return nil, err 77 | } 78 | return &key, nil 79 | } 80 | 81 | func (s *Store) GetByCid(cid string) (*Key, error) { 82 | txn, err := s.store.NewTransaction(true) 83 | if err != nil { 84 | return nil, err 85 | } 86 | defer txn.Discard() 87 | 88 | val, err := txn.Get(dsCid.ChildString(cid)) 89 | if err != nil { 90 | return nil, err 91 | } 92 | return s.Get(string(val)) 93 | } 94 | 95 | func (s *Store) Delete(name string) error { 96 | txn, err := s.store.NewTransaction(false) 97 | if err != nil { 98 | return err 99 | } 100 | defer txn.Discard() 101 | 102 | val, err := txn.Get(dsPrefix.ChildString(name)) 103 | if err != nil { 104 | return err 105 | } 106 | key, err := decode(val) 107 | if err != nil { 108 | return err 109 | } 110 | 111 | // Delete "indexes" 112 | if err := txn.Delete(dsCid.ChildString(key.Cid)); err != nil { 113 | return err 114 | } 115 | 116 | // Delete key value 117 | if err := txn.Delete(dsPrefix.ChildString(key.Name)); err != nil { 118 | return err 119 | } 120 | 121 | return txn.Commit() 122 | } 123 | -------------------------------------------------------------------------------- /ipns/store/store_test.go: -------------------------------------------------------------------------------- 1 | package store_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | "github.com/stretchr/testify/require" 8 | . "github.com/textileio/go-buckets/ipns/store" 9 | "github.com/textileio/go-threads/core/thread" 10 | "github.com/textileio/go-threads/db" 11 | ) 12 | 13 | func TestStore_Create(t *testing.T) { 14 | ds := db.NewTxMapDatastore() 15 | defer ds.Close() 16 | store := NewStore(ds) 17 | 18 | err := store.Create("foo", "cid", thread.NewRandomIDV1()) 19 | require.NoError(t, err) 20 | } 21 | 22 | func TestStore_Get(t *testing.T) { 23 | ds := db.NewTxMapDatastore() 24 | defer ds.Close() 25 | store := NewStore(ds) 26 | 27 | threadID := thread.NewRandomIDV1() 28 | err := store.Create("foo", "cid", threadID) 29 | require.NoError(t, err) 30 | 31 | key, err := store.Get("foo") 32 | require.NoError(t, err) 33 | assert.Equal(t, "foo", key.Name) 34 | assert.Equal(t, "cid", key.Cid) 35 | assert.Equal(t, threadID, key.ThreadID) 36 | } 37 | 38 | func TestStore_GetByCid(t *testing.T) { 39 | ds := db.NewTxMapDatastore() 40 | defer ds.Close() 41 | store := NewStore(ds) 42 | 43 | threadID := thread.NewRandomIDV1() 44 | err := store.Create("foo", "cid", threadID) 45 | require.NoError(t, err) 46 | 47 | key, err := store.GetByCid("cid") 48 | require.NoError(t, err) 49 | assert.Equal(t, "foo", key.Name) 50 | assert.Equal(t, "cid", key.Cid) 51 | assert.Equal(t, threadID, key.ThreadID) 52 | } 53 | 54 | func TestStore_Delete(t *testing.T) { 55 | ds := db.NewTxMapDatastore() 56 | defer ds.Close() 57 | store := NewStore(ds) 58 | 59 | err := store.Create("foo", "cid", thread.NewRandomIDV1()) 60 | require.NoError(t, err) 61 | 62 | err = store.Delete("foo") 63 | require.NoError(t, err) 64 | _, err = store.Get("foo") 65 | require.Error(t, err) 66 | } 67 | -------------------------------------------------------------------------------- /list.go: -------------------------------------------------------------------------------- 1 | package buckets 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/ipfs/interface-go-ipfs-core/path" 8 | "github.com/textileio/go-threads/core/did" 9 | core "github.com/textileio/go-threads/core/thread" 10 | ) 11 | 12 | // ListPath lists all paths under a path. 13 | func (b *Buckets) ListPath( 14 | ctx context.Context, 15 | thread core.ID, 16 | key string, 17 | identity did.Token, 18 | pth string, 19 | ) (*PathItem, *Bucket, error) { 20 | if err := thread.Validate(); err != nil { 21 | return nil, nil, fmt.Errorf("invalid thread id: %v", err) 22 | } 23 | pth = trimSlash(pth) 24 | instance, bpth, err := b.getBucketAndPath(ctx, thread, key, identity, pth) 25 | if err != nil { 26 | return nil, nil, err 27 | } 28 | item, err := b.listPath(ctx, instance, bpth) 29 | if err != nil { 30 | return nil, nil, err 31 | } 32 | log.Debugf("listed %s in %s", pth, key) 33 | return item, instanceToBucket(thread, instance), nil 34 | } 35 | 36 | // ListIPFSPath lists all paths under a path. 37 | func (b *Buckets) ListIPFSPath(ctx context.Context, pth string) (*PathItem, error) { 38 | log.Debugf("listed ipfs path %s", pth) 39 | return b.pathToItem(ctx, nil, path.New(pth), true) 40 | } 41 | -------------------------------------------------------------------------------- /local/access.go: -------------------------------------------------------------------------------- 1 | package local 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/textileio/go-buckets/collection" 8 | "github.com/textileio/go-threads/core/did" 9 | ) 10 | 11 | // PushPathAccessRoles updates path access roles by merging the pushed roles with existing roles 12 | // and returns the merged roles. 13 | // roles is a map of string marshaled public keys to path roles. A non-nil error is returned 14 | // if the map keys are not unmarshalable to public keys. 15 | // To delete a role for a public key, set its value to buckets.None. 16 | func (b *Bucket) PushPathAccessRoles( 17 | ctx context.Context, 18 | pth string, 19 | roles map[did.DID]collection.Role, 20 | ) (merged map[did.DID]collection.Role, err error) { 21 | ctx, err = b.authCtx(ctx, time.Hour) 22 | if err != nil { 23 | return 24 | } 25 | id, err := b.Thread() 26 | if err != nil { 27 | return 28 | } 29 | if err = b.c.PushPathAccessRoles(ctx, id, b.Key(), pth, roles); err != nil { 30 | return 31 | } 32 | return b.c.PullPathAccessRoles(ctx, id, b.Key(), pth) 33 | } 34 | 35 | // PullPathAccessRoles returns access roles for a path. 36 | func (b *Bucket) PullPathAccessRoles(ctx context.Context, pth string) (roles map[did.DID]collection.Role, err error) { 37 | ctx, err = b.authCtx(ctx, time.Hour) 38 | if err != nil { 39 | return 40 | } 41 | id, err := b.Thread() 42 | if err != nil { 43 | return 44 | } 45 | return b.c.PullPathAccessRoles(ctx, id, b.Key(), pth) 46 | } 47 | -------------------------------------------------------------------------------- /local/add.go: -------------------------------------------------------------------------------- 1 | package local 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | "strings" 9 | "time" 10 | 11 | "github.com/ipfs/go-cid" 12 | "github.com/ipfs/interface-go-ipfs-core/path" 13 | "github.com/textileio/go-buckets" 14 | "golang.org/x/sync/errgroup" 15 | ) 16 | 17 | // AddRemoteCid stages the Unixfs dag at cid in the local bucket, 18 | // optionally allowing the caller to select a merge strategy. 19 | func (b *Bucket) AddRemoteCid(ctx context.Context, c cid.Cid, dest string, opts ...AddOption) error { 20 | b.Lock() 21 | defer b.Unlock() 22 | ctx, err := b.authCtx(ctx, time.Hour) 23 | if err != nil { 24 | return err 25 | } 26 | args := &addOptions{} 27 | for _, opt := range opts { 28 | opt(args) 29 | } 30 | return b.mergeIpfsPath(ctx, path.IpfsPath(c), dest, args.merge, args.events) 31 | } 32 | 33 | func (b *Bucket) mergeIpfsPath( 34 | ctx context.Context, 35 | ipfsBasePth path.Path, 36 | dest string, 37 | merge SelectMergeFunc, 38 | events chan<- Event, 39 | ) error { 40 | ok, err := b.containsPath(dest) 41 | if err != nil { 42 | return err 43 | } else if !ok { 44 | return fmt.Errorf("destination %s is not in bucket path", dest) 45 | } 46 | 47 | folderReplace, toAdd, err := b.listMergePath(ctx, ipfsBasePth, "", dest, merge) 48 | if err != nil { 49 | return err 50 | } 51 | 52 | // Remove all the folders that were decided to be replaced. 53 | for _, fr := range folderReplace { 54 | if err := os.RemoveAll(fr); err != nil { 55 | return err 56 | } 57 | } 58 | 59 | // Add files that are missing, or were decided to be overwritten. 60 | if len(toAdd) > 0 { 61 | progress := handleAllPullProgress(toAdd, events) 62 | defer close(progress) 63 | 64 | eg, gctx := errgroup.WithContext(ctx) 65 | for _, o := range toAdd { 66 | o := o 67 | eg.Go(func() error { 68 | if gctx.Err() != nil { 69 | return nil 70 | } 71 | if err := os.Remove(o.path); err != nil && !os.IsNotExist(err) { 72 | return err 73 | } 74 | trimmedDest := strings.TrimPrefix(o.path, dest) 75 | return b.getIpfsFile( 76 | gctx, 77 | path.Join(ipfsBasePth, trimmedDest), 78 | o.path, 79 | o.size, 80 | o.cid, 81 | events, 82 | progress, 83 | ) 84 | }) 85 | } 86 | if err := eg.Wait(); err != nil { 87 | return err 88 | } 89 | } 90 | return nil 91 | } 92 | 93 | // listMergePath walks the local bucket and the remote IPFS UnixFS DAG asking 94 | // the client if wants to (replace, merge, ignore) matching folders, and if wants 95 | // to (overwrite, ignore) matching files. Any non-matching files or folders in the 96 | // IPFS UnixFS DAG will be added locally. 97 | // The first return value is a slice of paths of folders that were decided to be 98 | // replaced completely (not merged). The second return value are a list of files 99 | // that should be added locally. If one of them exist, can be understood that should 100 | // be overwritten. 101 | func (b *Bucket) listMergePath( 102 | ctx context.Context, 103 | ipfsBasePth path.Path, 104 | ipfsRelPath, dest string, 105 | merge SelectMergeFunc, 106 | ) ([]string, []object, error) { 107 | // List remote IPFS UnixFS path level 108 | rep, err := b.c.ListIpfsPath(ctx, path.Join(ipfsBasePth, ipfsRelPath)) 109 | if err != nil { 110 | return nil, nil, err 111 | } 112 | 113 | // If its a dir, ask if should be ignored, replaced, or merged. 114 | if rep.Item.IsDir { 115 | var replacedFolders []string 116 | var toAdd []object 117 | 118 | var folderExists bool 119 | 120 | localFolderPath := filepath.Join(dest, ipfsRelPath) 121 | if _, err := os.Stat(localFolderPath); err == nil { 122 | folderExists = true 123 | } 124 | 125 | if folderExists && merge != nil { 126 | ms, err := merge(fmt.Sprintf("Merge strategy for %s", localFolderPath), true) 127 | if err != nil { 128 | return nil, nil, err 129 | } 130 | switch ms { 131 | case Skip: 132 | return nil, nil, nil 133 | case Merge: 134 | break 135 | case Replace: 136 | replacedFolders = append(replacedFolders, localFolderPath) 137 | merge = nil 138 | } 139 | } 140 | for _, i := range rep.Item.Items { 141 | nestFolderReplace, nestAdd, err := b.listMergePath( 142 | ctx, 143 | ipfsBasePth, 144 | filepath.Join(ipfsRelPath, i.Name), 145 | dest, 146 | merge, 147 | ) 148 | if err != nil { 149 | return nil, nil, err 150 | } 151 | replacedFolders = append(replacedFolders, nestFolderReplace...) 152 | toAdd = append(toAdd, nestAdd...) 153 | } 154 | return replacedFolders, toAdd, nil 155 | } 156 | 157 | // If it's a file and it exists, confirm whether or not it should be overwritten. 158 | pth := filepath.Join(dest, ipfsRelPath) 159 | if _, err := os.Stat(pth); err == nil && merge != nil { 160 | ms, err := merge(fmt.Sprintf("Overwrite %s", pth), false) 161 | if err != nil { 162 | return nil, nil, err 163 | } 164 | switch ms { 165 | case Skip: 166 | return nil, nil, nil 167 | case Merge: 168 | return nil, nil, fmt.Errorf("cannot merge files") 169 | case Replace: 170 | break 171 | } 172 | } else if err != nil && !os.IsNotExist(err) { 173 | return nil, nil, err 174 | } 175 | 176 | c, err := cid.Decode(rep.Item.Cid) 177 | if err != nil { 178 | return nil, nil, err 179 | } 180 | o := object{path: pth, name: rep.Item.Name, size: rep.Item.Size, cid: c} 181 | return nil, []object{o}, nil 182 | } 183 | 184 | func (b *Bucket) getIpfsFile( 185 | ctx context.Context, 186 | ipfsPath path.Path, 187 | filePath string, 188 | size int64, 189 | c cid.Cid, 190 | events chan<- Event, 191 | progress chan<- int64, 192 | ) error { 193 | if err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil { 194 | return err 195 | } 196 | file, err := os.Create(filePath) 197 | if err != nil { 198 | return err 199 | } 200 | defer file.Close() 201 | 202 | prog, finish := handlePullProgress(progress, size) 203 | defer finish() 204 | if err := b.c.PullIpfsPath(ctx, ipfsPath, file, buckets.WithProgress(prog)); err != nil { 205 | return err 206 | } 207 | 208 | if events != nil { 209 | events <- Event{ 210 | Type: EventFileComplete, 211 | Path: filePath, 212 | Cid: c, 213 | Size: size, 214 | Complete: size, 215 | } 216 | } 217 | 218 | return nil 219 | } 220 | -------------------------------------------------------------------------------- /local/crypto.go: -------------------------------------------------------------------------------- 1 | package local 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "os" 8 | "time" 9 | 10 | "github.com/textileio/dcrypto" 11 | ) 12 | 13 | // EncryptLocalPath encrypts the file at path with key, writing the result to the writer. 14 | // Encryption is AES-CTR + AES-512 HMAC. 15 | func (b *Bucket) EncryptLocalPath(pth string, key []byte, w io.Writer) error { 16 | return encryptLocalPath(pth, key, dcrypto.NewEncrypter, w) 17 | } 18 | 19 | // DecryptLocalPath decrypts the file at path with key, writing the result to the writer. 20 | // Encryption is AES-CTR + AES-512 HMAC. 21 | func (b *Bucket) DecryptLocalPath(pth string, key []byte, w io.Writer) error { 22 | return decryptLocalPath(pth, key, dcrypto.NewDecrypter, w) 23 | } 24 | 25 | // EncryptLocalPathWithPassword encrypts the file at path with password, writing the result to the writer. 26 | // Encryption is AES-CTR + AES-512 HMAC. 27 | // https://godoc.org/golang.org/x/crypto/scrypt is used to derive the keys from the password. 28 | func (b *Bucket) EncryptLocalPathWithPassword(pth, password string, w io.Writer) error { 29 | return encryptLocalPath(pth, []byte(password), dcrypto.NewEncrypterWithPassword, w) 30 | } 31 | 32 | // DecryptLocalPathWithPassword decrypts the file at path with password, writing the result to the writer. 33 | // Encryption is AES-CTR + AES-512 HMAC. 34 | // https://godoc.org/golang.org/x/crypto/scrypt is used to derive the keys from the password. 35 | func (b *Bucket) DecryptLocalPathWithPassword(pth, password string, w io.Writer) error { 36 | return decryptLocalPath(pth, []byte(password), dcrypto.NewDecrypterWithPassword, w) 37 | } 38 | 39 | type encryptFunc func(io.Reader, []byte) (io.Reader, error) 40 | type decryptFunc func(io.Reader, []byte) (io.ReadCloser, error) 41 | 42 | func encryptLocalPath(pth string, keyOrPassword []byte, fn encryptFunc, w io.Writer) error { 43 | file, err := os.Open(pth) 44 | if err != nil { 45 | return err 46 | } 47 | defer file.Close() 48 | info, err := file.Stat() 49 | if err != nil { 50 | return err 51 | } 52 | if info.IsDir() { 53 | return fmt.Errorf("path %s is not a file", pth) 54 | } 55 | 56 | r, err := fn(file, keyOrPassword) 57 | if err != nil { 58 | return err 59 | } 60 | if _, err := io.Copy(w, r); err != nil { 61 | return err 62 | } 63 | return nil 64 | } 65 | 66 | func decryptLocalPath(pth string, keyOrPassword []byte, fn decryptFunc, w io.Writer) error { 67 | file, err := os.Open(pth) 68 | if err != nil { 69 | return err 70 | } 71 | defer file.Close() 72 | info, err := file.Stat() 73 | if err != nil { 74 | return err 75 | } 76 | if info.IsDir() { 77 | return fmt.Errorf("path %s is not a file", pth) 78 | } 79 | r, err := fn(file, keyOrPassword) 80 | if err != nil { 81 | return err 82 | } 83 | if _, err := io.Copy(w, r); err != nil { 84 | return err 85 | } 86 | return nil 87 | } 88 | 89 | // DecryptRemotePath decrypts the file at the remote path with password, writing the result to the writer. 90 | // Encryption is AES-CTR + AES-512 HMAC. 91 | func (b *Bucket) DecryptRemotePath(ctx context.Context, pth string, key []byte, w io.Writer) error { 92 | return b.decryptRemotePath(ctx, pth, key, dcrypto.NewDecrypter, w) 93 | } 94 | 95 | // DecryptRemotePathWithPassword decrypts the file at the remote path with password, writing the result to the writer. 96 | // Encryption is AES-CTR + AES-512 HMAC. 97 | // https://godoc.org/golang.org/x/crypto/scrypt is used to derive the keys from the password. 98 | func (b *Bucket) DecryptRemotePathWithPassword(ctx context.Context, pth, password string, w io.Writer) error { 99 | return b.decryptRemotePath(ctx, pth, []byte(password), dcrypto.NewDecrypterWithPassword, w) 100 | } 101 | 102 | func (b *Bucket) decryptRemotePath( 103 | ctx context.Context, 104 | pth string, 105 | keyOrPassword []byte, 106 | fn decryptFunc, 107 | w io.Writer, 108 | ) error { 109 | ctx, err := b.authCtx(ctx, time.Hour) 110 | if err != nil { 111 | return err 112 | } 113 | id, err := b.Thread() 114 | if err != nil { 115 | return err 116 | } 117 | errs := make(chan error) 118 | reader, writer := io.Pipe() 119 | go func() { 120 | defer writer.Close() 121 | if err := b.c.PullPath(ctx, id, b.Key(), pth, writer); err != nil { 122 | errs <- err 123 | } 124 | }() 125 | go func() { 126 | defer close(errs) 127 | r, err := fn(reader, keyOrPassword) 128 | if err != nil { 129 | errs <- err 130 | return 131 | } 132 | defer r.Close() 133 | if _, err := io.Copy(w, r); err != nil { 134 | errs <- err 135 | return 136 | } 137 | }() 138 | return <-errs 139 | } 140 | -------------------------------------------------------------------------------- /local/diff.go: -------------------------------------------------------------------------------- 1 | package local 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "path/filepath" 7 | "runtime" 8 | "strings" 9 | 10 | du "github.com/ipfs/go-merkledag/dagutils" 11 | aurora2 "github.com/logrusorgru/aurora" 12 | "github.com/textileio/go-buckets/cmd" 13 | "github.com/textileio/go-buckets/collection" 14 | ) 15 | 16 | var aurora = aurora2.NewAurora(runtime.GOOS != "windows") 17 | 18 | // Change describes a local bucket change. 19 | type Change struct { 20 | Type du.ChangeType 21 | Name string // Absolute file name 22 | Path string // File name relative to the bucket root 23 | Rel string // File name relative to the bucket current working directory 24 | } 25 | 26 | // ChangeType returns a string representation of a change type. 27 | func ChangeType(t du.ChangeType) string { 28 | switch t { 29 | case du.Mod: 30 | return "modified:" 31 | case du.Add: 32 | return "new file:" 33 | case du.Remove: 34 | return "deleted: " 35 | default: 36 | return "" 37 | } 38 | } 39 | 40 | // ChangeColor returns an appropriate color for the given change type. 41 | func ChangeColor(t du.ChangeType) func(arg interface{}) aurora2.Value { 42 | switch t { 43 | case du.Mod: 44 | return aurora.Yellow 45 | case du.Add: 46 | return aurora.Green 47 | case du.Remove: 48 | return aurora.Red 49 | default: 50 | return nil 51 | } 52 | } 53 | 54 | // DiffLocal returns a list of locally staged bucket file changes. 55 | func (b *Bucket) DiffLocal() ([]Change, error) { 56 | if b.repo == nil { 57 | return nil, ErrNotABucket 58 | } 59 | bp, err := b.Path() 60 | if err != nil { 61 | return nil, err 62 | } 63 | 64 | ctx, cancel := context.WithTimeout(context.Background(), cmd.Timeout) 65 | defer cancel() 66 | diff, err := b.repo.Diff(ctx, bp) 67 | if err != nil { 68 | return nil, err 69 | } 70 | var all []Change 71 | if len(diff) == 0 { 72 | return all, nil 73 | } 74 | for _, c := range diff { 75 | fp := filepath.Join(bp, c.Path) 76 | switch c.Type { 77 | case du.Mod, du.Add: 78 | names, err := b.walkPath(fp) 79 | if err != nil { 80 | return nil, err 81 | } 82 | for _, n := range names { 83 | p := strings.TrimPrefix(n, bp+string(os.PathSeparator)) 84 | r, err := filepath.Rel(b.cwd, n) 85 | if err != nil { 86 | return nil, err 87 | } 88 | all = append(all, Change{Type: c.Type, Name: n, Path: p, Rel: r}) 89 | } 90 | case du.Remove: 91 | r, err := filepath.Rel(b.cwd, fp) 92 | if err != nil { 93 | return nil, err 94 | } 95 | all = append(all, Change{Type: c.Type, Name: fp, Path: c.Path, Rel: r}) 96 | } 97 | } 98 | return all, nil 99 | } 100 | 101 | func (b *Bucket) walkPath(pth string) (names []string, err error) { 102 | err = filepath.Walk(pth, func(n string, info os.FileInfo, err error) error { 103 | if err != nil { 104 | return err 105 | } 106 | if !info.IsDir() { 107 | f := strings.TrimPrefix(n, pth+string(os.PathSeparator)) 108 | if Ignore(n) || 109 | f == collection.SeedName || 110 | strings.HasPrefix(f, b.conf.Dir) || 111 | strings.HasSuffix(f, patchExt) { 112 | return nil 113 | } 114 | names = append(names, n) 115 | } 116 | return nil 117 | }) 118 | if err != nil { 119 | return 120 | } 121 | return names, nil 122 | } 123 | -------------------------------------------------------------------------------- /local/list.go: -------------------------------------------------------------------------------- 1 | package local 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "path/filepath" 8 | "time" 9 | 10 | "github.com/ipfs/go-cid" 11 | pb "github.com/textileio/go-buckets/api/pb/buckets" 12 | ) 13 | 14 | var errEmptyItem = fmt.Errorf("item is empty") 15 | 16 | // BucketItem describes an item (file/directory) in a bucket. 17 | type BucketItem struct { 18 | Cid cid.Cid `json:"cid"` 19 | Name string `json:"name"` 20 | Path string `json:"path"` 21 | Size int64 `json:"size"` 22 | IsDir bool `json:"is_dir"` 23 | Items []BucketItem `json:"items"` 24 | ItemsCount int `json:"items_count"` 25 | } 26 | 27 | // ListRemotePath returns a list of all bucket items under path. 28 | func (b *Bucket) ListRemotePath(ctx context.Context, pth string) (items []BucketItem, err error) { 29 | pth = filepath.ToSlash(pth) 30 | if pth == "." || pth == "/" || pth == "./" { 31 | pth = "" 32 | } 33 | ctx, err = b.authCtx(ctx, time.Hour) 34 | if err != nil { 35 | return 36 | } 37 | id, err := b.Thread() 38 | if err != nil { 39 | return 40 | } 41 | rep, err := b.c.ListPath(ctx, id, b.Key(), pth) 42 | if err != nil { 43 | return 44 | } 45 | if len(rep.Item.Items) > 0 { 46 | items = make([]BucketItem, len(rep.Item.Items)) 47 | for j, k := range rep.Item.Items { 48 | ii, err := pbItemToItem(k) 49 | if err != nil { 50 | return items, err 51 | } 52 | items[j] = ii 53 | } 54 | } else if !rep.Item.IsDir { 55 | items = make([]BucketItem, 1) 56 | item, err := pbItemToItem(rep.Item) 57 | if err != nil { 58 | return items, err 59 | } 60 | items[0] = item 61 | } 62 | return items, nil 63 | } 64 | 65 | func pbItemToItem(pi *pb.PathItem) (item BucketItem, err error) { 66 | if pi.Cid == "" { 67 | return item, errEmptyItem 68 | } 69 | c, err := cid.Decode(pi.Cid) 70 | if err != nil { 71 | return 72 | } 73 | items := make([]BucketItem, len(pi.Items)) 74 | for j, k := range pi.Items { 75 | ii, err := pbItemToItem(k) 76 | if errors.Is(err, errEmptyItem) { 77 | items = nil 78 | break 79 | } else if err != nil { 80 | return item, err 81 | } 82 | items[j] = ii 83 | } 84 | return BucketItem{ 85 | Cid: c, 86 | Name: pi.Name, 87 | Path: pi.Path, 88 | Size: pi.Size, 89 | IsDir: pi.IsDir, 90 | Items: items, 91 | ItemsCount: int(pi.ItemsCount), 92 | }, nil 93 | } 94 | -------------------------------------------------------------------------------- /local/options.go: -------------------------------------------------------------------------------- 1 | package local 2 | 3 | import ( 4 | cid "github.com/ipfs/go-cid" 5 | ) 6 | 7 | type newOptions struct { 8 | name string 9 | private bool 10 | fromCid cid.Cid 11 | strategy InitStrategy 12 | events chan<- Event 13 | unfreeze bool 14 | } 15 | 16 | // NewOption is used when creating a new bucket. 17 | type NewOption func(*newOptions) 18 | 19 | // WithName sets a name for the bucket. 20 | func WithName(name string) NewOption { 21 | return func(args *newOptions) { 22 | args.name = name 23 | } 24 | } 25 | 26 | // WithPrivate specifies that an encryption key will be used for the bucket. 27 | func WithPrivate(private bool) NewOption { 28 | return func(args *newOptions) { 29 | args.private = private 30 | } 31 | } 32 | 33 | // WithCid indicates an inited bucket should be boostraped with a particular UnixFS DAG. 34 | func WithCid(c cid.Cid) NewOption { 35 | return func(args *newOptions) { 36 | args.fromCid = c 37 | } 38 | } 39 | 40 | // InitStrategy describes the type of init strategy. 41 | type InitStrategy int 42 | 43 | const ( 44 | // Hybrid indicates locally staged changes should be accepted, excluding deletions. 45 | Hybrid InitStrategy = iota 46 | // Soft indicates locally staged changes should be accepted, including deletions. 47 | Soft 48 | // Hard indicates locally staged changes should be discarded. 49 | Hard 50 | ) 51 | 52 | // WithStrategy allows for selecting the init strategy. Hybrid is the default. 53 | func WithStrategy(strategy InitStrategy) NewOption { 54 | return func(args *newOptions) { 55 | args.strategy = strategy 56 | } 57 | } 58 | 59 | // WithInitEvents allows the caller to receive events when pulling 60 | // files from an existing bucket on initialization. 61 | func WithInitEvents(ch chan<- Event) NewOption { 62 | return func(args *newOptions) { 63 | args.events = ch 64 | } 65 | } 66 | 67 | type pathOptions struct { 68 | confirm ConfirmDiffFunc 69 | force bool 70 | hard bool 71 | events chan<- Event 72 | } 73 | 74 | // PathOption is used when pushing or pulling bucket paths. 75 | type PathOption func(*pathOptions) 76 | 77 | // ConfirmDiffFunc is a caller-provided function which presents a list of bucket changes. 78 | type ConfirmDiffFunc func([]Change) bool 79 | 80 | // WithConfirm allows the caller to confirm a list of bucket changes. 81 | func WithConfirm(f ConfirmDiffFunc) PathOption { 82 | return func(args *pathOptions) { 83 | args.confirm = f 84 | } 85 | } 86 | 87 | // WithForce indicates all remote files should be pulled even if they already exist. 88 | func WithForce(b bool) PathOption { 89 | return func(args *pathOptions) { 90 | args.force = b 91 | } 92 | } 93 | 94 | // WithHard indicates locally staged changes should be discarded. 95 | func WithHard(b bool) PathOption { 96 | return func(args *pathOptions) { 97 | args.hard = b 98 | } 99 | } 100 | 101 | // WithEvents allows the caller to receive events when pushing or pulling files. 102 | func WithEvents(ch chan<- Event) PathOption { 103 | return func(args *pathOptions) { 104 | args.events = ch 105 | } 106 | } 107 | 108 | type addOptions struct { 109 | merge SelectMergeFunc 110 | events chan<- Event 111 | } 112 | 113 | // SelectMergeFunc is a caller-provided function which is used to select a merge strategy. 114 | type SelectMergeFunc func(description string, isDir bool) (MergeStrategy, error) 115 | 116 | // MergeStrategy describes the type of path merge strategy. 117 | type MergeStrategy string 118 | 119 | const ( 120 | // Skip skips the path merge. 121 | Skip MergeStrategy = "Skip" 122 | // Merge attempts to merge the paths (directories only). 123 | Merge = "Merge" 124 | // Replace replaces the old path with the new path. 125 | Replace = "Replace" 126 | ) 127 | 128 | // AddOption is used when staging a remote Unixfs dag cid in a local bucket. 129 | type AddOption func(*addOptions) 130 | 131 | // WithSelectMerge allows the caller to select the path merge strategy. 132 | func WithSelectMerge(f SelectMergeFunc) AddOption { 133 | return func(args *addOptions) { 134 | args.merge = f 135 | } 136 | } 137 | 138 | // WithAddEvents allows the caller to receive events when staging files from a remote Unixfs dag. 139 | func WithAddEvents(ch chan<- Event) AddOption { 140 | return func(args *addOptions) { 141 | args.events = ch 142 | } 143 | } 144 | 145 | type watchOptions struct { 146 | offline bool 147 | events chan<- Event 148 | } 149 | 150 | // WatchOption is used when watching a bucket for changes. 151 | type WatchOption func(*watchOptions) 152 | 153 | // WithOffline will keep watching for bucket changes while the local network is offline. 154 | func WithOffline(offline bool) WatchOption { 155 | return func(args *watchOptions) { 156 | args.offline = offline 157 | } 158 | } 159 | 160 | // WithWatchEvents allows the caller to receive events when watching a bucket for changes. 161 | func WithWatchEvents(ch chan<- Event) WatchOption { 162 | return func(args *watchOptions) { 163 | args.events = ch 164 | } 165 | } 166 | -------------------------------------------------------------------------------- /local/push.go: -------------------------------------------------------------------------------- 1 | package local 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "os" 7 | "path/filepath" 8 | "strings" 9 | "time" 10 | 11 | du "github.com/ipfs/go-merkledag/dagutils" 12 | "github.com/ipfs/interface-go-ipfs-core/path" 13 | "github.com/textileio/go-buckets" 14 | "github.com/textileio/go-threads/core/thread" 15 | ) 16 | 17 | // PushLocal pushes local files. 18 | // By default, only staged changes are pushed. See PathOption for more info. 19 | func (b *Bucket) PushLocal(ctx context.Context, opts ...PathOption) (roots Roots, err error) { 20 | b.Lock() 21 | defer b.Unlock() 22 | ctx, err = b.authCtx(ctx, time.Hour) 23 | if err != nil { 24 | return 25 | } 26 | id, err := b.Thread() 27 | if err != nil { 28 | return 29 | } 30 | args := &pathOptions{} 31 | for _, opt := range opts { 32 | opt(args) 33 | } 34 | 35 | diff, err := b.DiffLocal() 36 | if errors.Is(err, ErrNotABucket) { 37 | args.force = true 38 | } else if err != nil { 39 | return 40 | } 41 | bp, err := b.Path() 42 | if err != nil { 43 | return roots, err 44 | } 45 | if args.force { // Reset the diff to show all files as additions 46 | var reset []Change 47 | names, err := b.walkPath(bp) 48 | if err != nil { 49 | return roots, err 50 | } 51 | for _, n := range names { 52 | r, err := filepath.Rel(b.cwd, n) 53 | if err != nil { 54 | return roots, err 55 | } 56 | p := strings.TrimPrefix(n, bp+string(os.PathSeparator)) 57 | reset = append(reset, Change{Type: du.Add, Name: n, Path: p, Rel: r}) 58 | } 59 | // Add unique additions 60 | loop: 61 | for _, c := range reset { 62 | for _, x := range diff { 63 | if c.Path == x.Path { 64 | continue loop 65 | } 66 | } 67 | diff = append(diff, c) 68 | } 69 | } 70 | if len(diff) == 0 { 71 | return roots, ErrUpToDate 72 | } 73 | if args.confirm != nil { 74 | if ok := args.confirm(diff); !ok { 75 | return roots, ErrAborted 76 | } 77 | } 78 | 79 | r, err := b.Roots(ctx) 80 | if err != nil { 81 | return 82 | } 83 | xr := path.IpfsPath(r.Remote) 84 | var rm, add []Change 85 | key := b.Key() 86 | for _, c := range diff { 87 | switch c.Type { 88 | case du.Mod, du.Add: 89 | add = append(add, c) 90 | case du.Remove: 91 | rm = append(rm, c) 92 | } 93 | } 94 | xr, err = b.addFiles(ctx, id, key, xr, add, args.force, args.events) 95 | if err != nil { 96 | return roots, err 97 | } 98 | if len(rm) > 0 { 99 | for _, c := range rm { 100 | var err error 101 | xr, err = b.rmFile(ctx, id, key, xr, c, args.force, args.events) 102 | if err != nil { 103 | return roots, err 104 | } 105 | } 106 | } 107 | 108 | if b.repo != nil { 109 | if err := b.repo.Save(ctx); err != nil { 110 | return roots, err 111 | } 112 | rc, err := b.getRemoteRoot(ctx) 113 | if err != nil { 114 | return roots, err 115 | } 116 | if err := b.repo.SetRemotePath("", rc); err != nil { 117 | return roots, err 118 | } 119 | } 120 | return b.Roots(ctx) 121 | } 122 | 123 | type pendingFile struct { 124 | path string 125 | rel string 126 | } 127 | 128 | func (b *Bucket) addFiles( 129 | ctx context.Context, 130 | id thread.ID, 131 | key string, 132 | xroot path.Resolved, 133 | changes []Change, 134 | force bool, 135 | events chan<- Event, 136 | ) (path.Resolved, error) { 137 | progress := make(chan int64) 138 | defer close(progress) 139 | files := make(map[string]pendingFile) 140 | 141 | opts := []buckets.Option{buckets.WithProgress(progress)} 142 | if !force { 143 | opts = append(opts, buckets.WithFastForwardOnly(xroot)) 144 | } 145 | q, err := b.c.PushPaths(ctx, id, key, opts...) 146 | if err != nil { 147 | return nil, err 148 | } 149 | defer q.Close() 150 | 151 | for _, c := range changes { 152 | file := pendingFile{ 153 | path: c.Path, 154 | rel: c.Rel, 155 | } 156 | pth := filepath.ToSlash(c.Path) 157 | files[pth] = file 158 | if err := q.AddFile(file.path, c.Name); err != nil { 159 | return nil, err 160 | } 161 | } 162 | 163 | size := q.Size() 164 | go func() { 165 | for p := range progress { 166 | var u int64 167 | if p > size { 168 | u = size 169 | } else { 170 | u = p 171 | } 172 | if events != nil { 173 | events <- Event{ 174 | Type: EventProgress, 175 | Size: size, 176 | Complete: u, 177 | } 178 | } 179 | } 180 | }() 181 | 182 | var root path.Resolved 183 | for q.Next() { 184 | if q.Err() != nil { 185 | return nil, q.Err() 186 | } 187 | file := files[q.Current.Path] 188 | root = q.Current.Root 189 | 190 | if b.repo != nil { 191 | if err := b.repo.SetRemotePath(file.path, q.Current.Cid); err != nil { 192 | return nil, err 193 | } 194 | } 195 | 196 | if events != nil { 197 | events <- Event{ 198 | Type: EventFileComplete, 199 | Path: file.rel, 200 | Cid: q.Current.Cid, 201 | Size: q.Current.Size, 202 | } 203 | } 204 | } 205 | return root, nil 206 | } 207 | 208 | func (b *Bucket) rmFile( 209 | ctx context.Context, 210 | id thread.ID, 211 | key string, 212 | xroot path.Resolved, 213 | c Change, 214 | force bool, 215 | events chan<- Event, 216 | ) (path.Resolved, error) { 217 | var opts []buckets.Option 218 | if !force { 219 | opts = append(opts, buckets.WithFastForwardOnly(xroot)) 220 | } 221 | root, err := b.c.RemovePath(ctx, id, key, c.Path, opts...) 222 | if err != nil { 223 | if !strings.HasSuffix(err.Error(), "no link by that name") { 224 | return nil, err 225 | } 226 | } 227 | 228 | if b.repo != nil { 229 | if err := b.repo.RemovePath(ctx, c.Path); err != nil { 230 | return nil, err 231 | } 232 | } 233 | 234 | if events != nil { 235 | events <- Event{ 236 | Type: EventFileRemoved, 237 | Path: c.Rel, 238 | } 239 | } 240 | return root, nil 241 | } 242 | -------------------------------------------------------------------------------- /local/testdata/a/bar.txt: -------------------------------------------------------------------------------- 1 | Astonishment across the centuries the sky calls to us with pretty stories for which there's little good evidence courage of our questions billions upon billions? Another world great turbulent clouds a still more glorious dawn awaits courage of our questions Tunguska event how far away. Two ghostly white figures in coveralls and helmets are soflty dancing ship of the imagination star stuff harvesting star light from which we spring citizens of distant epochs something incredible is waiting to be known. -------------------------------------------------------------------------------- /local/testdata/a/foo.txt: -------------------------------------------------------------------------------- 1 | Radio telescope with pretty stories for which there's little good evidence the sky calls to us something incredible is waiting to be known Rig Veda science. From which we spring the only home we've ever known concept of the number one the carbon in our apple pies Orion's sword extraordinary claims require extraordinary evidence. The ash of stellar alchemy encyclopaedia galactica across the centuries with pretty stories for which there's little good evidence a very small stage in a vast cosmic arena a very small stage in a vast cosmic arena? -------------------------------------------------------------------------------- /local/testdata/a/one/baz.txt: -------------------------------------------------------------------------------- 1 | Orion's sword great turbulent clouds brain is the seed of intelligence made in the interiors of collapsing stars cosmos white dwarf. Rig Veda ship of the imagination shores of the cosmic ocean rich in heavy atoms shores of the cosmic ocean courage of our questions. Kindling the energy hidden in matter permanence of the stars the only home we've ever known how far away citizens of distant epochs shores of the cosmic ocean. -------------------------------------------------------------------------------- /local/testdata/a/one/buz.txt: -------------------------------------------------------------------------------- 1 | Extraplanetary as a patch of light encyclopaedia galactica cosmos the ash of stellar alchemy quasar. Intelligent beings vanquish the impossible hundreds of thousands gathered by gravity permanence of the stars Rig Veda? Extraordinary claims require extraordinary evidence courage of our questions invent the universe the only home we've ever known a very small stage in a vast cosmic arena made in the interiors of collapsing stars. Dream of the mind's eye two ghostly white figures in coveralls and helmets are soflty dancing a mote of dust suspended in a sunbeam the carbon in our apple pies the sky calls to us a still more glorious dawn awaits? -------------------------------------------------------------------------------- /local/testdata/a/one/two/boo.txt: -------------------------------------------------------------------------------- 1 | Billions upon billions something incredible is waiting to be known star stuff harvesting star light Sea of Tranquility of brilliant syntheses billions upon billions. The carbon in our apple pies are creatures of the cosmos tendrils of gossamer clouds vanquish the impossible rich in mystery a mote of dust suspended in a sunbeam. The ash of stellar alchemy a still more glorious dawn awaits how far away a very small stage in a vast cosmic arena invent the universe bits of moving fluff and billions upon billions upon billions upon billions upon billions upon billions upon billions. -------------------------------------------------------------------------------- /local/testdata/a/one/two/fuz.txt: -------------------------------------------------------------------------------- 1 | Shores of the cosmic ocean extraordinary claims require extraordinary evidence cosmic ocean as a patch of light the only home we've ever known courage of our questions. Hydrogen atoms rings of Uranus a very small stage in a vast cosmic arena with pretty stories for which there's little good evidence great turbulent clouds permanence of the stars. With pretty stories for which there's little good evidence a mote of dust suspended in a sunbeam a mote of dust suspended in a sunbeam hearts of the stars from which we spring globular star cluster. -------------------------------------------------------------------------------- /local/testdata/b/foo.txt: -------------------------------------------------------------------------------- 1 | Laws of physics prime number the carbon in our apple pies dream of the mind's eye emerged into consciousness finite but unbounded. Encyclopaedia galactica globular star cluster a very small stage in a vast cosmic arena Orion's sword something incredible is waiting to be known astonishment. Bits of moving fluff gathered by gravity tendrils of gossamer clouds birth two ghostly white figures in coveralls and helmets are soflty dancing citizens of distant epochs and billions upon billions upon billions upon billions upon billions upon billions upon billions. -------------------------------------------------------------------------------- /local/testdata/b/one/baz.txt: -------------------------------------------------------------------------------- 1 | Orion's sword great turbulent clouds brain is the seed of intelligence made in the interiors of collapsing stars cosmos white dwarf. Rig Veda ship of the imagination shores of the cosmic ocean rich in heavy atoms shores of the cosmic ocean courage of our questions. Kindling the energy hidden in matter permanence of the stars the only home we've ever known how far away citizens of distant epochs shores of the cosmic ocean. -------------------------------------------------------------------------------- /local/testdata/b/one/muz.txt: -------------------------------------------------------------------------------- 1 | Extraplanetary as a patch of light encyclopaedia galactica cosmos the ash of stellar alchemy quasar. Intelligent beings vanquish the impossible hundreds of thousands gathered by gravity permanence of the stars Rig Veda? Extraordinary claims require extraordinary evidence courage of our questions invent the universe the only home we've ever known a very small stage in a vast cosmic arena made in the interiors of collapsing stars. Dream of the mind's eye two ghostly white figures in coveralls and helmets are soflty dancing a mote of dust suspended in a sunbeam the carbon in our apple pies the sky calls to us a still more glorious dawn awaits? -------------------------------------------------------------------------------- /local/testdata/b/one/three/far.txt: -------------------------------------------------------------------------------- 1 | Decipherment take root and flourish white dwarf Hypatia something incredible is waiting to be known concept of the number one. Two ghostly white figures in coveralls and helmets are soflty dancing prime number courage of our questions the sky calls to us vanquish the impossible across the centuries. Descended from astronomers the sky calls to us rich in heavy atoms a still more glorious dawn awaits dream of the mind's eye descended from astronomers and billions upon billions upon billions upon billions upon billions upon billions upon billions. -------------------------------------------------------------------------------- /local/testdata/b/one/two/fuz.txt: -------------------------------------------------------------------------------- 1 | Shores of the cosmic ocean extraordinary claims require extraordinary evidence cosmic ocean as a patch of light the only home we've ever known courage of our questions. Hydrogen atoms rings of Uranus a very small stage in a vast cosmic arena with pretty stories for which there's little good evidence great turbulent clouds permanence of the stars. With pretty stories for which there's little good evidence a mote of dust suspended in a sunbeam a mote of dust suspended in a sunbeam hearts of the stars from which we spring globular star cluster. -------------------------------------------------------------------------------- /local/testdata/c/one.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/textileio/go-buckets/89def8d01abaafa36b18eaef8c2b78c871cfff8c/local/testdata/c/one.jpg -------------------------------------------------------------------------------- /local/testdata/c/two.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/textileio/go-buckets/89def8d01abaafa36b18eaef8c2b78c871cfff8c/local/testdata/c/two.jpg -------------------------------------------------------------------------------- /local/watch.go: -------------------------------------------------------------------------------- 1 | package local 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "strings" 7 | "time" 8 | 9 | "github.com/textileio/go-buckets" 10 | ) 11 | 12 | const ( 13 | fileSystemWatchInterval = time.Millisecond * 100 14 | reconnectInterval = time.Second * 5 15 | ) 16 | 17 | //// Watch watches for and auto-pushes local bucket changes at an interval, 18 | //// and listens for and auto-pulls remote changes as they arrive. 19 | //// Use the WithOffline option to keep watching during network interruptions. 20 | //// Returns a channel of watch connectivity states. 21 | //// Cancel context to stop watching. 22 | //func (b *Bucket) Watch(ctx context.Context, opts ...WatchOption) (<-chan cmd.WatchState, error) { 23 | // ctx, err := b.auth(ctx) 24 | // if err != nil { 25 | // return nil, err 26 | // } 27 | // args := &watchOptions{} 28 | // for _, opt := range opts { 29 | // opt(args) 30 | // } 31 | // if !args.offline { 32 | // return b.watchWhileConnected(ctx, args.events) 33 | // } 34 | // return cmd.Watch(ctx, func(ctx context.Context) (<-chan cmd.WatchState, error) { 35 | // return b.watchWhileConnected(ctx, args.events) 36 | // }, reconnectInterval) 37 | //} 38 | // 39 | //// watchWhileConnected will watch until context is canceled or an error occurs. 40 | //func (b *Bucket) watchWhileConnected(ctx context.Context, pevents chan<- Event) (<-chan cmd.WatchState, error) { 41 | // id, err := b.Thread() 42 | // if err != nil { 43 | // return nil, err 44 | // } 45 | // bp, err := b.Path() 46 | // if err != nil { 47 | // return nil, err 48 | // } 49 | // 50 | // state := make(chan cmd.WatchState) 51 | // go func() { 52 | // defer close(state) 53 | // w := watcher.New() 54 | // defer w.Close() 55 | // w.SetMaxEvents(1) 56 | // if err := w.AddRecursive(bp); err != nil { 57 | // state <- cmd.WatchState{Err: err, Aborted: true} 58 | // return 59 | // } 60 | // 61 | // // Start listening for remote changes 62 | // events, err := b.clients.Threads.Listen(ctx, id, []client.ListenOption{{ 63 | // Type: client.ListenAll, 64 | // InstanceID: b.Key(), 65 | // }}) 66 | // if err != nil { 67 | // state <- cmd.WatchState{Err: err, Aborted: !cmd.IsConnectionError(err)} 68 | // return 69 | // } 70 | // errs := make(chan error) 71 | // go func() { 72 | // for e := range events { 73 | // if e.Err != nil { 74 | // errs <- e.Err // events will close on error 75 | // } else if err := b.watchPull(ctx, pevents); err != nil { 76 | // errs <- err 77 | // return 78 | // } 79 | // } 80 | // }() 81 | // 82 | // // Start listening for local changes 83 | // go func() { 84 | // if err := w.Start(fileSystemWatchInterval); err != nil { 85 | // errs <- err 86 | // } 87 | // }() 88 | // go func() { 89 | // for { 90 | // select { 91 | // case <-w.Event: 92 | // if err := b.watchPush(ctx, pevents); err != nil { 93 | // errs <- err 94 | // } 95 | // case err := <-w.Error: 96 | // errs <- err 97 | // case <-w.Closed: 98 | // return 99 | // } 100 | // } 101 | // }() 102 | // 103 | // // Manually sync once on startup 104 | // if err := b.watchPush(ctx, pevents); err != nil { 105 | // state <- cmd.WatchState{Err: err, Aborted: !cmd.IsConnectionError(err)} 106 | // return 107 | // } 108 | // 109 | // // If we made it here, we must be online 110 | // state <- cmd.WatchState{State: cmd.Online} 111 | // 112 | // for { 113 | // select { 114 | // case err := <-errs: 115 | // state <- cmd.WatchState{Err: err, Aborted: !cmd.IsConnectionError(err)} 116 | // return 117 | // case <-ctx.Done(): 118 | // return 119 | // } 120 | // } 121 | // }() 122 | // return state, nil 123 | //} 124 | 125 | func (b *Bucket) watchPush(ctx context.Context, events chan<- Event) error { 126 | b.pushBlock <- struct{}{} 127 | defer func() { 128 | <-b.pushBlock 129 | }() 130 | if _, err := b.PushLocal(ctx, WithEvents(events)); errors.Is(err, ErrUpToDate) { 131 | return nil 132 | } else if err != nil && strings.Contains(err.Error(), buckets.ErrNonFastForward.Error()) { 133 | // Pull remote changes 134 | if _, err = b.PullRemote(ctx, WithEvents(events)); err != nil { 135 | return err 136 | } 137 | // Now try pushing again 138 | if _, err = b.PushLocal(ctx, WithEvents(events)); err != nil { 139 | return err 140 | } 141 | } else if err != nil { 142 | return err 143 | } 144 | return nil 145 | } 146 | 147 | func (b *Bucket) watchPull(ctx context.Context, events chan<- Event) error { 148 | select { 149 | case b.pushBlock <- struct{}{}: 150 | if _, err := b.PullRemote(ctx, WithEvents(events)); !errors.Is(err, ErrUpToDate) { 151 | <-b.pushBlock 152 | return err 153 | } 154 | <-b.pushBlock 155 | default: 156 | // Ignore if there's a push in progress 157 | } 158 | return nil 159 | } 160 | -------------------------------------------------------------------------------- /move.go: -------------------------------------------------------------------------------- 1 | package buckets 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | gopath "path" 7 | "strings" 8 | "time" 9 | 10 | "github.com/ipfs/interface-go-ipfs-core/path" 11 | "github.com/textileio/go-buckets/collection" 12 | "github.com/textileio/go-buckets/dag" 13 | "github.com/textileio/go-threads/core/did" 14 | core "github.com/textileio/go-threads/core/thread" 15 | ) 16 | 17 | // MovePath moves a path to a different location. 18 | // The destination path does not need to exist. 19 | // Currently, moving the root path is not possible. 20 | func (b *Buckets) MovePath( 21 | ctx context.Context, 22 | thread core.ID, 23 | key string, 24 | identity did.Token, 25 | root path.Resolved, 26 | fpth, tpth string, 27 | ) (int64, *Bucket, error) { 28 | txn, err := b.NewTxn(thread, key, identity) 29 | if err != nil { 30 | return 0, nil, err 31 | } 32 | defer txn.Close() 33 | return txn.MovePath(ctx, root, fpth, tpth) 34 | } 35 | 36 | // MovePath is Txn based MovePath. 37 | func (t *Txn) MovePath( 38 | ctx context.Context, 39 | root path.Resolved, 40 | fpth, tpth string, 41 | ) (int64, *Bucket, error) { 42 | fpth, err := parsePath(fpth) 43 | if err != nil { 44 | return 0, nil, err 45 | } 46 | if fpth == "" { 47 | // @todo: Enable move of root directory 48 | return 0, nil, fmt.Errorf("root cannot be moved") 49 | } 50 | tpth, err = parsePath(tpth) 51 | if err != nil { 52 | return 0, nil, err 53 | } 54 | // Paths are the same, nothing to do 55 | if fpth == tpth { 56 | return 0, nil, fmt.Errorf("path is destination") 57 | } 58 | 59 | instance, pth, err := t.b.getBucketAndPath(ctx, t.thread, t.key, t.identity, fpth) 60 | if err != nil { 61 | return 0, nil, fmt.Errorf("getting path: %v", err) 62 | } 63 | if root != nil && root.String() != instance.Path { 64 | return 0, nil, ErrNonFastForward 65 | } 66 | 67 | instance.UpdatedAt = time.Now().UnixNano() 68 | instance.SetMetadataAtPath(tpth, collection.Metadata{ 69 | UpdatedAt: instance.UpdatedAt, 70 | }) 71 | instance.UnsetMetadataWithPrefix(fpth + "/") 72 | if err := t.b.c.Verify(ctx, t.thread, instance, collection.WithIdentity(t.identity)); err != nil { 73 | return 0, nil, fmt.Errorf("verifying bucket update: %v", err) 74 | } 75 | 76 | fbpth, err := getBucketPath(instance, fpth) 77 | if err != nil { 78 | return 0, nil, err 79 | } 80 | fitem, err := t.b.pathToItem(ctx, instance, fbpth, false) 81 | if err != nil { 82 | return 0, nil, err 83 | } 84 | tbpth, err := getBucketPath(instance, tpth) 85 | if err != nil { 86 | return 0, nil, err 87 | } 88 | titem, err := t.b.pathToItem(ctx, instance, tbpth, false) 89 | if err == nil { 90 | if fitem.IsDir && !titem.IsDir { 91 | return 0, nil, fmt.Errorf("destination is not a directory") 92 | } 93 | if titem.IsDir { 94 | // from => to becomes new dir: 95 | // - "c" => "b" becomes "b/c" 96 | // - "a.jpg" => "b" becomes "b/a.jpg" 97 | tpth = gopath.Join(tpth, fitem.Name) 98 | } 99 | } 100 | 101 | pnode, err := dag.GetNodeAtPath(ctx, t.b.ipfs, pth, instance.GetLinkEncryptionKey()) 102 | if err != nil { 103 | return 0, nil, fmt.Errorf("getting node: %v", err) 104 | } 105 | 106 | var dirPath path.Resolved 107 | if instance.IsPrivate() { 108 | ctx, dirPath, err = dag.CopyDag(ctx, t.b.ipfs, instance, pnode, fpth, tpth) 109 | if err != nil { 110 | return 0, nil, fmt.Errorf("copying node: %v", err) 111 | } 112 | } else { 113 | ctx, dirPath, err = t.b.setPathFromExistingCid( 114 | ctx, 115 | instance, 116 | path.New(instance.Path), 117 | tpth, 118 | pnode.Cid(), 119 | nil, 120 | nil, 121 | ) 122 | if err != nil { 123 | return 0, nil, fmt.Errorf("copying path: %v", err) 124 | } 125 | } 126 | instance.Path = dirPath.String() 127 | 128 | // If "a/b" => "a/", cleanup only needed for priv 129 | if strings.HasPrefix(fpth, tpth) { 130 | if instance.IsPrivate() { 131 | ctx, dirPath, err = t.b.removePath(ctx, instance, fpth) 132 | if err != nil { 133 | return 0, nil, fmt.Errorf("removing path: %v", err) 134 | } 135 | instance.Path = dirPath.String() 136 | } 137 | 138 | if err := t.b.saveAndPublish(ctx, t.thread, t.identity, instance); err != nil { 139 | return 0, nil, err 140 | } 141 | 142 | log.Debugf("moved %s to %s", fpth, tpth) 143 | return dag.GetPinnedBytes(ctx), instanceToBucket(t.thread, instance), nil 144 | } 145 | 146 | if strings.HasPrefix(tpth, fpth) { 147 | // If "a/" => "a/b" cleanup each leaf in "a" that isn't "b" (skipping .textileseed) 148 | ppth := path.Join(path.New(instance.Path), fpth) 149 | item, err := t.b.listPath(ctx, instance, ppth) 150 | if err != nil { 151 | return 0, nil, fmt.Errorf("listing path: %v", err) 152 | } 153 | for _, chld := range item.Items { 154 | sp := trimSlash(movePathRegexp.ReplaceAllString(chld.Path, "")) 155 | if strings.Compare(chld.Name, collection.SeedName) == 0 || sp == tpth { 156 | continue 157 | } 158 | ctx, dirPath, err = t.b.removePath(ctx, instance, trimSlash(sp)) 159 | if err != nil { 160 | return 0, nil, fmt.Errorf("removing path: %v", err) 161 | } 162 | instance.Path = dirPath.String() 163 | } 164 | } else { 165 | // if a/ => b/ remove a 166 | ctx, dirPath, err = t.b.removePath(ctx, instance, fpth) 167 | if err != nil { 168 | return 0, nil, fmt.Errorf("removing path: %v", err) 169 | } 170 | instance.Path = dirPath.String() 171 | } 172 | 173 | if err := t.b.saveAndPublish(ctx, t.thread, t.identity, instance); err != nil { 174 | return 0, nil, err 175 | } 176 | 177 | log.Debugf("moved %s to %s", fpth, tpth) 178 | return dag.GetPinnedBytes(ctx), instanceToBucket(t.thread, instance), nil 179 | } 180 | -------------------------------------------------------------------------------- /options.go: -------------------------------------------------------------------------------- 1 | package buckets 2 | 3 | import ( 4 | c "github.com/ipfs/go-cid" 5 | "github.com/ipfs/interface-go-ipfs-core/path" 6 | core "github.com/textileio/go-threads/core/thread" 7 | ) 8 | 9 | // CreateOptions are used when creating a new bucket. 10 | type CreateOptions struct { 11 | Thread core.ID 12 | Name string 13 | Private bool 14 | Cid c.Cid 15 | } 16 | 17 | type CreateOption func(*CreateOptions) 18 | 19 | // WithThread specifies a thread for the bucket. 20 | // A new thread is created by default. 21 | func WithThread(thread core.ID) CreateOption { 22 | return func(args *CreateOptions) { 23 | args.Thread = thread 24 | } 25 | } 26 | 27 | // WithName sets a name for the bucket. 28 | func WithName(name string) CreateOption { 29 | return func(args *CreateOptions) { 30 | args.Name = name 31 | } 32 | } 33 | 34 | // WithPrivate specifies that an encryption key will be used for the bucket. 35 | func WithPrivate(private bool) CreateOption { 36 | return func(args *CreateOptions) { 37 | args.Private = private 38 | } 39 | } 40 | 41 | // WithCid indicates that an inited bucket should be boostraped from a UnixFS DAG. 42 | func WithCid(cid c.Cid) CreateOption { 43 | return func(args *CreateOptions) { 44 | args.Cid = cid 45 | } 46 | } 47 | 48 | // Options are used to perform bucket operations. 49 | type Options struct { 50 | Root path.Resolved 51 | Progress chan<- int64 52 | } 53 | 54 | type Option func(*Options) 55 | 56 | // WithFastForwardOnly instructs the remote to reject non-fast-forward updates by comparing root with the remote. 57 | func WithFastForwardOnly(root path.Resolved) Option { 58 | return func(args *Options) { 59 | args.Root = root 60 | } 61 | } 62 | 63 | // WithProgress writes progress updates to the given channel. 64 | func WithProgress(ch chan<- int64) Option { 65 | return func(args *Options) { 66 | args.Progress = ch 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /pinning/openapi/.openapi-generator-ignore: -------------------------------------------------------------------------------- 1 | # OpenAPI Generator Ignore 2 | # Generated by openapi-generator https://github.com/openapitools/openapi-generator 3 | 4 | # Use this file to prevent files from being overwritten by the generator. 5 | # The patterns follow closely to .gitignore or .dockerignore. 6 | 7 | # As an example, the C# client generator defines ApiClient.cs. 8 | # You can make changes and tell OpenAPI Generator to ignore just this file by uncommenting the following line: 9 | #ApiClient.cs 10 | 11 | # You can match any string of characters against a directory, file or extension with a single asterisk (*): 12 | #foo/*/qux 13 | # The above matches foo/bar/qux and foo/baz/qux, but not foo/bar/baz/qux 14 | 15 | # You can recursively match patterns against a directory, file or extension with a double asterisk (**): 16 | #foo/**/qux 17 | # This matches foo/bar/qux, foo/baz/qux, and foo/bar/baz/qux 18 | 19 | # You can also negate patterns with an exclamation (!). 20 | # For example, you can ignore all files in a docs folder with the file extension .md: 21 | #docs/*.md 22 | # Then explicitly reverse the ignore rule for a single file: 23 | #!docs/README.md 24 | -------------------------------------------------------------------------------- /pinning/openapi/.openapi-generator/FILES: -------------------------------------------------------------------------------- 1 | .openapi-generator-ignore 2 | Dockerfile 3 | api/openapi.yaml 4 | go/README.md 5 | go/api_pins.go 6 | go/model_failure.go 7 | go/model_failure_error.go 8 | go/model_pin.go 9 | go/model_pin_results.go 10 | go/model_pin_status.go 11 | go/model_status.go 12 | go/model_text_matching_strategy.go 13 | go/routers.go 14 | main.go 15 | -------------------------------------------------------------------------------- /pinning/openapi/.openapi-generator/VERSION: -------------------------------------------------------------------------------- 1 | 5.0.1 -------------------------------------------------------------------------------- /pinning/openapi/go/model_query.go: -------------------------------------------------------------------------------- 1 | package openapi 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // Query represents Pin query parameters. 8 | // This is derived from the openapi spec using https://github.com/deepmap/oapi-codegen, not 9 | // https://github.com/OpenAPITools/openapi-generator, which doens't output anything for the 10 | // listPins query body. 11 | type Query struct { 12 | // Cid can be used to filter by one or more Pin Cids. 13 | Cid []string `form:"cid" json:"cid,omitempty"` 14 | // Name can be used to filer by Pin name (by default case-sensitive, exact match). 15 | Name string `form:"name" json:"name,omitempty"` 16 | // Match can be used to customize the text matching strategy applied when Name is present. 17 | Match string `form:"match" json:"match,omitempty"` 18 | // Status can be used to filter by Pin status. 19 | Status string `form:"status" json:"status,omitempty"` 20 | // Before can by used to filter by before creation (queued) time. 21 | Before time.Time `form:"before" json:"before,omitempty"` 22 | // After can by used to filter by after creation (queued) time. 23 | After time.Time `form:"after" json:"after,omitempty"` 24 | // Limit specifies the max number of Pins to return. 25 | Limit int32 `form:"limit" json:"limit,omitempty"` 26 | } 27 | 28 | // QueryMeta can be used to filter results by Pin metadata. 29 | // This was pulled out of the openapi generated Query above because gin is not able to 30 | // auto parse the map string value sent by the generic pinning client, i.e., "meta=map[foo:one bar:two]". 31 | type QueryMeta map[string]string 32 | -------------------------------------------------------------------------------- /pull.go: -------------------------------------------------------------------------------- 1 | package buckets 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | 8 | ipfsfiles "github.com/ipfs/go-ipfs-files" 9 | "github.com/ipfs/interface-go-ipfs-core/path" 10 | "github.com/textileio/dcrypto" 11 | "github.com/textileio/go-buckets/dag" 12 | "github.com/textileio/go-threads/core/did" 13 | core "github.com/textileio/go-threads/core/thread" 14 | ) 15 | 16 | type pathReader struct { 17 | r io.Reader 18 | closers []io.Closer 19 | } 20 | 21 | func (r *pathReader) Read(p []byte) (int, error) { 22 | return r.r.Read(p) 23 | } 24 | 25 | func (r *pathReader) Close() error { 26 | // Close in reverse. 27 | for i := len(r.closers) - 1; i >= 0; i-- { 28 | if err := r.closers[i].Close(); err != nil { 29 | return err 30 | } 31 | } 32 | return nil 33 | } 34 | 35 | // PullPath returns a reader to a bucket path. 36 | func (b *Buckets) PullPath( 37 | ctx context.Context, 38 | thread core.ID, 39 | key string, 40 | identity did.Token, 41 | pth string, 42 | ) (io.ReadCloser, error) { 43 | if err := thread.Validate(); err != nil { 44 | return nil, fmt.Errorf("invalid thread id: %v", err) 45 | } 46 | pth = trimSlash(pth) 47 | instance, bpth, err := b.getBucketAndPath(ctx, thread, key, identity, pth) 48 | if err != nil { 49 | return nil, err 50 | } 51 | fileKey, err := instance.GetFileEncryptionKeyForPath(pth) 52 | if err != nil { 53 | return nil, err 54 | } 55 | 56 | var filePath path.Resolved 57 | if instance.IsPrivate() { 58 | buckPath, err := dag.NewResolvedPath(instance.Path) 59 | if err != nil { 60 | return nil, err 61 | } 62 | np, isDir, r, err := dag.GetNodesToPath(ctx, b.ipfs, buckPath, pth, instance.GetLinkEncryptionKey()) 63 | if err != nil { 64 | return nil, err 65 | } 66 | if r != "" { 67 | return nil, fmt.Errorf("could not resolve path: %s", bpth) 68 | } 69 | if isDir { 70 | return nil, fmt.Errorf("node is a directory") 71 | } 72 | fn := np[len(np)-1] 73 | filePath = path.IpfsPath(fn.New.Cid()) 74 | } else { 75 | filePath, err = b.ipfs.ResolvePath(ctx, bpth) 76 | if err != nil { 77 | return nil, err 78 | } 79 | } 80 | 81 | r := &pathReader{} 82 | node, err := b.ipfs.Unixfs().Get(ctx, filePath) 83 | if err != nil { 84 | return nil, err 85 | } 86 | r.closers = append(r.closers, node) 87 | 88 | file := ipfsfiles.ToFile(node) 89 | if file == nil { 90 | _ = r.Close() 91 | return nil, fmt.Errorf("node is a directory") 92 | } 93 | if fileKey != nil { 94 | dr, err := dcrypto.NewDecrypter(file, fileKey) 95 | if err != nil { 96 | _ = r.Close() 97 | return nil, err 98 | } 99 | r.closers = append(r.closers, dr) 100 | r.r = dr 101 | } else { 102 | r.r = file 103 | } 104 | 105 | log.Debugf("pulled %s from %s", pth, instance.Key) 106 | return r, nil 107 | } 108 | 109 | // PullIPFSPath returns a reader to an IPFS path. 110 | func (b *Buckets) PullIPFSPath(ctx context.Context, pth string) (io.ReadCloser, error) { 111 | node, err := b.ipfs.Unixfs().Get(ctx, path.New(pth)) 112 | if err != nil { 113 | return nil, err 114 | } 115 | file := ipfsfiles.ToFile(node) 116 | if file == nil { 117 | return nil, fmt.Errorf("node is a directory") 118 | } 119 | return file, nil 120 | } 121 | -------------------------------------------------------------------------------- /remove.go: -------------------------------------------------------------------------------- 1 | package buckets 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/ipfs/interface-go-ipfs-core/path" 8 | "github.com/textileio/go-buckets/collection" 9 | "github.com/textileio/go-buckets/dag" 10 | "github.com/textileio/go-threads/core/did" 11 | core "github.com/textileio/go-threads/core/thread" 12 | ) 13 | 14 | // RemovePath removed a path from a bucket. 15 | // All children under the path will be removed and unpinned. 16 | func (b *Buckets) RemovePath( 17 | ctx context.Context, 18 | thread core.ID, 19 | key string, 20 | identity did.Token, 21 | root path.Resolved, 22 | pth string, 23 | ) (int64, *Bucket, error) { 24 | txn, err := b.NewTxn(thread, key, identity) 25 | if err != nil { 26 | return 0, nil, err 27 | } 28 | defer txn.Close() 29 | return txn.RemovePath(ctx, root, pth) 30 | } 31 | 32 | // RemovePath is Txn based RemovePath. 33 | func (t *Txn) RemovePath(ctx context.Context, root path.Resolved, pth string) (int64, *Bucket, error) { 34 | pth, err := parsePath(pth) 35 | if err != nil { 36 | return 0, nil, err 37 | } 38 | 39 | instance, err := t.b.c.GetSafe(ctx, t.thread, t.key, collection.WithIdentity(t.identity)) 40 | if err != nil { 41 | return 0, nil, err 42 | } 43 | if root != nil && root.String() != instance.Path { 44 | return 0, nil, ErrNonFastForward 45 | } 46 | 47 | instance.UpdatedAt = time.Now().UnixNano() 48 | instance.UnsetMetadataWithPrefix(pth) 49 | if err := t.b.c.Verify(ctx, t.thread, instance, collection.WithIdentity(t.identity)); err != nil { 50 | return 0, nil, err 51 | } 52 | 53 | ctx, dirPath, err := t.b.removePath(ctx, instance, pth) 54 | if err != nil { 55 | return 0, nil, err 56 | } 57 | 58 | instance.Path = dirPath.String() 59 | if err := t.b.saveAndPublish(ctx, t.thread, t.identity, instance); err != nil { 60 | return 0, nil, err 61 | } 62 | 63 | log.Debugf("removed %s from %s", pth, t.key) 64 | return dag.GetPinnedBytes(ctx), instanceToBucket(t.thread, instance), nil 65 | } 66 | -------------------------------------------------------------------------------- /scripts/gen_js_protos.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eo pipefail 3 | 4 | wd="$(pwd -P)" 5 | 6 | js_paths=() 7 | while IFS= read -r -d $'\0'; do 8 | js_paths+=("$REPLY") 9 | done < <(find . -path "*/pb/javascript" ! -path "*/node_modules/*" -print0) 10 | 11 | echo installing dependencies 12 | for path in "${js_paths[@]}"; do 13 | cd "${path}" && npm install >/dev/null 2>&1 && cd "${wd}" 14 | done 15 | 16 | echo generating js-protos in api/pb/buckets/javascript 17 | ./buildtools/protoc/bin/protoc \ 18 | --proto_path=. \ 19 | --plugin=protoc-gen-ts=api/pb/buckets/javascript/node_modules/.bin/protoc-gen-ts \ 20 | --js_out=import_style=commonjs,binary:api/pb/buckets/javascript \ 21 | --ts_out=service=grpc-web:api/pb/buckets/javascript \ 22 | api/pb/buckets/buckets.proto 23 | -------------------------------------------------------------------------------- /scripts/protoc_gen_plugin.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eo pipefail 3 | 4 | # From https://github.com/bufbuild/buf/blob/master/make/go/scripts/protoc_gen_plugin.bash 5 | 6 | fail() { 7 | echo "$@" >&2 8 | exit 1 9 | } 10 | 11 | usage() { 12 | echo "usage: ${0} \ 13 | --proto_path=path/to/one \ 14 | --proto_path=path/to/two \ 15 | --proto_include_path=path/to/one \ 16 | --proto_include_path=path/to/two \ 17 | --plugin_name=go \ 18 | --plugin_out=gen/proto/go \ 19 | --plugin_opt=plugins=grpc" 20 | } 21 | 22 | check_flag_value_set() { 23 | if [ -z "${1}" ]; then 24 | usage 25 | exit 1 26 | fi 27 | } 28 | 29 | PROTO_PATHS=() 30 | PROTO_INCLUDE_PATHS=() 31 | PLUGIN_NAME= 32 | PLUGIN_OUT= 33 | PLUGIN_OPT= 34 | while test $# -gt 0; do 35 | case "${1}" in 36 | -h|--help) 37 | usage 38 | exit 0 39 | ;; 40 | --proto_path*) 41 | PROTO_PATHS+=("$(echo ${1} | sed -e 's/^[^=]*=//g')") 42 | shift 43 | ;; 44 | --proto_include_path*) 45 | PROTO_INCLUDE_PATHS+=("$(echo ${1} | sed -e 's/^[^=]*=//g')") 46 | shift 47 | ;; 48 | --plugin_name*) 49 | PLUGIN_NAME="$(echo ${1} | sed -e 's/^[^=]*=//g')" 50 | shift 51 | ;; 52 | --plugin_out*) 53 | PLUGIN_OUT="$(echo ${1} | sed -e 's/^[^=]*=//g')" 54 | shift 55 | ;; 56 | --plugin_opt*) 57 | PLUGIN_OPT="$(echo ${1} | sed -e 's/^[^=]*=//g')" 58 | shift 59 | ;; 60 | *) 61 | usage 62 | exit 1 63 | ;; 64 | esac 65 | done 66 | 67 | check_flag_value_set "${PROTO_PATHS[@]}" 68 | check_flag_value_set "${PLUGIN_NAME}" 69 | check_flag_value_set "${PLUGIN_OUT}" 70 | 71 | PROTOC_FLAGS=() 72 | for proto_path in "${PROTO_PATHS[@]}"; do 73 | PROTOC_FLAGS+=("--proto_path=${proto_path}") 74 | done 75 | for proto_path in "${PROTO_INCLUDE_PATHS[@]}"; do 76 | PROTOC_FLAGS+=("--proto_path=${proto_path}") 77 | done 78 | PROTOC_FLAGS+=("--${PLUGIN_NAME}_out=${PLUGIN_OUT}") 79 | if [ -n "${PLUGIN_OPT}" ]; then 80 | PROTOC_FLAGS+=("--${PLUGIN_NAME}_opt=${PLUGIN_OPT}") 81 | fi 82 | 83 | mkdir -p "${PLUGIN_OUT}" 84 | for proto_path in "${PROTO_PATHS[@]}"; do 85 | for dir in $(find "${proto_path}" -type f ! -path './buildtools/*' ! -path '*/node_modules/*' -name '*.proto' -print0 | xargs -0 -n1 dirname | sort | uniq); do 86 | echo protoc "${PROTOC_FLAGS[@]}" "$(find "${dir}" -name '*.proto' ! -path '*/node_modules/*')" 87 | ./buildtools/protoc/bin/protoc "${PROTOC_FLAGS[@]}" "$(find "${dir}" -name '*.proto' ! -path '*/node_modules/*')" 88 | done 89 | done 90 | -------------------------------------------------------------------------------- /scripts/publish_js_protos.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eo pipefail 3 | 4 | tag="latest" 5 | 6 | while getopts v:t:p: option 7 | do 8 | case "${option}" 9 | in 10 | v) version=${OPTARG};; 11 | t) token=${OPTARG};; 12 | p) 13 | if [[ "$OPTARG" == "true" ]]; 14 | then 15 | tag="next" 16 | fi 17 | ;; 18 | esac 19 | done 20 | 21 | [[ -z "$version" ]] && { echo "Please specify a new version, e.g., -v v1.0.0" ; exit 1; } 22 | [[ -z "$token" ]] && { echo "Please specify an NPM auth token, e.g., -t mytoken" ; exit 1; } 23 | 24 | wd="$(pwd -P)" 25 | 26 | js_paths=() 27 | while IFS= read -r -d $'\0'; do 28 | js_paths+=("$REPLY") 29 | done < <(find . -path "*/pb/javascript" ! -path "*/node_modules/*" -print0) 30 | 31 | echo installing dependencies 32 | npm install -g json >/dev/null 2>&1 33 | 34 | for path in "${js_paths[@]}"; do 35 | cd "${path}" 36 | json -I -f package.json -e "this.version=('$version').replace('v', '')" >/dev/null 2>&1 37 | echo publishing js-protos in "${path}" with version "${version}" and token "${token}" 38 | NODE_AUTH_TOKEN="${token}" npm publish --access=public --tag ${tag} 39 | json -I -f package.json -e "this.version=('0.0.0')" >/dev/null 2>&1 40 | cd "${wd}" 41 | done 42 | -------------------------------------------------------------------------------- /set.go: -------------------------------------------------------------------------------- 1 | package buckets 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | c "github.com/ipfs/go-cid" 8 | "github.com/ipfs/interface-go-ipfs-core/path" 9 | "github.com/textileio/go-buckets/collection" 10 | "github.com/textileio/go-buckets/dag" 11 | "github.com/textileio/go-threads/core/did" 12 | core "github.com/textileio/go-threads/core/thread" 13 | ) 14 | 15 | // SetPath pulls data from IPFS into a bucket path. 16 | func (b *Buckets) SetPath( 17 | ctx context.Context, 18 | thread core.ID, 19 | key string, 20 | identity did.Token, 21 | root path.Resolved, 22 | pth string, 23 | cid c.Cid, 24 | meta map[string]interface{}, 25 | ) (int64, *Bucket, error) { 26 | txn, err := b.NewTxn(thread, key, identity) 27 | if err != nil { 28 | return 0, nil, err 29 | } 30 | defer txn.Close() 31 | return txn.SetPath(ctx, root, pth, cid, meta) 32 | } 33 | 34 | // SetPath is Txn based SetPath. 35 | func (t *Txn) SetPath( 36 | ctx context.Context, 37 | root path.Resolved, 38 | pth string, 39 | cid c.Cid, 40 | meta map[string]interface{}, 41 | ) (int64, *Bucket, error) { 42 | instance, err := t.b.c.GetSafe(ctx, t.thread, t.key, collection.WithIdentity(t.identity)) 43 | if err != nil { 44 | return 0, nil, err 45 | } 46 | if root != nil && root.String() != instance.Path { 47 | return 0, nil, ErrNonFastForward 48 | } 49 | 50 | pth = trimSlash(pth) 51 | instance.UpdatedAt = time.Now().UnixNano() 52 | instance.SetMetadataAtPath(pth, collection.Metadata{ 53 | UpdatedAt: instance.UpdatedAt, 54 | Info: meta, 55 | }) 56 | instance.UnsetMetadataWithPrefix(pth + "/") 57 | 58 | if err := t.b.c.Verify(ctx, t.thread, instance, collection.WithIdentity(t.identity)); err != nil { 59 | return 0, nil, err 60 | } 61 | 62 | var linkKey, fileKey []byte 63 | if instance.IsPrivate() { 64 | linkKey = instance.GetLinkEncryptionKey() 65 | var err error 66 | fileKey, err = instance.GetFileEncryptionKeyForPath(pth) 67 | if err != nil { 68 | return 0, nil, err 69 | } 70 | } 71 | 72 | buckPath := path.New(instance.Path) 73 | ctx, dirPath, err := t.b.setPathFromExistingCid(ctx, instance, buckPath, pth, cid, linkKey, fileKey) 74 | if err != nil { 75 | return 0, nil, err 76 | } 77 | instance.Path = dirPath.String() 78 | if err := t.b.c.Save(ctx, t.thread, instance, collection.WithIdentity(t.identity)); err != nil { 79 | return 0, nil, err 80 | } 81 | 82 | log.Debugf("set %s to %s", pth, cid) 83 | return dag.GetPinnedBytes(ctx), instanceToBucket(t.thread, instance), nil 84 | } 85 | --------------------------------------------------------------------------------