├── .github ├── linters │ ├── .golangci.yml │ ├── .dockerfilelintrc │ ├── .hadolint.yaml │ └── .markdown-lint.yml ├── dependabot.yml ├── ISSUE_TEMPLATE │ ├── Bug_report.md │ └── Feature_request.md └── workflows │ ├── stale.yml │ ├── snapcraft.yml │ ├── testing.yml │ ├── linter.yml │ ├── codeql-analysis.yml │ ├── builder.yml │ └── release.yml ├── doc.go ├── .gitignore ├── snapcraft.yaml ├── LICENSE ├── go.mod ├── ipfs ├── options.go ├── ipfs.go └── ipfs_test.go ├── .golangci.yml ├── cmd └── rivet │ └── main.go ├── rivet.go ├── README.md ├── Makefile ├── rivet_test.go ├── install.sh └── go.sum /.github/linters/.golangci.yml: -------------------------------------------------------------------------------- 1 | ../../.golangci.yml -------------------------------------------------------------------------------- /.github/linters/.dockerfilelintrc: -------------------------------------------------------------------------------- 1 | rules: 2 | missing_tag: off 3 | -------------------------------------------------------------------------------- /.github/linters/.hadolint.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | ########################## 3 | ## Hadolint config file ## 4 | ########################## 5 | ignored: 6 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the MIT 3 | // license that can be found in the LICENSE file. 4 | 5 | /* 6 | Rivet is both a command-line tool and a Golang package for archiving 7 | webpages to IPFS. 8 | */ 9 | package rivet // import "github.com/wabarc/rivet" 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | 17 | /bin 18 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "gomod" 9 | directory: "/" 10 | schedule: 11 | interval: "daily" 12 | open-pull-requests-limit: 10 13 | 14 | - package-ecosystem: "github-actions" 15 | directory: "/" 16 | schedule: 17 | interval: "daily" 18 | 19 | -------------------------------------------------------------------------------- /snapcraft.yaml: -------------------------------------------------------------------------------- 1 | name: rivet 2 | 3 | version: 'git' 4 | 5 | summary: A toolkit makes it easier to archive webpages to IPFS. 6 | 7 | description: | 8 | Rivet is both a command-line tool and a Golang package for archiving webpages to IPFS. 9 | Website https://github.com/wabarc/rivet 10 | 11 | grade: stable 12 | confinement: strict 13 | compression: lzo 14 | base: core18 15 | 16 | parts: 17 | rivet: 18 | plugin: go 19 | source: https://github.com/wabarc/rivet.git 20 | go-importpath: github.com/wabarc/rivet/cmd/rivet 21 | build-packages: 22 | - build-essential 23 | 24 | apps: 25 | rivet: 26 | command: rivet 27 | plugs: 28 | - home 29 | - network 30 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/Bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: 🐛 Bug Report 3 | about: If something isn't working as expected 🤔. 4 | --- 5 | 6 | ## Bug Report 7 | 8 | **Current Behavior** 9 | A clear and concise description of the behavior. 10 | 11 | ```shell 12 | // Your error message 13 | ``` 14 | 15 | **Expected behavior/code** 16 | A clear and concise description of what you expected to happen (or code). 17 | 18 | **Environment** 19 | 20 | - Rivet version(s): [e.g. v0.8.0] 21 | - Golang version: [e.g. Go 1.16] 22 | - OS: [e.g. OSX 10.13.4, Windows 10] 23 | 24 | **Possible Solution** 25 | 26 | 27 | 28 | **Additional context/Screenshots** 29 | Add any other context about the problem here. If applicable, add screenshots to help explain. 30 | 31 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/Feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: 🚀 Feature Request 3 | about: I have a suggestion (and may want to implement it 🙂)! 4 | --- 5 | 6 | ## Feature Request 7 | 8 | **Is your feature request related to a problem? Please describe.** 9 | A clear and concise description of what the problem is. Ex. I have an issue when [...] 10 | 11 | **Describe the solution you'd like** 12 | A clear and concise description of what you want to happen. Add any considered drawbacks. 13 | 14 | **Describe alternatives you've considered** 15 | A clear and concise description of any alternative solutions or features you've considered. 16 | 17 | **Teachability, Documentation, Adoption, Migration Strategy** 18 | If you can, explain how users will be able to use this and possibly write out a version the docs. 19 | Maybe a screenshot or design? 20 | -------------------------------------------------------------------------------- /.github/workflows/stale.yml: -------------------------------------------------------------------------------- 1 | name: Stale 2 | 3 | on: 4 | schedule: 5 | - cron: "0 3 * * 6" 6 | 7 | permissions: 8 | issues: write 9 | pull-requests: write 10 | 11 | jobs: 12 | stale: 13 | name: Stale 14 | runs-on: ubuntu-latest 15 | steps: 16 | - name: Mark stale issues and pull requests 17 | uses: actions/stale@v5 18 | with: 19 | repo-token: ${{ github.token }} 20 | exempt-issue-labels: "enhancement,question,help wanted,bug" 21 | exempt-pr-labels: "need-help,WIP" 22 | stale-issue-message: "This issue is stale because it has been open 120 days with no activity. Remove stale label or comment or this will be closed in 5 days" 23 | stale-pr-message: 'It has been open 120 days with no activity. Remove stale label or comment or this will be closed in 5 days' 24 | days-before-stale: 120 25 | days-before-close: 5 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Wayback Archiver 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.github/linters/.markdown-lint.yml: -------------------------------------------------------------------------------- 1 | --- 2 | ########################### 3 | ########################### 4 | ## Markdown Linter rules ## 5 | ########################### 6 | ########################### 7 | 8 | # Linter rules doc: 9 | # - https://github.com/DavidAnson/markdownlint 10 | # 11 | # Note: 12 | # To comment out a single error: 13 | # 14 | # any violations you want 15 | # 16 | # 17 | 18 | ############### 19 | # Rules by id # 20 | ############### 21 | MD004: false # Unordered list style 22 | MD007: 23 | indent: 2 # Unordered list indentation 24 | MD013: 25 | line_length: 400 # Line length 80 is far to short 26 | MD026: 27 | punctuation: ".,;:!。，；:" # List of not allowed 28 | MD029: false # Ordered list item prefix 29 | MD033: false # Allow inline HTML 30 | MD034: false # Allow Bare URL 31 | MD036: false # Emphasis used instead of a heading 32 | MD041: false # Allow top-level heading first line 33 | 34 | ################# 35 | # Rules by tags # 36 | ################# 37 | blank_lines: false # Error on blank lines 38 | -------------------------------------------------------------------------------- /.github/workflows/snapcraft.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Wayback Archiver. All rights reserved. 2 | # Use of this source code is governed by the GNU GPL v3 3 | # license that can be found in the LICENSE file. 4 | # 5 | name: Snapcraft 6 | 7 | on: 8 | push: 9 | tags: 10 | - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10 11 | workflow_dispatch: 12 | 13 | permissions: write-all 14 | 15 | jobs: 16 | snapcraft: 17 | runs-on: ubuntu-latest 18 | steps: 19 | - name: Check out Git repository 20 | uses: actions/checkout@v3 21 | with: 22 | fetch-depth: 0 23 | 24 | - name: Set up QEMU 25 | uses: docker/setup-qemu-action@v1 26 | 27 | - name: Set env & Print rivet version 28 | shell: bash 29 | run: | 30 | version=$(git describe --tags `git rev-list --tags --max-count=1` | sed -e 's/v//g') 31 | sed -i "s/version: 'git'/version: '${version}'/g" snapcraft.yaml 32 | 33 | - id: build 34 | name: Build Snap 35 | uses: snapcore/action-build@v1 36 | with: 37 | snapcraft-channel: stable 38 | 39 | - name: Upload artifact 40 | uses: actions/upload-artifact@v3 41 | with: 42 | name: rivet-snap 43 | path: ${{ steps.build.outputs.snap }} 44 | 45 | - uses: snapcore/action-publish@v1 46 | name: Release Snap 47 | if: github.repository == 'wabarc/rivet' 48 | with: 49 | store_login: ${{ secrets.SNAPCRAFT_TOKEN }} 50 | snap: ${{ steps.build.outputs.snap }} 51 | release: stable 52 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/wabarc/rivet 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/cenkalti/backoff/v4 v4.2.1 7 | github.com/go-shiori/obelisk v0.0.0-20230316095823-42f6a2f99d9d 8 | github.com/ipfs/go-ipfs-api v0.6.0 9 | github.com/kennygrant/sanitize v1.2.4 10 | github.com/pkg/errors v0.9.1 11 | github.com/wabarc/helper v0.0.0-20230418130954-be7440352bcb 12 | github.com/wabarc/ipfs-pinner v1.1.1-0.20230502052510-dc378f9e202b 13 | ) 14 | 15 | require ( 16 | github.com/andybalholm/cascadia v1.3.2 // indirect 17 | github.com/benbjohnson/clock v1.3.3 // indirect 18 | github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect 19 | github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect 20 | github.com/fortytw2/leaktest v1.3.0 // indirect 21 | github.com/go-shiori/dom v0.0.0-20210627111528-4e4722cd0d65 // indirect 22 | github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect 23 | github.com/ipfs/boxo v0.8.1 // indirect 24 | github.com/ipfs/go-cid v0.4.1 // indirect 25 | github.com/klauspost/cpuid/v2 v2.2.4 // indirect 26 | github.com/libp2p/go-buffer-pool v0.1.0 // indirect 27 | github.com/libp2p/go-flow-metrics v0.1.0 // indirect 28 | github.com/libp2p/go-libp2p v0.27.1 // indirect 29 | github.com/mattn/go-isatty v0.0.18 // indirect 30 | github.com/minio/sha256-simd v1.0.0 // indirect 31 | github.com/mitchellh/go-homedir v1.1.0 // indirect 32 | github.com/mr-tron/base58 v1.2.0 // indirect 33 | github.com/multiformats/go-base32 v0.1.0 // indirect 34 | github.com/multiformats/go-base36 v0.2.0 // indirect 35 | github.com/multiformats/go-multiaddr v0.9.0 // indirect 36 | github.com/multiformats/go-multibase v0.2.0 // indirect 37 | github.com/multiformats/go-multicodec v0.9.0 // indirect 38 | github.com/multiformats/go-multihash v0.2.1 // indirect 39 | github.com/multiformats/go-multistream v0.4.1 // indirect 40 | github.com/multiformats/go-varint v0.0.7 // indirect 41 | github.com/sirupsen/logrus v1.9.0 // indirect 42 | github.com/spaolacci/murmur3 v1.1.0 // indirect 43 | github.com/tdewolff/parse/v2 v2.6.5 // indirect 44 | github.com/whyrusleeping/tar-utils v0.0.0-20201201191210-20a61371de5b // indirect 45 | github.com/ybbus/httpretry v1.0.2 // indirect 46 | golang.org/x/crypto v0.8.0 // indirect 47 | golang.org/x/net v0.9.0 // indirect 48 | golang.org/x/sync v0.1.0 // indirect 49 | golang.org/x/sys v0.7.0 // indirect 50 | golang.org/x/text v0.9.0 // indirect 51 | google.golang.org/protobuf v1.30.0 // indirect 52 | lukechampine.com/blake3 v1.1.7 // indirect 53 | mvdan.cc/xurls/v2 v2.5.0 // indirect 54 | ) 55 | -------------------------------------------------------------------------------- /ipfs/options.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the MIT 3 | // license that can be found in the LICENSE file. 4 | 5 | package ipfs 6 | 7 | import ( 8 | "net" 9 | "net/http" 10 | "strconv" 11 | 12 | shell "github.com/ipfs/go-ipfs-api" 13 | pinner "github.com/wabarc/ipfs-pinner" 14 | ) 15 | 16 | // PinningOption is a function type that modifies a Pinning struct by setting one of its fields. 17 | // Each PinningOption function takes a pointer to a Pinning struct and returns nothing. When 18 | // invoked, it modifies the Pinning struct by setting the appropriate field. 19 | type PinningOption func(*Pinning) 20 | 21 | // Mode sets the Mode field of a Pinning struct to the given mode. 22 | func Mode(m mode) PinningOption { 23 | return func(o *Pinning) { 24 | o.Mode = m 25 | } 26 | } 27 | 28 | // Host sets the Host field of a Pinning struct to the given host. 29 | func Host(h string) PinningOption { 30 | return func(o *Pinning) { 31 | o.Host = h 32 | } 33 | } 34 | 35 | // Port sets the Port field of a Pinning struct to the given port. 36 | func Port(p int) PinningOption { 37 | return func(o *Pinning) { 38 | o.Port = p 39 | } 40 | } 41 | 42 | // Uses sets the Pinner field of a Pinning struct to the given name of the pinner. 43 | func Uses(p string) PinningOption { 44 | return func(o *Pinning) { 45 | o.Pinner = p 46 | } 47 | } 48 | 49 | // Apikey sets the Apikey field of a Pinning struct to the given key. 50 | func Apikey(k string) PinningOption { 51 | return func(o *Pinning) { 52 | o.Apikey = k 53 | } 54 | } 55 | 56 | // Secret sets the Secret field of a Pinning struct to the given key. 57 | func Secret(s string) PinningOption { 58 | return func(o *Pinning) { 59 | o.Secret = s 60 | } 61 | } 62 | 63 | // Backoff sets the backoff field of a Pinning struct to the given boolean value. 64 | func Backoff(b bool) PinningOption { 65 | return func(o *Pinning) { 66 | o.backoff = b 67 | } 68 | } 69 | 70 | // Client sets the Client field of a Pinning struct to the given http.Client instance. 71 | func Client(c *http.Client) PinningOption { 72 | return func(o *Pinning) { 73 | o.Client = c 74 | } 75 | } 76 | 77 | // Options takes one or more PinningOptions and returns a Pinning struct has been configured 78 | // according to those options. 79 | func Options(options ...PinningOption) Pinning { 80 | var p Pinning 81 | for _, o := range options { 82 | o(&p) 83 | } 84 | if p.Mode == Local { 85 | p.shell = shell.NewShell(net.JoinHostPort(p.Host, strconv.Itoa(p.Port))) 86 | } 87 | if p.Mode == Remote && p.Pinner == "" { 88 | p.Pinner = pinner.Infura 89 | } 90 | return p 91 | } 92 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | --- 2 | ######################### 3 | ######################### 4 | ## Golang Linter rules ## 5 | ######################### 6 | ######################### 7 | 8 | # configure golangci-lint 9 | # see https://github.com/golangci/golangci-lint/blob/master/.golangci.example.yml 10 | run: 11 | # default concurrency is a available CPU number. 12 | # concurrency: 4 # explicitly omit this value to fully utilize available resources. 13 | deadline: 5m 14 | issues-exit-code: 1 15 | tests: false 16 | 17 | issues: 18 | exclude-rules: 19 | - path: _test\.go 20 | linters: 21 | - dupl 22 | - gosec 23 | - goconst 24 | linters: 25 | disable-all: true 26 | enable: 27 | - gosec 28 | - unconvert 29 | - gocyclo 30 | - goconst 31 | - goimports 32 | - gocritic 33 | - bodyclose 34 | - misspell 35 | - rowserrcheck 36 | - structcheck 37 | - stylecheck 38 | - typecheck 39 | - varcheck 40 | - unconvert 41 | - unparam 42 | - whitespace 43 | linters-settings: 44 | errcheck: 45 | # report about assignment of errors to blank identifier: `num, _ := strconv.Atoi(numStr)`; 46 | # default is false: such cases aren't reported by default. 47 | check-blank: true 48 | govet: 49 | # report about shadowed variables 50 | check-shadowing: true 51 | gocyclo: 52 | # minimal code complexity to report, 30 by default 53 | min-complexity: 15 54 | gosec: 55 | # To specify a set of rules to explicitly exclude. 56 | # Available rules: https://github.com/securego/gosec#available-rules 57 | excludes: 58 | - G108 59 | # To specify the configuration of rules. 60 | # The configuration of rules is not fully documented by gosec: 61 | # https://github.com/securego/gosec#configuration 62 | # https://github.com/securego/gosec/blob/569328eade2ccbad4ce2d0f21ee158ab5356a5cf/rules/rulelist.go#L60-L102 63 | config: 64 | misspell: 65 | # Correct spellings using locale preferences for US or UK. 66 | # Default is to use a neutral variety of English. 67 | # Setting locale to US will correct the British spelling of 'colour' to 'color'. 68 | locale: US 69 | ignore-words: 70 | - someword 71 | stylecheck: 72 | go: "1.16" 73 | # https://staticcheck.io/docs/options#checks 74 | checks: [ "all", "-ST1003", "-ST1008", "-ST1016" ] 75 | # https://staticcheck.io/docs/options#initialisms 76 | initialisms: [ "ACL", "API", "ASCII", "CPU", "CSS", "DNS", "EOF", "GUID", "HTML", "HTTP", "HTTPS", "ID", "IP", "JSON", "QPS", "RAM", "RPC", "SLA", "SMTP", "SQL", "SSH", "TCP", "TLS", "TTL", "UDP", "UI", "GID", "UID", "UUID", "URI", "URL", "UTF8", "VM", "XML", "XMPP", "XSRF", "XSS" ] 77 | -------------------------------------------------------------------------------- /.github/workflows/testing.yml: -------------------------------------------------------------------------------- 1 | name: Testing 2 | 3 | on: 4 | push: 5 | branches: 6 | - "*" 7 | paths: 8 | - "**/*.go" 9 | - "go.mod" 10 | - "go.sum" 11 | - ".github/workflows/testing.yml" 12 | - "Makefile" 13 | pull_request: 14 | branches: [ main ] 15 | types: [ opened, synchronize, reopened ] 16 | paths: 17 | - "**/*.go" 18 | - "go.mod" 19 | - "go.sum" 20 | - ".github/workflows/testing.yml" 21 | - "Makefile" 22 | workflow_dispatch: 23 | 24 | permissions: write-all 25 | 26 | jobs: 27 | testing: 28 | name: Testing 29 | runs-on: ${{ matrix.os }} 30 | strategy: 31 | fail-fast: false 32 | matrix: 33 | os: [ ubuntu-latest, macos-latest, windows-latest ] 34 | go: [ "1.18", "1.19" ] 35 | steps: 36 | - name: Set up Go ${{ matrix.go }}.x 37 | uses: actions/setup-go@v3 38 | with: 39 | go-version: ${{ matrix.go }} 40 | 41 | - name: Set up IPFS 42 | uses: ibnesayeed/setup-ipfs@fa9de9ebdf580cf20c588d867a2d62044f956495 43 | with: 44 | ipfs_version: "0.17.0" 45 | run_daemon: true 46 | 47 | - name: Check out code base 48 | if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' 49 | uses: actions/checkout@v3 50 | with: 51 | fetch-depth: 0 52 | 53 | - name: Check out code base 54 | if: github.event_name == 'pull_request' 55 | uses: actions/checkout@v3 56 | with: 57 | fetch-depth: 0 58 | ref: ${{ github.event.pull_request.head.sha }} 59 | 60 | - name: Cache go module 61 | uses: actions/cache@v3 62 | with: 63 | path: | 64 | ~/.cache/go-build 65 | ~/Library/Caches/go-build 66 | %LocalAppData%\go-build 67 | ~/go/pkg/mod 68 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 69 | restore-keys: ${{ runner.os }}-go- 70 | 71 | - name: Get dependencies 72 | run: | 73 | go get -v -t -d ./... 74 | 75 | - name: Run test 76 | env: 77 | IPFS_PINNER_PINATA_API_KEY: ${{ secrets.IPFS_PINNER_PINATA_API_KEY }} 78 | IPFS_PINNER_PINATA_SECRET_API_KEY: ${{ secrets.IPFS_PINNER_PINATA_SECRET_API_KEY }} 79 | run: | 80 | make test 81 | make test-cover 82 | 83 | - name: Upload coverage 84 | uses: actions/upload-artifact@v3 85 | with: 86 | name: coverage-${{ matrix.os }} 87 | path: coverage.* 88 | 89 | - name: Run integration test 90 | env: 91 | IPFS_PINNER_PINATA_API_KEY: ${{ secrets.IPFS_PINNER_PINATA_API_KEY }} 92 | IPFS_PINNER_PINATA_SECRET_API_KEY: ${{ secrets.IPFS_PINNER_PINATA_SECRET_API_KEY }} 93 | run: make test-integration 94 | -------------------------------------------------------------------------------- /cmd/rivet/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "net/url" 8 | "os" 9 | "sync" 10 | "time" 11 | 12 | "github.com/wabarc/rivet" 13 | "github.com/wabarc/rivet/ipfs" 14 | 15 | pinner "github.com/wabarc/ipfs-pinner" 16 | ) 17 | 18 | func main() { 19 | var ( 20 | mode string 21 | timeout uint 22 | // for local mode 23 | host string 24 | port int 25 | // for remote mode 26 | target string 27 | apikey string 28 | secret string 29 | ) 30 | 31 | flag.Usage = func() { 32 | fmt.Fprintf(os.Stdout, "Usage:\n\n") 33 | fmt.Fprintf(os.Stdout, " rivet [options] [url1] ... [urlN]\n\n") 34 | 35 | flag.PrintDefaults() 36 | } 37 | basePrint := func() { 38 | fmt.Print("A toolkit makes it easier to archive webpages to IPFS.\n\n") 39 | flag.Usage() 40 | fmt.Fprint(os.Stdout, "\n") 41 | } 42 | 43 | flag.StringVar(&mode, "m", "remote", "Pin mode, supports mode: local, remote, archive") 44 | flag.UintVar(&timeout, "timeout", 30, "Timeout for every input URL") 45 | flag.StringVar(&host, "host", "localhost", "IPFS node address") 46 | flag.IntVar(&port, "port", 5001, "IPFS node port") 47 | flag.StringVar(&target, "t", "infura", "IPFS pinner, supports pinners: infura, pinata, nftstorage, web3storage.") 48 | flag.StringVar(&apikey, "u", "", "Pinner apikey or username.") 49 | flag.StringVar(&secret, "p", "", "Pinner sceret or password.") 50 | flag.Parse() 51 | 52 | opts := []ipfs.PinningOption{ 53 | ipfs.Mode(ipfs.Remote), 54 | } 55 | if mode == "local" { 56 | opts = []ipfs.PinningOption{ 57 | ipfs.Mode(ipfs.Local), 58 | ipfs.Host(host), 59 | ipfs.Port(port), 60 | } 61 | } 62 | 63 | switch target { 64 | case pinner.Infura, pinner.Pinata, pinner.NFTStorage, pinner.Web3Storage: 65 | opts = append(opts, ipfs.Uses(target), ipfs.Apikey(apikey), ipfs.Secret(secret)) 66 | default: 67 | basePrint() 68 | fmt.Fprintln(os.Stderr, "Unknown target") 69 | os.Exit(0) 70 | } 71 | 72 | links := flag.Args() 73 | if len(links) < 1 { 74 | basePrint() 75 | fmt.Fprintln(os.Stderr, "link is missing") 76 | os.Exit(1) 77 | } 78 | 79 | ctx := context.Background() 80 | opt := ipfs.Options(opts...) 81 | toc := time.Duration(timeout) * time.Second 82 | var wg sync.WaitGroup 83 | for _, link := range links { 84 | wg.Add(1) 85 | go func(link string) { 86 | defer wg.Done() 87 | 88 | input, err := url.Parse(link) 89 | if err != nil { 90 | fmt.Fprintf(os.Stderr, "rivet: %v\n", err) 91 | return 92 | } 93 | 94 | reqctx, cancel := context.WithTimeout(ctx, toc) 95 | defer cancel() 96 | 97 | r := &rivet.Shaft{Hold: opt, ArchiveOnly: mode == "archive"} 98 | if dest, err := r.Wayback(reqctx, input); err != nil { 99 | fmt.Fprintf(os.Stderr, "rivet: %v\n", err) 100 | } else { 101 | fmt.Fprintf(os.Stdout, "%s %s\n", dest, link) 102 | } 103 | }(link) 104 | } 105 | wg.Wait() 106 | } 107 | -------------------------------------------------------------------------------- /.github/workflows/linter.yml: -------------------------------------------------------------------------------- 1 | name: Linter 2 | 3 | on: 4 | push: 5 | branches: 6 | - '**' 7 | pull_request: 8 | branches: 9 | - '**' 10 | types: [ opened, synchronize, reopened ] 11 | 12 | permissions: write-all 13 | 14 | jobs: 15 | lint: 16 | runs-on: ubuntu-latest 17 | steps: 18 | - name: Check out code base 19 | if: github.event_name == 'push' 20 | uses: actions/checkout@v3 21 | with: 22 | fetch-depth: 0 23 | 24 | - name: Check out code base 25 | if: github.event_name == 'pull_request' 26 | uses: actions/checkout@v3 27 | with: 28 | fetch-depth: 0 29 | ref: ${{ github.event.pull_request.head.sha }} 30 | 31 | - name: Lint Code Base 32 | uses: github/super-linter@v4 33 | env: 34 | DEFAULT_BRANCH: 'main' 35 | VALIDATE_MARKDOWN: true 36 | VALIDATE_DOCKERFILE: true 37 | VALIDATE_BASH: true 38 | VALIDATE_BASH_EXEC: true 39 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 40 | 41 | go: 42 | runs-on: ubuntu-latest 43 | steps: 44 | - name: Check out code base 45 | if: github.event_name == 'push' 46 | uses: actions/checkout@v3 47 | with: 48 | fetch-depth: 0 49 | 50 | - name: Check out code base 51 | if: github.event_name == 'pull_request' 52 | uses: actions/checkout@v3 53 | with: 54 | fetch-depth: 0 55 | ref: ${{ github.event.pull_request.head.sha }} 56 | 57 | - name: Golang linter 58 | uses: golangci/golangci-lint-action@v3.1.0 59 | 60 | shellcheck: 61 | runs-on: ubuntu-latest 62 | steps: 63 | - name: Check out code base 64 | if: github.event_name == 'push' 65 | uses: actions/checkout@v3 66 | with: 67 | fetch-depth: 0 68 | 69 | - name: Check out code base 70 | if: github.event_name == 'pull_request' 71 | uses: actions/checkout@v3 72 | with: 73 | fetch-depth: 0 74 | ref: ${{ github.event.pull_request.head.sha }} 75 | 76 | - name: Run shellcheck with reviewdog 77 | uses: reviewdog/action-shellcheck@v1 78 | 79 | misspell: 80 | runs-on: ubuntu-latest 81 | steps: 82 | - name: Check out code base 83 | if: github.event_name == 'push' 84 | uses: actions/checkout@v3 85 | with: 86 | fetch-depth: 0 87 | 88 | - name: Check out code base 89 | if: github.event_name == 'pull_request' 90 | uses: actions/checkout@v3 91 | with: 92 | fetch-depth: 0 93 | ref: ${{ github.event.pull_request.head.sha }} 94 | 95 | - name: Run misspell with reviewdog 96 | uses: reviewdog/action-misspell@v1 97 | 98 | alex: 99 | runs-on: ubuntu-latest 100 | steps: 101 | - name: Check out code base 102 | if: github.event_name == 'push' 103 | uses: actions/checkout@v3 104 | with: 105 | fetch-depth: 0 106 | 107 | - name: Check out code base 108 | if: github.event_name == 'pull_request' 109 | uses: actions/checkout@v3 110 | with: 111 | fetch-depth: 0 112 | ref: ${{ github.event.pull_request.head.sha }} 113 | 114 | - name: Run alex with reviewdog 115 | uses: reviewdog/action-alex@v1 116 | 117 | goreportcard: 118 | if: ${{ github.ref == 'refs/heads/main' }} 119 | runs-on: ubuntu-latest 120 | steps: 121 | - name: Run Go report card 122 | run: | 123 | path=$(curl -sf -X POST -F "repo=github.com/$GITHUB_REPOSITORY" https://goreportcard.com/checks | jq -r '.redirect') 124 | echo -e "\nSee report for https://goreportcard.com${path}" 125 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # ******** NOTE ******** 12 | 13 | name: "CodeQL" 14 | 15 | on: 16 | push: 17 | branches: [ main ] 18 | pull_request: 19 | # The branches below must be a subset of the branches above 20 | branches: [ main ] 21 | schedule: 22 | - cron: '33 23 * * 4' 23 | 24 | permissions: write-all 25 | 26 | jobs: 27 | analyze: 28 | name: Analyze 29 | runs-on: ubuntu-latest 30 | 31 | strategy: 32 | fail-fast: false 33 | matrix: 34 | language: [ 'go' ] 35 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] 36 | # Learn more: 37 | # https://docs.github.com/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed 38 | 39 | steps: 40 | - name: Check out code base 41 | if: github.event_name == 'push' || github.event_name == 'schedule' 42 | uses: actions/checkout@v3 43 | with: 44 | fetch-depth: 0 45 | 46 | - name: Check out code base 47 | if: github.event_name == 'pull_request' 48 | uses: actions/checkout@v3 49 | with: 50 | fetch-depth: 0 51 | ref: ${{ github.event.pull_request.head.sha }} 52 | 53 | # Initializes the CodeQL tools for scanning. 54 | - name: Initialize CodeQL 55 | uses: github/codeql-action/init@v2 56 | with: 57 | languages: ${{ matrix.language }} 58 | # If you wish to specify custom queries, you can do so here or in a config file. 59 | # By default, queries listed here will override any specified in a config file. 60 | # Prefix the list here with "+" to use these queries and those in the config file. 61 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 62 | 63 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 64 | # If this step fails, then you should remove it and run the build manually (see below) 65 | - name: Autobuild 66 | uses: github/codeql-action/autobuild@v2 67 | 68 | # ℹ️ Command-line programs to run using the OS shell. 69 | # 📚 https://git.io/JvXDl 70 | 71 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 72 | # and modify them (or add more) to build your code if your project 73 | # uses a compiled language 74 | 75 | #- run: | 76 | # make bootstrap 77 | # make release 78 | 79 | - name: Perform CodeQL Analysis 80 | uses: github/codeql-action/analyze@v2 81 | 82 | nancy: 83 | name: Sonatype Nancy 84 | runs-on: ubuntu-latest 85 | steps: 86 | - name: Check out code base 87 | if: github.event_name == 'push' || github.event_name == 'schedule' 88 | uses: actions/checkout@v3 89 | with: 90 | fetch-depth: 0 91 | 92 | - name: Check out code base 93 | if: github.event_name == 'pull_request' 94 | uses: actions/checkout@v3 95 | with: 96 | fetch-depth: 0 97 | ref: ${{ github.event.pull_request.head.sha }} 98 | 99 | - name: Set up Go 1.x 100 | uses: actions/setup-go@v3 101 | with: 102 | go-version: ^1.17 103 | 104 | - name: Write Go module list 105 | run: go list -json -m all > go.list 106 | 107 | - name: Perform Nancy 108 | uses: sonatype-nexus-community/nancy-github-action@main 109 | continue-on-error: true 110 | 111 | -------------------------------------------------------------------------------- /rivet.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the MIT 3 | // license that can be found in the LICENSE file. 4 | 5 | package rivet 6 | 7 | import ( 8 | "bytes" 9 | "context" 10 | "io" 11 | "io/ioutil" 12 | "net/http" 13 | "net/url" 14 | "os" 15 | "path/filepath" 16 | "regexp" 17 | "strings" 18 | "time" 19 | 20 | "github.com/go-shiori/obelisk" 21 | "github.com/kennygrant/sanitize" 22 | "github.com/pkg/errors" 23 | "github.com/wabarc/rivet/ipfs" 24 | ) 25 | 26 | // Shaft represents the rivet handler. 27 | type Shaft struct { 28 | // Client represents a http client. 29 | Client *http.Client 30 | 31 | // Hold specifies which IPFS mode to pin data through. 32 | Hold ipfs.Pinning 33 | 34 | // Next is a fallback pinning service. If the `Hold` 35 | // pinning service fails, it will be used. 36 | Next ipfs.Pinning 37 | 38 | // Do not store file on any IPFS node, just archive 39 | ArchiveOnly bool 40 | } 41 | 42 | // Wayback uses IPFS to archive webpages. 43 | func (s *Shaft) Wayback(ctx context.Context, input *url.URL) (cid string, err error) { 44 | name := sanitize.BaseName(input.Host) + sanitize.BaseName(input.Path) 45 | dir := "rivet-" + name 46 | if len(dir) > 255 { 47 | dir = dir[:254] 48 | } 49 | 50 | dir, err = ioutil.TempDir(os.TempDir(), dir+"-") 51 | if err != nil { 52 | return "", errors.Wrap(err, "create temp directory failed: "+dir) 53 | } 54 | defer os.RemoveAll(dir) 55 | 56 | uri := input.String() 57 | req := obelisk.Request{URL: uri, Input: inputFromContext(ctx)} 58 | arc := &obelisk.Archiver{ 59 | DisableJS: isDisableJS(uri), 60 | 61 | SkipResourceURLError: true, 62 | 63 | WrapDirectory: dir, 64 | RequestTimeout: 3 * time.Second, 65 | } 66 | if s.Client != nil { 67 | arc.Transport = s.Client.Transport 68 | } 69 | arc.Validate() 70 | 71 | content, _, err := arc.Archive(ctx, req) 72 | if err != nil { 73 | return "", errors.Wrap(err, "archive failed") 74 | } 75 | 76 | // For auto indexing in IPFS, the filename should be index.html. 77 | indexFile := filepath.Join(dir, "index.html") 78 | if s.ArchiveOnly { 79 | indexFile = name + ".html" 80 | } 81 | 82 | if err := ioutil.WriteFile(indexFile, content, 0600); err != nil { 83 | return "", errors.Wrap(err, "create index file failed") 84 | } 85 | 86 | if s.ArchiveOnly { 87 | return indexFile, nil 88 | } 89 | 90 | switch s.Hold.Mode { 91 | case ipfs.Local: 92 | cid, err = (&ipfs.Locally{Pinning: s.Hold}).PinDir(dir) 93 | case ipfs.Remote: 94 | cid, err = (&ipfs.Remotely{Pinning: s.Hold}).PinDir(dir) 95 | } 96 | if err != nil { 97 | // Try fallback pinning service 98 | switch s.Next.Mode { 99 | case ipfs.Local: 100 | cid, err = (&ipfs.Locally{Pinning: s.Next}).PinDir(dir) 101 | case ipfs.Remote: 102 | cid, err = (&ipfs.Remotely{Pinning: s.Next}).PinDir(dir) 103 | } 104 | if err != nil { 105 | return "", errors.Wrap(err, "pin failed") 106 | } 107 | } 108 | if cid == "" { 109 | return "", errors.New("cid empty") 110 | } 111 | 112 | return "https://ipfs.io/ipfs/" + cid, nil 113 | } 114 | 115 | type ctxKeyInput struct{} 116 | 117 | // WithInput permits to inject a webpage into a context by given input. 118 | func (s *Shaft) WithInput(ctx context.Context, input []byte) (c context.Context) { 119 | return context.WithValue(ctx, ctxKeyInput{}, input) 120 | } 121 | 122 | func inputFromContext(ctx context.Context) io.Reader { 123 | if b, ok := ctx.Value(ctxKeyInput{}).([]byte); ok { 124 | return bytes.NewReader(b) 125 | } 126 | return nil 127 | } 128 | 129 | func isDisableJS(link string) bool { 130 | // e.g. DISABLEJS_URIS=wikipedia.org|eff.org/tags 131 | uris := os.Getenv("DISABLEJS_URIS") 132 | if uris == "" { 133 | return false 134 | } 135 | 136 | regex := regexp.QuoteMeta(strings.ReplaceAll(uris, "|", "@@")) 137 | re := regexp.MustCompile(`(?m)` + strings.ReplaceAll(regex, "@@", "|")) 138 | 139 | return re.MatchString(link) 140 | } 141 | -------------------------------------------------------------------------------- /.github/workflows/builder.yml: -------------------------------------------------------------------------------- 1 | name: Builder 2 | 3 | on: 4 | push: 5 | branches: "*" 6 | paths: 7 | - "**/*.go" 8 | - "go.mod" 9 | - "go.sum" 10 | - "Makefile" 11 | - "build/**" 12 | - ".github/workflows/builder.yml" 13 | pull_request: 14 | branches: "*" 15 | paths: 16 | - "**/*.go" 17 | - "go.mod" 18 | - "go.sum" 19 | workflow_dispatch: 20 | 21 | env: 22 | PRODUCT: rivet 23 | 24 | permissions: write-all 25 | 26 | jobs: 27 | build: 28 | name: Build 29 | strategy: 30 | matrix: 31 | os: [ linux, freebsd, openbsd, dragonfly, windows, darwin ] 32 | arch: [ amd64, 386 ] 33 | include: 34 | - os: linux 35 | arch: arm 36 | arm: 5 37 | - os: linux 38 | arch: arm 39 | arm: 6 40 | - os: linux 41 | arch: arm 42 | arm: 7 43 | - os: linux 44 | arch: arm64 45 | - os: linux 46 | arch: mips 47 | mips: softfloat 48 | - os: linux 49 | arch: mips 50 | mips: hardfloat 51 | - os: linux 52 | arch: mipsle 53 | mipsle: softfloat 54 | - os: linux 55 | arch: mipsle 56 | mipsle: hardfloat 57 | - os: linux 58 | arch: mips64 59 | - os: linux 60 | arch: mips64le 61 | - os: linux 62 | arch: ppc64 63 | - os: linux 64 | arch: ppc64le 65 | - os: linux 66 | arch: s390x 67 | - os: windows 68 | arch: arm 69 | - os: windows 70 | arch: arm64 71 | - os: android 72 | arch: arm64 73 | - os: darwin 74 | arch: arm64 75 | - os: freebsd 76 | arch: arm64 77 | exclude: 78 | - os: darwin 79 | arch: 386 80 | - os: dragonfly 81 | arch: 386 82 | fail-fast: false 83 | runs-on: ubuntu-latest 84 | continue-on-error: true 85 | env: 86 | GOOS: ${{ matrix.os }} 87 | GOARCH: ${{ matrix.arch }} 88 | GOARM: ${{ matrix.arm }} 89 | GOMIPS: ${{ matrix.mips }} 90 | GOMIPS64: ${{ matrix.mips64 }} 91 | GOMIPSLE: ${{ matrix.mipsle }} 92 | steps: 93 | - name: Set up Go 1.x 94 | uses: actions/setup-go@v3 95 | with: 96 | go-version: ^1.17 97 | 98 | - name: Check out code base 99 | if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' 100 | uses: actions/checkout@v3 101 | with: 102 | fetch-depth: 0 103 | 104 | - name: Check out code base 105 | if: github.event_name == 'pull_request' 106 | uses: actions/checkout@v3 107 | with: 108 | fetch-depth: 0 109 | ref: ${{ github.event.pull_request.head.sha }} 110 | 111 | - name: Cache go module 112 | uses: actions/cache@v3 113 | with: 114 | path: ~/go/pkg/mod 115 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 116 | restore-keys: ${{ runner.os }}-go- 117 | 118 | - name: Get dependencies 119 | run: | 120 | go get -v -t -d ./... 121 | 122 | - name: Build binary 123 | id: builder 124 | run: | 125 | ARGS="${GOOS}-${GOARCH}" 126 | if [[ -n "${GOARM}" ]]; then 127 | ARGS="${ARGS}v${GOARM}" 128 | elif [[ -n "${GOMIPS}" ]]; then 129 | ARGS="${ARGS}-${GOMIPS}" 130 | elif [[ -n "${GOMIPS64}" ]]; then 131 | ARGS="${ARGS}-${GOMIPS64}" 132 | elif [[ -n "${GOMIPSLE}" ]]; then 133 | ARGS="${ARGS}-${GOMIPSLE}" 134 | fi 135 | make ${ARGS} 136 | echo "filename=${{ env.PRODUCT }}-${ARGS}" >> $GITHUB_OUTPUT 137 | 138 | - name: Upload binary artifacts 139 | uses: actions/upload-artifact@v3 140 | with: 141 | name: ${{ steps.builder.outputs.filename }} 142 | path: ./build/binary/${{ env.PRODUCT }}* 143 | if-no-files-found: error 144 | 145 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Rivet 2 | 3 | [![LICENSE](https://img.shields.io/github/license/wabarc/rivet.svg?color=green)](https://github.com/wabarc/rivet/blob/main/LICENSE) 4 | [![Go Report Card](https://goreportcard.com/badge/github.com/wabarc/rivet)](https://goreportcard.com/report/github.com/wabarc/rivet) 5 | [![GitHub Workflow Status](https://img.shields.io/github/workflow/status/wabarc/rivet/Builder?color=brightgreen)](https://github.com/wabarc/rivet/actions/workflows/builder.yml) 6 | [![Go Reference](https://img.shields.io/badge/godoc-reference-blue.svg)](https://pkg.go.dev/github.com/wabarc/rivet) 7 | [![Releases](https://img.shields.io/github/v/release/wabarc/rivet.svg?include_prereleases&color=blue)](https://github.com/wabarc/rivet/releases) 8 | 9 | Rivet is both a command-line tool and a Golang package for archiving webpages to IPFS. 10 | 11 | Supported Golang version: See [.github/workflows/testing.yml](./.github/workflows/testing.yml) 12 | 13 | ## Installation 14 | 15 | The simplest, cross-platform way is to download from [GitHub Releases](https://github.com/wabarc/rivet/releases) and place the executable file in your PATH. 16 | 17 | From source: 18 | 19 | ```sh 20 | go get -u github.com/wabarc/rivet/cmd/rivet 21 | ``` 22 | 23 | From GitHub Releases: 24 | 25 | ```sh 26 | sh <(wget https://github.com/wabarc/rivet/raw/main/install.sh -O-) 27 | ``` 28 | 29 | Using [Snapcraft](https://snapcraft.io/rivet) (on GNU/Linux) 30 | 31 | ```sh 32 | sudo snap install rivet 33 | ``` 34 | 35 | ## Usage 36 | 37 | ### Command line 38 | 39 | ```sh 40 | A toolkit makes it easier to archive webpages to IPFS. 41 | 42 | Usage: 43 | 44 | rivet [options] [url1] ... [urlN] 45 | 46 | -host string 47 | IPFS node address (default "localhost") 48 | -m string 49 | Pin mode, supports mode: local, remote, archive (default "remote") 50 | -p string 51 | Pinner sceret or password. 52 | -port int 53 | IPFS node port (default 5001) 54 | -t string 55 | IPFS pinner, supports pinners: infura, pinata, nftstorage, web3storage. (default "infura") 56 | -timeout uint 57 | Timeout for every input URL (default 30) 58 | -u string 59 | Pinner apikey or username. 60 | ``` 61 | 62 | #### Examples 63 | 64 | Stores data on local IPFS node. 65 | 66 | ```sh 67 | rivet -m local https://example.com https://example.org 68 | ``` 69 | 70 | Stores data to remote pinning services. 71 | 72 | ```sh 73 | rivet https://example.com 74 | ``` 75 | 76 | Or, specify a pinning service. 77 | 78 | ```sh 79 | rivet -t pinata -k your-apikey -s your-secret https://example.com 80 | ``` 81 | 82 | Or, stores file locally without any IPFS node. 83 | 84 | ```sh 85 | rivet -m archive https://example.com 86 | ``` 87 | 88 | ### Go package 89 | 90 | 91 | ```go 92 | package main 93 | 94 | import ( 95 | "context" 96 | "fmt" 97 | "net/url" 98 | 99 | "github.com/wabarc/ipfs-pinner" 100 | "github.com/wabarc/rivet" 101 | "github.com/wabarc/rivet/ipfs" 102 | ) 103 | 104 | func main() { 105 | opts := []ipfs.PinningOption{ 106 | ipfs.Mode(ipfs.Remote), 107 | ipfs.Uses(pinner.Infura), 108 | } 109 | p := ipfs.Options(opts...) 110 | r := &rivet.Shaft{Hold: p} 111 | l := "https://example.com" 112 | input, err := url.Parse(l) 113 | if err != nil { 114 | panic(err) 115 | } 116 | 117 | dst, err := r.Wayback(context.TODO(), input) 118 | if err != nil { 119 | panic(err) 120 | } 121 | fmt.Println(dst) 122 | } 123 | ``` 124 | 125 | 126 | ## F.A.Q 127 | 128 | ### Optional to disable JavaScript for some URI? 129 | 130 | If you prefer to disable JavaScript on saving webpage, you could add environmental variables `DISABLEJS_URIS` 131 | and set the values with the following formats: 132 | 133 | ```sh 134 | export DISABLEJS_URIS=wikipedia.org|eff.org/tags 135 | ``` 136 | 137 | It will disable JavaScript for domain of the `wikipedia.org` and path of the `eff.org/tags` if matching it. 138 | 139 | ## Credit 140 | 141 | Special thanks to [@RadhiFadlillah](https://github.com/RadhiFadlillah) for making [obelisk](https://github.com/go-shiori/obelisk), under which the crawling of the web is based. 142 | 143 | ## Contributing 144 | 145 | We encourage all contributions to this repository! Open an issue! Or open a Pull Request! 146 | 147 | ## License 148 | 149 | This software is released under the terms of the MIT. See the [LICENSE](https://github.com/wabarc/rivet/blob/main/LICENSE) file for details. 150 | 151 | -------------------------------------------------------------------------------- /ipfs/ipfs.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the MIT 3 | // license that can be found in the LICENSE file. 4 | 5 | package ipfs 6 | 7 | import ( 8 | "bytes" 9 | "net/http" 10 | "time" 11 | 12 | "github.com/cenkalti/backoff/v4" 13 | "github.com/pkg/errors" 14 | 15 | shell "github.com/ipfs/go-ipfs-api" 16 | pinner "github.com/wabarc/ipfs-pinner" 17 | ) 18 | 19 | type mode int 20 | 21 | const ( 22 | Remote mode = iota + 1 // Store files to pinning service 23 | Local // Store file on local IPFS node 24 | 25 | maxElapsedTime = time.Minute 26 | maxRetries = 3 27 | ) 28 | 29 | var _ Pinner = (*Locally)(nil) 30 | var _ Pinner = (*Remotely)(nil) 31 | 32 | // The HandlerFunc type is an adapter to allow the use of 33 | // ordinary functions as IPFS handlers. 34 | type HandlerFunc func(Pinner, interface{}) (string, error) 35 | 36 | // Pinner is an interface that wraps the Pin method. 37 | type Pinner interface { 38 | // Pin implements data transmission to the destination service by given buf. It 39 | // returns the content-id returned by the local IPFS server or a remote pinning service. 40 | Pin(buf []byte) (string, error) 41 | 42 | // Pin implements directory transmission to the destination service by given path. It 43 | // returns the content-id returned by the local IPFS server or a remote pinning service. 44 | PinDir(path string) (string, error) 45 | } 46 | 47 | // Locally embeds the Pinning struct, which provides configuration for pinning services 48 | // used for data storage. 49 | type Locally struct { 50 | Pinning 51 | } 52 | 53 | // Remotely embeds the Pinning struct, which provides configuration for pinning services 54 | // used for data storage. 55 | type Remotely struct { 56 | Pinning 57 | } 58 | 59 | // Pinning provides the configuration of pinning services that will be utilized for data storage. 60 | type Pinning struct { 61 | // It supports both daemon server and remote pinner, defaults to remote pinner. 62 | Mode mode 63 | 64 | Host string 65 | Port int 66 | shell *shell.Shell // Only for daemon mode 67 | 68 | // For pinner mode, which normally requires the apikey and secret of the pinning service. 69 | Pinner string 70 | Apikey string 71 | Secret string 72 | 73 | // Client represents a http client. 74 | Client *http.Client 75 | 76 | // Whether or not to use backoff stragty. 77 | backoff bool 78 | } 79 | 80 | // Pin implements putting the data to local IPFS node by given buf. It 81 | // returns content-id and an error. 82 | func (l *Locally) Pin(buf []byte) (cid string, err error) { 83 | action := func() error { 84 | cid, err = l.shell.Add(bytes.NewReader(buf), shell.Pin(true)) 85 | return err 86 | } 87 | err = l.doRetry(action) 88 | if err != nil { 89 | return "", errors.Wrap(err, "add file to IPFS failed") 90 | } 91 | return 92 | } 93 | 94 | // Pin implements putting the data to local IPFS node by given buf. It 95 | // returns content-id and an error. 96 | func (l *Locally) PinDir(path string) (cid string, err error) { 97 | action := func() error { 98 | cid, err = l.shell.AddDir(path) 99 | return err 100 | } 101 | err = l.doRetry(action) 102 | if err != nil { 103 | return "", errors.Wrap(err, "add directory to IPFS failed") 104 | } 105 | return 106 | } 107 | 108 | // Pin implements putting the data to destination pinning service by given buf. It 109 | // returns content-id and an error. 110 | func (r *Remotely) Pin(buf []byte) (cid string, err error) { 111 | action := func() error { 112 | cid, err = r.remotely().Pin(buf) 113 | return err 114 | } 115 | err = r.doRetry(action) 116 | return 117 | } 118 | 119 | // Pin implements putting the data to destination pinning service by given buf. It 120 | // returns content-id and an error. 121 | func (r *Remotely) PinDir(path string) (cid string, err error) { 122 | action := func() error { 123 | cid, err = r.remotely().Pin(path) 124 | return err 125 | } 126 | err = r.doRetry(action) 127 | return 128 | } 129 | 130 | func (r *Remotely) remotely() *pinner.Config { 131 | return &pinner.Config{ 132 | Pinner: r.Pinner, 133 | Apikey: r.Apikey, 134 | Secret: r.Secret, 135 | Client: r.Client, 136 | } 137 | } 138 | 139 | func (p *Pinning) doRetry(op backoff.Operation) error { 140 | if p.backoff { 141 | exp := backoff.NewExponentialBackOff() 142 | exp.MaxElapsedTime = maxElapsedTime 143 | bo := backoff.WithMaxRetries(exp, maxRetries) 144 | 145 | return backoff.Retry(op, bo) 146 | } 147 | 148 | return op() 149 | } 150 | -------------------------------------------------------------------------------- /ipfs/ipfs_test.go: -------------------------------------------------------------------------------- 1 | package ipfs 2 | 3 | import ( 4 | "fmt" 5 | "mime" 6 | "mime/multipart" 7 | "net/http" 8 | "net/url" 9 | "strconv" 10 | "strings" 11 | "testing" 12 | 13 | "github.com/wabarc/helper" 14 | "github.com/wabarc/ipfs-pinner" 15 | ) 16 | 17 | var ( 18 | apikey = "1234" 19 | secret = "abcd" 20 | badRequestJSON = `{}` 21 | unauthorizedJSON = `{}` 22 | pinHashJSON = `{ 23 | "hashToPin": "Qmaisz6NMhDB51cCvNWa1GMS7LU1pAxdF4Ld6Ft9kZEP2a" 24 | }` 25 | pinFileJSON = `{ 26 | "IpfsHash": "Qmaisz6NMhDB51cCvNWa1GMS7LU1pAxdF4Ld6Ft9kZEP2a", 27 | "PinSize": 1234, 28 | "Timestamp": "1979-01-01 00:00:00Z" 29 | }` 30 | ipfsCid = "Qmaisz6NMhDB51cCvNWa1GMS7LU1pAxdF4Ld6Ft9kZEP2a" 31 | addJSON = fmt.Sprintf(`{ 32 | "Bytes": 0, 33 | "Hash": "%s", 34 | "Name": "name", 35 | "Size": "1B" 36 | }`, ipfsCid) 37 | ) 38 | 39 | func handleResponse(w http.ResponseWriter, r *http.Request) { 40 | switch r.URL.Hostname() { 41 | case "api.pinata.cloud": 42 | authorization := r.Header.Get("Authorization") 43 | apiKey := r.Header.Get("pinata_api_key") 44 | apiSec := r.Header.Get("pinata_secret_api_key") 45 | switch { 46 | case apiKey != "" && apiSec != "": 47 | // access 48 | case authorization != "" && !strings.HasPrefix(authorization, "Bearer"): 49 | w.WriteHeader(http.StatusUnauthorized) 50 | _, _ = w.Write([]byte(unauthorizedJSON)) 51 | return 52 | default: 53 | w.WriteHeader(http.StatusUnauthorized) 54 | _, _ = w.Write([]byte(unauthorizedJSON)) 55 | return 56 | } 57 | 58 | switch r.URL.Path { 59 | case "/pinning/pinFileToIPFS": 60 | _ = r.ParseMultipartForm(32 << 20) 61 | _, params, parseErr := mime.ParseMediaType(r.Header.Get("Content-Type")) 62 | if parseErr != nil { 63 | w.WriteHeader(http.StatusBadRequest) 64 | _, _ = w.Write([]byte(badRequestJSON)) 65 | return 66 | } 67 | 68 | multipartReader := multipart.NewReader(r.Body, params["boundary"]) 69 | defer r.Body.Close() 70 | 71 | // Pin directory 72 | if multipartReader != nil && len(r.MultipartForm.File["file"]) > 1 { 73 | _, _ = w.Write([]byte(pinFileJSON)) 74 | return 75 | } 76 | // Pin file 77 | if multipartReader != nil && len(r.MultipartForm.File["file"]) == 1 { 78 | _, _ = w.Write([]byte(pinFileJSON)) 79 | return 80 | } 81 | case "/pinning/pinByHash": 82 | _, _ = w.Write([]byte(pinHashJSON)) 83 | return 84 | } 85 | } 86 | } 87 | 88 | func TestLocally(t *testing.T) { 89 | handleResponse := func(w http.ResponseWriter, r *http.Request) { 90 | switch r.URL.Path { 91 | case "/api/v0/add": 92 | w.WriteHeader(http.StatusOK) 93 | w.Header().Set("Content-Type", "application/json") 94 | _, _ = w.Write([]byte(addJSON)) 95 | } 96 | } 97 | 98 | _, mux, server := helper.MockServer() 99 | mux.HandleFunc("/", handleResponse) 100 | defer server.Close() 101 | 102 | u, _ := url.Parse(server.URL) 103 | host := u.Hostname() 104 | port, _ := strconv.Atoi(u.Port()) 105 | opts := []PinningOption{ 106 | Mode(Local), 107 | Host(host), 108 | Port(port), 109 | } 110 | 111 | p := Options(opts...) 112 | b := []byte(helper.RandString(6, "lower")) 113 | i, err := (&Locally{p}).Pin(b) 114 | if err != nil { 115 | t.Errorf("Unexpected pin data locally: %v", err) 116 | } 117 | if i != ipfsCid { 118 | t.Fatalf("Unexpected cid got %s instead of %s", i, ipfsCid) 119 | } 120 | } 121 | 122 | func TestRemotely(t *testing.T) { 123 | client, mux, server := helper.MockServer() 124 | mux.HandleFunc("/", handleResponse) 125 | defer server.Close() 126 | 127 | opts := []PinningOption{ 128 | Mode(Remote), 129 | Uses(pinner.Pinata), 130 | Apikey(apikey), 131 | Secret(secret), 132 | Client(client), 133 | } 134 | 135 | p := Options(opts...) 136 | b := []byte(helper.RandString(6, "lower")) 137 | _, err := (&Remotely{p}).Pin(b) 138 | if err != nil { 139 | t.Errorf("Unexpected pin data remotely: %v", err) 140 | } 141 | } 142 | 143 | func TestRateLimit(t *testing.T) { 144 | counter := 0 145 | handleResponse := func(w http.ResponseWriter, r *http.Request) { 146 | counter++ 147 | if counter <= maxRetries { 148 | _, _ = w.Write([]byte(``)) 149 | return 150 | } 151 | switch r.URL.Path { 152 | case "/api/v0/add": 153 | w.WriteHeader(http.StatusOK) 154 | w.Header().Set("Content-Type", "application/json") 155 | _, _ = w.Write([]byte(addJSON)) 156 | } 157 | } 158 | 159 | _, mux, server := helper.MockServer() 160 | mux.HandleFunc("/", handleResponse) 161 | defer server.Close() 162 | 163 | u, _ := url.Parse(server.URL) 164 | host := u.Hostname() 165 | port, _ := strconv.Atoi(u.Port()) 166 | opts := []PinningOption{ 167 | Mode(Local), 168 | Host(host), 169 | Port(port), 170 | Backoff(true), 171 | } 172 | 173 | p := Options(opts...) 174 | b := []byte(helper.RandString(6, "lower")) 175 | i, err := (&Locally{p}).Pin(b) 176 | if err != nil { 177 | t.Errorf("Unexpected pin data locally: %v", err) 178 | } 179 | if i != ipfsCid { 180 | t.Fatalf("Unexpected cid got %s instead of %s", i, ipfsCid) 181 | } 182 | } 183 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | export GO111MODULE = on 2 | export CGO_ENABLED = 0 3 | export GOPROXY = https://proxy.golang.org 4 | 5 | NAME = rivet 6 | REPO = github.com/wabarc/rivet 7 | BINDIR ?= ./build/binary 8 | PACKDIR ?= ./build/package 9 | LDFLAGS := $(shell echo "-X '${REPO}/version.Version=`git describe --tags --abbrev=0`'") 10 | LDFLAGS := $(shell echo "${LDFLAGS} -X '${REPO}/version.Commit=`git rev-parse --short HEAD`'") 11 | LDFLAGS := $(shell echo "${LDFLAGS} -X '${REPO}/version.BuildDate=`date +%FT%T%z`'") 12 | GOBUILD ?= go build -trimpath --ldflags "-s -w ${LDFLAGS} -buildid=" -v 13 | VERSION ?= $(shell git describe --tags `git rev-list --tags --max-count=1` | sed -e 's/v//g') 14 | GOFILES ?= $(wildcard ./cmd/rivet/*.go) 15 | PROJECT := github.com/wabarc/rivet 16 | PACKAGES ?= $(shell go list ./...) 17 | DOCKER ?= $(shell which docker || which podman) 18 | DOCKER_IMAGE := wabarc/rivet 19 | DEB_IMG_ARCH := amd64 20 | 21 | .DEFAULT_GOAL := help 22 | 23 | .PHONY: help 24 | help: ## show help message 25 | @awk 'BEGIN {FS = ":.*##"; printf "Usage:\n make \n\nTargets: \033[36m\033[0m\n"} /^[$$()% 0-9a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) 26 | 27 | PLATFORM_LIST = \ 28 | darwin-amd64 \ 29 | darwin-arm64 \ 30 | linux-386 \ 31 | linux-amd64 \ 32 | linux-armv5 \ 33 | linux-armv6 \ 34 | linux-armv7 \ 35 | linux-arm64 \ 36 | linux-mips-softfloat \ 37 | linux-mips-hardfloat \ 38 | linux-mipsle-softfloat \ 39 | linux-mipsle-hardfloat \ 40 | linux-mips64 \ 41 | linux-mips64le \ 42 | linux-ppc64 \ 43 | linux-ppc64le \ 44 | linux-s390x \ 45 | freebsd-386 \ 46 | freebsd-amd64 \ 47 | freebsd-arm64 \ 48 | openbsd-386 \ 49 | openbsd-amd64 \ 50 | dragonfly-amd64 \ 51 | android-arm64 52 | 53 | WINDOWS_ARCH_LIST = \ 54 | windows-386 \ 55 | windows-amd64 \ 56 | windows-arm \ 57 | windows-arm64 58 | 59 | .PHONY: \ 60 | all-arch \ 61 | tar_releases \ 62 | zip_releases \ 63 | releases \ 64 | clean \ 65 | test \ 66 | fmt \ 67 | rpm \ 68 | debian \ 69 | debian-packages \ 70 | docker-image 71 | 72 | .SECONDEXPANSION: 73 | %: ## Build binary, format: linux-amd64, darwin-arm64, full list: https://golang.org/doc/install/source#environment 74 | $(eval OS := $(shell echo $@ | cut -d'-' -f1)) 75 | $(eval ARM := $(shell echo $@ | cut -d'-' -f2 | grep arm | sed -e 's/arm64//' | tr -dc '[0-9]')) 76 | $(eval ARCH := $(shell echo $@ | cut -d'-' -f2 | sed -e 's/armv.*/arm/' | grep -v $(OS))) 77 | $(eval MIPS := $(shell echo $@ | cut -d'-' -f3)) 78 | $(if $(strip $(OS)),,$(error missing OS)) 79 | $(if $(strip $(ARCH)),,$(error missing ARCH)) 80 | GOOS="$(OS)" GOARCH="$(ARCH)" GOMIPS="$(MIPS)" GOARM="$(ARM)" $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ $(GOFILES) 81 | 82 | .PHONY: build 83 | build: ## Build binary for current OS 84 | $(GOBUILD) -o $(BINDIR)/$(NAME) $(GOFILES) 85 | 86 | .PHONY: linux-armv8 87 | linux-armv8: linux-arm64 88 | 89 | ifeq ($(TARGET),) 90 | tar_releases := $(addsuffix .gz, $(PLATFORM_LIST)) 91 | zip_releases := $(addsuffix .zip, $(WINDOWS_ARCH_LIST)) 92 | else 93 | ifeq ($(findstring windows,$(TARGET)),windows) 94 | zip_releases := $(addsuffix .zip, $(TARGET)) 95 | else 96 | tar_releases := $(addsuffix .gz, $(TARGET)) 97 | endif 98 | endif 99 | 100 | $(tar_releases): %.gz : % 101 | chmod +x $(BINDIR)/$(NAME)-$(basename $@) 102 | tar -czf $(PACKDIR)/$(NAME)-$(basename $@)-$(VERSION).tar.gz --transform "s/.*\///g" $(BINDIR)/$(NAME)-$(basename $@) LICENSE README.md 103 | 104 | $(zip_releases): %.zip : % 105 | @mv $(BINDIR)/$(NAME)-$(basename $@) $(BINDIR)/$(NAME)-$(basename $@).exe 106 | zip -m -j $(PACKDIR)/$(NAME)-$(basename $@)-$(VERSION).zip $(BINDIR)/$(NAME)-$(basename $@).exe LICENSE README.md 107 | 108 | all-arch: $(PLATFORM_LIST) $(WINDOWS_ARCH_LIST) ## Build binary for all architecture 109 | 110 | releases: $(tar_releases) $(zip_releases) ## Packaging all binaries 111 | 112 | clean: ## Clean workspace 113 | rm -f $(BINDIR)/* 114 | rm -f $(PACKDIR)/* 115 | rm -rf data-dir* coverage* bin *.out 116 | 117 | fmt: ## Format codebase 118 | @echo "-> Running go fmt" 119 | @go fmt $(PACKAGES) 120 | 121 | vet: ## Vet codebase 122 | @echo "-> Running go vet" 123 | @go vet $(PACKAGES) 124 | 125 | test: ## Run testing 126 | @echo "-> Running go test" 127 | @go clean -testcache 128 | @CGO_ENABLED=1 go test -v -race -cover -coverprofile=coverage.out -covermode=atomic -parallel=1 ./... 129 | 130 | test-integration: ## Run integration testing 131 | @echo 'mode: atomic' > coverage.out 132 | @go list ./... | xargs -n1 -I{} sh -c 'CGO_ENABLED=1 go test -race -tags=integration -covermode=atomic -coverprofile=coverage.tmp -coverpkg $(go list ./... | tr "\n" ",") {} && tail -n +2 coverage.tmp >> coverage.out || exit 255' 133 | @rm coverage.tmp 134 | 135 | test-cover: ## Collect code coverage 136 | @echo "-> Running go tool cover" 137 | @go tool cover -func=coverage.out 138 | @go tool cover -html=coverage.out -o coverage.html 139 | 140 | bench: ## Benchmark test 141 | @echo "-> Running benchmark" 142 | @go test -v -bench ./... 143 | 144 | profile: ## Test and profile 145 | @echo "-> Running profile" 146 | @go test -cpuprofile cpu.prof -memprofile mem.prof -v -bench ./... 147 | 148 | scan: ## Scan vulnerabilities 149 | @echo "-> Scanning vulnerabilities..." 150 | @go list -json -m all | $(DOCKER) run --rm -i sonatypecommunity/nancy sleuth --skip-update-check 151 | -------------------------------------------------------------------------------- /rivet_test.go: -------------------------------------------------------------------------------- 1 | package rivet 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "context" 7 | "image" 8 | "image/color" 9 | "image/png" 10 | "mime" 11 | "mime/multipart" 12 | "net/http" 13 | "net/url" 14 | "strings" 15 | "testing" 16 | 17 | "github.com/wabarc/helper" 18 | "github.com/wabarc/ipfs-pinner" 19 | "github.com/wabarc/rivet/ipfs" 20 | ) 21 | 22 | var ( 23 | apikey = "1234" 24 | secret = "abcd" 25 | badRequestJSON = `{}` 26 | unauthorizedJSON = `{}` 27 | pinHashJSON = `{ 28 | "hashToPin": "Qmaisz6NMhDB51cCvNWa1GMS7LU1pAxdF4Ld6Ft9kZEP2a" 29 | }` 30 | pinFileJSON = `{ 31 | "IpfsHash": "Qmaisz6NMhDB51cCvNWa1GMS7LU1pAxdF4Ld6Ft9kZEP2a", 32 | "PinSize": 1234, 33 | "Timestamp": "1979-01-01 00:00:00Z" 34 | }` 35 | content = ` 36 | 37 | Example Domain 38 | 39 | 40 | 41 |
42 |

Example Domain

43 |

This domain is for use in illustrative examples in documents. You may use this 44 | domain in literature without prior coordination or asking for permission.

45 |

More information...

46 |

47 |
48 | 49 | 50 | ` 51 | ) 52 | 53 | func genImage(height int) bytes.Buffer { 54 | width := 1024 55 | 56 | upLeft := image.Point{0, 0} 57 | lowRight := image.Point{width, height} 58 | 59 | img := image.NewRGBA(image.Rectangle{upLeft, lowRight}) 60 | 61 | // Colors are defined by Red, Green, Blue, Alpha uint8 values. 62 | cyan := color.RGBA{100, 200, 200, 0xff} 63 | 64 | // Set color for each pixel. 65 | for x := 0; x < width; x++ { 66 | for y := 0; y < height; y++ { 67 | switch { 68 | case x < width/2 && y < height/2: // upper left quadrant 69 | img.Set(x, y, cyan) 70 | case x >= width/2 && y >= height/2: // lower right quadrant 71 | img.Set(x, y, color.White) 72 | default: 73 | // Use zero value. 74 | } 75 | } 76 | } 77 | 78 | var b bytes.Buffer 79 | f := bufio.NewWriter(&b) 80 | png.Encode(f, img) // Encode as PNG. 81 | 82 | return b 83 | } 84 | 85 | func handleResponse(w http.ResponseWriter, r *http.Request) { 86 | switch r.URL.Hostname() { 87 | case "api.pinata.cloud": 88 | authorization := r.Header.Get("Authorization") 89 | apiKey := r.Header.Get("pinata_api_key") 90 | apiSec := r.Header.Get("pinata_secret_api_key") 91 | switch { 92 | case apiKey != "" && apiSec != "": 93 | // access 94 | case authorization != "" && !strings.HasPrefix(authorization, "Bearer"): 95 | w.WriteHeader(http.StatusUnauthorized) 96 | _, _ = w.Write([]byte(unauthorizedJSON)) 97 | return 98 | default: 99 | w.WriteHeader(http.StatusUnauthorized) 100 | _, _ = w.Write([]byte(unauthorizedJSON)) 101 | return 102 | } 103 | 104 | switch r.URL.Path { 105 | case "/pinning/pinFileToIPFS": 106 | _ = r.ParseMultipartForm(32 << 20) 107 | _, params, parseErr := mime.ParseMediaType(r.Header.Get("Content-Type")) 108 | if parseErr != nil { 109 | w.WriteHeader(http.StatusBadRequest) 110 | _, _ = w.Write([]byte(badRequestJSON)) 111 | return 112 | } 113 | 114 | multipartReader := multipart.NewReader(r.Body, params["boundary"]) 115 | defer r.Body.Close() 116 | 117 | // Pin directory 118 | if multipartReader != nil && len(r.MultipartForm.File["file"]) > 1 { 119 | _, _ = w.Write([]byte(pinFileJSON)) 120 | return 121 | } 122 | // Pin file 123 | if multipartReader != nil && len(r.MultipartForm.File["file"]) == 1 { 124 | _, _ = w.Write([]byte(pinFileJSON)) 125 | return 126 | } 127 | case "/pinning/pinByHash": 128 | _, _ = w.Write([]byte(pinHashJSON)) 129 | return 130 | } 131 | default: 132 | switch r.URL.Path { 133 | case "/": 134 | w.Header().Set("Content-Type", "text/html") 135 | _, _ = w.Write([]byte(content)) 136 | case "/image.png": 137 | buf := genImage(1024) 138 | w.Header().Set("Content-Type", "image/png") 139 | _, _ = w.Write(buf.Bytes()) 140 | } 141 | } 142 | } 143 | 144 | func TestWayback(t *testing.T) { 145 | client, mux, server := helper.MockServer() 146 | mux.HandleFunc("/", handleResponse) 147 | defer server.Close() 148 | 149 | opts := []ipfs.PinningOption{ 150 | ipfs.Mode(ipfs.Remote), 151 | ipfs.Uses(pinner.Pinata), 152 | ipfs.Apikey(apikey), 153 | ipfs.Secret(secret), 154 | ipfs.Client(client), 155 | } 156 | opt := ipfs.Options(opts...) 157 | 158 | link := server.URL 159 | r := &Shaft{Hold: opt, Client: client} 160 | input, err := url.Parse(link) 161 | if err != nil { 162 | t.Fatal(err) 163 | } 164 | 165 | _, err = r.Wayback(context.TODO(), input) 166 | if err != nil { 167 | t.Fatal(err) 168 | } 169 | } 170 | 171 | func TestWaybackWithInput(t *testing.T) { 172 | client, mux, server := helper.MockServer() 173 | mux.HandleFunc("/", handleResponse) 174 | defer server.Close() 175 | 176 | opts := []ipfs.PinningOption{ 177 | ipfs.Mode(ipfs.Remote), 178 | ipfs.Uses(pinner.Pinata), 179 | ipfs.Apikey(apikey), 180 | ipfs.Secret(secret), 181 | ipfs.Client(client), 182 | } 183 | opt := ipfs.Options(opts...) 184 | 185 | link := server.URL 186 | r := &Shaft{Hold: opt, Client: client} 187 | input, err := url.Parse(link) 188 | if err != nil { 189 | t.Fatal(err) 190 | } 191 | 192 | ctx := r.WithInput(context.TODO(), []byte(content)) 193 | _, err = r.Wayback(ctx, input) 194 | if err != nil { 195 | t.Fatal(err) 196 | } 197 | } 198 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10 7 | 8 | env: 9 | PRODUCT: rivet 10 | 11 | permissions: write-all 12 | 13 | jobs: 14 | build: 15 | name: Build 16 | strategy: 17 | matrix: 18 | os: [ linux, freebsd, openbsd, dragonfly, windows, darwin ] 19 | arch: [ amd64, 386 ] 20 | include: 21 | - os: linux 22 | arch: arm 23 | arm: 5 24 | - os: linux 25 | arch: arm 26 | arm: 6 27 | - os: linux 28 | arch: arm 29 | arm: 7 30 | - os: linux 31 | arch: arm64 32 | - os: linux 33 | arch: mips 34 | mips: softfloat 35 | - os: linux 36 | arch: mips 37 | mips: hardfloat 38 | - os: linux 39 | arch: mipsle 40 | mipsle: softfloat 41 | - os: linux 42 | arch: mipsle 43 | mipsle: hardfloat 44 | - os: linux 45 | arch: mips64 46 | - os: linux 47 | arch: mips64le 48 | - os: linux 49 | arch: ppc64 50 | - os: linux 51 | arch: ppc64le 52 | - os: linux 53 | arch: s390x 54 | - os: windows 55 | arch: arm 56 | - os: windows 57 | arch: arm64 58 | - os: android 59 | arch: arm64 60 | - os: darwin 61 | arch: arm64 62 | - os: freebsd 63 | arch: arm64 64 | exclude: 65 | - os: darwin 66 | arch: 386 67 | - os: dragonfly 68 | arch: 386 69 | fail-fast: false 70 | runs-on: ubuntu-latest 71 | env: 72 | GOOS: ${{ matrix.os }} 73 | GOARCH: ${{ matrix.arch }} 74 | GOARM: ${{ matrix.arm }} 75 | GOMIPS: ${{ matrix.mips }} 76 | GOMIPS64: ${{ matrix.mips64 }} 77 | GOMIPSLE: ${{ matrix.mipsle }} 78 | steps: 79 | - name: Check out code into the Go module directory 80 | uses: actions/checkout@v3 81 | with: 82 | fetch-depth: 0 83 | 84 | - name: Set up Go 1.x 85 | uses: actions/setup-go@v3 86 | with: 87 | go-version: ^1.17 88 | 89 | - name: Build fat binary 90 | id: builder 91 | run: | 92 | ARGS="${GOOS}-${GOARCH}" 93 | if [[ -n "${GOARM}" ]]; then 94 | ARGS="${ARGS}v${GOARM}" 95 | elif [[ -n "${GOMIPS}" ]]; then 96 | ARGS="${ARGS}-${GOMIPS}" 97 | elif [[ -n "${GOMIPS64}" ]]; then 98 | ARGS="${ARGS}-${GOMIPS64}" 99 | elif [[ -n "${GOMIPSLE}" ]]; then 100 | ARGS="${ARGS}-${GOMIPSLE}" 101 | fi 102 | make ${ARGS} 103 | echo "args=${ARGS}" >> $GITHUB_OUTPUT 104 | 105 | - name: Archive binary 106 | run: make TARGET=${{ steps.builder.outputs.args }} releases 107 | 108 | - name: Upload artifact 109 | uses: actions/upload-artifact@v3 110 | with: 111 | name: ${{ env.PRODUCT }} 112 | path: build/package/${{ env.PRODUCT }}* 113 | 114 | snapcraft: 115 | name: Build Snap 116 | runs-on: ubuntu-latest 117 | outputs: 118 | version: ${{ steps.env.outputs.version }} 119 | steps: 120 | - name: Check out code base 121 | uses: actions/checkout@v3 122 | with: 123 | fetch-depth: 0 124 | 125 | - name: Set up QEMU 126 | uses: docker/setup-qemu-action@v1 127 | 128 | - name: Set env & Print wayback version 129 | shell: bash 130 | id: env 131 | run: | 132 | version=$(git describe --tags `git rev-list --tags --max-count=1` | sed -e 's/v//g') 133 | sed -i "s/version: 'git'/version: '${version}'/g" snapcraft.yaml 134 | echo "version=${version}" >> $GITHUB_OUTPUT 135 | 136 | - id: build 137 | name: Build snap 138 | uses: snapcore/action-build@v1 139 | with: 140 | snapcraft-channel: stable 141 | 142 | - name: Upload artifact 143 | uses: actions/upload-artifact@v3 144 | with: 145 | name: ${{ env.PRODUCT }} 146 | path: ${{ steps.build.outputs.snap }} 147 | 148 | checksum: 149 | name: Get archived packages checksum 150 | runs-on: ubuntu-latest 151 | needs: [ build, snapcraft ] 152 | outputs: 153 | digest: ${{ steps.digest.outputs.result }} 154 | steps: 155 | - name: Download math result from build job 156 | uses: actions/download-artifact@v3 157 | with: 158 | name: ${{ env.PRODUCT }} 159 | path: . 160 | 161 | - name: Create all binary digest 162 | id: digest 163 | run: | 164 | digest=$(find *${{ env.PRODUCT }}* -type f -exec sha256sum {} +) 165 | output="${digest//$'%'/%25}" 166 | output="${output//$'\n'/%0A}" 167 | echo "result=${output}" >> $GITHUB_OUTPUT 168 | # Write digest to file 169 | version=${{ needs.snapcraft.outputs.version }} 170 | echo "${digest}" > "${{ env.PRODUCT }}-${version}-checksums.txt" 171 | 172 | - name: Upload artifact 173 | uses: actions/upload-artifact@v3 174 | with: 175 | name: ${{ env.PRODUCT }} 176 | path: ${{ env.PRODUCT }}-*-checksums.txt 177 | 178 | release: 179 | name: Create and upload release 180 | runs-on: ubuntu-latest 181 | needs: [build, checksum] 182 | steps: 183 | - name: Check out code base 184 | uses: actions/checkout@v3 185 | with: 186 | fetch-depth: 0 187 | 188 | - name: Generate Git log 189 | run: | 190 | git fetch origin +refs/tags/*:refs/tags/* 191 | echo "Current Tag: ${GITHUB_REF}" 192 | git checkout ${GITHUB_REF} -b release-log 193 | GITVER=$(git describe --tags) 194 | PREVVER=$(git describe --tags --abbrev=0 ${GITVER}~1) 195 | git log --oneline ${PREVVER}..${GITVER} > gittaglogs.txt 196 | MORE=$(echo "See full [CHANGELOG.md](https://github.com/${{ github.repository }}/blob/${{ github.sha }}/CHANGELOG.md)") 197 | echo -e "*Release ${GITVER}* #rivet\n" > release-note.md 198 | cut -c9- gittaglogs.txt | sed -e 's/^/- /' | sed -e 's/\"/\\"/g' >> release-note.md 199 | echo -e "\n${MORE}" | tee -a release-note.md gittaglogs.txt > /dev/null 200 | # Append digests 201 | echo ' 202 | **Digests in this release:** 203 | 204 | ``` 205 | ${{ needs.checksum.outputs.digest }} 206 | ``` 207 | ' >> gittaglogs.txt 208 | 209 | - name: Upload artifact 210 | uses: actions/upload-artifact@v3 211 | with: 212 | name: release-note 213 | path: release-note.md 214 | 215 | - name: Download math result from build and checksum jobs 216 | uses: actions/download-artifact@v3 217 | with: 218 | name: ${{ env.PRODUCT }} 219 | path: ${{ env.PRODUCT }} 220 | 221 | - name: Create Release 222 | uses: softprops/action-gh-release@v1 223 | if: startsWith(github.ref, 'refs/tags/') 224 | env: 225 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token 226 | with: 227 | body_path: gittaglogs.txt 228 | files: ${{ env.PRODUCT }}/*${{ env.PRODUCT }}* 229 | prerelease: true 230 | draft: false 231 | 232 | notification: 233 | if: github.repository == 'wabarc/rivet' 234 | name: Send Notification 235 | runs-on: ubuntu-latest 236 | needs: [release] 237 | steps: 238 | - name: Download artifact 239 | uses: actions/download-artifact@v3 240 | with: 241 | name: release-note 242 | path: . 243 | 244 | - name: Send release note to Telegram channel 245 | continue-on-error: true 246 | run: | 247 | TEXT="$(cat release-note.md)" 248 | echo -e "${TEXT}" 249 | curl --silent --output /dev/null --show-error --fail -X POST \ 250 | -H 'Content-Type: application/json' \ 251 | -d '{"chat_id": "${{ secrets.TELEGRAM_TO }}", "text": "'"${TEXT}"'", "parse_mode": "markdown"}' \ 252 | "https://api.telegram.org/bot${{ secrets.TELEGRAM_TOKEN }}/sendMessage" 253 | -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | # Code generated by godownloader. DO NOT EDIT. 4 | # 5 | 6 | usage() { 7 | this=$1 8 | cat </dev/null 148 | } 149 | echoerr() { 150 | echo "$@" 1>&2 151 | } 152 | # shellcheck disable=SC2317 153 | log_prefix() { 154 | echo "$0" 155 | } 156 | _logp=6 157 | log_set_priority() { 158 | _logp="$1" 159 | } 160 | log_priority() { 161 | if test -z "$1"; then 162 | echo "$_logp" 163 | return 164 | fi 165 | [ "$1" -le "$_logp" ] 166 | } 167 | log_tag() { 168 | case $1 in 169 | 0) echo "emerg" ;; 170 | 1) echo "alert" ;; 171 | 2) echo "crit" ;; 172 | 3) echo "err" ;; 173 | 4) echo "warning" ;; 174 | 5) echo "notice" ;; 175 | 6) echo "info" ;; 176 | 7) echo "debug" ;; 177 | *) echo "$1" ;; 178 | esac 179 | } 180 | log_debug() { 181 | log_priority 7 || return 0 182 | echoerr "$(log_prefix)" "$(log_tag 7)" "$@" 183 | } 184 | log_info() { 185 | log_priority 6 || return 0 186 | echoerr "$(log_prefix)" "$(log_tag 6)" "$@" 187 | } 188 | log_err() { 189 | log_priority 3 || return 0 190 | echoerr "$(log_prefix)" "$(log_tag 3)" "$@" 191 | } 192 | log_crit() { 193 | log_priority 2 || return 0 194 | echoerr "$(log_prefix)" "$(log_tag 2)" "$@" 195 | } 196 | uname_os() { 197 | os=$(uname -s | tr '[:upper:]' '[:lower:]') 198 | case "$os" in 199 | msys*) os="windows" ;; 200 | mingw*) os="windows" ;; 201 | cygwin*) os="windows" ;; 202 | win*) os="windows" ;; 203 | esac 204 | echo "$os" 205 | } 206 | uname_arch() { 207 | arch=$(uname -m) 208 | case $arch in 209 | x86_64) arch="amd64" ;; 210 | x86) arch="386" ;; 211 | i686) arch="386" ;; 212 | i386) arch="386" ;; 213 | aarch64) arch="arm64" ;; 214 | armv5*) arch="armv5" ;; 215 | armv6*) arch="armv6" ;; 216 | armv7*) arch="armv7" ;; 217 | esac 218 | echo "${arch}" 219 | } 220 | uname_os_check() { 221 | os=$(uname_os) 222 | case "$os" in 223 | darwin) return 0 ;; 224 | dragonfly) return 0 ;; 225 | freebsd) return 0 ;; 226 | linux) return 0 ;; 227 | android) return 0 ;; 228 | nacl) return 0 ;; 229 | netbsd) return 0 ;; 230 | openbsd) return 0 ;; 231 | plan9) return 0 ;; 232 | solaris) return 0 ;; 233 | windows) return 0 ;; 234 | esac 235 | log_crit "uname_os_check '$(uname -s)' got converted to '$os' which is not a GOOS value. Please file bug at https://github.com/client9/shlib" 236 | return 1 237 | } 238 | uname_arch_check() { 239 | arch=$(uname_arch) 240 | case "$arch" in 241 | 386) return 0 ;; 242 | amd64) return 0 ;; 243 | arm64) return 0 ;; 244 | armv5) return 0 ;; 245 | armv6) return 0 ;; 246 | armv7) return 0 ;; 247 | ppc64) return 0 ;; 248 | ppc64le) return 0 ;; 249 | mips) return 0 ;; 250 | mipsle) return 0 ;; 251 | mips64) return 0 ;; 252 | mips64le) return 0 ;; 253 | s390x) return 0 ;; 254 | amd64p32) return 0 ;; 255 | esac 256 | log_crit "uname_arch_check '$(uname -m)' got converted to '$arch' which is not a GOARCH value. Please file bug report at https://github.com/client9/shlib" 257 | return 1 258 | } 259 | untar() { 260 | tarball=$1 261 | case "${tarball}" in 262 | *.tar.gz | *.tgz) tar --no-same-owner -xzf "${tarball}" ;; 263 | *.tar) tar --no-same-owner -xf "${tarball}" ;; 264 | *.zip) unzip "${tarball}" ;; 265 | *) 266 | log_err "untar unknown archive format for ${tarball}" 267 | return 1 268 | ;; 269 | esac 270 | } 271 | http_download_curl() { 272 | local_file=$1 273 | source_url=$2 274 | header=$3 275 | if [ -z "$header" ]; then 276 | code=$(curl -w '%{http_code}' -sL -o "$local_file" "$source_url") 277 | else 278 | code=$(curl -w '%{http_code}' -sL -H "$header" -o "$local_file" "$source_url") 279 | fi 280 | if [ "$code" != "200" ]; then 281 | log_debug "http_download_curl received HTTP status $code" 282 | return 1 283 | fi 284 | return 0 285 | } 286 | http_download_wget() { 287 | local_file=$1 288 | source_url=$2 289 | header=$3 290 | if [ -z "$header" ]; then 291 | wget -q -O "$local_file" "$source_url" 292 | else 293 | wget -q --header "$header" -O "$local_file" "$source_url" 294 | fi 295 | } 296 | http_download() { 297 | log_debug "http_download $2" 298 | if is_command curl; then 299 | http_download_curl "$@" 300 | return 301 | elif is_command wget; then 302 | http_download_wget "$@" 303 | return 304 | fi 305 | log_crit "http_download unable to find wget or curl" 306 | return 1 307 | } 308 | http_copy() { 309 | tmp=$(mktemp) 310 | http_download "${tmp}" "$1" "$2" || return 1 311 | body=$(cat "$tmp") 312 | rm -f "${tmp}" 313 | echo "$body" 314 | } 315 | github_release() { 316 | owner_repo=$1 317 | version=$2 318 | test -z "$version" && version="latest" 319 | giturl="https://github.com/${owner_repo}/releases/${version}" 320 | json=$(http_copy "$giturl" "Accept:application/json") 321 | test -z "$json" && return 1 322 | version=$(echo "$json" | tr -s '\n' ' ' | sed 's/.*"tag_name":"//' | sed 's/".*//') 323 | test -z "$version" && return 1 324 | echo "$version" 325 | } 326 | hash_sha256() { 327 | TARGET=${1:-/dev/stdin} 328 | if is_command gsha256sum; then 329 | hash=$(gsha256sum "$TARGET") || return 1 330 | echo "$hash" | cut -d ' ' -f 1 331 | elif is_command sha256sum; then 332 | hash=$(sha256sum "$TARGET") || return 1 333 | echo "$hash" | cut -d ' ' -f 1 334 | elif is_command shasum; then 335 | hash=$(shasum -a 256 "$TARGET" 2>/dev/null) || return 1 336 | echo "$hash" | cut -d ' ' -f 1 337 | elif is_command openssl; then 338 | hash=$(openssl -dst openssl dgst -sha256 "$TARGET") || return 1 339 | echo "$hash" | cut -d ' ' -f a 340 | else 341 | log_crit "hash_sha256 unable to find command to compute sha-256 hash" 342 | return 1 343 | fi 344 | } 345 | hash_sha256_verify() { 346 | TARGET=$1 347 | checksums=$2 348 | if [ -z "$checksums" ]; then 349 | log_err "hash_sha256_verify checksum file not specified in arg2" 350 | return 1 351 | fi 352 | BASENAME=${TARGET##*/} 353 | want=$(grep "${BASENAME}" "${checksums}" 2>/dev/null | tr '\t' ' ' | cut -d ' ' -f 1) 354 | if [ -z "$want" ]; then 355 | log_err "hash_sha256_verify unable to find checksum for '${TARGET}' in '${checksums}'" 356 | return 1 357 | fi 358 | got=$(hash_sha256 "$TARGET") 359 | if [ "$want" != "$got" ]; then 360 | log_err "hash_sha256_verify checksum for '$TARGET' did not verify ${want} vs $got" 361 | return 1 362 | fi 363 | } 364 | cat /dev/null <