├── integrations ├── ritool │ ├── .gitignore │ ├── main.go │ ├── group.go │ ├── ldbcid.go │ ├── head.go │ └── claims.go ├── web │ ├── ribswebapp │ │ ├── build │ │ │ ├── robots.txt │ │ │ ├── icon │ │ │ │ ├── ico_gg.png │ │ │ │ ├── ico_gr.png │ │ │ │ ├── ico_rg.png │ │ │ │ ├── ico_rr.png │ │ │ │ ├── ico_gr.svg │ │ │ │ ├── ico_gg.svg │ │ │ │ ├── ico_rr.svg │ │ │ │ ├── ico_rg.svg │ │ │ │ └── ico.svg │ │ │ ├── manifest.json │ │ │ ├── asset-manifest.json │ │ │ ├── index.html │ │ │ └── static │ │ │ │ └── css │ │ │ │ └── main.d7b9c3ff.css │ │ ├── public │ │ │ ├── robots.txt │ │ │ ├── icon │ │ │ │ ├── ico_gg.png │ │ │ │ ├── ico_gr.png │ │ │ │ ├── ico_rg.png │ │ │ │ ├── ico_rr.png │ │ │ │ ├── ico_gr.svg │ │ │ │ ├── ico_gg.svg │ │ │ │ ├── ico_rr.svg │ │ │ │ ├── ico_rg.svg │ │ │ │ └── ico.svg │ │ │ ├── manifest.json │ │ │ └── index.html │ │ ├── .gitignore │ │ ├── src │ │ │ ├── index.css │ │ │ ├── routes │ │ │ │ ├── Root │ │ │ │ │ └── RpcStatus.js │ │ │ │ ├── Providers.css │ │ │ │ ├── Provider.css │ │ │ │ ├── Deal.css │ │ │ │ ├── Status.css │ │ │ │ ├── Root.js │ │ │ │ ├── Groups.css │ │ │ │ ├── Root.css │ │ │ │ ├── Content.js │ │ │ │ ├── Repair.js │ │ │ │ ├── Group.js │ │ │ │ └── Providers.js │ │ │ ├── index.js │ │ │ └── helpers │ │ │ │ ├── rpc.js │ │ │ │ └── fmt.js │ │ └── package.json │ ├── ribsweb.go │ └── rpc.go └── kuri │ ├── cmd │ └── kuri │ │ └── main.go │ └── ribsplugin │ └── metacleanup.go ├── version.json ├── doc ├── deals.png ├── ribsweb.png └── architecture.png ├── .gitignore ├── .github └── workflows │ ├── go-test-config.json │ ├── tagpush.yml │ ├── releaser.yml │ ├── go-check.yml │ ├── go-test.yml │ └── release-check.yml ├── go.work ├── FUNDING.json ├── tools ├── cleanup_pebble │ └── README.md └── dump_pebble │ └── dump_pebble.go ├── Makefile ├── ributil ├── boosttypes │ ├── transports.ipldsch │ └── transports.go ├── mahttp.go ├── cached_query.go ├── primebs.go ├── memwatch.go ├── retrydb.go ├── boostnet │ └── transports.go ├── parcommp.go ├── minratewriter_test.go └── robusthttp.go ├── LICENSE-APACHE ├── gen └── main.go ├── cidgravity ├── cidgravity.go ├── get_best_available_providers.go └── get_deal_states.go ├── carlog ├── mhlist.go ├── idx_bsst.go └── idx_level.go ├── LICENSE-MIT ├── rbdeal ├── deal_params.go ├── deal_diag.go ├── external.go ├── wallet.go └── external_http_path.go ├── bsst └── bsst_test.go ├── rbstor ├── index_metered.go ├── diag.go ├── basic_test.go ├── group_worker.go ├── group_storage.go └── group_finalize.go ├── iface_rbmeta.go ├── rbmeta └── api-exemple.md ├── iface_ribs.go └── iface_rbs.go /integrations/ritool/.gitignore: -------------------------------------------------------------------------------- 1 | ritool 2 | -------------------------------------------------------------------------------- /version.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "" 3 | } 4 | -------------------------------------------------------------------------------- /doc/deals.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CIDgravity/gw/HEAD/doc/deals.png -------------------------------------------------------------------------------- /doc/ribsweb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CIDgravity/gw/HEAD/doc/ribsweb.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /build.sh 2 | /deploy.sh 3 | kuri 4 | ritool 5 | gwcfg 6 | settings.env 7 | -------------------------------------------------------------------------------- /doc/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CIDgravity/gw/HEAD/doc/architecture.png -------------------------------------------------------------------------------- /.github/workflows/go-test-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "skip32bit": true, 3 | "skipRace": true 4 | } 5 | -------------------------------------------------------------------------------- /go.work: -------------------------------------------------------------------------------- 1 | go 1.24.0 2 | 3 | toolchain go1.24.2 4 | 5 | use ( 6 | . 7 | integrations/kuri 8 | ) 9 | -------------------------------------------------------------------------------- /integrations/web/ribswebapp/build/robots.txt: -------------------------------------------------------------------------------- 1 | # https://www.robotstxt.org/robotstxt.html 2 | User-agent: * 3 | Disallow: 4 | -------------------------------------------------------------------------------- /integrations/web/ribswebapp/public/robots.txt: -------------------------------------------------------------------------------- 1 | # https://www.robotstxt.org/robotstxt.html 2 | User-agent: * 3 | Disallow: 4 | -------------------------------------------------------------------------------- /integrations/web/ribswebapp/build/icon/ico_gg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CIDgravity/gw/HEAD/integrations/web/ribswebapp/build/icon/ico_gg.png -------------------------------------------------------------------------------- /integrations/web/ribswebapp/build/icon/ico_gr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CIDgravity/gw/HEAD/integrations/web/ribswebapp/build/icon/ico_gr.png -------------------------------------------------------------------------------- /integrations/web/ribswebapp/build/icon/ico_rg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CIDgravity/gw/HEAD/integrations/web/ribswebapp/build/icon/ico_rg.png -------------------------------------------------------------------------------- /integrations/web/ribswebapp/build/icon/ico_rr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CIDgravity/gw/HEAD/integrations/web/ribswebapp/build/icon/ico_rr.png -------------------------------------------------------------------------------- /FUNDING.json: -------------------------------------------------------------------------------- 1 | { 2 | "drips": { 3 | "filecoin": { 4 | "ownedBy": "0xF0BA30D239121bcef596bca00C1A739201c16714" 5 | } 6 | } 7 | } 8 | 9 | -------------------------------------------------------------------------------- /integrations/web/ribswebapp/public/icon/ico_gg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CIDgravity/gw/HEAD/integrations/web/ribswebapp/public/icon/ico_gg.png -------------------------------------------------------------------------------- /integrations/web/ribswebapp/public/icon/ico_gr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CIDgravity/gw/HEAD/integrations/web/ribswebapp/public/icon/ico_gr.png -------------------------------------------------------------------------------- /integrations/web/ribswebapp/public/icon/ico_rg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CIDgravity/gw/HEAD/integrations/web/ribswebapp/public/icon/ico_rg.png -------------------------------------------------------------------------------- /integrations/web/ribswebapp/public/icon/ico_rr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CIDgravity/gw/HEAD/integrations/web/ribswebapp/public/icon/ico_rr.png -------------------------------------------------------------------------------- /tools/cleanup_pebble/README.md: -------------------------------------------------------------------------------- 1 | This tool is made to remove entries in the cidgravity-gw peble db that are not in a group anymore 2 | 3 | ``` 4 | go mod init 5 | go mod tidy 6 | go build 7 | ``` 8 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: kuboribs gwcfg 2 | 3 | kuboribs: 4 | go build -o kuri ./integrations/kuri/cmd/kuri 5 | .PHONY: kuboribs 6 | 7 | gwcfg: 8 | go build -o gwcfg ./integrations/gwcfg/main.go 9 | .PHONY: gwcfg 10 | clean: 11 | rm -f kuri gwcfg -------------------------------------------------------------------------------- /.github/workflows/tagpush.yml: -------------------------------------------------------------------------------- 1 | name: Tag Push Checker 2 | 3 | on: 4 | push: 5 | tags: 6 | - v* 7 | 8 | permissions: 9 | contents: read 10 | issues: write 11 | 12 | concurrency: 13 | group: ${{ github.workflow }}-${{ github.ref }} 14 | cancel-in-progress: true 15 | 16 | jobs: 17 | releaser: 18 | uses: pl-strflt/uci/.github/workflows/tagpush.yml@v0.0 19 | -------------------------------------------------------------------------------- /.github/workflows/releaser.yml: -------------------------------------------------------------------------------- 1 | name: Releaser 2 | 3 | on: 4 | push: 5 | paths: [ 'version.json' ] 6 | workflow_dispatch: 7 | 8 | permissions: 9 | contents: write 10 | 11 | concurrency: 12 | group: ${{ github.workflow }}-${{ github.sha }} 13 | cancel-in-progress: true 14 | 15 | jobs: 16 | releaser: 17 | uses: pl-strflt/uci/.github/workflows/releaser.yml@v0.0 18 | -------------------------------------------------------------------------------- /integrations/web/ribswebapp/build/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "short_name": "RIBSWeb", 3 | "name": "RIBSWeb", 4 | "icons": [ 5 | { 6 | "src": "favicon.ico", 7 | "sizes": "64x64 32x32 24x24 16x16", 8 | "type": "image/x-icon" 9 | } 10 | ], 11 | "start_url": ".", 12 | "display": "standalone", 13 | "theme_color": "#000000", 14 | "background_color": "#ffffff" 15 | } 16 | -------------------------------------------------------------------------------- /integrations/web/ribswebapp/public/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "short_name": "RIBSWeb", 3 | "name": "RIBSWeb", 4 | "icons": [ 5 | { 6 | "src": "favicon.ico", 7 | "sizes": "64x64 32x32 24x24 16x16", 8 | "type": "image/x-icon" 9 | } 10 | ], 11 | "start_url": ".", 12 | "display": "standalone", 13 | "theme_color": "#000000", 14 | "background_color": "#ffffff" 15 | } 16 | -------------------------------------------------------------------------------- /integrations/web/ribswebapp/.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | 8 | # testing 9 | /coverage 10 | 11 | # misc 12 | .DS_Store 13 | .env.local 14 | .env.development.local 15 | .env.test.local 16 | .env.production.local 17 | 18 | npm-debug.log* 19 | yarn-debug.log* 20 | yarn-error.log* 21 | -------------------------------------------------------------------------------- /integrations/web/ribswebapp/src/index.css: -------------------------------------------------------------------------------- 1 | body { 2 | margin: 0; 3 | font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', 4 | 'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue', 5 | sans-serif; 6 | -webkit-font-smoothing: antialiased; 7 | -moz-osx-font-smoothing: grayscale; 8 | } 9 | 10 | code { 11 | font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New', 12 | monospace; 13 | } 14 | -------------------------------------------------------------------------------- /.github/workflows/go-check.yml: -------------------------------------------------------------------------------- 1 | name: Go Checks 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: ["main"] 7 | workflow_dispatch: 8 | 9 | permissions: 10 | contents: read 11 | 12 | concurrency: 13 | group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} 14 | cancel-in-progress: true 15 | 16 | jobs: 17 | go-check: 18 | uses: pl-strflt/uci/.github/workflows/go-check.yml@v0.0 19 | -------------------------------------------------------------------------------- /integrations/web/ribswebapp/build/asset-manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "files": { 3 | "main.css": "/static/css/main.d7b9c3ff.css", 4 | "main.js": "/static/js/main.d0637968.js", 5 | "index.html": "/index.html", 6 | "main.d7b9c3ff.css.map": "/static/css/main.d7b9c3ff.css.map", 7 | "main.d0637968.js.map": "/static/js/main.d0637968.js.map" 8 | }, 9 | "entrypoints": [ 10 | "static/css/main.d7b9c3ff.css", 11 | "static/js/main.d0637968.js" 12 | ] 13 | } -------------------------------------------------------------------------------- /ributil/boosttypes/transports.ipldsch: -------------------------------------------------------------------------------- 1 | # Defines the response to a query asking which transport protocols a 2 | # Storage Provider supports 3 | type Multiaddr bytes 4 | 5 | type Protocol struct { 6 | # The name of the transport protocol 7 | # Known protocols: "libp2p", "http", "https" 8 | Name String 9 | # The addresses of the endpoint in multiaddr format 10 | Addresses [Multiaddr] 11 | } 12 | 13 | type QueryResponse struct { 14 | Protocols [Protocol] 15 | } 16 | -------------------------------------------------------------------------------- /integrations/ritool/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/urfave/cli/v2" 5 | "os" 6 | ) 7 | 8 | func main() { 9 | app := cli.App{ 10 | Name: "ribs", 11 | Usage: "ribs repository manipulation commands", 12 | 13 | Commands: []*cli.Command{ 14 | headCmd, 15 | carlogCmd, 16 | idxLevelCmd, 17 | ldbcidCmd, 18 | groupCmd, 19 | claimsExtendCmd, 20 | }, 21 | } 22 | 23 | if err := app.Run(os.Args); err != nil { 24 | panic(err) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /.github/workflows/go-test.yml: -------------------------------------------------------------------------------- 1 | name: Go Test 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: ["main"] 7 | workflow_dispatch: 8 | 9 | permissions: 10 | contents: read 11 | 12 | concurrency: 13 | group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} 14 | cancel-in-progress: true 15 | 16 | jobs: 17 | go-test: 18 | uses: pl-strflt/uci/.github/workflows/go-test.yml@v0.0 19 | with: 20 | go-versions: '["this"]' 21 | -------------------------------------------------------------------------------- /.github/workflows/release-check.yml: -------------------------------------------------------------------------------- 1 | name: Release Checker 2 | 3 | on: 4 | pull_request_target: 5 | paths: [ 'version.json' ] 6 | types: [ opened, synchronize, reopened, labeled, unlabeled ] 7 | workflow_dispatch: 8 | 9 | permissions: 10 | contents: write 11 | pull-requests: write 12 | 13 | concurrency: 14 | group: ${{ github.workflow }}-${{ github.ref }} 15 | cancel-in-progress: true 16 | 17 | jobs: 18 | release-check: 19 | uses: pl-strflt/uci/.github/workflows/release-check.yml@v0.0 20 | -------------------------------------------------------------------------------- /LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at 2 | 3 | http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 6 | -------------------------------------------------------------------------------- /gen/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/lotus-web3/ribs/bsst" 8 | "github.com/lotus-web3/ribs/carlog" 9 | 10 | gen "github.com/whyrusleeping/cbor-gen" 11 | ) 12 | 13 | func main() { 14 | err := gen.WriteMapEncodersToFile("./carlog/cbor_gen.go", "carlog", carlog.Head{}) 15 | if err != nil { 16 | fmt.Println(err) 17 | os.Exit(1) 18 | } 19 | 20 | err = gen.WriteMapEncodersToFile("./bsst/cbor_gen.go", "bsst", bsst.BSSTHeader{}) 21 | if err != nil { 22 | fmt.Println(err) 23 | os.Exit(1) 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /cidgravity/cidgravity.go: -------------------------------------------------------------------------------- 1 | package cidgravity 2 | 3 | import ( 4 | "sync" 5 | "github.com/lotus-web3/ribs/configuration" 6 | "golang.org/x/sync/semaphore" 7 | logging "github.com/ipfs/go-log/v2" 8 | ) 9 | 10 | var log = logging.Logger("ribs:cidg") 11 | 12 | type CIDGravity struct { 13 | lk sync.Mutex 14 | sem *semaphore.Weighted 15 | } 16 | 17 | func (cidg *CIDGravity) init() error { 18 | cidg.lk.Lock() 19 | defer cidg.lk.Unlock() 20 | if cidg.sem == nil { 21 | cfg := configuration.GetConfig() 22 | cidg.sem = semaphore.NewWeighted(cfg.CidGravity.MaxConns) 23 | } 24 | return nil 25 | } 26 | -------------------------------------------------------------------------------- /integrations/web/ribswebapp/src/routes/Root/RpcStatus.js: -------------------------------------------------------------------------------- 1 | import React, { useEffect, useState } from "react"; 2 | import RibsRPC from "../../helpers/rpc"; 3 | 4 | function RpcStatus() { 5 | const [status, setStatus] = useState("Connecting..."); 6 | 7 | useEffect(() => { 8 | RibsRPC.onConnect(() => { 9 | setStatus("RPC: OK"); 10 | }); 11 | 12 | RibsRPC.onDisconnect(() => { 13 | setStatus("RPC: Disconnected"); 14 | }); 15 | }, []); 16 | 17 | return
{status}
; 18 | } 19 | 20 | export default RpcStatus; 21 | -------------------------------------------------------------------------------- /integrations/web/ribswebapp/build/index.html: -------------------------------------------------------------------------------- 1 | RIBSWeb
-------------------------------------------------------------------------------- /ributil/mahttp.go: -------------------------------------------------------------------------------- 1 | package ributil 2 | 3 | import ( 4 | "github.com/ipni/go-libipni/maurl" 5 | "github.com/multiformats/go-multiaddr" 6 | "golang.org/x/xerrors" 7 | "net/url" 8 | ) 9 | 10 | func MaddrsToUrl(addrs []multiaddr.Multiaddr) (*url.URL, error) { 11 | var err error 12 | var url *url.URL 13 | for _, addr := range addrs { 14 | url, err = maurl.ToURL(addr) 15 | if err == nil && url != nil { 16 | return url, nil 17 | } 18 | } 19 | if err == nil && url == nil { 20 | return nil, xerrors.New("no valid multiaddrs") 21 | } 22 | // we have to do this because we get ws and wss from maurl.ToURL 23 | if url != nil && !(url.Scheme == "http" || url.Scheme == "https") { 24 | return nil, xerrors.New("no valid HTTP multiaddrs") 25 | } 26 | return url, err 27 | } 28 | -------------------------------------------------------------------------------- /integrations/web/ribswebapp/src/routes/Providers.css: -------------------------------------------------------------------------------- 1 | .Providers { 2 | margin: 0 auto; 3 | width: 100%; 4 | } 5 | 6 | .providers-table { 7 | width: 100%; 8 | border-collapse: collapse; 9 | margin-top: 1rem; 10 | } 11 | 12 | .providers-table th, 13 | .providers-table td { 14 | padding: 0.5rem; 15 | text-align: left; 16 | border-bottom: 1px solid #ccc; 17 | } 18 | 19 | .providers-table th { 20 | background-color: #f5f5f5; 21 | font-weight: bold; 22 | } 23 | 24 | .providers-table tbody tr:nth-child(odd) { 25 | background-color: #f9f9f9; 26 | } 27 | 28 | .stat-gray { 29 | color: #777; 30 | } 31 | 32 | .providers-ask { 33 | font-size: 12px; 34 | font-family: monospace; 35 | } 36 | 37 | .providers-nodeal-longtime { 38 | color: #f44336; 39 | } 40 | -------------------------------------------------------------------------------- /integrations/ritool/group.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/urfave/cli/v2" 6 | "strconv" 7 | ) 8 | 9 | var groupCmd = &cli.Command{ 10 | Name: "group", 11 | Usage: "Group utils", 12 | Subcommands: []*cli.Command{ 13 | groupNumToIdCmd, 14 | }, 15 | } 16 | 17 | var groupNumToIdCmd = &cli.Command{ 18 | Name: "num-to-id", 19 | Usage: "Convert group number to group ID", 20 | ArgsUsage: "[group number]", 21 | Action: func(c *cli.Context) error { 22 | if c.NArg() != 1 { 23 | return cli.Exit("Invalid number of arguments", 1) 24 | } 25 | 26 | gi, err := strconv.ParseInt(c.Args().First(), 10, 64) 27 | if err != nil { 28 | return cli.Exit("Invalid group number", 1) 29 | } 30 | 31 | fmt.Println(strconv.FormatInt(gi, 32)) 32 | 33 | return nil 34 | }, 35 | } 36 | -------------------------------------------------------------------------------- /integrations/web/ribswebapp/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | RIBSWeb 11 | 12 | 13 | 14 | 15 | 16 | 17 |
18 | 19 | 20 | -------------------------------------------------------------------------------- /integrations/kuri/cmd/kuri/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "fmt" 6 | 7 | "github.com/ipfs/kubo/cmd/ipfs/kubo" 8 | "github.com/ipfs/kubo/plugin/loader" 9 | 10 | "github.com/lotus-web3/ribs/ributil" 11 | "github.com/lotus-web3/ribs/configuration" 12 | kuboribs "github.com/lotus_web3/ribs/integrations/kuri/ribsplugin" 13 | ) 14 | 15 | func main() { 16 | os.Exit(mainRet()) 17 | } 18 | 19 | func mainRet() (exitCode int) { 20 | mw := ributil.MemoryWatchdog() 21 | defer mw() 22 | 23 | if err := configuration.LoadConfig(); err != nil { 24 | fmt.Fprintln(os.Stderr, "Configuration load failed: %w\n", err) 25 | } else { 26 | fmt.Fprintln(os.Stderr, "Configuration loaded\n") 27 | } 28 | return kubo.Start(kubo.BuildEnv(func(loader *loader.PluginLoader) error { 29 | return loader.Load(kuboribs.Plugin) 30 | })) 31 | } 32 | -------------------------------------------------------------------------------- /integrations/web/ribswebapp/src/routes/Provider.css: -------------------------------------------------------------------------------- 1 | .provider-meta-table { 2 | width: auto; 3 | } 4 | 5 | .provider-deals-error-col { 6 | color: #f44336; 7 | 8 | overflow: hidden; /* Hides overflow */ 9 | } 10 | 11 | .provider-deals-error-col pre { 12 | white-space: pre-wrap; /* Allows the content to wrap */ 13 | word-wrap: break-word; /* Breaks words if they're too long */ 14 | overflow: auto; /* Adds a scrollbar if content overflows */ 15 | max-height: 100%; /* Ensures the pre doesn't grow beyond its container */ 16 | } 17 | 18 | .prov-progress-container { 19 | display: flex; 20 | flex-direction: column; 21 | } 22 | 23 | .provider-deals-nowrap { 24 | white-space: nowrap; 25 | } 26 | 27 | .prov-text-with-progress { 28 | margin-bottom: 1px; /* Adjust as needed for spacing between text and progress bar */ 29 | } 30 | -------------------------------------------------------------------------------- /ributil/cached_query.go: -------------------------------------------------------------------------------- 1 | package ributil 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | ) 7 | 8 | type CachedQuery[T any] struct { 9 | cached T 10 | 11 | lk sync.Mutex 12 | 13 | refreshInterval time.Duration 14 | lastRefresh time.Time 15 | 16 | refreshFunc func() (T, error) 17 | } 18 | 19 | func NewCachedQuery[T any](refreshInterval time.Duration, refreshFunc func() (T, error)) *CachedQuery[T] { 20 | return &CachedQuery[T]{ 21 | refreshInterval: refreshInterval, 22 | refreshFunc: refreshFunc, 23 | } 24 | } 25 | 26 | func (cq *CachedQuery[T]) Get() (T, error) { 27 | cq.lk.Lock() 28 | defer cq.lk.Unlock() 29 | 30 | if time.Since(cq.lastRefresh) > cq.refreshInterval { 31 | var err error 32 | cq.cached, err = cq.refreshFunc() 33 | if err != nil { 34 | return cq.cached, err 35 | } 36 | cq.lastRefresh = time.Now() 37 | } 38 | 39 | return cq.cached, nil 40 | } 41 | -------------------------------------------------------------------------------- /integrations/web/ribswebapp/src/routes/Deal.css: -------------------------------------------------------------------------------- 1 | .Deal { 2 | background-color: #f5f5f5; 3 | padding: 1rem; 4 | border-radius: 5px; 5 | box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1), 0 4px 6px rgba(0, 0, 0, 0.1); 6 | font-size: 0.8rem; 7 | height: 100px; 8 | display: grid; 9 | grid-template-rows: repeat(4, 1fr); 10 | gap: 2px; 11 | align-items: center; 12 | position: relative; 13 | transition: all 0.3s ease; 14 | } 15 | 16 | .Deal:hover { 17 | box-shadow: 0 6px 12px rgba(0, 0, 0, 0.1), 0 12px 24px rgba(0, 0, 0, 0.1); 18 | transform: translateY(-3px); 19 | } 20 | 21 | .deal-failed { 22 | background: #ffd6cc; 23 | } 24 | 25 | .deal-sealed { 26 | background: #ddffdd; 27 | } 28 | 29 | .deal-sealed-retr { 30 | background: #99ff99; 31 | } 32 | 33 | .deal-err { 34 | text-overflow: ellipsis; 35 | overflow: hidden; 36 | display: block; 37 | } -------------------------------------------------------------------------------- /carlog/mhlist.go: -------------------------------------------------------------------------------- 1 | package carlog 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "os" 7 | 8 | "github.com/multiformats/go-multihash" 9 | ) 10 | 11 | func SaveMHList(filePath string, list []multihash.Multihash) error { 12 | var buf bytes.Buffer 13 | 14 | for _, c := range list { 15 | buf.Write(c) 16 | } 17 | 18 | return os.WriteFile(filePath, buf.Bytes(), 0644) 19 | } 20 | 21 | func LoadMHList(filePath string) ([]multihash.Multihash, error) { 22 | data, err := os.ReadFile(filePath) 23 | if err != nil { 24 | return nil, err 25 | } 26 | 27 | cidList := make([]multihash.Multihash, 0) 28 | buf := bytes.NewBuffer(data) 29 | mr := multihash.NewReader(buf) 30 | 31 | for { 32 | c, err := mr.ReadMultihash() 33 | if err != nil { 34 | if err == io.EOF { 35 | break 36 | } 37 | return nil, err 38 | } 39 | 40 | cidList = append(cidList, c) 41 | } 42 | 43 | return cidList, nil 44 | } 45 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /integrations/web/ribswebapp/build/icon/ico_gr.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 12 | 14 | 16 | 23 | 29 | 36 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /integrations/web/ribswebapp/public/icon/ico_gr.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 12 | 14 | 16 | 23 | 29 | 36 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /integrations/web/ribswebapp/src/routes/Status.css: -------------------------------------------------------------------------------- 1 | .Status { 2 | padding: 1rem; 3 | } 4 | 5 | .status-grid { 6 | display: grid; 7 | grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); 8 | grid-gap: 1rem; 9 | } 10 | 11 | .status-grid > div { 12 | background-color: #f5f5f5; 13 | padding: 1.5rem; 14 | border-radius: 5px; 15 | box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1), 0 4px 6px rgba(0, 0, 0, 0.1); 16 | transition: all 0.3s ease; 17 | } 18 | 19 | .status-grid > div:hover { 20 | box-shadow: 0 6px 12px rgba(0, 0, 0, 0.1), 0 12px 24px rgba(0, 0, 0, 0.1); 21 | transform: translateY(-3px); 22 | } 23 | 24 | .status-grid h2 { 25 | margin-bottom: 0.5rem; 26 | } 27 | 28 | @media screen and (max-width: 768px) { 29 | .status-grid { 30 | grid-template-columns: 1fr; 31 | } 32 | } 33 | 34 | .compact-table { 35 | width: 100%; 36 | border-collapse: collapse; 37 | font-size: 0.9rem; 38 | } 39 | 40 | .compact-table th, 41 | .compact-table td { 42 | padding: 0.3rem 0.5rem; 43 | text-align: left; 44 | border-bottom: 1px solid #e0e0e0; 45 | } 46 | 47 | .compact-table tr:last-child td { 48 | border-bottom: none; 49 | } 50 | 51 | .important-metric { 52 | color: #4caf50; 53 | font-weight: bold; 54 | } -------------------------------------------------------------------------------- /integrations/web/ribswebapp/build/icon/ico_gg.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 12 | 14 | 17 | 24 | 30 | 37 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /integrations/web/ribswebapp/build/icon/ico_rr.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 12 | 14 | 17 | 24 | 30 | 37 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /integrations/web/ribswebapp/public/icon/ico_gg.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 12 | 14 | 17 | 24 | 30 | 37 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /integrations/web/ribswebapp/public/icon/ico_rr.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 12 | 14 | 17 | 24 | 30 | 37 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /integrations/web/ribswebapp/build/icon/ico_rg.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 12 | 14 | 17 | 24 | 30 | 37 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /integrations/web/ribswebapp/public/icon/ico_rg.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 12 | 14 | 17 | 24 | 30 | 37 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /integrations/web/ribswebapp/src/index.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ReactDOM from 'react-dom/client'; 3 | import { 4 | createBrowserRouter, 5 | RouterProvider, 6 | } from "react-router-dom"; 7 | import './index.css'; 8 | 9 | import Root from './routes/Root'; 10 | import Status from './routes/Status'; 11 | import Groups from './routes/Groups'; 12 | import Group from "./routes/Group"; 13 | import Providers from './routes/Providers'; 14 | import Provider from './routes/Provider'; 15 | import Content from "./routes/Content"; 16 | import Repair from "./routes/Repair"; 17 | 18 | const router = createBrowserRouter([ 19 | { 20 | path: "/", 21 | element: , 22 | children: [ 23 | { path: "/", element: }, 24 | { path: "groups", element: }, 25 | { path: "groups/:groupKey", element: }, 26 | { path: "providers", element: }, 27 | { path: "provider/:providerID", element: }, 28 | { path: "content", element: }, 29 | { path: "repair", element: }, 30 | ], 31 | }, 32 | ]); 33 | 34 | const root = ReactDOM.createRoot(document.getElementById('root')); 35 | root.render( 36 | 37 | 38 | 39 | ); 40 | -------------------------------------------------------------------------------- /rbdeal/deal_params.go: -------------------------------------------------------------------------------- 1 | package rbdeal 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/filecoin-project/go-state-types/abi" 7 | "github.com/filecoin-project/lotus/chain/types" 8 | ) 9 | 10 | const mFil = 1_000_000_000_000_000 11 | 12 | var ( 13 | maxVerifPrice float64 = 0 14 | 15 | // 2 mFil/gib/mo is roughly cloud cost currently 16 | maxPrice float64 = (1 * mFil) / 2 / 60 / 24 / 30.436875 17 | 18 | // piece size range ribs is aiming for 19 | minPieceSize = 32 << 30 20 | maxPieceSize = 64 << 30 21 | 22 | dealPublishFinality abi.ChainEpoch = 60 23 | 24 | dealDownloadTimeout = time.Hour * 24 * 4 25 | ) 26 | 27 | // deal checker 28 | const clientReadDeadline = 10 * time.Second 29 | const clientWriteDeadline = 10 * time.Second 30 | 31 | var ParallelDealChecks = 10 32 | 33 | var minDatacap = types.NewInt(192 << 30) 34 | 35 | // deal transfers 36 | 37 | // todo this definitely needs to be configurable by the user 38 | var minTransferMbps = 8 // at 10 Mbps, a 32 GiB piece takes ~7 hours to transfer 39 | var linkSpeedMbps = 1000 // 1 Gbps 40 | 41 | //lint:ignore U1000 To be revisited in the context of S3 offloading. 42 | var maxTransfers = linkSpeedMbps / minTransferMbps * 8 / 10 // 80% for safety margin 43 | 44 | // time the sp has to start the first transfer, and for data to not be flowing 45 | // also transfer rate check interval 46 | var transferIdleTimeout = 120 * time.Minute 47 | 48 | var maxTransferRetries = 10000 49 | -------------------------------------------------------------------------------- /integrations/web/ribswebapp/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ribswebapp", 3 | "version": "0.1.0", 4 | "private": true, 5 | "dependencies": { 6 | "@testing-library/jest-dom": "^5.16.5", 7 | "@testing-library/react": "^13.4.0", 8 | "@testing-library/user-event": "^13.5.0", 9 | "jsonrpc-websocket-client": "^0.7.2", 10 | "localforage": "^1.10.0", 11 | "match-sorter": "^6.3.1", 12 | "react": "^18.2.0", 13 | "react-copy-to-clipboard": "^5.1.0", 14 | "react-dom": "^18.2.0", 15 | "react-router-dom": "^6.9.0", 16 | "react-scripts": "5.0.1", 17 | "recharts": "^2.7.2", 18 | "rpc-websockets": "^7.5.1", 19 | "sort-by": "^1.2.0", 20 | "url": "^0.11.0", 21 | "web-vitals": "^2.1.4" 22 | }, 23 | "scripts": { 24 | "start": "react-scripts start", 25 | "build": "react-scripts build", 26 | "test": "react-scripts test", 27 | "eject": "react-scripts eject" 28 | }, 29 | "eslintConfig": { 30 | "extends": [ 31 | "react-app", 32 | "react-app/jest" 33 | ] 34 | }, 35 | "browserslist": { 36 | "production": [ 37 | ">0.2%", 38 | "not dead", 39 | "not op_mini all" 40 | ], 41 | "development": [ 42 | "last 1 chrome version", 43 | "last 1 firefox version", 44 | "last 1 safari version" 45 | ] 46 | }, 47 | "main": "index.js", 48 | "keywords": [], 49 | "author": "", 50 | "license": "ISC", 51 | "description": "" 52 | } 53 | -------------------------------------------------------------------------------- /carlog/idx_bsst.go: -------------------------------------------------------------------------------- 1 | package carlog 2 | 3 | import ( 4 | "github.com/lotus-web3/ribs/bsst" 5 | mh "github.com/multiformats/go-multihash" 6 | "golang.org/x/xerrors" 7 | ) 8 | 9 | type BSSTIndex struct { 10 | bsi *bsst.BSST 11 | } 12 | 13 | func (b *BSSTIndex) Has(c []mh.Multihash) ([]bool, error) { 14 | return b.bsi.Has(c) 15 | } 16 | 17 | func (b *BSSTIndex) Get(c []mh.Multihash) ([]int64, error) { 18 | return b.bsi.Get(c) 19 | } 20 | 21 | func (b *BSSTIndex) Close() error { 22 | return b.bsi.Close() 23 | } 24 | 25 | func (b *BSSTIndex) Entries() (int64, error) { 26 | panic("not supported") 27 | } 28 | 29 | func (b *BSSTIndex) List(f func(c mh.Multihash, offs []int64) error) error { 30 | panic("not supported") 31 | } 32 | 33 | func OpenBSSTIndex(path string) (*BSSTIndex, error) { 34 | bss, err := bsst.Open(path) 35 | if err != nil { 36 | return nil, err 37 | } 38 | return &BSSTIndex{ 39 | bsi: bss, 40 | }, nil 41 | } 42 | 43 | type IndexSource interface { 44 | bsst.Source 45 | Entries() (int64, error) 46 | } 47 | 48 | func CreateBSSTIndex(path string, index IndexSource) (*BSSTIndex, error) { 49 | ents, err := index.Entries() 50 | if err != nil { 51 | return nil, xerrors.Errorf("getting level index entries: %w", err) 52 | } 53 | bss, err := bsst.Create(path, ents, index) 54 | if err != nil { 55 | return nil, xerrors.Errorf("bsst create: %w", err) 56 | } 57 | return &BSSTIndex{ 58 | bsi: bss, 59 | }, nil 60 | } 61 | 62 | var _ ReadableIndex = (*BSSTIndex)(nil) 63 | -------------------------------------------------------------------------------- /ributil/boosttypes/transports.go: -------------------------------------------------------------------------------- 1 | package boosttypes 2 | 3 | import ( 4 | _ "embed" 5 | "fmt" 6 | 7 | "github.com/ipld/go-ipld-prime/node/bindnode" 8 | bindnoderegistry "github.com/ipld/go-ipld-prime/node/bindnode/registry" 9 | "github.com/multiformats/go-multiaddr" 10 | ) 11 | 12 | type Protocol struct { 13 | // The name of the transport protocol eg "libp2p" or "http" 14 | Name string 15 | // The address of the endpoint in multiaddr format 16 | Addresses []multiaddr.Multiaddr 17 | } 18 | 19 | type QueryResponse struct { 20 | Protocols []Protocol 21 | } 22 | 23 | //go:embed transports.ipldsch 24 | var embedSchema []byte 25 | 26 | func multiAddrFromBytes(b []byte) (interface{}, error) { 27 | ma, err := multiaddr.NewMultiaddrBytes(b) 28 | if err != nil { 29 | return nil, err 30 | } 31 | return &ma, err 32 | } 33 | 34 | func multiAddrToBytes(iface interface{}) ([]byte, error) { 35 | ma, ok := iface.(*multiaddr.Multiaddr) 36 | if !ok { 37 | return nil, fmt.Errorf("expected *Multiaddr value") 38 | } 39 | 40 | return (*ma).Bytes(), nil 41 | } 42 | 43 | var BindnodeRegistry = bindnoderegistry.NewRegistry() 44 | 45 | func init() { 46 | var dummyMa multiaddr.Multiaddr 47 | var bindnodeOptions = []bindnode.Option{ 48 | bindnode.TypedBytesConverter(&dummyMa, multiAddrFromBytes, multiAddrToBytes), 49 | } 50 | if err := BindnodeRegistry.RegisterType((*QueryResponse)(nil), string(embedSchema), "QueryResponse", bindnodeOptions...); err != nil { 51 | panic(err.Error()) 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /bsst/bsst_test.go: -------------------------------------------------------------------------------- 1 | package bsst 2 | 3 | import ( 4 | "encoding/binary" 5 | "path/filepath" 6 | "testing" 7 | 8 | mh "github.com/multiformats/go-multihash" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | type testSource struct { 13 | n int64 14 | } 15 | 16 | func (t *testSource) List(f func(c mh.Multihash, offs []int64) error) error { 17 | for i := int64(0); i < t.n; i++ { 18 | var ib [8]byte 19 | binary.LittleEndian.PutUint64(ib[:], uint64(i)) 20 | 21 | h, err := mh.Sum(ib[:], mh.SHA2_256, -1) 22 | if err != nil { 23 | return err 24 | } 25 | 26 | if err := f(h, []int64{i | 0x7faa_0000_c000_0000}); err != nil { 27 | return err 28 | } 29 | } 30 | 31 | return nil 32 | } 33 | 34 | var _ Source = (*testSource)(nil) 35 | 36 | func TestBSSTCreate(t *testing.T) { 37 | 38 | n := int64(13421280) 39 | 40 | bsst, err := Create(filepath.Join(t.TempDir(), "/a.bsst"), n, &testSource{n: n}) 41 | require.NoError(t, err) 42 | t.Cleanup(func() { require.NoError(t, bsst.Close()) }) 43 | 44 | var got int64 45 | 46 | err = (&testSource{n: n}).List(func(c mh.Multihash, offs []int64) error { 47 | h, err := bsst.Has([]mh.Multihash{c}) 48 | require.NoError(t, err) 49 | if !h[0] { 50 | require.True(t, h[0]) // todo probably full buckets 51 | } 52 | 53 | r, err := bsst.Get([]mh.Multihash{c}) 54 | require.NoError(t, err) 55 | require.Equal(t, offs[0], r[0]) 56 | 57 | got++ 58 | return nil 59 | }) 60 | require.NoError(t, err) 61 | require.Equal(t, n, got) 62 | } 63 | -------------------------------------------------------------------------------- /rbstor/index_metered.go: -------------------------------------------------------------------------------- 1 | package rbstor 2 | 3 | import ( 4 | "context" 5 | "sync/atomic" 6 | 7 | iface "github.com/lotus-web3/ribs" 8 | "github.com/multiformats/go-multihash" 9 | ) 10 | 11 | type MeteredIndex struct { 12 | sub iface.Index 13 | 14 | reads, writes int64 15 | } 16 | 17 | func (m *MeteredIndex) Close() error { 18 | return m.sub.Close() 19 | } 20 | 21 | func (m *MeteredIndex) EstimateSize(ctx context.Context) (int64, error) { 22 | return m.sub.EstimateSize(ctx) 23 | } 24 | 25 | func (m *MeteredIndex) GetGroups(ctx context.Context, mh []multihash.Multihash, cb func(cidx int, gk iface.GroupKey) (more bool, err error)) error { 26 | atomic.AddInt64(&m.reads, int64(len(mh))) 27 | return m.sub.GetGroups(ctx, mh, cb) 28 | } 29 | 30 | func (m *MeteredIndex) GetSizes(ctx context.Context, mh []multihash.Multihash, cb func([]int32) error) error { 31 | atomic.AddInt64(&m.reads, int64(len(mh))) 32 | return m.sub.GetSizes(ctx, mh, cb) 33 | } 34 | 35 | func (m *MeteredIndex) AddGroup(ctx context.Context, mh []multihash.Multihash, sizes []int32, group iface.GroupKey) error { 36 | atomic.AddInt64(&m.writes, int64(len(mh))) 37 | return m.sub.AddGroup(ctx, mh, sizes, group) 38 | } 39 | 40 | func (m *MeteredIndex) Sync(ctx context.Context) error { 41 | return m.sub.Sync(ctx) 42 | } 43 | 44 | func (m *MeteredIndex) DropGroup(ctx context.Context, mh []multihash.Multihash, group iface.GroupKey) error { 45 | atomic.AddInt64(&m.writes, int64(len(mh))) 46 | return m.sub.DropGroup(ctx, mh, group) 47 | } 48 | 49 | func NewMeteredIndex(sub iface.Index) *MeteredIndex { 50 | return &MeteredIndex{sub: sub} 51 | } 52 | 53 | var _ iface.Index = &MeteredIndex{} 54 | -------------------------------------------------------------------------------- /integrations/web/ribswebapp/src/routes/Root.js: -------------------------------------------------------------------------------- 1 | import './Root.css'; 2 | import { Link, Outlet, useLocation } from "react-router-dom"; 3 | import RpcStatus from "./Root/RpcStatus"; 4 | 5 | function Root() { 6 | const location = useLocation(); 7 | 8 | const formatPath = () => { 9 | if (location.pathname === "/") { 10 | return "Status"; 11 | } 12 | return location.pathname.replace("/", ""); 13 | }; 14 | 15 | return ( 16 |
17 |
18 |
RIBS
19 |
{formatPath()}
20 |
21 |
22 |
23 |
24 | Status 25 |
26 |
27 | Groups 28 |
29 |
30 | Providers 31 |
32 |
33 | Content 34 |
35 |
36 | Repair 37 |
38 |
39 |
40 | 41 |
42 |
43 | ); 44 | } 45 | 46 | export default Root; -------------------------------------------------------------------------------- /iface_rbmeta.go: -------------------------------------------------------------------------------- 1 | package ribs 2 | 3 | import ( 4 | "go.mongodb.org/mongo-driver/bson/primitive" 5 | ) 6 | type FileMetadata struct { 7 | // All as ref, so they are optionnal in usage 8 | Id *primitive.ObjectID `bson:"_id"` 9 | User *string `bson:"user"` 10 | ParentPath *string `bson:"parent"` 11 | Filename *string `bson:"name"` 12 | Cid *string `bson:"cid"` 13 | File *bool `bson:"file"` 14 | Size *uint64 `bson:"size"` 15 | StartTime *int64 `bson:"start_ts"` 16 | EndTime *int64 `bson:"end_ts"` 17 | CleanupRequired *bool `bson:"cleanup_required"` 18 | Groups *[]int64 `bson:"groups"` 19 | } 20 | type ChildInfo struct { 21 | Cid string `bson:"cid"` 22 | Size uint64 `bson:"size"` 23 | } 24 | type ChildMetadata struct { 25 | Id *primitive.ObjectID `bson:"_id"` 26 | Cid *string `bson:"cid"` 27 | Childs map[string]ChildInfo `bson:"childs"` 28 | } 29 | type DirectoryItem struct { 30 | Filename string 31 | Cid string 32 | } 33 | 34 | type MetaExplorer interface { 35 | ListChilds(cid string) (map[string]ChildInfo, error) 36 | ListGroups(string) ([]int64, error) 37 | } 38 | 39 | type MetadataDB interface { 40 | /* Functions that should be fast, done directly as we write */ 41 | WriteFile(filepath string, c string, size uint64) error 42 | WriteDir(filepath string, c string) error 43 | Remove(filepath string) error 44 | Rename(oldName, newName string) error 45 | /* Cleanup functions */ 46 | LaunchCleanupLoop(MetaExplorer) error 47 | LaunchServer() error 48 | 49 | /* Query Functions */ 50 | ListFiles(user string, path string) ([]DirectoryItem, error) 51 | GetFileInfo(user string, parent string, name string, ts *int64) (*FileMetadata, error) 52 | } 53 | 54 | -------------------------------------------------------------------------------- /integrations/web/ribsweb.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "context" 5 | "embed" 6 | "fmt" 7 | "net" 8 | "net/http" 9 | _ "net/http/pprof" 10 | "path/filepath" 11 | txtempl "text/template" 12 | 13 | logging "github.com/ipfs/go-log/v2" 14 | "github.com/lotus-web3/ribs" 15 | ) 16 | 17 | var log = logging.Logger("ribsweb") 18 | 19 | //go:embed ribswebapp/build 20 | var dres embed.FS 21 | 22 | type RIBSWeb struct { 23 | ribs ribs.RIBS 24 | } 25 | 26 | func (ri *RIBSWeb) Index(w http.ResponseWriter, r *http.Request) { 27 | // todo there has to be something better than this horrid hack 28 | _, err := dres.ReadFile(filepath.Join("ribswebapp", "build", r.URL.Path)) 29 | if err == nil && r.URL.Path != "/" { 30 | r.URL.Path = filepath.Join("ribswebapp", "build", r.URL.Path) 31 | http.FileServer(http.FS(dres)).ServeHTTP(w, r) 32 | return 33 | } 34 | 35 | log.Debugw("index", "path", r.URL.Path) 36 | 37 | tpl, err := txtempl.New("index.html").ParseFS(dres, "ribswebapp/build/index.html") 38 | if err != nil { 39 | fmt.Println(err) 40 | http.Error(w, err.Error(), http.StatusInternalServerError) 41 | return 42 | } 43 | w.Header().Set("Content-Type", "text/html") 44 | data := map[string]interface{}{} 45 | if err := tpl.Execute(w, data); err != nil { 46 | http.Error(w, err.Error(), 500) 47 | return 48 | } 49 | } 50 | 51 | func Serve(ctx context.Context, listen string, ribs ribs.RIBS) error { 52 | handlers := &RIBSWeb{ 53 | ribs: ribs, 54 | } 55 | 56 | rpc, closer, err := MakeRPCServer(ctx, ribs) 57 | if err != nil { 58 | return err 59 | } 60 | defer closer() 61 | 62 | mux := http.NewServeMux() 63 | mux.HandleFunc("/", handlers.Index) 64 | 65 | mux.Handle("/rpc/v0", rpc) 66 | 67 | mux.Handle("/debug/", http.DefaultServeMux) 68 | 69 | server := &http.Server{Addr: listen, Handler: mux, BaseContext: func(_ net.Listener) context.Context { return ctx }} 70 | return server.ListenAndServe() 71 | } 72 | -------------------------------------------------------------------------------- /ributil/primebs.go: -------------------------------------------------------------------------------- 1 | package ributil 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | 8 | "github.com/filecoin-project/lotus/blockstore" 9 | blocks "github.com/ipfs/go-block-format" 10 | "github.com/ipfs/go-cid" 11 | "github.com/ipld/go-ipld-prime" 12 | "github.com/ipld/go-ipld-prime/linking" 13 | ipldstorage "github.com/ipld/go-ipld-prime/storage" 14 | ) 15 | 16 | type IpldStoreWrapper struct { 17 | BS blockstore.Blockstore 18 | } 19 | 20 | func (b *IpldStoreWrapper) Get(ctx context.Context, key string) ([]byte, error) { 21 | keyCid, err := cid.Cast([]byte(key)) 22 | if err != nil { 23 | return nil, fmt.Errorf("bad CID key: %w", err) 24 | } 25 | 26 | bk, err := b.BS.Get(ctx, keyCid) 27 | if err != nil { 28 | return nil, err 29 | } 30 | 31 | return bk.RawData(), nil 32 | } 33 | 34 | func (b *IpldStoreWrapper) Has(ctx context.Context, key string) (bool, error) { 35 | keyCid, err := cid.Cast([]byte(key)) 36 | if err != nil { 37 | return false, fmt.Errorf("bad CID key: %w", err) 38 | } 39 | 40 | return b.BS.Has(ctx, keyCid) 41 | } 42 | 43 | func (b *IpldStoreWrapper) Put(ctx context.Context, key string, content []byte) error { 44 | keyCid, err := cid.Cast([]byte(key)) 45 | if err != nil { 46 | return fmt.Errorf("bad CID key: %w", err) 47 | } 48 | 49 | bk, err := blocks.NewBlockWithCid(content, keyCid) 50 | if err != nil { 51 | return fmt.Errorf("bad block: %w", err) 52 | } 53 | 54 | return b.BS.Put(ctx, bk) 55 | } 56 | 57 | // BlockWriteOpener returns a BlockWriteOpener that operates on this storage. 58 | func (b *IpldStoreWrapper) BlockWriteOpener() linking.BlockWriteOpener { 59 | return func(lctx linking.LinkContext) (io.Writer, linking.BlockWriteCommitter, error) { 60 | wr, wrcommit, err := ipldstorage.PutStream(lctx.Ctx, b) 61 | return wr, func(lnk ipld.Link) error { 62 | return wrcommit(lnk.Binary()) 63 | }, err 64 | } 65 | } 66 | 67 | var _ ipldstorage.WritableStorage = (*IpldStoreWrapper)(nil) 68 | var _ ipldstorage.ReadableStorage = (*IpldStoreWrapper)(nil) 69 | -------------------------------------------------------------------------------- /integrations/web/ribswebapp/src/routes/Groups.css: -------------------------------------------------------------------------------- 1 | .group { 2 | background-color: #f5f5f5; 3 | padding: 1rem; 4 | border-radius: 4px; 5 | box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1), 0 4px 6px rgba(0, 0, 0, 0.1); 6 | margin-bottom: 1rem; 7 | 8 | display: grid; 9 | grid-template-columns: repeat(auto-fill, minmax(300px, 1fr)); 10 | align-items: end; 11 | gap: 1rem; 12 | } 13 | 14 | .group h3 { 15 | display: flex; 16 | justify-content: space-between; 17 | } 18 | 19 | .group p { 20 | margin: 0.5rem 0; 21 | line-height: 1.1; 22 | } 23 | 24 | .group-state { 25 | font-weight: normal; 26 | font-size: 1rem; 27 | } 28 | 29 | .progress-bar { 30 | background-color: #e0e0e0; 31 | height: 8px; 32 | border-radius: 4px; 33 | overflow: hidden; 34 | } 35 | 36 | .thin-bar { 37 | height: 3px; 38 | margin-top: 1px; 39 | } 40 | 41 | .progress-bar__fill { 42 | background-color: #4caf50; 43 | height: 100%; 44 | } 45 | 46 | .progress-bar__fill-red { 47 | background-color: #f44336; 48 | } 49 | 50 | .group-info { 51 | grid-row: span 2; 52 | height: 200px; 53 | 54 | background-color: #f5f5f5; 55 | padding: 1rem; 56 | border-radius: 5px; 57 | box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1), 0 4px 6px rgba(0, 0, 0, 0.1); 58 | align-items: center; 59 | position: relative; 60 | transition: all 0.3s ease; 61 | line-height: 1.1; 62 | } 63 | 64 | .group-info p { 65 | margin: 0 0; 66 | line-height: 1; 67 | } 68 | 69 | .deal-counts { 70 | margin-bottom: 1rem; 71 | } 72 | 73 | .deal-counts-seal { 74 | color: #4caf50; 75 | font-weight: bold; 76 | } 77 | 78 | .deal-counts-err { 79 | color: #f44336; 80 | font-weight: bold; 81 | } 82 | 83 | .deal-counts-start { 84 | color: #2196f3; 85 | } 86 | 87 | .deal-error { 88 | color: #f44336; 89 | } 90 | 91 | .pagination-btn { 92 | margin-right: 5px; 93 | margin-bottom: 3px; 94 | 95 | color: unset; 96 | background-color: unset; 97 | border: solid 1px #4caf50; 98 | border-radius: 3px; 99 | } 100 | -------------------------------------------------------------------------------- /ributil/memwatch.go: -------------------------------------------------------------------------------- 1 | package ributil 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/raulk/go-watchdog" 7 | ) 8 | 9 | func MemoryWatchdog() (cleanup func()) { 10 | // configure heap profile capture so that one is captured per episode where 11 | // utilization climbs over 90% of the limit. A maximum of 10 heapdumps 12 | // will be captured during life of this process. 13 | watchdog.HeapProfileMaxCaptures = 10 14 | watchdog.HeapProfileThreshold = 0.9 15 | 16 | policy := watchdog.NewWatermarkPolicy(0.50, 0.60, 0.70, 0.85, 0.90, 0.925, 0.95) 17 | 18 | // Try to initialize a watchdog in the following order of precedence: 19 | // 1. If a max heap limit has been provided, initialize a heap-driven watchdog. 20 | // 2. Else, try to initialize a cgroup-driven watchdog. 21 | // 3. Else, try to initialize a system-driven watchdog. 22 | // 4. Else, log a warning that the system is flying solo, and return. 23 | 24 | var onExit []func() 25 | 26 | addStopHook := func(stopFn func()) { 27 | onExit = append(onExit, stopFn) 28 | } 29 | 30 | cleanup = func() { 31 | for _, fn := range onExit { 32 | fn() 33 | } 34 | } 35 | 36 | // 1. If user has set max heap limit, apply it. 37 | if maxHeap := uint64(0); maxHeap != 0 { 38 | const minGOGC = 10 39 | err, stopFn := watchdog.HeapDriven(maxHeap, minGOGC, policy) 40 | if err == nil { 41 | log.Infof("initialized heap-driven watchdog; max heap: %d bytes", maxHeap) 42 | addStopHook(stopFn) 43 | return 44 | } 45 | log.Warnf("failed to initialize heap-driven watchdog; err: %s", err) 46 | log.Warnf("trying a cgroup-driven watchdog") 47 | } 48 | 49 | // 2. cgroup-driven watchdog. 50 | err, stopFn := watchdog.CgroupDriven(5*time.Second, policy) 51 | if err == nil { 52 | log.Infof("initialized cgroup-driven watchdog") 53 | addStopHook(stopFn) 54 | return 55 | } 56 | log.Warnf("failed to initialize cgroup-driven watchdog; err: %s", err) 57 | log.Warnf("trying a system-driven watchdog") 58 | 59 | // 3. system-driven watchdog. 60 | err, stopFn = watchdog.SystemDriven(0, 5*time.Second, policy) // 0 calculates the limit automatically. 61 | if err == nil { 62 | log.Infof("initialized system-driven watchdog") 63 | addStopHook(stopFn) 64 | return 65 | } 66 | 67 | // 4. log the failure 68 | log.Warnf("failed to initialize system-driven watchdog; err: %s", err) 69 | log.Warnf("system running without a memory watchdog") 70 | return 71 | } 72 | -------------------------------------------------------------------------------- /rbmeta/api-exemple.md: -------------------------------------------------------------------------------- 1 | ## Basic file info 2 | ``` 3 | $ curl -s -X POST http://localhost:9011/file-info -d '{"filePath": "/"}' | jq . 4 | { 5 | "success": true, 6 | "result": { 7 | "file": { 8 | "cid": "QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn" 9 | } 10 | } 11 | } 12 | $ curl -s -X POST http://localhost:9011/file-info -d '{"filePath": "/non-existent"}' | jq . 13 | { 14 | "success": false, 15 | "error": "File not found" 16 | } 17 | ``` 18 | 19 | 20 | ## detailed file info 21 | 22 | ``` 23 | success: 24 | error: # only if success is false 25 | result: # only if success is true 26 | file: 27 | cid: # File CID to retrieve directly on IPFS 28 | details: # Only if verbose: true is passed 29 | state: 30 | retrievableCopies: # mininum retrievableCopies of the different groups 31 | groups: 32 | - state: 33 | id: # block PieceCID if available 34 | retrievableCopies: 35 | deals: # only if in a suffisent state to have some 36 | - provider: 37 | state: 38 | dealId: 39 | endEpoch: 40 | isRetrievable: 41 | ``` 42 | 43 | #### States 44 | 45 | ##### DealState 46 | 47 | ``` 48 | # Pretty self describing 49 | DealStateProposed = "proposed" 50 | DealStatePublished = "published" 51 | DealStateActive = "active" 52 | ``` 53 | 54 | ##### Group State 55 | 56 | ``` 57 | # Group status from ribs internal naming 58 | GroupStateWritable = "writable" 59 | GroupStateFull = "full" 60 | GroupStateVRCARDone = "VRCARDone" 61 | GroupStateReadyForDeals = "ready for deals" 62 | GroupStateOffloaded = "offloaded" 63 | GroupStateReload = "reload" 64 | ``` 65 | 66 | ##### File State 67 | 68 | ``` 69 | # if at least 1 group not yet ready (writable/full/VRCAR done) 70 | FileStateStaging = "staging" 71 | # if all groups are ready 72 | FileStateOffloading = "offloading" 73 | # if all groups have at least 1 active deal 74 | FileStatePartiallyOffload = "partially offloaded" 75 | # If all groups are offloaded 76 | FileStateOffloaded = "offloaded" 77 | ``` 78 | -------------------------------------------------------------------------------- /rbstor/diag.go: -------------------------------------------------------------------------------- 1 | package rbstor 2 | 3 | import ( 4 | "context" 5 | "sync/atomic" 6 | 7 | iface "github.com/lotus-web3/ribs" 8 | "golang.org/x/xerrors" 9 | ) 10 | 11 | func (r *rbs) StorageDiag() iface.RBSDiag { 12 | return r 13 | } 14 | 15 | func (r *rbs) Groups() ([]iface.GroupKey, error) { 16 | return r.db.Groups() 17 | } 18 | 19 | func (r *rbs) GroupMeta(gk iface.GroupKey) (iface.GroupMeta, error) { 20 | m, err := r.db.GroupMeta(gk) 21 | if err != nil { 22 | return iface.GroupMeta{}, xerrors.Errorf("get group meta: %w", err) 23 | } 24 | 25 | r.lk.Lock() 26 | g, ok := r.openGroups[gk] 27 | r.lk.Unlock() 28 | 29 | if ok { 30 | m.ReadBlocks = g.readBlocks.Load() 31 | m.ReadBytes = g.readSize.Load() 32 | m.WriteBlocks = g.writeBlocks.Load() 33 | m.WriteBytes = g.writeSize.Load() 34 | } 35 | 36 | return m, nil 37 | } 38 | 39 | func (r *rbs) GetGroupStats() (*iface.GroupStats, error) { 40 | gs, err := r.db.GetGroupStats() 41 | if err != nil { 42 | return nil, err 43 | } 44 | 45 | r.lk.Lock() 46 | gs.OpenGroups = len(r.openGroups) 47 | gs.OpenWritable = len(r.writableGroups) 48 | r.lk.Unlock() 49 | 50 | return gs, nil 51 | } 52 | 53 | func (r *rbs) GroupIOStats() iface.GroupIOStats { 54 | r.lk.Lock() 55 | defer r.lk.Unlock() 56 | 57 | // first update global counters 58 | for _, group := range r.openGroups { 59 | readBlocks := group.readBlocks.Load() 60 | readSize := group.readSize.Load() 61 | writeBlocks := group.writeBlocks.Load() 62 | writeSize := group.writeSize.Load() 63 | 64 | r.grpReadBlocks += readBlocks - group.readBlocksSnap 65 | r.grpReadSize += readSize - group.readSizeSnap 66 | r.grpWriteBlocks += writeBlocks - group.writeBlocksSnap 67 | r.grpWriteSize += writeSize - group.writeSizeSnap 68 | 69 | group.readBlocksSnap = readBlocks 70 | group.readSizeSnap = readSize 71 | group.writeBlocksSnap = writeBlocks 72 | group.writeSizeSnap = writeSize 73 | } 74 | 75 | // then return the global counters 76 | stats := iface.GroupIOStats{ 77 | ReadBlocks: r.grpReadBlocks, 78 | ReadBytes: r.grpReadSize, 79 | WriteBlocks: r.grpWriteBlocks, 80 | WriteBytes: r.grpWriteSize, 81 | } 82 | 83 | return stats 84 | } 85 | 86 | func (r *rbs) TopIndexStats(ctx context.Context) (iface.TopIndexStats, error) { 87 | s, err := r.index.EstimateSize(ctx) 88 | if err != nil { 89 | return iface.TopIndexStats{}, xerrors.Errorf("estimate size: %w", err) 90 | } 91 | 92 | return iface.TopIndexStats{ 93 | Entries: s, 94 | Writes: atomic.LoadInt64(&r.index.writes), 95 | Reads: atomic.LoadInt64(&r.index.reads), 96 | }, nil 97 | } 98 | 99 | func (r *rbs) WorkerStats() iface.WorkerStats { 100 | return iface.WorkerStats{ 101 | Available: r.workersAvail.Load(), 102 | InFinalize: r.workersFinalizing.Load(), 103 | InCommP: r.workersCommP.Load(), 104 | InReload: r.workersFinDataReload.Load(), 105 | TaskQueue: int64(len(r.tasks)), 106 | CommPBytes: globalCommpBytes.Load(), 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /integrations/web/ribswebapp/src/helpers/rpc.js: -------------------------------------------------------------------------------- 1 | const RPC = require("rpc-websockets").Client; 2 | 3 | class RibsRPC { 4 | static instance = null; 5 | static connectCallbacks = []; 6 | static disconnectCallbacks = []; 7 | 8 | static createInstance() { 9 | const protocol = window.location.protocol === "https:" ? "wss://" : "ws://"; 10 | const hostname = window.location.hostname; 11 | const port = window.location.port ? `:${window.location.port}` : ""; // Include port if specified 12 | const rpcInstance = new RPC(`${protocol}${hostname}${port}/rpc/v0`); 13 | 14 | RibsRPC.connectCallbacks.forEach((callback) => { 15 | rpcInstance.on("open", callback); 16 | }); 17 | 18 | RibsRPC.disconnectCallbacks.forEach((callback) => { 19 | rpcInstance.on("close", callback); 20 | }); 21 | 22 | return rpcInstance; 23 | } 24 | 25 | static get() { 26 | if (RibsRPC.instance === null) { 27 | RibsRPC.instance = RibsRPC.createInstance(); 28 | RibsRPC.autoReconnect(); 29 | } 30 | return RibsRPC.instance; 31 | } 32 | 33 | static async call(method, params) { 34 | const maxRetries = 5; 35 | let delay = 1000; // 1 second in milliseconds 36 | 37 | for (let i = 0; i < maxRetries; i++) { 38 | try { 39 | return await RibsRPC.get().call('RIBS.' + method, params); 40 | } catch (error) { 41 | if (i === maxRetries - 1) { 42 | throw error; // Re-throw the error if we've reached the maximum number of retries 43 | } 44 | await new Promise(resolve => setTimeout(resolve, delay)); 45 | } 46 | } 47 | } 48 | 49 | static async callFil(method, params) { 50 | const maxRetries = 5; 51 | let delay = 1000; // 1 second in milliseconds 52 | 53 | for (let i = 0; i < maxRetries; i++) { 54 | try { 55 | return await RibsRPC.get().call('Filecoin.' + method, params); 56 | } catch (error) { 57 | if (i === maxRetries - 1) { 58 | throw error; // Re-throw the error if we've reached the maximum number of retries 59 | } 60 | await new Promise(resolve => setTimeout(resolve, delay)); 61 | } 62 | } 63 | } 64 | 65 | static onConnect(callback) { 66 | RibsRPC.connectCallbacks.push(callback); 67 | RibsRPC.get().on("open", callback); 68 | } 69 | 70 | static onDisconnect(callback) { 71 | RibsRPC.disconnectCallbacks.push(callback); 72 | RibsRPC.get().on("close", callback); 73 | } 74 | 75 | static autoReconnect() { 76 | RibsRPC.get().on("close", () => { 77 | setTimeout(() => { 78 | RibsRPC.instance = RibsRPC.createInstance(); 79 | RibsRPC.autoReconnect(); 80 | }, 5000); // Reconnect every 5 seconds 81 | }); 82 | } 83 | } 84 | 85 | export default RibsRPC; 86 | -------------------------------------------------------------------------------- /integrations/ritool/ldbcid.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/ipfs/go-cid" 7 | "github.com/ipfs/go-datastore" 8 | leveldb "github.com/ipfs/go-ds-leveldb" 9 | "github.com/mitchellh/go-homedir" 10 | "github.com/urfave/cli/v2" 11 | "golang.org/x/xerrors" 12 | ) 13 | 14 | var ldbcidCmd = &cli.Command{ 15 | Name: "ldb-cid", 16 | Subcommands: []*cli.Command{ 17 | ldbCidReadCmd, 18 | ldbCidWriteCmd, 19 | }, 20 | } 21 | 22 | var ldbCidReadCmd = &cli.Command{ 23 | Name: "read", 24 | Usage: "Read a CID from the local database", 25 | ArgsUsage: "[leveldb path] [key]", 26 | Action: func(cctx *cli.Context) error { 27 | if cctx.Args().Len() != 2 { 28 | return xerrors.Errorf("must pass leveldb path and key") 29 | } 30 | 31 | ldbPath := cctx.Args().Get(0) 32 | key := cctx.Args().Get(1) 33 | 34 | ldbPath, err := homedir.Expand(ldbPath) 35 | if err != nil { 36 | return xerrors.Errorf("expand homedir: %w", err) 37 | } 38 | 39 | ldb, err := leveldb.NewDatastore(ldbPath, nil) 40 | if err != nil { 41 | return xerrors.Errorf("open leveldb: %w", err) 42 | } 43 | 44 | c, err := ldb.Get(context.Background(), datastore.NewKey(key)) 45 | if err != nil { 46 | return xerrors.Errorf("get key: %w", err) 47 | } 48 | 49 | cc, err := cid.Cast(c) 50 | if err != nil { 51 | return xerrors.Errorf("cast cid: %w", err) 52 | } 53 | 54 | fmt.Println(cc.String()) 55 | 56 | return nil 57 | }, 58 | } 59 | 60 | var ldbCidWriteCmd = &cli.Command{ 61 | Name: "write", 62 | Usage: "Write a CID to the local database", 63 | ArgsUsage: "[leveldb path] [key] [cid]", 64 | Action: func(cctx *cli.Context) error { 65 | if cctx.Args().Len() != 3 { 66 | return xerrors.Errorf("must pass leveldb path, key, and cid") 67 | } 68 | 69 | ldbPath := cctx.Args().Get(0) 70 | key := cctx.Args().Get(1) 71 | cidStr := cctx.Args().Get(2) 72 | 73 | ldbPath, err := homedir.Expand(ldbPath) 74 | if err != nil { 75 | return xerrors.Errorf("expand homedir: %w", err) 76 | } 77 | 78 | ldb, err := leveldb.NewDatastore(ldbPath, nil) 79 | if err != nil { 80 | return xerrors.Errorf("open leveldb: %w", err) 81 | } 82 | 83 | { 84 | // print old 85 | 86 | c, err := ldb.Get(context.Background(), datastore.NewKey(key)) 87 | if err != nil { 88 | if err == datastore.ErrNotFound { 89 | goto afterPrint 90 | } 91 | return xerrors.Errorf("get key: %w", err) 92 | } 93 | 94 | cc, err := cid.Cast(c) 95 | if err != nil { 96 | return xerrors.Errorf("cast cid: %w", err) 97 | } 98 | 99 | fmt.Println("Old:", cc.String()) 100 | } 101 | afterPrint: 102 | 103 | cc, err := cid.Decode(cidStr) 104 | if err != nil { 105 | return xerrors.Errorf("decode cid: %w", err) 106 | } 107 | 108 | if err := ldb.Put(context.Background(), datastore.NewKey(key), cc.Bytes()); err != nil { 109 | return xerrors.Errorf("put key: %w", err) 110 | } 111 | 112 | return nil 113 | }, 114 | } 115 | -------------------------------------------------------------------------------- /integrations/web/ribswebapp/src/routes/Root.css: -------------------------------------------------------------------------------- 1 | * { 2 | box-sizing: border-box; 3 | } 4 | 5 | body { 6 | margin: 0; 7 | font-family: Arial, sans-serif; 8 | } 9 | 10 | /* Root layout */ 11 | .Root { 12 | display: grid; 13 | grid-template-columns: 200px 1fr; 14 | grid-template-rows: 50px 1fr; 15 | grid-template-areas: 16 | "head head" 17 | "nav body"; 18 | height: 100vh; 19 | } 20 | 21 | /* Responsive layout for small screens */ 22 | @media (max-width: 768px) { 23 | .Root { 24 | grid-template-columns: 1fr; 25 | grid-template-rows: 50px auto 1fr; 26 | grid-template-areas: 27 | "head" 28 | "nav" 29 | "body"; 30 | } 31 | } 32 | 33 | .Root-head { 34 | grid-area: head; 35 | display: flex; 36 | justify-content: space-between; 37 | align-items: center; 38 | background-color: #4CAF50; 39 | padding: 0 20px; 40 | color: white; 41 | box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1), 0 4px 6px rgba(0, 0, 0, 0.1); 42 | } 43 | 44 | .Root-head-logoname { 45 | font-size: 24px; 46 | font-weight: bold; 47 | margin-right: 10px; 48 | } 49 | 50 | .Root-head-path { 51 | font-size: 18px; 52 | flex-grow: 1; 53 | margin-right: 10px; 54 | } 55 | 56 | .Root-head-state { 57 | font-size: 14px; 58 | } 59 | 60 | .Root-nav { 61 | grid-area: nav; 62 | background-color: #f8f8f8; 63 | display: flex; 64 | flex-direction: column; 65 | padding: 20px; 66 | 67 | border-right: 1px solid rgba(0, 0, 0, 0.1); 68 | box-shadow: 2px 0 4px rgba(0, 0, 0, 0.1), 4px 0 6px rgba(0, 0, 0, 0.1); 69 | } 70 | 71 | /* Responsive layout for small screens */ 72 | @media (max-width: 768px) { 73 | .Root-nav { 74 | display: grid; 75 | grid-template-columns: repeat(auto-fill, minmax(100px, 1fr)); 76 | gap: 20px; 77 | padding: 20px 10px; 78 | } 79 | } 80 | 81 | .Root-nav-item { 82 | margin-bottom: 10px; 83 | } 84 | 85 | .Root-nav-item a { 86 | text-decoration: none; 87 | color: #333; 88 | transition: color 0.3s ease; 89 | } 90 | 91 | .Root-nav-item a:hover { 92 | color: #4CAF50; 93 | } 94 | 95 | .Root-nav-item a:hover { 96 | color: #4CAF50; 97 | } 98 | 99 | .Root-body { 100 | grid-area: body; 101 | padding: 20px; 102 | } 103 | 104 | button, .button-ish { 105 | background-color: #4caf50; 106 | border: none; 107 | border-radius: 3px; 108 | color: white; 109 | cursor: pointer; 110 | font-size: 1rem; 111 | margin-bottom: 1rem; 112 | padding: 8px 16px; 113 | text-align: center; 114 | text-decoration: none; 115 | display: inline-block; 116 | transition-duration: 0.4s; 117 | } 118 | 119 | button:hover, .button-ish { 120 | background-color: #45a049; 121 | color: white; 122 | } 123 | 124 | .button-sm { 125 | padding: 1px 2px; 126 | font-size: 0.9rem; 127 | margin-bottom: 0; 128 | margin-left: 1px; 129 | } 130 | 131 | a { 132 | color: #333; 133 | text-decoration: none; 134 | transition: color 0.3s ease; 135 | } 136 | 137 | a:hover { 138 | color: #4CAF50; 139 | } 140 | 141 | th { 142 | white-space: nowrap; 143 | } 144 | -------------------------------------------------------------------------------- /integrations/ritool/head.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "os" 9 | 10 | "github.com/lotus-web3/ribs/carlog" 11 | "github.com/urfave/cli/v2" 12 | "golang.org/x/xerrors" 13 | ) 14 | 15 | var headCmd = &cli.Command{ 16 | Name: "head", 17 | Usage: "Head commands", 18 | Subcommands: []*cli.Command{ 19 | headToJsonCmd, 20 | fromJsonCmd, 21 | }, 22 | } 23 | 24 | var headToJsonCmd = &cli.Command{ 25 | Name: "to-json", 26 | Usage: "read a head file into a json file", 27 | ArgsUsage: "[head file]", 28 | Action: func(c *cli.Context) error { 29 | if c.NArg() != 1 { 30 | return cli.Exit("Invalid number of arguments", 1) 31 | } 32 | 33 | headFile, err := os.Open(c.Args().First()) 34 | if err != nil { 35 | return xerrors.Errorf("open head file: %w", err) 36 | } 37 | 38 | // read head 39 | var headBuf [carlog.HeadSize]byte 40 | n, err := headFile.ReadAt(headBuf[:], 0) 41 | if err != nil { 42 | return xerrors.Errorf("HEAD READ ERROR: %w", err) 43 | } 44 | if n != len(headBuf) { 45 | return xerrors.Errorf("bad head read bytes (%d bytes)", n) 46 | } 47 | 48 | var h carlog.Head 49 | if err := h.UnmarshalCBOR(bytes.NewBuffer(headBuf[:])); err != nil { 50 | return xerrors.Errorf("unmarshal head: %w", err) 51 | } 52 | 53 | hjson, err := json.MarshalIndent(h, "", " ") 54 | if err != nil { 55 | return xerrors.Errorf("marshal head: %w", err) 56 | } 57 | 58 | fmt.Println(string(hjson)) 59 | 60 | return nil 61 | }, 62 | } 63 | 64 | var fromJsonCmd = &cli.Command{ 65 | Name: "from-json", 66 | Usage: "write json from stdin into a head file", 67 | ArgsUsage: "[output head file]", 68 | Action: func(c *cli.Context) error { 69 | if c.NArg() != 1 { 70 | return cli.Exit("Invalid number of arguments. Requires only output head file.", 1) 71 | } 72 | 73 | // Read the entire JSON from stdin 74 | jsonData, err := io.ReadAll(os.Stdin) 75 | if err != nil { 76 | return xerrors.Errorf("read json from stdin: %w", err) 77 | } 78 | 79 | var h carlog.Head 80 | if err := json.Unmarshal(jsonData, &h); err != nil { 81 | return xerrors.Errorf("unmarshal json: %w", err) 82 | } 83 | 84 | // Convert struct to CBOR format 85 | var buf bytes.Buffer 86 | if err := h.MarshalCBOR(&buf); err != nil { 87 | return xerrors.Errorf("marshal to cbor: %w", err) 88 | } 89 | 90 | // Open the head file for writing 91 | headFile, err := os.Create(c.Args().Get(0)) 92 | if err != nil { 93 | return xerrors.Errorf("open head file for writing: %w", err) 94 | } 95 | defer headFile.Close() 96 | 97 | // Write CBOR data to the head file 98 | if _, err := headFile.Write(buf.Bytes()); err != nil { 99 | return xerrors.Errorf("write to head file: %w", err) 100 | } 101 | 102 | pad := 512 - buf.Len() 103 | if pad < 0 { 104 | return xerrors.Errorf("head file too large") 105 | } 106 | 107 | // Pad the head file to 512 bytes 108 | if _, err := headFile.Write(make([]byte, pad)); err != nil { 109 | return xerrors.Errorf("pad head file: %w", err) 110 | } 111 | 112 | fmt.Printf("Successfully written to %s\n", c.Args().Get(0)) 113 | 114 | return nil 115 | }, 116 | } 117 | -------------------------------------------------------------------------------- /ributil/retrydb.go: -------------------------------------------------------------------------------- 1 | package ributil 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "strings" 7 | "time" 8 | ) 9 | 10 | // RetryDB retries 'database is locked' errors 11 | type RetryDB struct { 12 | db *sql.DB 13 | } 14 | 15 | func NewRetryDB(db *sql.DB) *RetryDB { 16 | return &RetryDB{db: db} 17 | } 18 | 19 | func isDbLockedError(err error) bool { 20 | return strings.Contains(err.Error(), "database is locked") 21 | } 22 | 23 | // Exec 24 | func (d *RetryDB) Exec(query string, args ...interface{}) (sql.Result, error) { 25 | var err error 26 | var result sql.Result 27 | for { 28 | result, err = d.db.Exec(query, args...) 29 | if err == nil { 30 | return result, nil 31 | } 32 | if !isDbLockedError(err) { 33 | return nil, err 34 | } 35 | 36 | log.Warnw("database is locked, retrying", "query", query, "args", args, "err", err) 37 | time.Sleep(50 * time.Millisecond) 38 | } 39 | } 40 | 41 | // ExecContext 42 | func (d *RetryDB) ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error) { 43 | var err error 44 | var result sql.Result 45 | for { 46 | result, err = d.db.ExecContext(ctx, query, args...) 47 | if err == nil { 48 | return result, nil 49 | } 50 | if !isDbLockedError(err) { 51 | return nil, err 52 | } 53 | 54 | log.Warnw("database is locked, retrying", "query", query, "args", args, "err", err) 55 | time.Sleep(50 * time.Millisecond) 56 | } 57 | } 58 | 59 | // QueryRow 60 | func (d *RetryDB) QueryRow(query string, args ...interface{}) *sql.Row { 61 | var err error 62 | var row *sql.Row 63 | for { 64 | row = d.db.QueryRow(query, args...) 65 | err = row.Err() 66 | if err == nil { 67 | return row 68 | } 69 | if !isDbLockedError(err) { 70 | return row 71 | } 72 | 73 | log.Warnw("database is locked, retrying", "query", query, "args", args, "err", err) 74 | time.Sleep(50 * time.Millisecond) 75 | } 76 | } 77 | 78 | // Query 79 | func (d *RetryDB) Query(query string, args ...interface{}) (*sql.Rows, error) { 80 | var err error 81 | var rows *sql.Rows 82 | for { 83 | rows, err = d.db.Query(query, args...) 84 | if err == nil { 85 | return rows, nil 86 | } 87 | if !isDbLockedError(err) { 88 | return nil, err 89 | } 90 | 91 | log.Warnw("database is locked, retrying", "query", query, "args", args, "err", err) 92 | time.Sleep(50 * time.Millisecond) 93 | } 94 | } 95 | 96 | // QueryContext 97 | func (d *RetryDB) QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error) { 98 | var err error 99 | var rows *sql.Rows 100 | for { 101 | rows, err = d.db.QueryContext(ctx, query, args...) 102 | if err == nil { 103 | return rows, nil 104 | } 105 | if !isDbLockedError(err) { 106 | return nil, err 107 | } 108 | 109 | log.Warnw("database is locked, retrying", "query", query, "args", args, "err", err) 110 | time.Sleep(50 * time.Millisecond) 111 | } 112 | } 113 | 114 | // Begin 115 | func (d *RetryDB) Begin() (*sql.Tx, error) { 116 | // todo retryTx 117 | 118 | var err error 119 | var tx *sql.Tx 120 | for { 121 | tx, err = d.db.Begin() 122 | if err == nil { 123 | return tx, nil 124 | } 125 | if !isDbLockedError(err) { 126 | return nil, err 127 | } 128 | 129 | log.Warnw("database is locked, retrying", "err", err) 130 | time.Sleep(50 * time.Millisecond) 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /rbstor/basic_test.go: -------------------------------------------------------------------------------- 1 | package rbstor 2 | 3 | import ( 4 | "context" 5 | "encoding/binary" 6 | "fmt" 7 | "io/fs" 8 | "path/filepath" 9 | "testing" 10 | "time" 11 | 12 | blocks "github.com/ipfs/go-block-format" 13 | iface "github.com/lotus-web3/ribs" 14 | "github.com/multiformats/go-multihash" 15 | "github.com/stretchr/testify/require" 16 | ) 17 | 18 | func TestBasic(t *testing.T) { 19 | t.Skip() 20 | td := t.TempDir() 21 | t.Cleanup(func() { 22 | if err := filepath.Walk(td, func(path string, info fs.FileInfo, err error) error { 23 | t.Log(path) 24 | return nil 25 | }); err != nil { 26 | t.Fatal(err) 27 | } 28 | }) 29 | 30 | ctx := context.Background() 31 | 32 | ri, err := Open(td) 33 | require.NoError(t, err) 34 | 35 | sess := ri.Session(ctx) 36 | 37 | wb := sess.Batch(ctx) 38 | 39 | b := blocks.NewBlock([]byte("hello world")) 40 | h := b.Cid().Hash() 41 | 42 | err = wb.Put(ctx, []blocks.Block{b}) 43 | require.NoError(t, err) 44 | 45 | err = wb.Flush(ctx) 46 | require.NoError(t, err) 47 | 48 | err = sess.View(ctx, []multihash.Multihash{h}, func(i int, b []byte) { 49 | require.Equal(t, 0, i) 50 | require.Equal(t, b, []byte("hello world")) 51 | }) 52 | require.NoError(t, err) 53 | 54 | require.NoError(t, ri.Close()) 55 | } 56 | 57 | func TestFullGroup(t *testing.T) { 58 | t.Skip("worker gate needed to re-enable this test") 59 | maxGroupSize = 100 << 20 60 | 61 | td := t.TempDir() 62 | t.Cleanup(func() { 63 | if err := filepath.Walk(td, func(path string, info fs.FileInfo, err error) error { 64 | t.Log(path) 65 | return nil 66 | }); err != nil { 67 | t.Fatal(err) 68 | } 69 | }) 70 | 71 | ctx := context.Background() 72 | // TODO there is no more worker gate; make this play nice with tests 73 | // workerGate := make(chan struct{}, 1) 74 | // ri, err := Open(td, WithWorkerGate(workerGate)) 75 | ri, err := Open(td) 76 | require.NoError(t, err) 77 | 78 | sess := ri.Session(ctx) 79 | 80 | wb := sess.Batch(ctx) 81 | 82 | var h multihash.Multihash 83 | 84 | for i := 0; i < 500; i++ { 85 | var blk [200_000]byte 86 | binary.BigEndian.PutUint64(blk[:], uint64(i)) 87 | 88 | b := blocks.NewBlock(blk[:]) 89 | h = b.Cid().Hash() 90 | 91 | err = wb.Put(ctx, []blocks.Block{b}) 92 | require.NoError(t, err) 93 | 94 | err = wb.Flush(ctx) 95 | require.NoError(t, err) 96 | } 97 | 98 | gs, err := ri.StorageDiag().GroupMeta(1) 99 | require.NoError(t, err) 100 | require.Equal(t, iface.GroupStateFull, gs.State) 101 | 102 | err = sess.View(ctx, []multihash.Multihash{h}, func(i int, b []byte) { 103 | require.Equal(t, 0, i) 104 | //require.Equal(t, b, []byte("hello world")) 105 | }) 106 | require.NoError(t, err) 107 | 108 | //workerGate <- struct{}{} // trigger a worker to run for one cycle 109 | 110 | require.Eventually(t, func() bool { 111 | gs, err := ri.StorageDiag().GroupMeta(1) 112 | require.NoError(t, err) 113 | fmt.Println("state now ", gs.State) 114 | return gs.State == iface.GroupStateVRCARDone 115 | }, 10*time.Second, 40*time.Millisecond) 116 | 117 | //workerGate <- struct{}{} // trigger a worker to allow processing close 118 | 119 | /*f, err := os.OpenFile("/tmp/ri.car", os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) 120 | require.NoError(t, err) 121 | defer f.Close() 122 | 123 | err = ri.(*rbs).openGroups[1].writeCar(f) 124 | require.NoError(t, err)*/ 125 | 126 | require.NoError(t, ri.Close()) 127 | } 128 | -------------------------------------------------------------------------------- /rbdeal/deal_diag.go: -------------------------------------------------------------------------------- 1 | package rbdeal 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/libp2p/go-libp2p/core/host" 7 | 8 | "github.com/filecoin-project/go-jsonrpc" 9 | "github.com/filecoin-project/lotus/api" 10 | "github.com/filecoin-project/lotus/api/client" 11 | iface "github.com/lotus-web3/ribs" 12 | ) 13 | 14 | func (r *ribs) DealDiag() iface.RIBSDiag { 15 | return r 16 | } 17 | 18 | func (r *ribs) CrawlState() iface.CrawlState { 19 | cs := r.crawlState.Load() 20 | if cs == nil { 21 | return iface.CrawlState{ 22 | State: "disabled", 23 | } 24 | } 25 | 26 | return *cs 27 | } 28 | 29 | func (r *ribs) ReachableProviders() []iface.ProviderMeta { 30 | return r.db.ReachableProviders() 31 | } 32 | 33 | func (r *ribs) ProviderInfo(id int64) (iface.ProviderInfo, error) { 34 | return r.db.ProviderInfo(id) 35 | } 36 | 37 | func (r *ribs) DealSummary() (iface.DealSummary, error) { 38 | return r.db.DealSummary() 39 | } 40 | 41 | func (r *ribs) GroupDeals(gk iface.GroupKey) ([]iface.DealMeta, error) { 42 | return r.db.GroupDeals(gk) 43 | } 44 | 45 | func (r *ribs) StagingStats() (iface.StagingStats, error) { 46 | return iface.StagingStats{ 47 | UploadBytes: r.s3UploadBytes.Load(), 48 | UploadStarted: r.s3UploadStarted.Load(), 49 | UploadDone: r.s3UploadDone.Load(), 50 | UploadErr: r.s3UploadErr.Load(), 51 | Redirects: r.s3Redirects.Load(), 52 | ReadReqs: r.s3ReadReqs.Load(), 53 | ReadBytes: r.s3ReadBytes.Load(), 54 | }, nil 55 | } 56 | 57 | func (r *ribs) Filecoin(ctx context.Context) (api.Gateway, jsonrpc.ClientCloser, error) { 58 | gw, closer, err := client.NewGatewayRPCV1(ctx, r.lotusRPCAddr, nil) 59 | if err != nil { 60 | panic(err) 61 | } 62 | 63 | return gw, closer, nil 64 | } 65 | 66 | func getLibP2PInfoForHost(h host.Host) iface.Libp2pInfo { 67 | if h == nil { 68 | return iface.Libp2pInfo{ 69 | Listen: []string{}, 70 | PeerID: "n/a", 71 | } 72 | } 73 | 74 | out := iface.Libp2pInfo{ 75 | PeerID: h.ID().String(), 76 | Peers: len(h.Network().Peers()), 77 | } 78 | 79 | for _, ma := range h.Network().ListenAddresses() { 80 | out.Listen = append(out.Listen, ma.String()) 81 | } 82 | 83 | return out 84 | } 85 | 86 | func (r *ribs) P2PNodes(ctx context.Context) (map[string]iface.Libp2pInfo, error) { 87 | out := map[string]iface.Libp2pInfo{} 88 | 89 | out["main"] = getLibP2PInfoForHost(r.host) 90 | out["crawl"] = getLibP2PInfoForHost(r.crawlHost) 91 | out["lassie"] = getLibP2PInfoForHost(r.retrHost) 92 | 93 | return out, nil 94 | } 95 | 96 | func (r *ribs) RetrChecker() iface.RetrCheckerStats { 97 | return iface.RetrCheckerStats{ 98 | ToDo: r.rckToDo.Load(), 99 | Started: r.rckStarted.Load(), 100 | Success: r.rckSuccess.Load(), 101 | Fail: r.rckFail.Load(), 102 | SuccessAll: r.rckSuccessAll.Load(), 103 | FailAll: r.rckFailAll.Load(), 104 | } 105 | } 106 | 107 | func (r *ribs) RetrievableDealCounts() ([]iface.DealCountStats, error) { 108 | return r.db.GetRetrievableDealStats() 109 | } 110 | 111 | func (r *ribs) SealedDealCounts() ([]iface.DealCountStats, error) { 112 | return r.db.GetSealedDealStats() 113 | } 114 | 115 | func (r *ribs) RepairQueue() (iface.RepairQueueStats, error) { 116 | return r.db.GetRepairStats() 117 | } 118 | 119 | func (r *ribs) RepairStats() (map[int]iface.RepairJob, error) { 120 | r.repairStatsLk.Lock() 121 | defer r.repairStatsLk.Unlock() 122 | 123 | out := map[int]iface.RepairJob{} 124 | for k, v := range r.repairStats { 125 | out[k] = *v 126 | } 127 | 128 | return out, nil 129 | } 130 | -------------------------------------------------------------------------------- /integrations/kuri/ribsplugin/metacleanup.go: -------------------------------------------------------------------------------- 1 | package kuboribs 2 | 3 | import ( 4 | "context" 5 | "sort" 6 | "strings" 7 | 8 | "github.com/ipfs/go-cid" 9 | format "github.com/ipfs/go-ipld-format" 10 | 11 | "github.com/ipfs/boxo/ipld/merkledag" 12 | 13 | "github.com/lotus-web3/ribs" 14 | //"golang.org/x/xerrors" 15 | ) 16 | 17 | type ExplorerInfo struct { 18 | dag format.DAGService 19 | rbs ribs.Storage 20 | } 21 | func (e ExplorerInfo) getNode(c string) (*merkledag.ProtoNode, error) { 22 | c_, err := cid.Decode(c) 23 | if err != nil { 24 | log.Errorw("Failed to decode CID", "cid", c) 25 | return nil, err 26 | } 27 | rnd, err := e.dag.Get(context.TODO(), c_) 28 | if err != nil { 29 | log.Errorw("error loading CID from DAG", "error", err, "cid", c) 30 | if strings.HasPrefix(err.Error(), "block was not found locally") { 31 | return nil, nil 32 | } 33 | return nil, err 34 | } 35 | 36 | pbnd, ok := rnd.(*merkledag.ProtoNode) 37 | if !ok { 38 | log.Warnw("error loading CID: not ProtoNode", "cid", c) 39 | return nil, nil 40 | } 41 | return pbnd, nil 42 | } 43 | func (e ExplorerInfo) ListChilds(c string) (map[string]ribs.ChildInfo, error) { 44 | log.Debugw("ListChilds", "cid", c) 45 | node, err := e.getNode(c) 46 | if err != nil { 47 | log.Errorw("Failed to decode CID", "cid", c) 48 | return nil, err 49 | } 50 | ret := make(map[string]ribs.ChildInfo) 51 | if node != nil { 52 | for _, child := range node.Links() { 53 | if child.Name != "" { 54 | ret[child.Name] = ribs.ChildInfo{ 55 | Cid: child.Cid.String(), 56 | Size: child.Size, 57 | } 58 | } 59 | } 60 | } 61 | return ret, nil 62 | } 63 | func (e ExplorerInfo) addGroups(c cid.Cid, grps map[int64]bool) (map[int64]bool, error) { 64 | groups, err := e.rbs.FindHashes(context.TODO(), c.Hash()) 65 | if err != nil { 66 | log.Errorw("Failed to find hash for CID", "cid", c) 67 | return nil, err 68 | } 69 | for _, grp := range groups { 70 | grps[grp] = true 71 | } 72 | return grps, nil 73 | } 74 | func (e ExplorerInfo) ListGroups(c string) ([]int64, error) { 75 | log.Debugw("ListGroups", "cid", c) 76 | node, err := e.getNode(c) 77 | if err != nil { 78 | log.Errorw("Failed to decode CID", "cid", c) 79 | return nil, err 80 | } 81 | grps := make(map[int64]bool) 82 | if node != nil { 83 | grps, err = e.addGroups(node.Cid(), grps) 84 | if err != nil { 85 | log.Errorw("Failed to find hash for root CID", "cid", c) 86 | return nil, err 87 | } 88 | for _, child := range node.Links() { 89 | grps, err = e.addGroups(child.Cid, grps) 90 | if err != nil { 91 | log.Errorw("Failed to find hash for CID", "cid", child.Cid.String(), "fileCid", c) 92 | return nil, err 93 | } 94 | } 95 | } else { // TODO: mutualize cid decode, and get some code here 96 | c_, err := cid.Decode(c) 97 | if err != nil { 98 | log.Errorw("Failed to decode CID", "cid", c) 99 | return nil, err 100 | } 101 | grps, err = e.addGroups(c_, grps) 102 | if err != nil { 103 | log.Errorw("Failed to find hash for root CID", "cid", c) 104 | return nil, err 105 | } 106 | } 107 | var ret []int64 108 | for grp := range grps { 109 | ret = append(ret, grp) 110 | } 111 | sort.Slice(ret, func(i, j int) bool { return ret[i] < ret[j] }) 112 | return ret, nil 113 | } 114 | 115 | func StartMeta(/* lc fx.Lifecycle, */mdb ribs.MetadataDB, r ribs.RIBS, dag format.DAGService) { 116 | explorer := ExplorerInfo{ 117 | dag: dag, 118 | rbs: r.Storage(), 119 | } 120 | mdb.LaunchCleanupLoop(explorer) 121 | mdb.LaunchServer() 122 | } 123 | -------------------------------------------------------------------------------- /cidgravity/get_best_available_providers.go: -------------------------------------------------------------------------------- 1 | package cidgravity 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "io/ioutil" 9 | "net/http" 10 | "time" 11 | "github.com/lotus-web3/ribs/configuration" 12 | ) 13 | 14 | type CIDgravityGetBestAvailableProvidersRequest struct { 15 | PieceCid string `json:"pieceCid"` 16 | Provider string `json:"provider"` 17 | StartEpoch uint64 `json:"startEpoch"` 18 | Duration uint64 `json:"duration"` 19 | StoragePricePerEpoch string `json:"storagePricePerEpoch"` 20 | ProviderCollateral string `json:"providerCollateral"` 21 | VerifiedDeal *bool `json:"verifiedDeal"` 22 | TransferSize uint64 `json:"transferSize"` 23 | TransferType string `json:"transferType"` 24 | RemoveUnsealedCopy *bool `json:"removeUnsealedCopy"` 25 | } 26 | 27 | type CIDgravityAPIError struct { 28 | Code string `json:"code"` 29 | Message string `json:"message"` 30 | } 31 | 32 | type CIDgravityAPIResult struct { 33 | Reason *string `json:"reason"` 34 | Providers []string `json:"providers"` 35 | } 36 | 37 | type CIDgravityAPIResponse struct { 38 | Error CIDgravityAPIError `json:"error"` 39 | Result *CIDgravityAPIResult `json:"result"` 40 | } 41 | 42 | func (cidg *CIDGravity) GetBestAvailableProviders(params CIDgravityGetBestAvailableProvidersRequest) ([]string, error) { 43 | log.Debugw("GetBestAvailableProviders", "params", params) 44 | cidg.init() 45 | 46 | cfg := configuration.GetConfig() 47 | 48 | // Define params 49 | method := "POST" 50 | authToken := cfg.CidGravity.ApiToken 51 | endpoint := cfg.CidGravity.ApiEndpointGetProviders 52 | 53 | // Parse params for request body 54 | var requestBody = new(bytes.Buffer) 55 | err := json.NewEncoder(requestBody).Encode(params) 56 | 57 | if err != nil { 58 | return nil, err 59 | } 60 | 61 | // Create HTTP request 62 | req, err := http.NewRequest(method, endpoint, requestBody) 63 | 64 | if err != nil { 65 | return nil, err 66 | } 67 | 68 | // Add authorization header 69 | req.Header.Set("X-API-KEY", authToken) 70 | 71 | // Create HTTP client 72 | // This will also define the request timeout to 30 seconds 73 | client := http.Client{ 74 | Transport: &http.Transport{ 75 | DisableKeepAlives: true, 76 | MaxIdleConnsPerHost: -1, 77 | }, 78 | Timeout: 30 * time.Second, 79 | } 80 | 81 | if err := cidg.sem.Acquire(context.TODO(), 1); err != nil { 82 | return nil, fmt.Errorf("Failed to acquire semaphore: %w", err) 83 | } 84 | // Send the request 85 | resp, err := client.Do(req) 86 | cidg.sem.Release(1) 87 | 88 | if err != nil { 89 | return nil, err 90 | } 91 | 92 | // Defer closing the response body 93 | defer resp.Body.Close() 94 | 95 | // Read the response content 96 | responseBody, err := ioutil.ReadAll(resp.Body) 97 | 98 | if err != nil { 99 | return nil, err 100 | } 101 | 102 | // Parse response to a valid response struct 103 | var result CIDgravityAPIResponse 104 | err = json.Unmarshal(responseBody, &result) 105 | 106 | if err != nil { 107 | return nil, err 108 | } 109 | 110 | // result object can contain both response message or error depending on request status code 111 | // If status code is not 200, return the object, but with an error that can be checked in main function 112 | if resp.StatusCode != 200 { 113 | log.Errorw("GetBestAvailableProviders", "code", resp.StatusCode, "result", result) 114 | return []string{result.Error.Code, result.Error.Message}, fmt.Errorf("status code is not 200") 115 | } 116 | 117 | log.Debugw("GetBestAvailableProviders", "result", result.Result) 118 | if result.Result == nil { 119 | return nil, fmt.Errorf("Failed to parse cidgravity providers") 120 | } 121 | return result.Result.Providers, nil 122 | } 123 | -------------------------------------------------------------------------------- /rbstor/group_worker.go: -------------------------------------------------------------------------------- 1 | package rbstor 2 | 3 | import ( 4 | "context" 5 | 6 | iface "github.com/lotus-web3/ribs" 7 | ) 8 | 9 | func (r *rbs) groupWorker(i int) { 10 | r.workersAvail.Add(1) 11 | defer r.workersAvail.Add(-1) 12 | 13 | for { 14 | select { 15 | case task := <-r.tasks: 16 | r.workerExecTask(task) 17 | case <-r.close: 18 | close(r.workerClosed[i]) 19 | return 20 | } 21 | } 22 | } 23 | 24 | func (r *rbs) workerExecTask(toExec task) { 25 | switch toExec.tt { 26 | case taskTypeFinalize: 27 | r.workersFinalizing.Add(1) 28 | defer r.workersFinalizing.Add(-1) 29 | 30 | r.lk.Lock() 31 | g, ok := r.openGroups[toExec.group] 32 | if !ok { 33 | r.lk.Unlock() 34 | log.Errorw("group not open", "group", toExec.group, "toExec", toExec) 35 | return 36 | } 37 | 38 | r.lk.Unlock() 39 | err := g.Finalize(context.TODO()) 40 | if err != nil { 41 | log.Errorw("finalizing group", "error", err, "group", toExec.group) 42 | } 43 | 44 | r.sendSub(toExec.group, iface.GroupStateFull, iface.GroupStateVRCARDone) 45 | 46 | log.Debugw("finalize fallthrough to genCommP", "group", toExec.group) 47 | fallthrough 48 | 49 | case taskTypeGenCommP: 50 | r.workersCommP.Add(1) 51 | defer r.workersCommP.Add(-1) 52 | 53 | r.lk.Lock() 54 | g, ok := r.openGroups[toExec.group] 55 | r.lk.Unlock() 56 | if !ok { 57 | log.Errorw("group not open", "group", toExec.group, "toExec", toExec) 58 | return 59 | } 60 | 61 | err := g.GenCommP() // todo do in finalize... 62 | if err != nil { 63 | log.Errorw("generating commP", "group", toExec.group, "err", err) 64 | } 65 | 66 | r.sendSub(toExec.group, iface.GroupStateVRCARDone, iface.GroupStateLocalReadyForDeals) 67 | case taskTypeFinDataReload: 68 | r.workersFinDataReload.Add(1) 69 | defer r.workersFinDataReload.Add(-1) 70 | 71 | r.lk.Lock() 72 | g, ok := r.openGroups[toExec.group] 73 | r.lk.Unlock() 74 | if !ok { 75 | log.Errorw("group not open", "group", toExec.group, "toExec", toExec) 76 | return 77 | } 78 | 79 | err := g.FinDataReload(context.TODO()) 80 | if err != nil { 81 | log.Errorw("finishing data reload", "group", toExec.group, "err", err) 82 | } 83 | 84 | r.sendSub(toExec.group, iface.GroupStateReload, iface.GroupStateLocalReadyForDeals) 85 | } 86 | } 87 | 88 | func (r *rbs) Subscribe(sub iface.GroupSub) { 89 | r.subLk.Lock() 90 | defer r.subLk.Unlock() 91 | 92 | r.subs = append(r.subs, sub) 93 | } 94 | 95 | func (r *rbs) resumeGroups(ctx context.Context) { 96 | gs, err := r.db.GroupStates() 97 | if err != nil { 98 | panic(err) 99 | } 100 | 101 | for g, st := range gs { 102 | switch st { 103 | case iface.GroupStateFull, iface.GroupStateVRCARDone, iface.GroupStateLocalReadyForDeals: 104 | if err := r.withReadableGroup(ctx, g, func(g *Group) error { 105 | return nil 106 | }); err != nil { 107 | log.Errorw("failed to resume group", "group", g, "err", err) 108 | return 109 | } 110 | } 111 | } 112 | } 113 | 114 | func (r *rbs) resumeGroup(group iface.GroupKey) { 115 | sendTask := func(tt taskType) { 116 | go func() { 117 | r.tasks <- task{ 118 | tt: tt, 119 | group: group, 120 | } 121 | }() 122 | } 123 | 124 | r.sendSub(group, r.openGroups[group].state, r.openGroups[group].state) 125 | 126 | switch r.openGroups[group].state { 127 | case iface.GroupStateWritable: // nothing to do 128 | case iface.GroupStateFull: 129 | sendTask(taskTypeFinalize) 130 | case iface.GroupStateVRCARDone: 131 | sendTask(taskTypeGenCommP) 132 | case iface.GroupStateLocalReadyForDeals: 133 | case iface.GroupStateOffloaded: 134 | case iface.GroupStateReload: 135 | sendTask(taskTypeFinDataReload) 136 | } 137 | } 138 | 139 | func (r *rbs) sendSub(group iface.GroupKey, old, new iface.GroupState) { 140 | r.subLk.Lock() 141 | defer r.subLk.Unlock() 142 | 143 | for _, sub := range r.subs { 144 | sub(group, old, new) 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /integrations/web/ribswebapp/src/routes/Content.js: -------------------------------------------------------------------------------- 1 | import React, {useEffect, useState} from 'react'; 2 | import RibsRPC from "../helpers/rpc"; 3 | import { formatBytesBinary, formatNum, epochToDate, epochToDuration } from "../helpers/fmt"; 4 | import { Group } from "./Groups"; 5 | 6 | function Content() { 7 | const [cid, setCid] = useState(''); 8 | const [groupIds, setGroupIds] = useState(null); 9 | const [headHeight, setHeadHeight] = useState(0); 10 | const [expandedGroupId, setExpandedGroupId] = useState(null); 11 | 12 | const [error, setError] = useState(null); 13 | 14 | const findCid = async () => { 15 | if (cid !== '') { 16 | try { 17 | const result = await RibsRPC.call("FindCid", [{"/": cid}]); 18 | setError(null); 19 | setGroupIds(result); 20 | setExpandedGroupId(0); 21 | } catch (error) { 22 | setError(error); 23 | setGroupIds(null); 24 | return; 25 | } 26 | } 27 | }; 28 | 29 | const fetchHead = async () => { 30 | try { 31 | const head = await RibsRPC.callFil("ChainHead"); 32 | setHeadHeight(head.Height); 33 | } catch (error) { 34 | console.error("Error fetching head:", error); 35 | } 36 | }; 37 | 38 | useEffect(() => { 39 | fetchHead(); 40 | const intervalId = setInterval(fetchHead, 5000); 41 | 42 | return () => { 43 | clearInterval(intervalId); 44 | }; 45 | }, []); 46 | 47 | const handleGroupClick = (id) => { 48 | if (id === expandedGroupId) { 49 | setExpandedGroupId(null); 50 | } else { 51 | setExpandedGroupId(id); 52 | } 53 | } 54 | 55 | return ( 56 |
57 |
58 |

Inspect Content

59 |
60 |
61 | 62 | setCid(e.target.value)} /> 63 |
64 | {groupIds === null ? null : groupIds.length > 0 ? ( 65 |
66 |

Group IDs

67 |
CID: {cid}
68 |
    69 | {groupIds.map((id, index) => ( 70 |
  • handleGroupClick(index)} style={{margin: '1em 0', cursor: 'pointer', padding: '1em', borderRadius: '10px', boxShadow: '0px 0px 10px rgba(0, 0, 0, 0.1)'}}> 71 | {index !== expandedGroupId && <>Group {id}} 72 | {index === expandedGroupId && } 73 |
    • 74 | ))} 75 |
76 |
77 | ) : ( 78 |
79 |

Not Found

80 |
81 | )} 82 | {error !== null && ( 83 |
84 |

Error

85 |
86 | {error.message} 87 |
88 |
89 | )} 90 |
91 | ); 92 | } 93 | 94 | export default Content; 95 | -------------------------------------------------------------------------------- /ributil/boostnet/transports.go: -------------------------------------------------------------------------------- 1 | package boostnet 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/filecoin-project/go-fil-markets/shared" 9 | logging "github.com/ipfs/go-log/v2" 10 | "github.com/ipld/go-ipld-prime/codec/dagcbor" 11 | "github.com/libp2p/go-libp2p/core/host" 12 | "github.com/libp2p/go-libp2p/core/network" 13 | "github.com/libp2p/go-libp2p/core/peer" 14 | "github.com/libp2p/go-libp2p/core/protocol" 15 | types "github.com/lotus-web3/ribs/ributil/boosttypes" 16 | ) 17 | 18 | var clog = logging.Logger("boost:lp2p:tspt:client") 19 | var slog = logging.Logger("boost:lp2p:tspt") 20 | 21 | // TransportsProtocolID is the protocol for querying which retrieval transports 22 | // the Storage Provider supports (http, libp2p, etc) 23 | const TransportsProtocolID = protocol.ID("/fil/retrieval/transports/1.0.0") 24 | 25 | // TransportsListener listens for incoming queries over libp2p 26 | type TransportsListener struct { 27 | host host.Host 28 | protocols []types.Protocol 29 | } 30 | 31 | const streamReadDeadline = 30 * time.Second 32 | const streamWriteDeadline = 30 * time.Second 33 | 34 | // QueryClientOption is an option for configuring the libp2p storage deal client 35 | type QueryClientOption func(*TransportsClient) 36 | 37 | // RetryParameters changes the default parameters around connection reopening 38 | func RetryParameters(minDuration time.Duration, maxDuration time.Duration, attempts float64, backoffFactor float64) QueryClientOption { 39 | return func(c *TransportsClient) { 40 | c.retryStream.SetOptions(shared.RetryParameters(minDuration, maxDuration, attempts, backoffFactor)) 41 | } 42 | } 43 | 44 | // TransportsClient sends retrieval queries over libp2p 45 | type TransportsClient struct { 46 | retryStream *shared.RetryStream 47 | } 48 | 49 | func NewTransportsClient(h host.Host, options ...QueryClientOption) *TransportsClient { 50 | c := &TransportsClient{ 51 | retryStream: shared.NewRetryStream(h), 52 | } 53 | for _, option := range options { 54 | option(c) 55 | } 56 | return c 57 | } 58 | 59 | // SendQuery sends a retrieval query over a libp2p stream to the peer 60 | func (c *TransportsClient) SendQuery(ctx context.Context, id peer.ID) (*types.QueryResponse, error) { 61 | clog.Debugw("query", "peer", id) 62 | 63 | // Create a libp2p stream to the provider 64 | s, err := c.retryStream.OpenStream(ctx, id, []protocol.ID{TransportsProtocolID}) 65 | if err != nil { 66 | return nil, err 67 | } 68 | 69 | defer s.Close() // nolint 70 | 71 | // Set a deadline on reading from the stream so it doesn't hang 72 | _ = s.SetReadDeadline(time.Now().Add(streamReadDeadline)) 73 | defer s.SetReadDeadline(time.Time{}) // nolint 74 | 75 | // Read the response from the stream 76 | queryResponsei, err := types.BindnodeRegistry.TypeFromReader(s, (*types.QueryResponse)(nil), dagcbor.Decode) 77 | if err != nil { 78 | return nil, fmt.Errorf("reading query response: %w", err) 79 | } 80 | queryResponse := queryResponsei.(*types.QueryResponse) 81 | 82 | clog.Debugw("response", "peer", id) 83 | 84 | return queryResponse, nil 85 | } 86 | 87 | func NewTransportsListener(h host.Host, protos []types.Protocol) *TransportsListener { 88 | return &TransportsListener{ 89 | host: h, 90 | protocols: protos, 91 | } 92 | } 93 | 94 | func (p *TransportsListener) Start() { 95 | p.host.SetStreamHandler(TransportsProtocolID, p.handleNewQueryStream) 96 | } 97 | 98 | func (p *TransportsListener) Stop() { 99 | p.host.RemoveStreamHandler(TransportsProtocolID) 100 | } 101 | 102 | // Called when the client opens a libp2p stream 103 | func (l *TransportsListener) handleNewQueryStream(s network.Stream) { 104 | defer s.Close() 105 | 106 | slog.Debugw("query", "peer", s.Conn().RemotePeer()) 107 | 108 | response := types.QueryResponse{Protocols: l.protocols} 109 | 110 | // Set a deadline on writing to the stream so it doesn't hang 111 | _ = s.SetWriteDeadline(time.Now().Add(streamWriteDeadline)) 112 | defer s.SetWriteDeadline(time.Time{}) // nolint 113 | 114 | // Write the response to the client 115 | err := types.BindnodeRegistry.TypeToWriter(&response, s, dagcbor.Encode) 116 | if err != nil { 117 | slog.Infow("error writing query response", "peer", s.Conn().RemotePeer(), "err", err) 118 | return 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /carlog/idx_level.go: -------------------------------------------------------------------------------- 1 | package carlog 2 | 3 | import ( 4 | "encoding/binary" 5 | 6 | "github.com/multiformats/go-multihash" 7 | "github.com/syndtr/goleveldb/leveldb" 8 | "github.com/syndtr/goleveldb/leveldb/errors" 9 | "github.com/syndtr/goleveldb/leveldb/filter" 10 | "github.com/syndtr/goleveldb/leveldb/opt" 11 | "github.com/syndtr/goleveldb/leveldb/storage" 12 | "golang.org/x/xerrors" 13 | ) 14 | 15 | type LevelDBIndex struct { 16 | *leveldb.DB 17 | } 18 | 19 | func OpenLevelDBIndex(path string, create bool) (*LevelDBIndex, error) { 20 | o := &opt.Options{ 21 | OpenFilesCacheCapacity: 500, 22 | ErrorIfExist: create, 23 | ErrorIfMissing: !create, 24 | Compression: opt.NoCompression, // this data is quite dense 25 | Filter: filter.NewBloomFilter(10), 26 | // todo NoSync 27 | } 28 | 29 | var err error 30 | var db *leveldb.DB 31 | if path == "" { 32 | db, err = leveldb.Open(storage.NewMemStorage(), o) 33 | } else { 34 | db, err = leveldb.OpenFile(path, o) 35 | if errors.IsCorrupted(err) && !o.GetReadOnly() { 36 | db, err = leveldb.RecoverFile(path, o) 37 | } 38 | } 39 | if err != nil { 40 | return nil, err 41 | } 42 | 43 | return &LevelDBIndex{ 44 | db, 45 | }, nil 46 | } 47 | 48 | func (l *LevelDBIndex) Has(c []multihash.Multihash) ([]bool, error) { 49 | out := make([]bool, len(c)) 50 | var err error 51 | 52 | for i, m := range c { 53 | out[i], err = l.DB.Has(m, nil) 54 | if err != nil { 55 | return nil, err 56 | } 57 | } 58 | 59 | return out, nil 60 | } 61 | 62 | func (l *LevelDBIndex) Put(c []multihash.Multihash, offs []int64) error { 63 | batch := leveldb.MakeBatch(len(c) * 64) 64 | for i, m := range c { 65 | if offs[i] == -1 { 66 | continue 67 | } 68 | var buf [8]byte 69 | binary.LittleEndian.PutUint64(buf[:], uint64(offs[i])) 70 | batch.Put(m, buf[:]) 71 | } 72 | return l.DB.Write(batch, nil) 73 | } 74 | 75 | // Get returns offsets to data, -1 if not found 76 | func (l *LevelDBIndex) Get(c []multihash.Multihash) ([]int64, error) { 77 | out := make([]int64, len(c)) 78 | var err error 79 | 80 | for i, m := range c { 81 | v, err := l.DB.Get(m, nil) 82 | switch err { 83 | case nil: 84 | case leveldb.ErrNotFound: 85 | out[i] = -1 86 | continue 87 | default: 88 | return nil, xerrors.Errorf("index get: %w", err) 89 | } 90 | 91 | if len(v) != 8 { 92 | return nil, xerrors.Errorf("invalid value length") 93 | } 94 | out[i] = int64(binary.LittleEndian.Uint64(v)) 95 | } 96 | 97 | return out, err 98 | } 99 | 100 | func (l *LevelDBIndex) Entries() (int64, error) { 101 | // todo is super mega shit, keep a count in jbob 102 | it := l.DB.NewIterator(nil, nil) 103 | defer it.Release() 104 | 105 | var count int64 106 | for it.Next() { 107 | count++ 108 | } 109 | 110 | return count, nil 111 | } 112 | 113 | func (l *LevelDBIndex) List(f func(c multihash.Multihash, offs []int64) error) error { 114 | it := l.DB.NewIterator(nil, nil) 115 | defer it.Release() 116 | 117 | for it.Next() { 118 | if len(it.Value()) != 8 { 119 | return xerrors.Errorf("invalid value length") 120 | } 121 | offs := int64(binary.LittleEndian.Uint64(it.Value())) 122 | if err := f(it.Key(), []int64{offs}); err != nil { 123 | return err 124 | } 125 | } 126 | 127 | return nil 128 | } 129 | 130 | func (l *LevelDBIndex) ToTruncate(atOrAbove int64) ([]multihash.Multihash, error) { 131 | var mhashes []multihash.Multihash 132 | it := l.DB.NewIterator(nil, nil) 133 | defer it.Release() 134 | 135 | for it.Next() { 136 | if len(it.Value()) != 8 { 137 | return nil, xerrors.Errorf("invalid value length") 138 | } 139 | offs, _ := fromOffsetLen(int64(binary.LittleEndian.Uint64(it.Value()))) 140 | if offs >= atOrAbove { 141 | keyCopy := make([]byte, len(it.Key())) 142 | copy(keyCopy, it.Key()) 143 | 144 | mhashes = append(mhashes, keyCopy) 145 | } 146 | } 147 | return mhashes, nil 148 | } 149 | 150 | func (l *LevelDBIndex) Del(c []multihash.Multihash) error { 151 | batch := leveldb.MakeBatch(len(c) * 64) // todo this can probably be way smaller 152 | for _, m := range c { 153 | batch.Delete(m) 154 | } 155 | 156 | return l.DB.Write(batch, &opt.WriteOptions{Sync: true}) 157 | } 158 | 159 | // todo sync 160 | 161 | func (l *LevelDBIndex) Close() error { 162 | return l.DB.Close() 163 | } 164 | 165 | var _ WritableIndex = &LevelDBIndex{} 166 | -------------------------------------------------------------------------------- /ributil/parcommp.go: -------------------------------------------------------------------------------- 1 | package ributil 2 | 3 | import ( 4 | "math/bits" 5 | "runtime" 6 | 7 | "github.com/filecoin-project/go-commp-utils/nonffi" 8 | commcid "github.com/filecoin-project/go-fil-commcid" 9 | commp "github.com/filecoin-project/go-fil-commp-hashhash" 10 | 11 | "github.com/ipfs/go-cid" 12 | "golang.org/x/xerrors" 13 | 14 | "github.com/filecoin-project/go-state-types/abi" 15 | 16 | "github.com/filecoin-project/go-commp-utils/zerocomm" 17 | ) 18 | 19 | type DataCIDSize struct { 20 | PayloadSize int64 21 | PieceSize abi.PaddedPieceSize 22 | PieceCID cid.Cid 23 | } 24 | 25 | const commPBufPad = abi.PaddedPieceSize(8 << 20) 26 | const CommPBuf = abi.UnpaddedPieceSize(commPBufPad - (commPBufPad / 128)) // can't use .Unpadded() for const 27 | 28 | type ciderr struct { 29 | c cid.Cid 30 | err error 31 | } 32 | 33 | type DataCidWriter struct { 34 | len int64 35 | buf [CommPBuf]byte 36 | leaves []chan ciderr 37 | 38 | tbufs [][CommPBuf]byte 39 | throttle chan int 40 | } 41 | 42 | func (w *DataCidWriter) Write(p []byte) (int, error) { 43 | if w.throttle == nil { 44 | w.throttle = make(chan int, runtime.NumCPU()) 45 | for i := 0; i < cap(w.throttle); i++ { 46 | w.throttle <- i 47 | } 48 | } 49 | if w.tbufs == nil { 50 | w.tbufs = make([][CommPBuf]byte, cap(w.throttle)) 51 | } 52 | 53 | n := len(p) 54 | for len(p) > 0 { 55 | buffered := int(w.len % int64(len(w.buf))) 56 | toBuffer := len(w.buf) - buffered 57 | if toBuffer > len(p) { 58 | toBuffer = len(p) 59 | } 60 | 61 | copied := copy(w.buf[buffered:], p[:toBuffer]) 62 | p = p[copied:] 63 | w.len += int64(copied) 64 | 65 | if copied > 0 && w.len%int64(len(w.buf)) == 0 { 66 | leaf := make(chan ciderr, 1) 67 | bufIdx := <-w.throttle 68 | copy(w.tbufs[bufIdx][:], w.buf[:]) 69 | 70 | go func() { 71 | defer func() { 72 | w.throttle <- bufIdx 73 | }() 74 | 75 | cc := new(commp.Calc) 76 | _, _ = cc.Write(w.tbufs[bufIdx][:]) 77 | p, _, _ := cc.Digest() 78 | l, _ := commcid.PieceCommitmentV1ToCID(p) 79 | leaf <- ciderr{ 80 | c: l, 81 | err: nil, 82 | } 83 | }() 84 | 85 | w.leaves = append(w.leaves, leaf) 86 | } 87 | } 88 | return n, nil 89 | } 90 | 91 | func (w *DataCidWriter) Sum() (DataCIDSize, error) { 92 | // process last non-zero leaf if exists 93 | lastLen := w.len % int64(len(w.buf)) 94 | rawLen := w.len 95 | 96 | leaves := make([]cid.Cid, len(w.leaves)) 97 | for i, leaf := range w.leaves { 98 | r := <-leaf 99 | if r.err != nil { 100 | return DataCIDSize{}, xerrors.Errorf("processing leaf %d: %w", i, r.err) 101 | } 102 | leaves[i] = r.c 103 | } 104 | 105 | // process remaining bit of data 106 | if lastLen != 0 { 107 | if len(leaves) != 0 { 108 | copy(w.buf[lastLen:], make([]byte, int(int64(CommPBuf)-lastLen))) 109 | lastLen = int64(CommPBuf) 110 | } 111 | 112 | cc := new(commp.Calc) 113 | _, _ = cc.Write(w.buf[:lastLen]) 114 | pb, pps, _ := cc.Digest() 115 | p, _ := commcid.PieceCommitmentV1ToCID(pb) 116 | 117 | if abi.PaddedPieceSize(pps).Unpadded() < CommPBuf { // special case for pieces smaller than 16MiB 118 | return DataCIDSize{ 119 | PayloadSize: w.len, 120 | PieceSize: abi.PaddedPieceSize(pps), 121 | PieceCID: p, 122 | }, nil 123 | } 124 | 125 | leaves = append(leaves, p) 126 | } 127 | 128 | // pad with zero pieces to power-of-two size 129 | fillerLeaves := (1 << (bits.Len(uint(len(leaves) - 1)))) - len(leaves) 130 | for i := 0; i < fillerLeaves; i++ { 131 | leaves = append(leaves, zerocomm.ZeroPieceCommitment(CommPBuf)) 132 | } 133 | 134 | if len(leaves) == 1 { 135 | return DataCIDSize{ 136 | PayloadSize: rawLen, 137 | PieceSize: abi.PaddedPieceSize(len(leaves)) * commPBufPad, 138 | PieceCID: leaves[0], 139 | }, nil 140 | } 141 | 142 | pieces := make([]abi.PieceInfo, len(leaves)) 143 | for i, leaf := range leaves { 144 | pieces[i] = abi.PieceInfo{ 145 | Size: commPBufPad, 146 | PieceCID: leaf, 147 | } 148 | } 149 | 150 | p, err := nonffi.GenerateUnsealedCID(abi.RegisteredSealProof_StackedDrg64GiBV1, pieces) 151 | if err != nil { 152 | return DataCIDSize{}, xerrors.Errorf("generating unsealed CID: %w", err) 153 | } 154 | 155 | return DataCIDSize{ 156 | PayloadSize: rawLen, 157 | PieceSize: abi.PaddedPieceSize(len(leaves)) * commPBufPad, 158 | PieceCID: p, 159 | }, nil 160 | } 161 | -------------------------------------------------------------------------------- /tools/dump_pebble/dump_pebble.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/binary" 5 | "encoding/json" 6 | "flag" 7 | "fmt" 8 | "log" 9 | "os" 10 | 11 | "github.com/cockroachdb/pebble" 12 | "github.com/multiformats/go-multihash" 13 | "github.com/ipfs/go-cid" 14 | ) 15 | 16 | // Struct for storing multihash and type 17 | type MHInfo struct { 18 | MH string `json:"mh"` 19 | Type string `json:"type"` 20 | } 21 | 22 | // Struct for storing CID version 1 and 0 23 | type CIDDetails struct { 24 | V1 string `json:"v1"` 25 | V0 *string `json:"v0,omitempty"` // Use pointer to handle null values 26 | Type string `json:"type"` 27 | } 28 | 29 | // Main struct to hold all the data 30 | type CIDInfo struct { 31 | Group uint64 `json:"group,omitempty"` 32 | Size uint32 `json:"size,omitempty"` 33 | Prefix string `json:"prefix"` 34 | MH MHInfo `json:"mh"` 35 | CID CIDDetails `json:"cid"` 36 | } 37 | 38 | func main() { 39 | // Standard flag parsing 40 | indexPath := flag.String("index", "", "Path to the pebble index") 41 | flag.Parse() 42 | 43 | // Set the log output to stderr 44 | log.SetOutput(os.Stderr) 45 | 46 | // Ensure required flags are provided 47 | if *indexPath == "" { 48 | log.Fatal("ERROR: Missing required flags: --index") 49 | } 50 | 51 | // Open the Pebble DB 52 | db, err := pebble.Open(*indexPath, &pebble.Options{}) 53 | if err != nil { 54 | // Explicit error message and program halt if opening Pebble DB fails 55 | log.Fatalf("ERROR: Failed to open Pebble DB at path %s: %s", *indexPath, err) 56 | } 57 | 58 | // Ensure that the DB gets closed when done 59 | defer func() { 60 | if err := db.Close(); err != nil { 61 | log.Printf("WARNING: Failed to close Pebble DB: %s", err) 62 | } 63 | }() 64 | 65 | // Iterate through all keys 66 | iter := db.NewIter(nil) 67 | defer iter.Close() // Close the iterator once we're done 68 | 69 | for iter.First(); iter.Valid(); iter.Next() { 70 | key := iter.Key() 71 | val := iter.Value() 72 | var mhBytes []byte 73 | var group uint64 74 | var size uint32 75 | var prefix []byte 76 | 77 | // Extract data for i: entries 78 | if len(key) > 2 && key[0] == 'i' && key[1] == ':' { 79 | prefix = []byte("i:") 80 | groupBytes := key[len(key)-8:] 81 | group = binary.BigEndian.Uint64(groupBytes) 82 | mhBytes = key[len(prefix) : len(key)-8] 83 | 84 | // Extract data for s: entries 85 | } else if len(key) > 2 && key[0] == 's' && key[1] == ':' { 86 | prefix = []byte("s:") 87 | mhBytes = key[len(prefix):] 88 | sizeBytes := key[:4] 89 | size = binary.BigEndian.Uint32(sizeBytes) 90 | groupBytes := val[4:] 91 | if len(groupBytes) != 8 { 92 | log.Printf("WARNING: Expected 8 bytes for group, but got %d bytes for key: %x", len(groupBytes), key) 93 | } else { 94 | group = binary.BigEndian.Uint64(groupBytes) 95 | } 96 | } 97 | 98 | 99 | mh, err := multihash.Cast(mhBytes) 100 | if err != nil { 101 | log.Printf("WARNING: Failed to cast multihash for key %s, mhBytes: %x, error: %s", key, mhBytes, err) 102 | continue 103 | } 104 | 105 | mhinfo, err := multihash.Decode(mhBytes) 106 | if err != nil { 107 | log.Printf("WARNING: Failed to decode multihash: %s", err) 108 | continue 109 | } 110 | 111 | // Generate the CID v1 112 | v1 := cid.NewCidV1(cid.Raw, mh) 113 | 114 | // Generate CID v0 if the multihash is SHA2-256 115 | var v0 *string 116 | if mhinfo.Code == multihash.SHA2_256 { 117 | v0Cid := cid.NewCidV0(mh) 118 | v0Str := v0Cid.String() 119 | v0 = &v0Str 120 | } 121 | 122 | // Format and Print 123 | cidInfo := CIDInfo{ 124 | Group: group, 125 | Prefix: string(prefix), 126 | Size: size, 127 | MH: MHInfo{ 128 | MH: mh.String(), 129 | Type: multihash.Codes[mhinfo.Code], 130 | }, 131 | CID: CIDDetails{ 132 | V1: v1.String(), 133 | V0: v0, 134 | Type: multihash.Codes[mhinfo.Code], 135 | }, 136 | } 137 | jsonData, err := json.Marshal(cidInfo) 138 | if err != nil { 139 | log.Printf("WARNING: Failed to marshal JSON: %s", err) 140 | continue 141 | } 142 | fmt.Println(string(jsonData)) 143 | } 144 | } 145 | 146 | -------------------------------------------------------------------------------- /iface_ribs.go: -------------------------------------------------------------------------------- 1 | package ribs 2 | 3 | import ( 4 | "context" 5 | "io" 6 | 7 | "github.com/filecoin-project/go-address" 8 | "github.com/filecoin-project/go-jsonrpc" 9 | "github.com/filecoin-project/go-state-types/abi" 10 | "github.com/filecoin-project/lotus/api" 11 | "github.com/ipfs/go-cid" 12 | ) 13 | 14 | type RIBS interface { 15 | RBS 16 | 17 | Wallet() Wallet 18 | DealDiag() RIBSDiag 19 | 20 | MetaDB() MetadataDB 21 | 22 | io.Closer 23 | } 24 | 25 | type RIBSDiag interface { 26 | //CarUploadStats() UploadStats 27 | DealSummary() (DealSummary, error) 28 | GroupDeals(gk GroupKey) ([]DealMeta, error) 29 | 30 | ProviderInfo(id int64) (ProviderInfo, error) 31 | CrawlState() CrawlState 32 | ReachableProviders() []ProviderMeta 33 | 34 | RetrStats() (RetrStats, error) 35 | 36 | StagingStats() (StagingStats, error) 37 | 38 | Filecoin(context.Context) (api.Gateway, jsonrpc.ClientCloser, error) 39 | 40 | P2PNodes(ctx context.Context) (map[string]Libp2pInfo, error) 41 | 42 | RetrChecker() RetrCheckerStats 43 | 44 | RetrievableDealCounts() ([]DealCountStats, error) 45 | SealedDealCounts() ([]DealCountStats, error) 46 | 47 | RepairQueue() (RepairQueueStats, error) 48 | RepairStats() (map[int]RepairJob, error) 49 | } 50 | 51 | type RepairQueueStats struct { 52 | Total, Assigned int 53 | } 54 | 55 | type RepairJob struct { 56 | GroupKey GroupKey 57 | 58 | State RepairJobState 59 | 60 | FetchProgress, FetchSize int64 61 | FetchUrl string 62 | } 63 | 64 | type RepairJobState string 65 | 66 | const ( 67 | RepairJobStateFetching RepairJobState = "fetching" 68 | RepairJobStateVerifying RepairJobState = "verifying" 69 | RepairJobStateImporting RepairJobState = "importing" 70 | ) 71 | 72 | type DealCountStats struct { 73 | Count int 74 | Groups int 75 | } 76 | 77 | type RetrCheckerStats struct { 78 | ToDo int64 79 | Started int64 80 | Success int64 81 | Fail int64 82 | SuccessAll int64 83 | FailAll int64 84 | } 85 | 86 | type Libp2pInfo struct { 87 | PeerID string 88 | 89 | Listen []string 90 | 91 | Peers int 92 | } 93 | 94 | type StagingStats struct { 95 | UploadBytes, UploadStarted, UploadDone, UploadErr, Redirects, ReadReqs, ReadBytes int64 96 | } 97 | 98 | type RetrStats struct { 99 | Success, Bytes, Fail, CacheHit, CacheMiss, Active int64 100 | HTTPTries, HTTPSuccess, HTTPBytes int64 101 | } 102 | 103 | type UploadStats struct { 104 | ByGroup map[GroupKey]*GroupUploadStats 105 | 106 | LastTotalBytes int64 107 | } 108 | 109 | type GroupUploadStats struct { 110 | ActiveRequests int 111 | UploadBytes int64 112 | } 113 | 114 | type DealMeta struct { 115 | UUID string 116 | Provider int64 117 | 118 | Sealed, Failed, Rejected bool 119 | 120 | StartEpoch, EndEpoch, StartTime int64 121 | 122 | Status string 123 | SealStatus string 124 | Error string 125 | DealID int64 126 | 127 | BytesRecv int64 128 | TxSize int64 129 | PubCid string 130 | 131 | RetrTTFBMs int64 132 | RetrSuccess, RetrFail int64 133 | NoRecentSuccess bool 134 | } 135 | 136 | type Wallet interface { 137 | WalletInfo() (WalletInfo, error) 138 | 139 | MarketAdd(ctx context.Context, amount abi.TokenAmount) (cid.Cid, error) 140 | MarketWithdraw(ctx context.Context, amount abi.TokenAmount) (cid.Cid, error) 141 | 142 | Withdraw(ctx context.Context, amount abi.TokenAmount, to address.Address) (cid.Cid, error) 143 | } 144 | 145 | type WalletInfo struct { 146 | Addr, IDAddr string 147 | 148 | DataCap string 149 | 150 | Balance string 151 | MarketBalance string 152 | MarketLocked string 153 | 154 | MarketBalanceDetailed api.MarketBalance 155 | } 156 | 157 | type CrawlState struct { 158 | State string 159 | 160 | At, Reachable, Total int64 161 | Boost, BBswap, BHttp int64 162 | } 163 | 164 | type DealSummary struct { 165 | NonFailed, InProgress, Done, Failed int64 166 | 167 | TotalDataSize, TotalDealSize int64 168 | StoredDataSize, StoredDealSize int64 169 | } 170 | 171 | type ProviderInfo struct { 172 | Meta ProviderMeta 173 | RecentDeals []DealMeta 174 | } 175 | 176 | type ProviderMeta struct { 177 | ID int64 178 | PingOk bool 179 | 180 | BoostDeals bool 181 | BoosterHttp bool 182 | BoosterBitswap bool 183 | 184 | IndexedSuccess int64 185 | IndexedFail int64 186 | 187 | DealStarted int64 188 | DealSuccess int64 189 | DealFail int64 190 | DealRejected int64 191 | 192 | MostRecentDealStart int64 193 | 194 | // price in fil/gib/epoch 195 | AskPrice float64 196 | AskVerifiedPrice float64 197 | 198 | AskMinPieceSize float64 199 | AskMaxPieceSize float64 200 | 201 | RetrievDeals, UnretrievDeals int64 202 | } 203 | -------------------------------------------------------------------------------- /integrations/web/rpc.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "context" 5 | "runtime" 6 | 7 | "github.com/filecoin-project/go-address" 8 | "github.com/filecoin-project/go-jsonrpc" 9 | "github.com/filecoin-project/go-state-types/abi" 10 | "github.com/ipfs/go-cid" 11 | 12 | "github.com/lotus-web3/ribs" 13 | ) 14 | 15 | type RIBSRpc struct { 16 | ribs ribs.RIBS 17 | } 18 | 19 | func (rc *RIBSRpc) WalletInfo(ctx context.Context) (ribs.WalletInfo, error) { 20 | return rc.ribs.Wallet().WalletInfo() 21 | } 22 | 23 | func (rc *RIBSRpc) WalletMarketAdd(ctx context.Context, amt abi.TokenAmount) (cid.Cid, error) { 24 | return rc.ribs.Wallet().MarketAdd(ctx, amt) 25 | } 26 | 27 | func (rc *RIBSRpc) WalletMarketWithdraw(ctx context.Context, amt abi.TokenAmount) (cid.Cid, error) { 28 | return rc.ribs.Wallet().MarketWithdraw(ctx, amt) 29 | } 30 | 31 | func (rc *RIBSRpc) WalletWithdraw(ctx context.Context, amt abi.TokenAmount, to address.Address) (cid.Cid, error) { 32 | return rc.ribs.Wallet().Withdraw(ctx, amt, to) 33 | } 34 | 35 | func (rc *RIBSRpc) Groups(ctx context.Context) ([]ribs.GroupKey, error) { 36 | return rc.ribs.StorageDiag().Groups() 37 | } 38 | 39 | func (rc *RIBSRpc) FindCid(ctx context.Context, hash cid.Cid) ([]ribs.GroupKey, error) { 40 | return rc.ribs.Storage().FindHashes(ctx, hash.Hash()) 41 | } 42 | 43 | func (rc *RIBSRpc) GroupMeta(ctx context.Context, group ribs.GroupKey) (ribs.GroupMeta, error) { 44 | return rc.ribs.StorageDiag().GroupMeta(group) 45 | } 46 | 47 | func (rc *RIBSRpc) GroupDeals(ctx context.Context, group ribs.GroupKey) ([]ribs.DealMeta, error) { 48 | return rc.ribs.DealDiag().GroupDeals(group) 49 | } 50 | 51 | func (rc *RIBSRpc) CrawlState(ctx context.Context) (ribs.CrawlState, error) { 52 | return rc.ribs.DealDiag().CrawlState(), nil 53 | } 54 | 55 | func (rc *RIBSRpc) CarUploadStats(ctx context.Context) (ribs.UploadStats, error) { 56 | //return rc.ribs.DealDiag().CarUploadStats(), nil 57 | return ribs.UploadStats{}, nil 58 | } 59 | 60 | func (rc *RIBSRpc) ReachableProviders(ctx context.Context) ([]ribs.ProviderMeta, error) { 61 | return rc.ribs.DealDiag().ReachableProviders(), nil 62 | } 63 | 64 | func (rc *RIBSRpc) ProviderInfo(ctx context.Context, id int64) (ribs.ProviderInfo, error) { 65 | return rc.ribs.DealDiag().ProviderInfo(id) 66 | } 67 | 68 | func (rc *RIBSRpc) DealSummary(ctx context.Context) (ribs.DealSummary, error) { 69 | return rc.ribs.DealDiag().DealSummary() 70 | } 71 | 72 | func (rc *RIBSRpc) RetrStats(ctx context.Context) (ribs.RetrStats, error) { 73 | return rc.ribs.DealDiag().RetrStats() 74 | } 75 | 76 | func (rc *RIBSRpc) StagingStats(ctx context.Context) (ribs.StagingStats, error) { 77 | return rc.ribs.DealDiag().StagingStats() 78 | } 79 | 80 | func (rc *RIBSRpc) TopIndexStats(ctx context.Context) (ribs.TopIndexStats, error) { 81 | return rc.ribs.StorageDiag().TopIndexStats(ctx) 82 | } 83 | 84 | func (rc *RIBSRpc) GroupIOStats(ctx context.Context) (ribs.GroupIOStats, error) { 85 | return rc.ribs.StorageDiag().GroupIOStats(), nil 86 | } 87 | 88 | func (rc *RIBSRpc) GetGroupStats(ctx context.Context) (*ribs.GroupStats, error) { 89 | return rc.ribs.StorageDiag().GetGroupStats() 90 | } 91 | 92 | func (rc *RIBSRpc) RuntimeStats(ctx context.Context) (runtime.MemStats, error) { 93 | var out runtime.MemStats 94 | runtime.ReadMemStats(&out) 95 | return out, nil 96 | } 97 | 98 | func (rc *RIBSRpc) RetrChecker() (ribs.RetrCheckerStats, error) { 99 | return rc.ribs.DealDiag().RetrChecker(), nil 100 | } 101 | 102 | func (rc *RIBSRpc) P2PNodes(ctx context.Context) (map[string]ribs.Libp2pInfo, error) { 103 | return rc.ribs.DealDiag().P2PNodes(ctx) 104 | } 105 | 106 | func (rc *RIBSRpc) WorkerStats(ctx context.Context) (ribs.WorkerStats, error) { 107 | return rc.ribs.StorageDiag().WorkerStats(), nil 108 | } 109 | 110 | func (rc *RIBSRpc) RetrievableDealCounts(ctx context.Context) ([]ribs.DealCountStats, error) { 111 | return rc.ribs.DealDiag().RetrievableDealCounts() 112 | } 113 | 114 | func (rc *RIBSRpc) SealedDealCounts(ctx context.Context) ([]ribs.DealCountStats, error) { 115 | return rc.ribs.DealDiag().SealedDealCounts() 116 | } 117 | 118 | func (rc *RIBSRpc) RepairQueue() (ribs.RepairQueueStats, error) { 119 | return rc.ribs.DealDiag().RepairQueue() 120 | } 121 | 122 | func (rc *RIBSRpc) RepairStats() (map[int]ribs.RepairJob, error) { 123 | return rc.ribs.DealDiag().RepairStats() 124 | } 125 | 126 | func MakeRPCServer(ctx context.Context, ribs ribs.RIBS) (*jsonrpc.RPCServer, jsonrpc.ClientCloser, error) { 127 | hnd := &RIBSRpc{ribs: ribs} 128 | 129 | fgw, closer, err := ribs.DealDiag().Filecoin(ctx) 130 | if err != nil { 131 | return nil, nil, err 132 | } 133 | 134 | sv := jsonrpc.NewServer() 135 | sv.Register("RIBS", hnd) 136 | sv.Register("Filecoin", fgw) 137 | 138 | return sv, closer, nil 139 | } 140 | -------------------------------------------------------------------------------- /cidgravity/get_deal_states.go: -------------------------------------------------------------------------------- 1 | package cidgravity 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "io/ioutil" 9 | "net/http" 10 | "time" 11 | "github.com/lotus-web3/ribs/configuration" 12 | "github.com/filecoin-project/go-state-types/abi" 13 | ) 14 | 15 | type Cid struct { 16 | Root string `json:"/" cidgravity:"required"` 17 | } 18 | type CIDgravityDealProposalStatus struct { 19 | PieceCid Cid `json:"PieceCID"` 20 | PieceSize uint64 `json:"PieceSize"` 21 | VerifiedDeal bool `json:"VerifiedDeal"` 22 | Client string `json:"Client"` 23 | Provider string `json:"Provider"` 24 | Label string `json:"Label"` 25 | StartEpoch abi.ChainEpoch `json:"StartEpoch"` 26 | EndEpoch abi.ChainEpoch `json:"EndEpoch"` 27 | StoragePricePerEpoch string `json:"StoragePricePerEpoch"` 28 | ProviderCollateral string `json:"ProviderCollateral"` 29 | ClientCollateral string `json:"ClientCollateral"` 30 | } 31 | type CIDgravityDealProposalState struct { 32 | Status string `json:"Status"` 33 | PublishedEpoch abi.ChainEpoch `json:"publishedEpoch"` 34 | OnChainStartEpoch abi.ChainEpoch `json:"onChainStartEpoch"` 35 | OnChainEndEpoch abi.ChainEpoch `json:"onChainEndEpoch"` 36 | } 37 | type CIDgravityDealStatus struct { 38 | Proposal CIDgravityDealProposalStatus `json:"proposal"` 39 | State CIDgravityDealProposalState `json:"state"` 40 | LastUpdate float64 `json:"lastUpdate"` 41 | } 42 | type CIDgravityDealStatesAPIResponse struct { 43 | Error CIDgravityAPIError `json:"error"` 44 | Next *string `json:"next"` 45 | Result map[abi.DealID]CIDgravityDealStatus `json:"result"` 46 | } 47 | type CIDgravityDealStatusRequest struct { 48 | Next *string `json:"next"` 49 | SortBy *string `json:"sortBy"` 50 | } 51 | 52 | func (cidg *CIDGravity) getDealStates(client *http.Client, token string, states *map[abi.DealID]CIDgravityDealStatus) error { 53 | cfg := configuration.GetConfig() 54 | 55 | method := "POST" 56 | endpoint := cfg.CidGravity.ApiEndpointGetDeals 57 | requestParams := CIDgravityDealStatusRequest{} 58 | 59 | for { 60 | // Parse params for request body 61 | var requestBody = new(bytes.Buffer) 62 | err := json.NewEncoder(requestBody).Encode(requestParams) 63 | 64 | if err != nil { 65 | return err 66 | } 67 | 68 | // Create HTTP request 69 | req, err := http.NewRequest(method, endpoint, requestBody) 70 | 71 | if err != nil { 72 | return err 73 | } 74 | 75 | // Add authorization header 76 | req.Header.Set("X-API-KEY", token) 77 | 78 | 79 | // Send the request 80 | resp, err := client.Do(req) 81 | 82 | if err != nil { 83 | return err 84 | } 85 | 86 | // Defer closing the response body 87 | defer resp.Body.Close() 88 | 89 | // Read the response content 90 | responseBody, err := ioutil.ReadAll(resp.Body) 91 | 92 | if err != nil { 93 | return err 94 | } 95 | 96 | // Parse response to a valid response struct 97 | var result CIDgravityDealStatesAPIResponse 98 | err = json.Unmarshal(responseBody, &result) 99 | 100 | if err != nil { 101 | return err 102 | } 103 | 104 | // result object can contain both response message or error depending on request status code 105 | // If status code is not 200, return the object, but with an error that can be checked in main function 106 | if resp.StatusCode != 200 { 107 | return fmt.Errorf("status code is not 200") 108 | } 109 | for k, v := range result.Result { 110 | (*states)[k] = v 111 | } 112 | if result.Next == nil { 113 | break 114 | } 115 | requestParams.Next = result.Next 116 | } 117 | return nil 118 | } 119 | 120 | func (cidg *CIDGravity) GetDealStates(ctx context.Context) (map[abi.DealID]CIDgravityDealStatus, error) { 121 | cidg.init() 122 | 123 | states := map[abi.DealID]CIDgravityDealStatus{} 124 | 125 | cfg := configuration.GetConfig() 126 | 127 | authToken := cfg.CidGravity.ApiToken 128 | 129 | log.Debugw("getDealStates") 130 | 131 | if err := cidg.sem.Acquire(ctx, 1); err != nil { 132 | return nil, fmt.Errorf("Failed to acquire semaphore: %w", err) 133 | } 134 | defer cidg.sem.Release(1) 135 | 136 | // Create HTTP client 137 | // This will also define the request timeout to 30 seconds 138 | client := http.Client{ 139 | Transport: &http.Transport{ 140 | MaxIdleConnsPerHost: 1, 141 | }, 142 | Timeout: 30 * time.Second, 143 | } 144 | if err := cidg.getDealStates(&client, authToken, &states); err != nil { 145 | return nil, err 146 | } 147 | for _, token := range cfg.CidGravity.AltTokens { 148 | if err := cidg.getDealStates(&client, token, &states); err != nil { 149 | return nil, err 150 | } 151 | } 152 | log.Debugw("getDealStates", "states", len(states)) 153 | return states, nil 154 | } 155 | -------------------------------------------------------------------------------- /ributil/minratewriter_test.go: -------------------------------------------------------------------------------- 1 | package ributil 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | func TestRateEnforcingWriter(t *testing.T) { 11 | t.Run("should write without error when rate is above minimum", func(t *testing.T) { 12 | var buf bytes.Buffer 13 | 14 | rc := NewRateCounters[int](MinAvgGlobalLogPeerRate(1024, 1000)).Get(0) 15 | rew := NewRateEnforcingWriter(&buf, rc, 50*time.Millisecond) 16 | defer rew.Done() 17 | 18 | data := make([]byte, 1024) 19 | time.Sleep(50 * time.Millisecond) 20 | n, err := rew.Write(data) 21 | if err != nil { 22 | t.Fatalf("expected no error, got: %v", err) 23 | } 24 | if n != len(data) { 25 | t.Fatalf("expected to write %d bytes, wrote %d", len(data), n) 26 | } 27 | }) 28 | 29 | t.Run("should write with error when rate is below minimum", func(t *testing.T) { 30 | var buf deadlineWriter 31 | rc := NewRateCounters[int](MinAvgGlobalLogPeerRate(1024, 1000)).Get(0) 32 | rew := NewRateEnforcingWriter(&buf, rc, 50*time.Millisecond) 33 | defer rew.Done() 34 | 35 | data := make([]byte, 1024) 36 | _, err := rew.Write(data) 37 | if err != nil { 38 | t.Fatal(err) 39 | } 40 | 41 | time.Sleep(60 * time.Millisecond) // Increase the sleep duration to make sure the rate is below the minimum 42 | n, err := rew.Write(data) 43 | t.Log(err) 44 | if !errors.Is(err, rew.writeError) { 45 | t.Fatalf("expected error, got: %v", err) 46 | } 47 | if n != 0 || buf.buf.Len() != 1024 { 48 | t.Fatalf("expected to write 0 bytes, wrote %d", n) 49 | } 50 | }) 51 | 52 | t.Run("should set write deadline on the underlying writer", func(t *testing.T) { 53 | var buf deadlineWriter 54 | rc := NewRateCounters[int](MinAvgGlobalLogPeerRate(1024, 1000)).Get(0) 55 | rew := NewRateEnforcingWriter(&buf, rc, 50*time.Millisecond) 56 | defer rew.Done() 57 | 58 | data := make([]byte, 1024) 59 | _, err := rew.Write(data) 60 | if err != nil { 61 | t.Fatalf("unexpected error: %v", err) 62 | } 63 | 64 | if buf.writeDeadline.IsZero() { 65 | t.Fatal("expected write deadline to be set") 66 | } 67 | }) 68 | } 69 | 70 | type deadlineWriter struct { 71 | buf bytes.Buffer 72 | writeDeadline time.Time 73 | } 74 | 75 | func (d *deadlineWriter) Write(p []byte) (n int, err error) { 76 | return d.buf.Write(p) 77 | } 78 | 79 | func (d *deadlineWriter) SetWriteDeadline(t time.Time) error { 80 | d.writeDeadline = t 81 | return nil 82 | } 83 | 84 | func TestRateEnforcingReader(t *testing.T) { 85 | t.Run("should read without error when rate is above minimum", func(t *testing.T) { 86 | data := make([]byte, 1024) 87 | buf := bytes.NewBuffer(data) 88 | 89 | rc := NewRateCounters[int](MinAvgGlobalLogPeerRate(1024, 1000)).Get(0) 90 | rer := NewRateEnforcingReader(buf, rc, 50*time.Millisecond) 91 | defer rer.Done() 92 | 93 | readData := make([]byte, 1024) 94 | time.Sleep(50 * time.Millisecond) 95 | n, err := rer.Read(readData) 96 | if err != nil { 97 | t.Fatalf("expected no error, got: %v", err) 98 | } 99 | if n != len(data) { 100 | t.Fatalf("expected to read %d bytes, read %d", len(data), n) 101 | } 102 | }) 103 | 104 | t.Run("should read with error when rate is below minimum", func(t *testing.T) { 105 | data := make([]byte, 1024) 106 | buf := bytes.NewBuffer(data) 107 | 108 | rc := NewRateCounters[int](MinAvgGlobalLogPeerRate(1024, 1000)).Get(0) 109 | rer := NewRateEnforcingReader(buf, rc, 50*time.Millisecond) 110 | defer rer.Done() 111 | 112 | readData := make([]byte, 1024) 113 | _, err := rer.Read(readData) 114 | if err != nil { 115 | t.Fatal(err) 116 | } 117 | 118 | time.Sleep(60 * time.Millisecond) // Increase the sleep duration to make sure the rate is below the minimum 119 | n, err := rer.Read(readData) 120 | t.Log(err) 121 | if !errors.Is(err, rer.readError) { 122 | t.Fatalf("expected error, got: %v", err) 123 | } 124 | if n != 0 { 125 | t.Fatalf("expected to read 0 bytes, read %d", n) 126 | } 127 | }) 128 | 129 | t.Run("should support SetReadDeadline on the underlying reader", func(t *testing.T) { 130 | var buf deadlineReader 131 | buf.buf = bytes.NewBuffer(make([]byte, 2000)) 132 | 133 | rc := NewRateCounters[int](MinAvgGlobalLogPeerRate(1024, 1000)).Get(0) 134 | rer := NewRateEnforcingReader(&buf, rc, 50*time.Millisecond) 135 | defer rer.Done() 136 | 137 | data := make([]byte, 1024) 138 | _, err := rer.Read(data) 139 | if err != nil { 140 | t.Fatalf("unexpected error: %v", err) 141 | } 142 | 143 | if buf.readDeadline.IsZero() { 144 | t.Fatal("expected read deadline to be set") 145 | } 146 | }) 147 | } 148 | 149 | type deadlineReader struct { 150 | buf *bytes.Buffer 151 | readDeadline time.Time 152 | } 153 | 154 | func (d *deadlineReader) Read(p []byte) (n int, err error) { 155 | return d.buf.Read(p) 156 | } 157 | 158 | func (d *deadlineReader) SetReadDeadline(t time.Time) error { 159 | d.readDeadline = t 160 | return nil 161 | } 162 | -------------------------------------------------------------------------------- /integrations/web/ribswebapp/src/helpers/fmt.js: -------------------------------------------------------------------------------- 1 | export function formatBytesBinary(bytes) { 2 | const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB']; 3 | let l = 0, n = parseInt(bytes, 10) || 0; 4 | while (n >= 1024 && ++l) { 5 | n = n / 1024; 6 | } 7 | return (n.toFixed(l > 0 ? 2 : 0) + ' ' + units[l]); 8 | } 9 | 10 | export function formatBitsBinary(bytes) { 11 | const units = ['bps', 'Kbps', 'Mbps', 'Gbps', 'Tbps', 'Pbps', 'Ebps', 'Zbps', 'Ybps']; 12 | let l = 0, n = parseInt(bytes*8, 10) || 0; 13 | while (n >= 1024 && ++l) { 14 | n = n / 1024; 15 | } 16 | return (n.toFixed(l > 0 ? 2 : 0) + ' ' + units[l]); 17 | } 18 | 19 | export function formatNum(nm, fracDigits = 0) { 20 | const units = ['', 'k', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y']; 21 | let l = 0, n = parseFloat(nm) || 0; 22 | while (n >= 1000 && ++l) { 23 | n = n / 1000; 24 | } 25 | return (n.toFixed(l > 0 ? fracDigits : 0) + ' ' + units[l]).trim(); 26 | } 27 | 28 | export function formatNum6(bytes) { 29 | const units = ['', 'k', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y']; 30 | let l = 0, n = parseInt(bytes, 10) || 0; 31 | while (n >= 1000 && ++l) { 32 | n = n / 1000; 33 | } 34 | 35 | const intPart = Math.floor(n); 36 | const decimalPart = Math.floor((n - intPart) * 1000); 37 | const formattedDecimal = decimalPart > 0 ? `.${decimalPart}` : ''; 38 | 39 | return (intPart + formattedDecimal + ' ' + units[l]); 40 | } 41 | 42 | export function calcEMA(currentValue, prevEMA, smoothingFactor) { 43 | return smoothingFactor * currentValue + (1 - smoothingFactor) * prevEMA; 44 | } 45 | 46 | export function formatPercent(num) { 47 | return (num * 100).toFixed(1) + '%'; 48 | } 49 | 50 | export function formatFil(n) { 51 | if (n === 0) { 52 | return '0'; 53 | } 54 | 55 | const units = ['aFIL', 'fFIL', 'pFIL', 'nFIL', 'uFIL', 'mFIL', 'FIL']; 56 | let l = 0; 57 | while (l+1 < units.length && n >= 1000) { 58 | l++; 59 | n = n / 1000; 60 | } 61 | 62 | return (n.toFixed(n < 10 && l > 0 ? 1 : 0) + ' ' + units[l]); 63 | } 64 | 65 | export function epochToDuration(epochs) { 66 | const epochDuration = 30; // 30 seconds per epoch 67 | const totalSeconds = Math.abs(epochs) * epochDuration; 68 | 69 | const daysInMonth = 30; // Approximate number of days in a month 70 | const months = Math.floor(totalSeconds / (daysInMonth * 24 * 60 * 60)); 71 | const days = Math.floor((totalSeconds % (daysInMonth * 24 * 60 * 60)) / (24 * 60 * 60)); 72 | const hours = Math.floor((totalSeconds % (24 * 60 * 60)) / (60 * 60)); 73 | const minutes = Math.floor((totalSeconds % (60 * 60)) / 60); 74 | 75 | let duration = ""; 76 | if (months > 0) { 77 | duration += `${months}mo `; 78 | } 79 | if (days > 0) { 80 | duration += `${days}d `; 81 | } 82 | if (hours > 0) { 83 | duration += `${hours}h `; 84 | } 85 | if (minutes > 0) { 86 | duration += `${minutes}m`; 87 | } 88 | return epochs < 0 ? `${duration.trim()} ago` : `in ${duration.trim()}`; 89 | } 90 | 91 | export function epochToDate(epochs, referenceDate = new Date(Date.UTC(2020, 9, 15, 22, 0, 0))) { 92 | const epochDuration = 30; // 30 seconds per epoch 93 | const totalSeconds = epochs * epochDuration; 94 | 95 | const newDate = new Date(referenceDate.getTime() + totalSeconds * 1000).toISOString().split('T')[0]; 96 | return newDate; 97 | } 98 | 99 | export const avgMonthDays = 30.436875; 100 | export const epochToMonth = (60/30) * 60 * 24 * avgMonthDays; 101 | 102 | export function formatTimestamp(unixTimestamp, dateOnly) { 103 | const date = new Date(unixTimestamp * 1000); 104 | const formattedDate = date.toISOString().split('T')[0]; 105 | 106 | if (dateOnly) { 107 | return formattedDate; 108 | } 109 | 110 | const now = new Date(); 111 | let diff = date - now; 112 | let prefix = '', suffix = ''; 113 | 114 | if (diff < 0) { 115 | suffix = " ago"; 116 | diff = -diff; 117 | } else { 118 | prefix = "in "; 119 | } 120 | 121 | const daysInMonth = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]; 122 | const isLeapYear = (year) => (year % 4 === 0 && year % 100 !== 0) || year % 400 === 0; 123 | if (isLeapYear(date.getFullYear())) daysInMonth[1] = 29; 124 | 125 | const months = Math.floor(diff / (daysInMonth[date.getMonth()] * 24 * 60 * 60 * 1000)); 126 | diff -= months * daysInMonth[date.getMonth()] * 24 * 60 * 60 * 1000; 127 | 128 | const days = Math.floor(diff / (24 * 60 * 60 * 1000)); 129 | diff -= days * 24 * 60 * 60 * 1000; 130 | 131 | const hours = Math.floor(diff / (60 * 60 * 1000)); 132 | diff -= hours * 60 * 60 * 1000; 133 | 134 | const minutes = Math.floor(diff / (60 * 1000)); 135 | 136 | return `${formattedDate}, ${prefix}${months ? `${months}mo ` : ''}${days ? `${days}d ` : ''}${hours}h ${minutes}m${suffix}`; 137 | } 138 | 139 | -------------------------------------------------------------------------------- /ributil/robusthttp.go: -------------------------------------------------------------------------------- 1 | package ributil 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | logging "github.com/ipfs/go-log/v2" 7 | "golang.org/x/xerrors" 8 | "io" 9 | "net" 10 | "net/http" 11 | "time" 12 | ) 13 | 14 | type robustHttpResponse struct { 15 | getRC func() *RateCounter 16 | 17 | url string 18 | 19 | cur io.Reader 20 | curCloser io.Closer 21 | atOff, dataSize int64 22 | } 23 | 24 | func init() { 25 | logging.SetLogLevel("ributil", "DEBUG") 26 | } 27 | 28 | var maxRetryCount = 15 29 | 30 | func (r *robustHttpResponse) Read(p []byte) (n int, err error) { 31 | defer func() { 32 | r.atOff += int64(n) 33 | }() 34 | 35 | for i := 0; i < maxRetryCount; i++ { 36 | if r.cur == nil { 37 | log.Debugw("Current response is nil, starting new request") 38 | 39 | if err := r.startReq(); err != nil { 40 | log.Errorw("Error in startReq", "error", err, "i", i) 41 | time.Sleep(1 * time.Second) 42 | continue 43 | } 44 | } 45 | 46 | n, err = r.cur.Read(p) 47 | if err == io.EOF { 48 | r.curCloser.Close() 49 | r.cur = nil 50 | log.Errorw("EOF reached in Read", "bytesRead", n) 51 | return n, err 52 | } 53 | if err != nil { 54 | log.Errorw("Read error", "error", err) 55 | r.curCloser.Close() 56 | r.cur = nil 57 | 58 | if n > 0 { 59 | return n, nil 60 | } 61 | 62 | log.Errorw("robust http read error, will retry", "err", err, "i", i) 63 | continue 64 | } 65 | if n == 0 { 66 | r.curCloser.Close() 67 | r.cur = nil 68 | log.Errorw("Read 0 bytes", "bytesRead", n) 69 | return 0, xerrors.Errorf("read 0 bytes") 70 | } 71 | 72 | return n, nil 73 | } 74 | 75 | return 0, xerrors.Errorf("http read failed after %d retries", maxRetryCount) 76 | } 77 | 78 | func (r *robustHttpResponse) Close() error { 79 | log.Debug("Entering function Close") 80 | if r.curCloser != nil { 81 | return r.curCloser.Close() 82 | } 83 | log.Warnw("Exiting Close with no current closer") 84 | return nil 85 | } 86 | 87 | func (r *robustHttpResponse) startReq() error { 88 | log.Debugw("Entering function startReq", "url", r.url) 89 | dialer := &net.Dialer{ 90 | Timeout: 20 * time.Second, 91 | } 92 | 93 | var nc net.Conn 94 | 95 | client := &http.Client{ 96 | Transport: &http.Transport{ 97 | DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { 98 | log.Debugw("DialContext called", "network", network, "addr", addr) 99 | conn, err := dialer.DialContext(ctx, network, addr) 100 | if err != nil { 101 | log.Errorw("DialContext error", "error", err) 102 | return nil, err 103 | } 104 | 105 | nc = conn 106 | 107 | // Set a deadline for the whole operation, including reading the response 108 | if err := conn.SetReadDeadline(time.Now().Add(30 * time.Second)); err != nil { 109 | log.Errorw("SetReadDeadline error", "error", err) 110 | return nil, xerrors.Errorf("set deadline: %w", err) 111 | } 112 | 113 | return conn, nil 114 | }, 115 | }, 116 | } 117 | 118 | req, err := http.NewRequest("GET", r.url, nil) 119 | if err != nil { 120 | log.Errorw("failed to create request", "err", err) 121 | return xerrors.Errorf("failed to create request") 122 | } 123 | 124 | req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", r.atOff, r.dataSize-1)) 125 | 126 | log.Debugw("Before sending HTTP request", "url", r.url, "cr", fmt.Sprintf("bytes=%d-%d", r.atOff, r.dataSize)) 127 | resp, err := client.Do(req) 128 | if err != nil { 129 | log.Errorw("Error in client.Do", "error", err) 130 | return xerrors.Errorf("do request: %w", err) 131 | } 132 | 133 | if resp.StatusCode != http.StatusPartialContent && resp.StatusCode != http.StatusOK { 134 | log.Errorw("Unexpected HTTP status", "status", resp.StatusCode) 135 | resp.Body.Close() 136 | return xerrors.Errorf("http status: %d", resp.StatusCode) 137 | } 138 | 139 | if nc == nil { 140 | log.Errorw("Connection is nil after client.Do") 141 | resp.Body.Close() 142 | return xerrors.Errorf("nc was nil") 143 | } 144 | 145 | var reqTxIdleTimeout = 4 * time.Second 146 | 147 | dlRead := &readerDeadliner{ 148 | Reader: resp.Body, 149 | setDeadline: nc.SetReadDeadline, 150 | } 151 | 152 | rc := r.getRC() 153 | rw := NewRateEnforcingReader(dlRead, rc, reqTxIdleTimeout) 154 | 155 | r.cur = rw 156 | r.curCloser = funcCloser(func() error { 157 | log.Debugw("Closing response body") 158 | rc.release() 159 | return resp.Body.Close() 160 | }) 161 | 162 | log.Debugw("Exiting startReq with success") 163 | return nil 164 | } 165 | 166 | type funcCloser func() error 167 | 168 | func (fc funcCloser) Close() error { 169 | return fc() 170 | } 171 | 172 | func RobustGet(url string, dataSize int64, rcf func() *RateCounter) io.ReadCloser { 173 | return &robustHttpResponse{ 174 | getRC: rcf, 175 | url: url, 176 | dataSize: dataSize, 177 | } 178 | } 179 | 180 | type readerDeadliner struct { 181 | io.Reader 182 | setDeadline func(time.Time) error 183 | } 184 | 185 | func (rd *readerDeadliner) SetReadDeadline(t time.Time) error { 186 | return rd.setDeadline(t) 187 | } 188 | -------------------------------------------------------------------------------- /rbstor/group_storage.go: -------------------------------------------------------------------------------- 1 | package rbstor 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | iface "github.com/lotus-web3/ribs" 8 | "golang.org/x/xerrors" 9 | "github.com/lotus-web3/ribs/configuration" 10 | ) 11 | 12 | func (r *rbs) createGroup(ctx context.Context) (iface.GroupKey, *Group, error) { 13 | if err := r.ensureSpaceForGroup(ctx); err != nil { 14 | return 0, nil, xerrors.Errorf("ensure space for group: %w", err) 15 | } 16 | 17 | selectedGroup, err := r.db.CreateGroup() 18 | if err != nil { 19 | return iface.UndefGroupKey, nil, xerrors.Errorf("creating group: %w", err) 20 | } 21 | 22 | g, err := r.openGroup(ctx, selectedGroup, 0, 0, 0, iface.GroupStateWritable, true) 23 | if err != nil { 24 | return iface.UndefGroupKey, nil, xerrors.Errorf("opening group: %w", err) 25 | } 26 | 27 | return selectedGroup, g, nil 28 | } 29 | 30 | func (r *rbs) openGroup(ctx context.Context, group iface.GroupKey, blocks, bytes, jbhead int64, state iface.GroupState, create bool) (*Group, error) { 31 | g, err := OpenGroup(ctx, r.db, r.index, &r.staging, group, blocks, bytes, jbhead, r.root, state, create) 32 | if err != nil { 33 | return nil, xerrors.Errorf("opening group: %w", err) 34 | } 35 | 36 | if state == iface.GroupStateWritable { 37 | r.writableGroups[group] = g 38 | } 39 | r.openGroups[group] = g 40 | 41 | return g, nil 42 | } 43 | 44 | func (r *rbs) withWritableGroup(ctx context.Context, prefer iface.GroupKey, cb func(group *Group) error) (selectedGroup iface.GroupKey, err error) { 45 | r.lk.Lock() 46 | defer r.lk.Unlock() 47 | 48 | r.writeLk.Lock() 49 | defer r.writeLk.Unlock() 50 | 51 | defer func() { 52 | if err != nil || selectedGroup == iface.UndefGroupKey { 53 | return 54 | } 55 | // if the group was filled, drop it from writableGroups and start finalize 56 | if r.writableGroups[selectedGroup].state != iface.GroupStateWritable { 57 | delete(r.writableGroups, selectedGroup) 58 | 59 | r.tasks <- task{ 60 | tt: taskTypeFinalize, 61 | group: selectedGroup, 62 | } 63 | } 64 | }() 65 | 66 | // todo prefer 67 | for g, grp := range r.writableGroups { 68 | return g, cb(grp) 69 | } 70 | 71 | // no writable groups, try to open one 72 | 73 | selectedGroup = iface.UndefGroupKey 74 | { 75 | var blocks, bytes, jbhead int64 76 | var state iface.GroupState 77 | 78 | selectedGroup, blocks, bytes, jbhead, state, err = r.db.GetWritableGroup() 79 | if err != nil { 80 | return iface.UndefGroupKey, xerrors.Errorf("finding writable groups: %w", err) 81 | } 82 | 83 | if selectedGroup != iface.UndefGroupKey { 84 | g, err := r.openGroup(ctx, selectedGroup, blocks, bytes, jbhead, state, false) 85 | if err != nil { 86 | return iface.UndefGroupKey, xerrors.Errorf("opening group: %w", err) 87 | } 88 | 89 | return selectedGroup, cb(g) 90 | } 91 | } 92 | 93 | // no writable groups, create one 94 | 95 | selectedGroup, g, err := r.createGroup(ctx) 96 | if err != nil { 97 | return iface.UndefGroupKey, xerrors.Errorf("creating group: %w", err) 98 | } 99 | 100 | return selectedGroup, cb(g) 101 | } 102 | 103 | func (r *rbs) withReadableGroup(ctx context.Context, group iface.GroupKey, cb func(group *Group) error) (err error) { 104 | r.lk.Lock() 105 | 106 | // todo prefer 107 | if r.openGroups[group] != nil { 108 | r.lk.Unlock() 109 | return cb(r.openGroups[group]) 110 | } 111 | 112 | // not open, open it 113 | 114 | blocks, bytes, jbhead, state, err := r.db.OpenGroup(group) 115 | if err != nil { 116 | r.lk.Unlock() 117 | return xerrors.Errorf("getting group metadata: %w", err) 118 | } 119 | 120 | g, err := r.openGroup(ctx, group, blocks, bytes, jbhead, state, false) 121 | if err != nil { 122 | r.lk.Unlock() 123 | return xerrors.Errorf("opening group: %w", err) 124 | } 125 | 126 | r.resumeGroup(group) 127 | 128 | r.lk.Unlock() 129 | return cb(g) 130 | } 131 | 132 | func (r *rbs) ensureSpaceForGroup(ctx context.Context) error { 133 | localCount, err := r.db.CountNonOffloadedGroups() 134 | if err != nil { 135 | return xerrors.Errorf("counting non-offloaded groups: %w", err) 136 | } 137 | 138 | cfg := configuration.GetConfig() 139 | if localCount < cfg.Ribs.MaxLocalGroupCount { 140 | return nil 141 | } 142 | 143 | var offloadCandidate iface.GroupKey 144 | for { 145 | offloadCandidate, err = r.db.GetOffloadCandidate() 146 | if err != nil { 147 | return xerrors.Errorf("getting offload candidate: %w", err) 148 | } 149 | 150 | if offloadCandidate != iface.UndefGroupKey { 151 | break 152 | } 153 | 154 | log.Errorw("no offload candidate, waiting for space", "localCount", localCount) 155 | 156 | // wait 1 min, then try again 157 | r.lk.Unlock() 158 | 159 | select { 160 | case <-ctx.Done(): 161 | r.lk.Lock() 162 | return ctx.Err() 163 | case <-time.After(time.Minute): 164 | } 165 | 166 | r.lk.Lock() 167 | } 168 | 169 | log.Errorw("local space full, offloading group", "group", offloadCandidate) 170 | 171 | // release read side 172 | r.lk.Unlock() 173 | defer r.lk.Lock() 174 | 175 | return r.withReadableGroup(ctx, offloadCandidate, func(g *Group) error { 176 | err := g.offloadStaging() 177 | return err 178 | }) 179 | } 180 | -------------------------------------------------------------------------------- /integrations/web/ribswebapp/src/routes/Repair.js: -------------------------------------------------------------------------------- 1 | import React, {useEffect, useState, useRef} from 'react'; 2 | import RibsRPC from "../helpers/rpc"; 3 | import { formatBytesBinary, formatNum, epochToDate, epochToDuration, calcEMA } from "../helpers/fmt"; 4 | import { Group } from "./Groups"; 5 | import {Link} from "react-router-dom"; 6 | 7 | export default function Repair() { 8 | const [workerStates, setWorkerStates] = useState({}); 9 | const [queueStats, setQueueStats] = useState({}); 10 | const prevWorkerStatesRef = useRef({}); 11 | const prevFetchTimeRef = useRef(Date.now()); 12 | const rateEMARef = useRef({}); 13 | 14 | const smoothingFactor = 1 / 10; 15 | 16 | const fetchWorkerStates = async () => { 17 | try { 18 | const result = await RibsRPC.call("RepairStats", []); 19 | setWorkerStates(result); 20 | 21 | const qStats = await RibsRPC.call("RepairQueue", []); 22 | setQueueStats(qStats); 23 | } catch (error) { 24 | console.error("Error fetching worker states:", error); 25 | } 26 | }; 27 | 28 | useEffect(() => { 29 | const prevWorkerStates = prevWorkerStatesRef.current; 30 | const currentTime = Date.now(); 31 | const elapsedTime = (currentTime - prevFetchTimeRef.current) / 1000; 32 | 33 | for (const [key, workerState] of Object.entries(workerStates)) { 34 | const prevWorkerState = prevWorkerStates[key] || {}; 35 | if (prevWorkerState.FetchProgress === undefined) continue; 36 | 37 | const fetchProgressDelta = workerState.FetchProgress - prevWorkerState.FetchProgress; 38 | const currentFetchRate = fetchProgressDelta / elapsedTime; 39 | 40 | rateEMARef.current[key] = calcEMA( 41 | currentFetchRate, 42 | rateEMARef.current[key] || 0, 43 | smoothingFactor 44 | ); 45 | } 46 | 47 | prevFetchTimeRef.current = currentTime; 48 | prevWorkerStatesRef.current = { ...workerStates }; 49 | }, [workerStates]); 50 | 51 | useEffect(() => { 52 | fetchWorkerStates(); 53 | const intervalId = setInterval(fetchWorkerStates, 1000); 54 | 55 | return () => { 56 | clearInterval(intervalId); 57 | }; 58 | }, []); 59 | 60 | return ( 61 |
62 |
63 |

Repair

64 |
65 |
66 |

Repair Queue

67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 |
TotalAssigned
{queueStats.Total} Group(s){queueStats.Assigned} Group(s)
81 |
82 |
83 |

Worker States

84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | {Object.keys(workerStates).map((key) => { 98 | const workerState = workerStates[key]; 99 | const fetchRate = rateEMARef.current[key] || 0; 100 | return ( 101 | 102 | 103 | 104 | 105 | 106 | 107 | 112 | 113 | 114 | ); 115 | })} 116 | 117 |
WorkerGroupStateFetch SizeFetch RateFetch ProgressFetch URL
{key}{workerState.GroupKey}{workerState.State}{formatBytesBinary(workerState.FetchSize)} / {formatBytesBinary(workerState.FetchProgress)} {formatNum(workerState.FetchProgress / (workerState.FetchSize+1) * 100, 2)}%{formatBytesBinary(fetchRate)}/s 108 |
109 |
110 |
111 | 		{workerState.FetchUrl}
118 |
119 |
120 | ); 121 | } 122 | -------------------------------------------------------------------------------- /integrations/web/ribswebapp/build/static/css/main.d7b9c3ff.css: -------------------------------------------------------------------------------- 1 | body{-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Roboto,Oxygen,Ubuntu,Cantarell,Fira Sans,Droid Sans,Helvetica Neue,sans-serif}code{font-family:source-code-pro,Menlo,Monaco,Consolas,Courier New,monospace}*{box-sizing:border-box}body{font-family:Arial,sans-serif;margin:0}.Root{display:grid;grid-template-areas:"head head" "nav body";grid-template-columns:200px 1fr;grid-template-rows:50px 1fr;height:100vh}@media (max-width:768px){.Root{grid-template-areas:"head" "nav" "body";grid-template-columns:1fr;grid-template-rows:50px auto 1fr}}.Root-head{align-items:center;background-color:#4caf50;box-shadow:0 2px 4px rgba(0,0,0,.1),0 4px 6px rgba(0,0,0,.1);color:#fff;display:flex;grid-area:head;justify-content:space-between;padding:0 20px}.Root-head-logoname{font-size:24px;font-weight:700;margin-right:10px}.Root-head-path{flex-grow:1;font-size:18px;margin-right:10px}.Root-head-state{font-size:14px}.Root-nav{background-color:#f8f8f8;border-right:1px solid rgba(0,0,0,.1);box-shadow:2px 0 4px rgba(0,0,0,.1),4px 0 6px rgba(0,0,0,.1);display:flex;flex-direction:column;grid-area:nav;padding:20px}@media (max-width:768px){.Root-nav{grid-gap:20px;display:grid;gap:20px;grid-template-columns:repeat(auto-fill,minmax(100px,1fr));padding:20px 10px}}.Root-nav-item{margin-bottom:10px}.Root-nav-item a{color:#333;text-decoration:none;transition:color .3s ease}.Root-nav-item a:hover{color:#4caf50}.Root-body{grid-area:body;padding:20px}.button-ish,button{background-color:#4caf50;border:none;border-radius:3px;color:#fff;cursor:pointer;display:inline-block;font-size:1rem;margin-bottom:1rem;padding:8px 16px;text-align:center;text-decoration:none;transition-duration:.4s}.button-ish,button:hover{background-color:#45a049;color:#fff}.button-sm{font-size:.9rem;margin-bottom:0;margin-left:1px;padding:1px 2px}a{color:#333;text-decoration:none;transition:color .3s ease}a:hover{color:#4caf50}th{white-space:nowrap}.Status{padding:1rem}.status-grid{grid-gap:1rem;display:grid;grid-template-columns:repeat(auto-fit,minmax(300px,1fr))}.status-grid>div{background-color:#f5f5f5;border-radius:5px;box-shadow:0 2px 4px rgba(0,0,0,.1),0 4px 6px rgba(0,0,0,.1);padding:1.5rem;transition:all .3s ease}.status-grid>div:hover{box-shadow:0 6px 12px rgba(0,0,0,.1),0 12px 24px rgba(0,0,0,.1);-webkit-transform:translateY(-3px);transform:translateY(-3px)}.status-grid h2{margin-bottom:.5rem}@media screen and (max-width:768px){.status-grid{grid-template-columns:1fr}}.compact-table{border-collapse:collapse;font-size:.9rem;width:100%}.compact-table td,.compact-table th{border-bottom:1px solid #e0e0e0;padding:.3rem .5rem;text-align:left}.compact-table tr:last-child td{border-bottom:none}.important-metric{color:#4caf50;font-weight:700}.group{grid-gap:1rem;align-items:end;background-color:#f5f5f5;border-radius:4px;box-shadow:0 2px 4px rgba(0,0,0,.1),0 4px 6px rgba(0,0,0,.1);display:grid;gap:1rem;grid-template-columns:repeat(auto-fill,minmax(300px,1fr));margin-bottom:1rem;padding:1rem}.group h3{display:flex;justify-content:space-between}.group p{line-height:1.1;margin:.5rem 0}.group-state{font-size:1rem;font-weight:400}.progress-bar{background-color:#e0e0e0;border-radius:4px;height:8px;overflow:hidden}.thin-bar{height:3px;margin-top:1px}.progress-bar__fill{background-color:#4caf50;height:100%}.progress-bar__fill-red{background-color:#f44336}.group-info{align-items:center;background-color:#f5f5f5;border-radius:5px;box-shadow:0 2px 4px rgba(0,0,0,.1),0 4px 6px rgba(0,0,0,.1);grid-row:span 2;height:200px;line-height:1.1;padding:1rem;position:relative;transition:all .3s ease}.group-info p{line-height:1;margin:0}.deal-counts{margin-bottom:1rem}.deal-counts-seal{color:#4caf50;font-weight:700}.deal-counts-err{color:#f44336;font-weight:700}.deal-counts-start{color:#2196f3}.deal-error{color:#f44336}.pagination-btn{background-color:initial;border:1px solid #4caf50;border-radius:3px;color:inherit;margin-bottom:3px;margin-right:5px}.Deal{grid-gap:2px;align-items:center;background-color:#f5f5f5;border-radius:5px;box-shadow:0 2px 4px rgba(0,0,0,.1),0 4px 6px rgba(0,0,0,.1);display:grid;font-size:.8rem;gap:2px;grid-template-rows:repeat(4,1fr);height:100px;padding:1rem;position:relative;transition:all .3s ease}.Deal:hover{box-shadow:0 6px 12px rgba(0,0,0,.1),0 12px 24px rgba(0,0,0,.1);-webkit-transform:translateY(-3px);transform:translateY(-3px)}.deal-failed{background:#ffd6cc}.deal-sealed{background:#dfd}.deal-sealed-retr{background:#9f9}.deal-err{display:block;overflow:hidden;text-overflow:ellipsis}.Providers{margin:0 auto;width:100%}.providers-table{border-collapse:collapse;margin-top:1rem;width:100%}.providers-table td,.providers-table th{border-bottom:1px solid #ccc;padding:.5rem;text-align:left}.providers-table th{background-color:#f5f5f5;font-weight:700}.providers-table tbody tr:nth-child(odd){background-color:#f9f9f9}.stat-gray{color:#777}.providers-ask{font-family:monospace;font-size:12px}.providers-nodeal-longtime{color:#f44336}.provider-meta-table{width:auto}.provider-deals-error-col{color:#f44336;overflow:hidden}.provider-deals-error-col pre{word-wrap:break-word;max-height:100%;overflow:auto;white-space:pre-wrap}.prov-progress-container{display:flex;flex-direction:column}.provider-deals-nowrap{white-space:nowrap}.prov-text-with-progress{margin-bottom:1px} 2 | /*# sourceMappingURL=main.d7b9c3ff.css.map*/ -------------------------------------------------------------------------------- /integrations/web/ribswebapp/src/routes/Group.js: -------------------------------------------------------------------------------- 1 | import React, { useState, useEffect } from "react"; 2 | import RibsRPC from "../helpers/rpc"; 3 | import { formatBytesBinary, formatNum, epochToDate, epochToDuration } from "../helpers/fmt"; 4 | import "./Groups.css"; 5 | import "./Deal.css"; 6 | import {Deal, GroupStateWritable, GroupStateOffloaded, groupStateText} from "./Groups"; 7 | import {useParams} from "react-router-dom"; 8 | 9 | export default function Group() { 10 | let { groupKey } = useParams(); 11 | // groupkey to int 12 | groupKey = parseInt(groupKey) 13 | 14 | const [headHeight, setHeadHeight] = useState(0); 15 | const [group, setGroup] = useState({ 16 | Deals: [], 17 | }); 18 | 19 | let refreshing = false; 20 | 21 | const fetchGroup = async () => { 22 | try { 23 | if (refreshing) return; 24 | refreshing = true; 25 | 26 | let now = Date.now(); 27 | let sinceLastRefresh = now - group.lastRefresh; 28 | 29 | if (group.State === GroupStateWritable && sinceLastRefresh < 500) { 30 | // writable groups at most every 500ms 31 | return; 32 | } else if (group.State === GroupStateOffloaded && sinceLastRefresh < 120000) { 33 | // offloaded groups at most every 120s 34 | return; 35 | } else if (group.State !== GroupStateWritable && sinceLastRefresh < 10000) { 36 | // non-writable groups at most every 10s 37 | return; 38 | } 39 | 40 | console.log("fetching group", groupKey, sinceLastRefresh, refreshing) 41 | 42 | let meta = await RibsRPC.call("GroupMeta", [groupKey]); 43 | let deals = await RibsRPC.call("GroupDeals", [groupKey]); 44 | 45 | now = Date.now(); 46 | 47 | refreshing = false; 48 | let groupData = { ...meta, GroupKey: groupKey, Deals: deals, lastRefresh: now }; 49 | 50 | setGroup(groupData); 51 | 52 | const head = await RibsRPC.callFil("ChainHead"); 53 | setHeadHeight(head.Height); 54 | } catch (error) { 55 | console.error("Error fetching group:", error); 56 | } 57 | }; 58 | 59 | useEffect(() => { 60 | fetchGroup(); 61 | const intervalId = setInterval(fetchGroup, 500); 62 | 63 | return () => { 64 | clearInterval(intervalId); 65 | }; 66 | }, [groupKey]); 67 | 68 | const renderProgressBar = (bytes, maxBytes) => { 69 | const percentage = (bytes / maxBytes) * 100; 70 | return ( 71 |
72 |
76 |
77 | ); 78 | }; 79 | 80 | const dealCounts = group.Deals.reduce( 81 | (counts, deal) => { 82 | if (deal.Failed) counts.errors++; 83 | else if (deal.Sealed) counts.sealed++; 84 | else counts.started++; 85 | 86 | return counts; 87 | }, 88 | { started: 0, sealed: 0, errors: 0 } 89 | ); 90 | 91 | const dealsToDisplay = group.Deals 92 | 93 | return ( 94 |
95 |
96 |

97 | Group {group.GroupKey} 98 |

99 | State: {groupStateText[group.State]} 100 |

101 | Blocks: {formatNum(group.Blocks)} / {formatNum(group.MaxBlocks)} 102 |

103 | {group.State === GroupStateWritable && renderProgressBar(group.Blocks, group.MaxBlocks) } 104 |

105 | Bytes: {formatBytesBinary(group.Bytes)} /{" "} 106 | {formatBytesBinary(group.MaxBytes)} 107 |

108 | {group.State === GroupStateWritable && renderProgressBar(group.Bytes, group.MaxBytes) } 109 |
110 | {dealCounts.sealed > 0 && {dealCounts.sealed} Sealed | } 111 | {dealCounts.started > 0 && {dealCounts.started} Started | } 112 | {dealCounts.errors > 0 && {dealCounts.errors} Errored} 113 |
114 |
PieceCID: {group.PieceCID} [filecoin.tools]
115 |
RootCID: {group.RootCID}
116 |
117 |
118 | {dealsToDisplay.length > 0 && ( 119 | <> 120 | {dealsToDisplay.map((deal) => ( 121 | 122 | ))} 123 | 124 | )} 125 |
126 |
127 | ); 128 | } -------------------------------------------------------------------------------- /rbdeal/external.go: -------------------------------------------------------------------------------- 1 | package rbdeal 2 | 3 | import ( 4 | "context" 5 | iface "github.com/lotus-web3/ribs" 6 | "golang.org/x/xerrors" 7 | "io" 8 | ) 9 | 10 | type CarSource func(context.Context, iface.GroupKey, func(int64), io.Writer) error 11 | 12 | type ExternalOffloader interface { 13 | maybeInitExternal(r *ribs) (bool, error) 14 | GetModuleName() string 15 | EnsureExternalPush(gid iface.GroupKey, src CarSource) error 16 | GetGroupExternalURL(gid iface.GroupKey, lpath string) (*string, error) 17 | CleanExternal(gid iface.GroupKey, lpath string) error 18 | ReadCar(ctx context.Context, group iface.GroupKey, path string, off int64, size int64) (io.ReadCloser, error) 19 | ReadCarFile(ctx context.Context, group iface.GroupKey) (io.ReadSeekCloser, error) 20 | } 21 | 22 | func (r *ribs) maybeInitExternal() error { 23 | modules := []ExternalOffloader{ 24 | &S3OffloadInfo{}, 25 | &LocalWebInfo{}, 26 | } 27 | for _, module := range modules { 28 | found, err := module.maybeInitExternal(r) 29 | if err != nil { 30 | return err 31 | } 32 | if found { 33 | log.Infow("XYZ: External module configured", "name", module.GetModuleName()) 34 | r.externalOffloader = module 35 | return nil 36 | } 37 | } 38 | 39 | r.RBS.StagingStorage().InstallStagingProvider(&ribsStagingProvider{r: r}) 40 | 41 | return nil 42 | } 43 | 44 | func (r *ribs) maybeEnsureEnsureExternalPush(gid iface.GroupKey) error { 45 | mname, err := r.db.NeedExternalModule() 46 | if err != nil { 47 | return xerrors.Errorf("XYZ: External: fail to check if external required: %w", err) 48 | } 49 | if mname != nil { 50 | if r.externalOffloader == nil { 51 | return nil 52 | } 53 | if lmod := r.externalOffloader.GetModuleName(); lmod != *mname { 54 | return xerrors.Errorf("XYZ: External: need %s module, %s loaded", mname, lmod) 55 | } 56 | } 57 | module, _, err := r.db.GetExternalPath(gid) 58 | if err != nil { 59 | return xerrors.Errorf("XYZ: External: fail to get ext path: %w", err) 60 | } 61 | if module != nil { 62 | // already pushed 63 | return nil 64 | } 65 | 66 | if r.externalOffloader == nil { 67 | return nil 68 | } 69 | 70 | return (r.externalOffloader).EnsureExternalPush(gid, r.RBS.Storage().ReadCar) 71 | } 72 | 73 | func (r *ribs) maybeGetExternalURL(gid iface.GroupKey) (*string, error) { 74 | module, path, err := r.db.GetExternalPath(gid) 75 | if err != nil { 76 | return nil, xerrors.Errorf("XYZ: External: fail to get ext path: %w", err) 77 | } 78 | if module == nil { 79 | return nil, nil 80 | } 81 | if r.externalOffloader == nil { 82 | return nil, xerrors.Errorf("XYZ: External: offloaded to %s, but no module loaded", *module) 83 | } 84 | if lmod := r.externalOffloader.GetModuleName(); lmod != *module { 85 | return nil, xerrors.Errorf("XYZ: External: offloaded to %s, but %s module loaded", *module, lmod) 86 | } 87 | return r.externalOffloader.GetGroupExternalURL(gid, *path) 88 | } 89 | 90 | func (r *ribs) cleanupExternalOffload(gid iface.GroupKey) error { 91 | module, epath, err := r.db.GetExternalPath(gid) 92 | if err != nil { 93 | return xerrors.Errorf("XYZ: LocalWeb: Failed to get external path from db %w", err) 94 | } 95 | if epath == nil { 96 | // nothing to clean 97 | return nil 98 | } 99 | 100 | if r.externalOffloader == nil { 101 | return xerrors.Errorf("XYZ: External: offloaded to %s, but no module loaded", *module) 102 | } 103 | if lmod := r.externalOffloader.GetModuleName(); lmod != *module { 104 | return xerrors.Errorf("XYZ: External: offloaded to %s, but %s module loaded", *module, lmod) 105 | } 106 | return r.externalOffloader.CleanExternal(gid, *epath) 107 | } 108 | 109 | type ribsStagingProvider struct { 110 | r *ribs 111 | } 112 | 113 | // HasCar implements ribs.StagingStorageProvider. 114 | func (r *ribsStagingProvider) HasCar(ctx context.Context, group iface.GroupKey) (bool, error) { 115 | module, _, err := r.r.db.GetExternalPath(group) 116 | if err != nil { 117 | return false, xerrors.Errorf("XYZ: External: fail to get ext path: %w", err) 118 | } 119 | return module != nil, nil 120 | } 121 | 122 | // ReadCar implements ribs.StagingStorageProvider. 123 | func (r *ribsStagingProvider) ReadCar(ctx context.Context, group iface.GroupKey, off int64, size int64) (io.ReadCloser, error) { 124 | module, path, err := r.r.db.GetExternalPath(group) 125 | if err != nil { 126 | return nil, xerrors.Errorf("XYZ: External: fail to get ext path: %w", err) 127 | } 128 | 129 | if module == nil { 130 | return nil, xerrors.Errorf("XYZ: External: group %d has no external path", group) 131 | } 132 | 133 | if path == nil { 134 | return nil, xerrors.Errorf("XYZ: External: group %d has no external path", group) 135 | } 136 | 137 | if r.r.externalOffloader == nil { 138 | return nil, xerrors.Errorf("XYZ: External: offloaded to %s, but no module loaded", *module) 139 | } 140 | 141 | if lmod := r.r.externalOffloader.GetModuleName(); lmod != *module { 142 | return nil, xerrors.Errorf("XYZ: External: offloaded to %s, but %s module loaded", *module, lmod) 143 | } 144 | 145 | return r.r.externalOffloader.ReadCar(ctx, group, *path, off, size) 146 | } 147 | 148 | // Upload implements ribs.StagingStorageProvider. 149 | func (r *ribsStagingProvider) Upload(ctx context.Context, group iface.GroupKey, size int64, src func(writer io.Writer) error) error { 150 | return (r.r.externalOffloader).EnsureExternalPush(group, func(ctx context.Context, gk iface.GroupKey, f func(int64), w io.Writer) error { 151 | return src(w) 152 | }) 153 | } 154 | -------------------------------------------------------------------------------- /integrations/web/ribswebapp/build/icon/ico.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 16 | 36 | 43 | 50 | 57 | 64 | 65 | 67 | 71 | 78 | 84 | 91 | 98 | 99 | 103 | 110 | 116 | 123 | 130 | 131 | 135 | 142 | 148 | 155 | 162 | 163 | 167 | 174 | 180 | 187 | 194 | 195 | 196 | -------------------------------------------------------------------------------- /integrations/web/ribswebapp/public/icon/ico.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 16 | 36 | 43 | 50 | 57 | 64 | 65 | 67 | 71 | 78 | 84 | 91 | 98 | 99 | 103 | 110 | 116 | 123 | 130 | 131 | 135 | 142 | 148 | 155 | 162 | 163 | 167 | 174 | 180 | 187 | 194 | 195 | 196 | -------------------------------------------------------------------------------- /rbdeal/wallet.go: -------------------------------------------------------------------------------- 1 | package rbdeal 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/filecoin-project/go-state-types/big" 8 | "github.com/filecoin-project/lotus/api" 9 | "github.com/filecoin-project/lotus/api/client" 10 | "github.com/filecoin-project/lotus/chain/actors" 11 | marketactor "github.com/filecoin-project/lotus/chain/actors/builtin/market" 12 | "github.com/filecoin-project/lotus/chain/types" 13 | iface "github.com/lotus-web3/ribs" 14 | "golang.org/x/xerrors" 15 | 16 | "github.com/ipfs/go-cid" 17 | 18 | "github.com/filecoin-project/go-address" 19 | "github.com/filecoin-project/go-state-types/abi" 20 | "github.com/lotus-web3/ribs/configuration" 21 | ) 22 | 23 | func (r *ribs) MarketAdd(ctx context.Context, amount abi.TokenAmount) (cid.Cid, error) { 24 | r.msgSendLk.Lock() 25 | defer r.msgSendLk.Unlock() 26 | 27 | r.marketFundsLk.Lock() 28 | defer r.marketFundsLk.Unlock() 29 | 30 | gw, closer, err := client.NewGatewayRPCV1(ctx, r.lotusRPCAddr, nil) 31 | if err != nil { 32 | panic(err) 33 | } 34 | defer closer() 35 | 36 | w, err := r.wallet.GetDefault() 37 | if err != nil { 38 | return cid.Undef, xerrors.Errorf("getting default wallet: %w", err) 39 | } 40 | 41 | params, err := actors.SerializeParams(&w) 42 | if err != nil { 43 | return cid.Undef, err 44 | } 45 | 46 | m := &types.Message{ 47 | To: marketactor.Address, 48 | From: w, 49 | Value: amount, 50 | Method: marketactor.Methods.AddBalance, 51 | Params: params, 52 | } 53 | 54 | nc, err := gw.MpoolGetNonce(ctx, w) 55 | if err != nil { 56 | return cid.Cid{}, xerrors.Errorf("mpool get gas: %w", err) 57 | } 58 | 59 | m.Nonce = nc 60 | 61 | m, err = gw.GasEstimateMessageGas(ctx, m, nil, types.EmptyTSK) 62 | if err != nil { 63 | return cid.Cid{}, xerrors.Errorf("gas estimate message gas: %w", err) 64 | } 65 | 66 | sig, err := r.wallet.WalletSign(ctx, w, m.Cid().Bytes(), api.MsgMeta{ 67 | Type: api.MTChainMsg, 68 | }) 69 | if err != nil { 70 | return cid.Cid{}, xerrors.Errorf("signing message: %w", err) 71 | } 72 | 73 | sm := &types.SignedMessage{ 74 | Message: *m, 75 | Signature: *sig, 76 | } 77 | 78 | c, aerr := gw.MpoolPush(ctx, sm) 79 | if aerr != nil { 80 | return cid.Undef, aerr 81 | } 82 | 83 | log.Infow("add market balance", "cid", c, "amount", amount) 84 | 85 | return c, nil 86 | } 87 | 88 | func (r *ribs) MarketWithdraw(ctx context.Context, amount abi.TokenAmount) (cid.Cid, error) { 89 | //TODO implement me 90 | panic("implement me") 91 | } 92 | 93 | func (r *ribs) Withdraw(ctx context.Context, amount abi.TokenAmount, to address.Address) (cid.Cid, error) { 94 | //TODO implement me 95 | panic("implement me") 96 | } 97 | 98 | func (r *ribs) watchMarket(ctx context.Context) { 99 | defer close(r.marketWatchClosed) 100 | cfg := configuration.GetConfig() 101 | 102 | if !cfg.Wallet.AutoMarketBalance.GreaterThan(types.NewInt(0)) { 103 | log.Infow("AutoMarketBalance at 0, no need to watch market.") 104 | return 105 | } 106 | 107 | for { 108 | select { 109 | case <-r.close: 110 | return 111 | default: 112 | } 113 | 114 | i, err := r.WalletInfo() 115 | if err != nil { 116 | goto cooldown 117 | } 118 | { 119 | avail := types.BigSub(i.MarketBalanceDetailed.Escrow, i.MarketBalanceDetailed.Locked) 120 | 121 | if avail.GreaterThan(cfg.Wallet.MinMarketBalance) { 122 | goto cooldown 123 | } 124 | 125 | log.Infow("market balance low, topping up") 126 | 127 | toAdd := big.Sub(cfg.Wallet.AutoMarketBalance, avail) 128 | 129 | c, err := r.MarketAdd(ctx, toAdd) 130 | if err != nil { 131 | log.Errorw("error adding market funds", "error", err) 132 | goto cooldown 133 | } 134 | 135 | log.Infow("AUTO-ADDED MARKET FUNDS", "amount", types.FIL(toAdd), "msg", c) 136 | } 137 | 138 | cooldown: 139 | select { 140 | case <-r.close: 141 | return 142 | case <-time.After(2 * cfg.Wallet.UpgradeInterval): 143 | } 144 | } 145 | } 146 | 147 | func _must(err error, msgAndArgs ...interface{}) { 148 | if err != nil { 149 | if len(msgAndArgs) == 0 { 150 | panic(err) 151 | } 152 | panic(xerrors.Errorf(msgAndArgs[0].(string)+": %w", err)) 153 | } 154 | } 155 | 156 | func (r *ribs) WalletInfo() (iface.WalletInfo, error) { 157 | r.marketFundsLk.Lock() 158 | defer r.marketFundsLk.Unlock() 159 | 160 | cfg := configuration.GetConfig() 161 | if r.cachedWalletInfo != nil && time.Since(r.lastWalletInfoUpdate) < cfg.Wallet.UpgradeInterval { 162 | return *r.cachedWalletInfo, nil 163 | } 164 | 165 | addr, err := r.wallet.GetDefault() 166 | if err != nil { 167 | return iface.WalletInfo{}, xerrors.Errorf("get default wallet: %w", err) 168 | } 169 | 170 | ctx := context.TODO() 171 | 172 | gw, closer, err := client.NewGatewayRPCV1(ctx, r.lotusRPCAddr, nil) 173 | if err != nil { 174 | panic(err) 175 | } 176 | defer closer() 177 | 178 | b, err := gw.WalletBalance(ctx, addr) 179 | if err != nil { 180 | return iface.WalletInfo{}, xerrors.Errorf("get wallet balance: %w", err) 181 | } 182 | 183 | mb, err := gw.StateMarketBalance(ctx, addr, types.EmptyTSK) 184 | if err != nil { 185 | return iface.WalletInfo{}, xerrors.Errorf("get market balance: %w", err) 186 | } 187 | 188 | dc, err := gw.StateVerifiedClientStatus(ctx, addr, types.EmptyTSK) 189 | if err != nil { 190 | return iface.WalletInfo{}, xerrors.Errorf("get verified client status: %w", err) 191 | } 192 | 193 | id, err := gw.StateLookupID(ctx, addr, types.EmptyTSK) 194 | if err != nil { 195 | return iface.WalletInfo{}, xerrors.Errorf("get address id: %w", err) 196 | } 197 | 198 | wi := iface.WalletInfo{ 199 | Addr: addr.String(), 200 | IDAddr: id.String(), 201 | Balance: types.FIL(b).Short(), 202 | MarketBalance: types.FIL(mb.Escrow).Short(), 203 | MarketLocked: types.FIL(mb.Locked).Short(), 204 | MarketBalanceDetailed: mb, 205 | } 206 | 207 | if dc != nil { 208 | wi.DataCap = types.SizeStr(*dc) 209 | } 210 | 211 | r.cachedWalletInfo = &wi 212 | r.lastWalletInfoUpdate = time.Now() 213 | 214 | return wi, nil 215 | } 216 | -------------------------------------------------------------------------------- /rbdeal/external_http_path.go: -------------------------------------------------------------------------------- 1 | package rbdeal 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "os" 8 | "path" 9 | 10 | "github.com/google/uuid" 11 | iface "github.com/lotus-web3/ribs" 12 | "github.com/lotus-web3/ribs/configuration" 13 | "github.com/mitchellh/go-homedir" 14 | "golang.org/x/xerrors" 15 | ) 16 | 17 | type LocalWebInfo struct { 18 | name string 19 | path string 20 | url string 21 | r *ribs 22 | } 23 | 24 | const EXTERNAL_LOCALWEB = "local-web" 25 | 26 | func getLocalWebPath() (string, error) { 27 | cfg := configuration.GetConfig() 28 | p := cfg.External.Localweb.Path 29 | 30 | if p == "" { 31 | rdir, err := homedir.Expand(cfg.Ribs.DataDir) 32 | if err != nil { 33 | return "", xerrors.Errorf("XYZ: LocalWeb: failed to expand data dir: %w", err) 34 | } 35 | p = path.Join(rdir, "cardata") 36 | } 37 | 38 | if err := os.MkdirAll(p, 0755); err != nil { 39 | return "", xerrors.Errorf("XYZ: LocalWeb: failed to create cardata dir: %w", err) 40 | } 41 | 42 | return p, nil 43 | } 44 | 45 | func (lwi *LocalWebInfo) maybeInitExternal(r *ribs) (bool, error) { 46 | var err error 47 | cfg := configuration.GetConfig() 48 | 49 | lwi.name = EXTERNAL_LOCALWEB 50 | lwi.path, err = getLocalWebPath() 51 | if err != nil { 52 | return false, xerrors.Errorf("XYZ: LocalWeb: failed to get local web path: %w", err) 53 | } 54 | lwi.url = cfg.External.Localweb.Url 55 | 56 | lwi.r = r 57 | if lwi.url == "" { 58 | if lwi.path == "" && cfg.External.Localweb.BuiltinServer { 59 | return false, xerrors.Errorf("XYZ: LocalWeb: todo: path is not set, builtin server is enabled but url is not set: '%s' & '%s'", lwi.path, lwi.url) 60 | } 61 | return false, nil 62 | } 63 | 64 | return true, nil 65 | } 66 | 67 | func (lwi *LocalWebInfo) GetModuleName() string { 68 | return lwi.name 69 | } 70 | func setSizeNoop(_ int64) {} 71 | 72 | func (lwi *LocalWebInfo) EnsureExternalPush(gid iface.GroupKey, src CarSource) error { 73 | // generate random uuid 74 | fname := fmt.Sprintf("%d-%s.car", gid, uuid.New().String()) 75 | // ensure we don't have a file by that name on target 76 | target, err := getLocalWebPath() 77 | if err != nil { 78 | return xerrors.Errorf("XYZ: LocalWeb: failed to get local web path: %w", err) 79 | } 80 | target = path.Join(target, fname) 81 | if _, err := os.Stat(target); err == nil { 82 | return xerrors.Errorf("XYZ: LocalWeb: Random generated UUID for localweb cach already exists: %s", fname) 83 | } 84 | // Write the carfile there 85 | carfile, err := os.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666) 86 | if err != nil { 87 | return xerrors.Errorf("XYZ: opening carfile: %w", err) 88 | } 89 | 90 | ctx := context.TODO() 91 | 92 | ctx, cancel := context.WithCancel(ctx) 93 | defer cancel() 94 | 95 | err = src(ctx, gid, setSizeNoop, carfile) 96 | if err != nil { 97 | return xerrors.Errorf("XYZ: failed to dump carfile: %w", err) 98 | } 99 | if err := carfile.Sync(); err != nil { 100 | return xerrors.Errorf("XYZ: failed to sync carfile: %w", err) 101 | } 102 | if err := carfile.Close(); err != nil { 103 | return xerrors.Errorf("XYZ: failed to close carfile: %w", err) 104 | } 105 | 106 | // store file name in db 107 | err = lwi.r.db.AddExternalPath(gid, lwi.name, fname) 108 | if err != nil { 109 | return xerrors.Errorf("XYZ: LocalWeb: Failed to store localpath: %w", err) 110 | } 111 | 112 | return nil 113 | } 114 | func (lwi *LocalWebInfo) GetGroupExternalURL(gid iface.GroupKey, lpath string) (*string, error) { 115 | url := fmt.Sprintf("%s/%s", lwi.url, lpath) 116 | return &url, nil 117 | } 118 | 119 | func (lwi *LocalWebInfo) CleanExternal(gid iface.GroupKey, lpath string) error { 120 | // remove file in target 121 | target, err := getLocalWebPath() 122 | if err != nil { 123 | return xerrors.Errorf("XYZ: LocalWeb: failed to get local web path: %w", err) 124 | } 125 | target = path.Join(target, lpath) 126 | if err := os.Remove(target); err != nil { 127 | return xerrors.Errorf("XYZ: External: failed to remove carfile %s for group %d: %w", target, gid, err) 128 | } 129 | // remove entry from db 130 | if err := lwi.r.db.DropExternalPath(gid); err != nil { 131 | return xerrors.Errorf("XYZ: External: failed to remove external path from db for group %d: %w", gid, err) 132 | } 133 | return nil 134 | } 135 | 136 | func (lwi *LocalWebInfo) ReadCar(ctx context.Context, group iface.GroupKey, pathStr string, off int64, size int64) (io.ReadCloser, error) { 137 | target, err := getLocalWebPath() 138 | if err != nil { 139 | return nil, xerrors.Errorf("XYZ: LocalWeb: failed to get local web path: %w", err) 140 | } 141 | target = path.Join(target, pathStr) 142 | file, err := os.Open(target) 143 | if err != nil { 144 | return nil, xerrors.Errorf("XYZ: LocalWeb: failed to open carfile %s for group %d: %w", target, group, err) 145 | } 146 | 147 | // Seek to the offset 148 | _, err = file.Seek(off, io.SeekStart) 149 | if err != nil { 150 | file.Close() 151 | return nil, xerrors.Errorf("XYZ: LocalWeb: failed to seek carfile %s for group %d: %w", target, group, err) 152 | } 153 | 154 | // Wrap the file in an io.LimitedReader to restrict to 'size' bytes 155 | lr := io.LimitReader(file, size) 156 | 157 | // Return a ReadCloser that closes the underlying file when closed 158 | return struct { 159 | io.Reader 160 | io.Closer 161 | }{ 162 | Reader: lr, 163 | Closer: file, 164 | }, nil 165 | } 166 | 167 | func (lwi *LocalWebInfo) ReadCarFile(ctx context.Context, group iface.GroupKey) (io.ReadSeekCloser, error) { 168 | target, err := getLocalWebPath() 169 | if err != nil { 170 | return nil, xerrors.Errorf("XYZ: LocalWeb: failed to get local web path: %w", err) 171 | } 172 | mod, pstr, err := lwi.r.db.GetExternalPath(group) 173 | if err != nil { 174 | return nil, xerrors.Errorf("XYZ: LocalWeb: failed to get external path for group %d: %w", group, err) 175 | } 176 | if mod == nil || *mod != lwi.name { 177 | return nil, xerrors.Errorf("XYZ: LocalWeb: external path for group %d is not a local web path", group) 178 | } 179 | 180 | target = path.Join(target, *pstr) 181 | file, err := os.Open(target) 182 | if err != nil { 183 | return nil, xerrors.Errorf("XYZ: LocalWeb: failed to open carfile %s for group %d: %w", target, group, err) 184 | } 185 | 186 | return file, nil 187 | } 188 | -------------------------------------------------------------------------------- /integrations/ritool/claims.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "github.com/filecoin-project/go-address" 7 | "github.com/filecoin-project/go-state-types/abi" 8 | "github.com/filecoin-project/go-state-types/big" 9 | verifregtypes13 "github.com/filecoin-project/go-state-types/builtin/v13/verifreg" 10 | "github.com/filecoin-project/go-state-types/builtin/v9/verifreg" 11 | "github.com/filecoin-project/lotus/build" 12 | "github.com/filecoin-project/lotus/chain/actors" 13 | verifreg2 "github.com/filecoin-project/lotus/chain/actors/builtin/verifreg" 14 | "github.com/filecoin-project/lotus/chain/types" 15 | cliutil "github.com/filecoin-project/lotus/cli/util" 16 | "github.com/filecoin-project/lotus/lib/must" 17 | _ "github.com/mattn/go-sqlite3" 18 | "github.com/urfave/cli/v2" 19 | "golang.org/x/xerrors" 20 | "strings" 21 | ) 22 | 23 | var claimsExtendCmd = &cli.Command{ 24 | Name: "claims-to-extend", 25 | Usage: "Extend claims", 26 | ArgsUsage: "[ribs db]", 27 | Flags: []cli.Flag{ 28 | &cli.Int64Flag{ 29 | Name: "client-id", 30 | Required: true, 31 | }, 32 | }, 33 | Action: func(c *cli.Context) error { 34 | rdb, err := sql.Open("sqlite3", (c.Args().First())) 35 | if err != nil { 36 | return xerrors.Errorf("open db: %w", err) 37 | } 38 | 39 | client := abi.ActorID(c.Int64("client-id")) 40 | 41 | must.One(rdb.Exec("PRAGMA journal_mode=WAL;")) 42 | must.One(rdb.Exec("PRAGMA synchronous = normal")) 43 | 44 | var provs []address.Address 45 | 46 | { 47 | // SELECT DISTINCT provider_addr FROM deals WHERE sealed=1 48 | rows, err := rdb.Query("SELECT DISTINCT provider_addr FROM deals WHERE sealed=1") 49 | if err != nil { 50 | return xerrors.Errorf("query: %w", err) 51 | } 52 | defer rows.Close() 53 | 54 | for rows.Next() { 55 | var prov uint64 56 | if err := rows.Scan(&prov); err != nil { 57 | return xerrors.Errorf("scan: %w", err) 58 | } 59 | provs = append(provs, must.One(address.NewIDAddress(prov))) 60 | } 61 | if err := rows.Err(); err != nil { 62 | return xerrors.Errorf("rows: %w", err) 63 | } 64 | if err := rows.Close(); err != nil { 65 | return xerrors.Errorf("close: %w", err) 66 | } 67 | } 68 | 69 | fmt.Printf("Getting claims for %d providers\n", len(provs)) 70 | 71 | chain, closer, err := cliutil.GetGatewayAPI(c) 72 | if err != nil { 73 | return xerrors.Errorf("getting gateway api: %w", err) 74 | } 75 | defer closer() 76 | 77 | ctx := cliutil.ReqContext(c) 78 | 79 | claimOurs := func(claim verifreg.Claim) bool { 80 | return claim.Client == client 81 | } 82 | 83 | var nclaims int 84 | claims := map[address.Address]map[verifreg.ClaimId]verifreg.Claim{} 85 | 86 | for n, prov := range provs { 87 | fmt.Printf("Getting claims for %s (%d/%d)\n", prov, n, len(provs)) 88 | allProvClaims, err := chain.StateGetClaims(ctx, prov, types.EmptyTSK) 89 | if err != nil { 90 | return xerrors.Errorf("getting claims: %w", err) 91 | } 92 | 93 | for claimID, claim := range allProvClaims { 94 | if !claimOurs(claim) { 95 | continue 96 | } 97 | 98 | if _, ok := claims[prov]; !ok { 99 | claims[prov] = map[verifreg.ClaimId]verifreg.Claim{} 100 | } 101 | claims[prov][claimID] = claim 102 | nclaims++ 103 | } 104 | } 105 | 106 | fmt.Printf("Got %d claims, creating messages\n", nclaims) 107 | 108 | tmax := abi.ChainEpoch(verifregtypes13.MaximumVerifiedAllocationTerm) 109 | params := verifreg.ExtendClaimTermsParams{} 110 | 111 | var totalGas int64 112 | totalFee := big.Zero() 113 | 114 | var mkMessage func(params verifreg.ExtendClaimTermsParams) (*types.Message, error) 115 | mkMessage = func(params verifreg.ExtendClaimTermsParams) (*types.Message, error) { 116 | clientAddr, err := address.NewIDAddress(uint64(client)) 117 | if err != nil { 118 | return nil, err 119 | } 120 | 121 | enc, err := actors.SerializeParams(¶ms) 122 | if err != nil { 123 | return nil, err 124 | } 125 | 126 | m := &types.Message{ 127 | To: verifreg2.Address, 128 | From: clientAddr, 129 | Method: verifreg2.Methods.ExtendClaimTerms, 130 | Params: enc, 131 | Value: types.NewInt(0), 132 | } 133 | 134 | m, err = chain.GasEstimateMessageGas(ctx, m, nil, types.EmptyTSK) 135 | 136 | if (err != nil && strings.Contains(err.Error(), "call ran out of gas")) || m.GasLimit >= build.BlockGasLimit*4/5 { 137 | // estimate two messages 138 | p1 := params 139 | p2 := params 140 | 141 | p1.Terms = p1.Terms[:len(p1.Terms)/2] 142 | p2.Terms = p2.Terms[len(p2.Terms)/2:] 143 | 144 | _, err := mkMessage(p1) 145 | if err != nil { 146 | return nil, err 147 | } 148 | _, err = mkMessage(p2) 149 | if err != nil { 150 | return nil, err 151 | } 152 | return nil, nil 153 | } 154 | 155 | if err != nil { 156 | return nil, err 157 | } 158 | 159 | fmt.Printf("Estimate: %d Exts GasLimit=%d (%02d%% blk lim), GasFeeCap=%s, GasPremium=%s, Fee=%s\n", len(params.Terms), 160 | m.GasLimit, m.GasLimit*100/build.BlockGasLimit, m.GasFeeCap, m.GasPremium, types.FIL(m.RequiredFunds())) 161 | 162 | totalGas += m.GasLimit 163 | totalFee = types.BigAdd(totalFee, m.RequiredFunds()) 164 | 165 | return m, nil 166 | } 167 | 168 | for provider, cls := range claims { 169 | mid, err := address.IDFromAddress(provider) 170 | if err != nil { 171 | return xerrors.Errorf("getting provider id: %w", err) 172 | } 173 | 174 | for claimId, claim := range cls { 175 | if claim.TermMax == tmax { 176 | continue 177 | } 178 | 179 | params.Terms = append(params.Terms, verifreg.ClaimTerm{ 180 | Provider: abi.ActorID(mid), 181 | ClaimId: claimId, 182 | TermMax: tmax, 183 | }) 184 | 185 | if len(params.Terms) >= 1000 { 186 | if _, err := mkMessage(params); err != nil { 187 | return err 188 | } 189 | params.Terms = nil 190 | } 191 | } 192 | } 193 | 194 | if len(params.Terms) > 0 { 195 | if _, err := mkMessage(params); err != nil { 196 | return err 197 | } 198 | } 199 | 200 | fmt.Printf("Total gas: %d (%.2f blks)\n", totalGas, float64(totalGas)/float64(build.BlockGasLimit)) 201 | fmt.Printf("Total fee: %s\n", types.FIL(totalFee)) 202 | 203 | return nil 204 | }, 205 | } 206 | -------------------------------------------------------------------------------- /rbstor/group_finalize.go: -------------------------------------------------------------------------------- 1 | package rbstor 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "sync/atomic" 7 | "time" 8 | 9 | commcid "github.com/filecoin-project/go-fil-commcid" 10 | "github.com/ipfs/go-cid" 11 | iface "github.com/lotus-web3/ribs" 12 | "github.com/lotus-web3/ribs/carlog" 13 | "github.com/lotus-web3/ribs/ributil" 14 | "golang.org/x/xerrors" 15 | ) 16 | 17 | var globalCommpBytes atomic.Int64 18 | 19 | func (m *Group) Finalize(ctx context.Context) error { 20 | m.dataLk.Lock() 21 | defer m.dataLk.Unlock() 22 | 23 | if m.state != iface.GroupStateFull { 24 | return xerrors.Errorf("group not in state for finalization: %d", m.state) 25 | } 26 | 27 | if err := m.jb.MarkReadOnly(); err != nil && err != carlog.ErrReadOnly { 28 | return xerrors.Errorf("mark read-only: %w", err) 29 | } 30 | 31 | if err := m.jb.Finalize(ctx); err != nil { 32 | return xerrors.Errorf("finalize jbob: %w", err) 33 | } 34 | 35 | if err := m.advanceState(ctx, iface.GroupStateVRCARDone); err != nil { 36 | return xerrors.Errorf("mark level index dropped: %w", err) 37 | } 38 | 39 | return nil 40 | } 41 | 42 | func (m *Group) GenCommP() error { 43 | if m.state != iface.GroupStateVRCARDone { 44 | return xerrors.Errorf("group not in state for generating top CAR: %d", m.state) 45 | } 46 | 47 | cc := new(ributil.DataCidWriter) 48 | 49 | start := time.Now() 50 | 51 | commStatWr := &rateStatWriter{ 52 | w: cc, 53 | st: &globalCommpBytes, 54 | } 55 | defer commStatWr.done() 56 | 57 | carSize, root, err := m.writeCar(commStatWr) 58 | if err != nil { 59 | return xerrors.Errorf("write car: %w", err) 60 | } 61 | 62 | sum, err := cc.Sum() 63 | if err != nil { 64 | return xerrors.Errorf("sum car (size: %d): %w", carSize, err) 65 | } 66 | 67 | log.Infow("generated commP", "duration", time.Since(start), "commP", sum.PieceCID, "pps", sum.PieceSize, "mbps", float64(carSize)/time.Since(start).Seconds()/1024/1024) 68 | 69 | p, _ := commcid.CIDToDataCommitmentV1(sum.PieceCID) 70 | 71 | if err := m.setCommP(context.Background(), iface.GroupStateLocalReadyForDeals, p, int64(sum.PieceSize), root, carSize); err != nil { 72 | return xerrors.Errorf("set commP: %w", err) 73 | } 74 | 75 | return nil 76 | } 77 | 78 | func (m *Group) LoadFilCar(ctx context.Context, f io.Reader, sz int64) error { 79 | if m.state != iface.GroupStateOffloaded { 80 | return xerrors.Errorf("can't offload group in state %d", m.state) 81 | } 82 | 83 | if err := m.jb.LoadData(ctx, f, sz); err != nil { 84 | return xerrors.Errorf("load carlog data: %w", err) 85 | } 86 | 87 | if err := m.advanceState(context.Background(), iface.GroupStateReload); err != nil { 88 | return xerrors.Errorf("marking group as offloaded: %w", err) 89 | } 90 | 91 | return nil 92 | } 93 | 94 | func (m *Group) FinDataReload(ctx context.Context) error { 95 | log.Infow("FIN DATA RELOAD") 96 | 97 | if m.state != iface.GroupStateReload { 98 | return xerrors.Errorf("group not in state for finishing data reload: %d", m.state) 99 | } 100 | 101 | gm, err := m.db.GroupMeta(m.id) 102 | if err != nil { 103 | return xerrors.Errorf("getting group meta: %w", err) 104 | } 105 | 106 | if gm.DealCarSize == nil { 107 | return xerrors.Errorf("deal car size is nil!") 108 | } 109 | 110 | if err := m.jb.FinDataReload(context.Background(), gm.Blocks, *gm.DealCarSize); err != nil { 111 | return xerrors.Errorf("carlog finalize data reload: %w", err) 112 | } 113 | 114 | log.Infow("finished data reload", "group", m.id) 115 | 116 | if err := m.advanceState(ctx, iface.GroupStateLocalReadyForDeals); err != nil { 117 | return xerrors.Errorf("mark level index dropped: %w", err) 118 | } 119 | 120 | return nil 121 | } 122 | 123 | func (m *Group) advanceState(ctx context.Context, st iface.GroupState) error { 124 | m.dblk.Lock() 125 | defer m.dblk.Unlock() 126 | 127 | m.state = st 128 | 129 | // todo enter failed state on error 130 | return m.db.SetGroupState(ctx, m.id, st) 131 | } 132 | 133 | func (m *Group) setCommP(ctx context.Context, state iface.GroupState, commp []byte, paddedPieceSize int64, root cid.Cid, carSize int64) error { 134 | m.dblk.Lock() 135 | defer m.dblk.Unlock() 136 | 137 | m.state = state 138 | 139 | // todo enter failed state on error 140 | return m.db.SetCommP(ctx, m.id, state, commp, paddedPieceSize, root, carSize) 141 | } 142 | 143 | // offload completely removes local data 144 | func (m *Group) offload() error { 145 | m.offloaded.Store(1) 146 | // m.jb.Offload will wait for any in-progress writes to finish 147 | 148 | m.dataLk.Lock() 149 | defer m.dataLk.Unlock() 150 | 151 | if m.state != iface.GroupStateLocalReadyForDeals { 152 | return xerrors.Errorf("can't offload group in state %d", m.state) 153 | } 154 | 155 | if err := m.advanceState(context.Background(), iface.GroupStateOffloaded); err != nil { 156 | return xerrors.Errorf("marking group as offloaded: %w", err) 157 | } 158 | 159 | // TODO Offloading state 160 | 161 | err := m.jb.Offload() 162 | if err != nil && err != carlog.ErrAlreadyOffloaded { 163 | return xerrors.Errorf("offloading carlog: %w", err) 164 | } 165 | 166 | if err := m.db.WriteOffloadEntry(m.id); err != nil { 167 | return xerrors.Errorf("write offload entry: %w", err) 168 | } 169 | 170 | return nil 171 | } 172 | 173 | // offloadStaging removes local data, reads will be redirected to staging 174 | func (m *Group) offloadStaging() error { 175 | m.dataLk.Lock() 176 | defer m.dataLk.Unlock() 177 | 178 | if m.state != iface.GroupStateLocalReadyForDeals { 179 | return xerrors.Errorf("can't offload group in state %d", m.state) 180 | } 181 | 182 | err := m.jb.OffloadData() 183 | if err != nil { 184 | return xerrors.Errorf("offloading carlog data: %w", err) 185 | } 186 | 187 | if err := m.db.WriteOffloadEntry(m.id); err != nil { 188 | return xerrors.Errorf("write offload entry: %w", err) 189 | } 190 | 191 | return nil 192 | } 193 | 194 | type rateStatWriter struct { 195 | w io.Writer 196 | 197 | st *atomic.Int64 198 | t int64 199 | } 200 | 201 | func (r *rateStatWriter) Write(p []byte) (n int, err error) { 202 | n, err = r.w.Write(p) 203 | 204 | r.t += int64(n) 205 | if r.t > 1<<23 { 206 | r.st.Add(r.t) 207 | r.t = 0 208 | } 209 | 210 | return 211 | } 212 | 213 | func (r *rateStatWriter) done() { 214 | r.st.Add(r.t) 215 | r.t = 0 216 | } 217 | 218 | var _ io.Writer = &rateStatWriter{} 219 | -------------------------------------------------------------------------------- /iface_rbs.go: -------------------------------------------------------------------------------- 1 | package ribs 2 | 3 | import ( 4 | "context" 5 | "io" 6 | 7 | blocks "github.com/ipfs/go-block-format" 8 | "github.com/ipfs/go-cid" 9 | "github.com/multiformats/go-multihash" 10 | ) 11 | 12 | type GroupKey = int64 13 | 14 | const UndefGroupKey = GroupKey(-1) 15 | 16 | // User 17 | 18 | type RBS interface { 19 | Start() error 20 | 21 | Session(ctx context.Context) Session 22 | Storage() Storage 23 | StorageDiag() RBSDiag 24 | 25 | // ExternalStorage manages offloaded data 26 | ExternalStorage() RBSExternalStorage 27 | 28 | // StagingStorage manages staged data (full non-replicated data) 29 | StagingStorage() RBSStagingStorage 30 | 31 | io.Closer 32 | } 33 | 34 | // Batch groups operations, NOT thread safe 35 | type Batch interface { 36 | // View is like See Session.View, and all constraints apply, but with batch 37 | // operations applied 38 | // todo: is this useful, is this making things too complicated? is this disabling some optimisations? 39 | //View(ctx context.Context, c []cid.Cid, cb func(cidx int, data []byte)) error 40 | 41 | // Put queues writes to the blockstore 42 | Put(ctx context.Context, b []blocks.Block) error 43 | 44 | // Unlink makes a blocks not retrievable from the blockstore 45 | // NOTE: this method is best-effort. Data may not be removed immediately, 46 | // and it may be retrievable even after the operation is committed 47 | // In case of conflicts, Put operation will be preferred over Unlink 48 | Unlink(ctx context.Context, c []multihash.Multihash) error 49 | 50 | // Flush commits data to the blockstore. The batch can be reused after commit 51 | Flush(ctx context.Context) error 52 | 53 | // todo? Fork(ctx) (Batch,error) for threaded 54 | } 55 | 56 | // Session groups correlated IO operations; thread safa 57 | type Session interface { 58 | // View attempts to read a list of cids 59 | // NOTE: 60 | // * Callback calls can happen out of order 61 | // * Callback calls can happen in parallel 62 | // * Callback will not be called for indexes where data is not found 63 | // * Callback `data` must not be referenced after the function returns 64 | // If the data is to be used after returning from the callback, it MUST be copied. 65 | View(ctx context.Context, c []multihash.Multihash, cb func(cidx int, data []byte)) error 66 | 67 | // -1 means not found 68 | GetSize(ctx context.Context, c []multihash.Multihash, cb func([]int32) error) error 69 | 70 | Batch(ctx context.Context) Batch 71 | } 72 | 73 | type Storage interface { 74 | FindHashes(ctx context.Context, hashes multihash.Multihash) ([]GroupKey, error) 75 | 76 | ReadCar(ctx context.Context, group GroupKey, sz func(int64), out io.Writer) error 77 | 78 | // HashSample returns a sample of hashes from the group saved when the group was finalized 79 | HashSample(ctx context.Context, group GroupKey) ([]multihash.Multihash, error) 80 | 81 | DescibeGroup(ctx context.Context, group GroupKey) (GroupDesc, error) 82 | 83 | Offload(ctx context.Context, group GroupKey) error 84 | 85 | LoadFilCar(ctx context.Context, group GroupKey, f io.Reader, sz int64) error 86 | 87 | Subscribe(GroupSub) 88 | } 89 | 90 | type GroupDesc struct { 91 | RootCid, PieceCid cid.Cid 92 | CarSize int64 93 | } 94 | 95 | type OffloadLoader interface { 96 | View(ctx context.Context, g GroupKey, c []multihash.Multihash, cb func(cidx int, data []byte)) error 97 | } 98 | 99 | type GroupSub func(group GroupKey, from, to GroupState) 100 | 101 | type RBSDiag interface { 102 | Groups() ([]GroupKey, error) 103 | GroupMeta(gk GroupKey) (GroupMeta, error) 104 | 105 | TopIndexStats(context.Context) (TopIndexStats, error) 106 | GetGroupStats() (*GroupStats, error) 107 | GroupIOStats() GroupIOStats 108 | 109 | WorkerStats() WorkerStats 110 | } 111 | 112 | type WorkerStats struct { 113 | Available, InFinalize, InCommP, InReload int64 114 | TaskQueue int64 115 | 116 | CommPBytes int64 117 | } 118 | 119 | /* Deal diag */ 120 | 121 | type GroupMeta struct { 122 | State GroupState 123 | 124 | MaxBlocks int64 125 | MaxBytes int64 126 | 127 | Blocks int64 128 | Bytes int64 129 | 130 | ReadBlocks, ReadBytes int64 131 | WriteBlocks, WriteBytes int64 132 | 133 | PieceCID, RootCID string 134 | 135 | DealCarSize *int64 // todo move to DescribeGroup 136 | } 137 | 138 | type GroupStats struct { 139 | GroupCount int64 140 | TotalDataSize int64 141 | NonOffloadedDataSize int64 142 | OffloadedDataSize int64 143 | 144 | OpenGroups, OpenWritable int 145 | } 146 | 147 | type GroupIOStats struct { 148 | ReadBlocks, ReadBytes int64 149 | WriteBlocks, WriteBytes int64 150 | } 151 | 152 | type TopIndexStats struct { 153 | Entries int64 154 | Writes, Reads int64 155 | } 156 | 157 | /* Storage internal */ 158 | 159 | // Index is the top level index, thread safe 160 | type Index interface { 161 | // GetGroups gets group ids for the multihashes 162 | GetGroups(ctx context.Context, mh []multihash.Multihash, cb func(cidx int, gk GroupKey) (more bool, err error)) error 163 | GetSizes(ctx context.Context, mh []multihash.Multihash, cb func([]int32) error) error 164 | 165 | AddGroup(ctx context.Context, mh []multihash.Multihash, sizes []int32, group GroupKey) error 166 | 167 | Sync(ctx context.Context) error 168 | DropGroup(ctx context.Context, mh []multihash.Multihash, group GroupKey) error 169 | EstimateSize(ctx context.Context) (int64, error) 170 | 171 | io.Closer 172 | } 173 | 174 | type GroupState int // todo move to rbstore? 175 | 176 | const ( 177 | GroupStateWritable GroupState = iota 178 | GroupStateFull 179 | GroupStateVRCARDone 180 | 181 | GroupStateLocalReadyForDeals 182 | GroupStateOffloaded 183 | 184 | GroupStateReload 185 | ) 186 | 187 | type RBSExternalStorage interface { 188 | InstallProvider(ExternalStorageProvider) 189 | } 190 | 191 | type ExternalStorageProvider interface { 192 | FetchBlocks(ctx context.Context, group GroupKey, mh []multihash.Multihash, cb func(cidx int, data []byte)) error 193 | } 194 | 195 | type RBSStagingStorage interface { 196 | InstallStagingProvider(StagingStorageProvider) 197 | } 198 | 199 | type StagingStorageProvider interface { 200 | Upload(ctx context.Context, group GroupKey, size int64, src func(writer io.Writer) error) error 201 | ReadCar(ctx context.Context, group GroupKey, off, size int64) (io.ReadCloser, error) 202 | HasCar(ctx context.Context, group GroupKey) (bool, error) 203 | } 204 | -------------------------------------------------------------------------------- /integrations/web/ribswebapp/src/routes/Providers.js: -------------------------------------------------------------------------------- 1 | import React, { useState, useEffect } from "react"; 2 | import RibsRPC from "../helpers/rpc"; 3 | import { formatBytesBinary, formatFil, epochToMonth, formatTimestamp } from "../helpers/fmt"; 4 | import "./Providers.css"; 5 | import {Link} from "react-router-dom"; 6 | 7 | function Providers() { 8 | const [providers, setProviders] = useState([]); 9 | 10 | const fetchProviders = async () => { 11 | try { 12 | const providerData = await RibsRPC.call("ReachableProviders"); 13 | setProviders(providerData.reverse()); 14 | } catch (error) { 15 | console.error("Error fetching providers:", error); 16 | } 17 | }; 18 | 19 | useEffect(() => { 20 | fetchProviders(); 21 | const intervalId = setInterval(fetchProviders, 5000); 22 | 23 | return () => { 24 | clearInterval(intervalId); 25 | }; 26 | }, []); 27 | 28 | const calculateDealPercentage = (part, total) => { 29 | if(total === 0){ 30 | return 0; 31 | } 32 | 33 | return Math.round((part / total) * 100); 34 | }; 35 | 36 | const generateCSVData = (providers) => { 37 | let csvRows = []; 38 | // Add headers 39 | csvRows.push("ID,DealStarted,DealSuccess,DealFail,DealRejected,MostRecentDealStart,RetrievPercent"); 40 | 41 | // Add data 42 | providers.forEach((provider) => { 43 | csvRows.push( 44 | `f0${provider.ID},${provider.DealStarted},${provider.DealSuccess},${provider.DealFail},${provider.DealRejected},${formatTimestamp(provider.MostRecentDealStart, true)},${calculateDealPercentage(provider.RetrievDeals,provider.UnretrievDeals+provider.RetrievDeals)}` 45 | ); 46 | }); 47 | 48 | return csvRows.join("\n"); 49 | }; 50 | 51 | const downloadCSV = () => { 52 | const csvData = generateCSVData(providers); 53 | const blob = new Blob([csvData], { type: "text/csv" }); 54 | const url = window.URL.createObjectURL(blob); 55 | const a = document.createElement("a"); 56 | a.setAttribute("hidden", ""); 57 | a.setAttribute("href", url); 58 | a.setAttribute("download", "providers.csv"); 59 | document.body.appendChild(a); 60 | a.click(); 61 | document.body.removeChild(a); 62 | }; 63 | 64 | return ( 65 |
66 |

Providers

67 | 68 | 69 | 70 | 71 | 72 | 73 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | {providers.map((provider, i) => ( 88 | 89 | 90 | 91 | 95 | 98 | 99 | 105 | 111 | 117 | 123 | 126 | 127 | ))} 128 | 129 |
Address 74 |
Piece Sizes
75 |
Price
76 | 		FeaturesStartedRejectedFailedSealedRetrievLast Deal Start
{i+1}.f0{provider.ID} 92 |
{`${formatBytesBinary(provider.AskMinPieceSize)} to ${formatBytesBinary(provider.AskMaxPieceSize)}`}
93 |
{`${formatFil(provider.AskPrice * epochToMonth)} (${formatFil(provider.AskVerifiedPrice * epochToMonth)})`}
94 | 		96 | {`${provider.BoosterHttp ? "http " : ""} ${provider.BoosterBitswap ? "bitswap" : ""}`.trim()} 97 | {provider.DealStarted} 100 | {provider.DealRejected} 101 |
102 |
103 |
104 | 		106 | {provider.DealFail} 107 |
108 |
109 |
110 | 		112 | {provider.DealSuccess} 113 |
114 |
115 |
116 | 		118 | {provider.RetrievDeals} ({calculateDealPercentage(provider.RetrievDeals, provider.UnretrievDeals+provider.RetrievDeals)}%) 119 |
120 |
121 |
122 | 		80000000 ? " providers-nodeal-longtime" : "") }> 124 | {provider.MostRecentDealStart === 0 ? "Never" : formatTimestamp(provider.MostRecentDealStart)} 125 |
130 |
131 | ); 132 | } 133 | 134 | export default Providers; 135 | --------------------------------------------------------------------------------