├── .env.example ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── custom.md │ └── feature_request.md └── workflows │ └── codeql.yml ├── .gitignore ├── .gitmodules ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── api ├── admin.go ├── deal.go ├── debug_profiler.go ├── health_check.go ├── metrics.go ├── node.go ├── open_info.go ├── open_stats.go ├── repair_retry.go ├── router.go ├── stats.go └── websocket.go ├── cmd ├── admin.go ├── car.go ├── commp.go ├── daemon.go ├── deal.go ├── delta_cmd_node.go ├── sp.go ├── status.go └── wallet.go ├── config └── config.go ├── core ├── commp.go ├── commp_test.go ├── deal_status.go ├── fastcommp.go ├── job.go ├── job_test.go ├── libp2p.go ├── libp2p_test.go ├── miner_assignment.go ├── node.go ├── repair.go ├── repair_test.go ├── replication.go ├── stats.go ├── stats_test.go ├── status_logger.go ├── wallet.go ├── wallet_test.go └── websocket.go ├── docker-compose.yml ├── docker ├── docker-compose.yml └── run.sh ├── docs ├── README.md ├── cli.md ├── content-deal-status.md ├── deal-metadata.md ├── deployment-modes.md ├── generate-swagger.md ├── getting-estuary-api-key.md ├── getting-started-run-delta-on-calibnet.md ├── getting-started-run-delta.md ├── getting-started-use-delta.md ├── make-batch-import-deal.md ├── make-e2e-deal.md ├── make-import-deal.md ├── manage-wallets.md ├── metrics-collection-telemetry.md ├── node-information.md ├── offline-signing.md ├── open-stats-info.md ├── process-flow-piece-commitment-compute.md ├── process-flow-storage-deal.md ├── repair-retry.md ├── running-delta-docker.md └── swagger │ ├── docs.go │ ├── swagger.json │ └── swagger.yaml ├── go.mod ├── go.sum ├── jobs ├── clean_up.go ├── clean_up_content.go ├── data_transfer_restart.go ├── data_transfer_status.go ├── deal_status_check.go ├── instance_meta.go ├── log_event.go ├── miner_check.go ├── piece_commp_compute.go ├── processor.go ├── repair.go ├── retry.go └── storage_deal_maker.go ├── main.go ├── metrics └── metrics.go ├── models ├── base.go ├── batch_import_content.go ├── content.go ├── content_deal.go ├── content_deal_proposal.go ├── content_deal_proposal_parameters.go ├── content_miner.go ├── content_wallet.go ├── database.go ├── instance_meta.go ├── log_event.go ├── log_models.go ├── piece_commitment.go ├── repair_request.go └── wallet.go ├── plugins ├── interface.go └── registry.go └── utils ├── chunk.go ├── constants.go ├── encode.go ├── epoch_height.go ├── file.go ├── model_utils.go └── status_change.go /.env.example: -------------------------------------------------------------------------------- 1 | # Node info 2 | NODE_NAME=delta-node 3 | NODE_DESCRIPTION=Experimental Deal Maker 4 | NODE_TYPE=delta-main 5 | 6 | # Database configuration 7 | MODE=standalone # HA 8 | DB_DSN=delta.db 9 | DELTA_AUTH=[REDACTED] 10 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **Delta Info** 14 | Please provide the list of delta uuids. You can get this info by calling the endpoint: 15 | ``` 16 | http://host/open/node/uuids 17 | ``` 18 | 19 | 20 | **To Reproduce** 21 | Steps to reproduce the behavior: 22 | 1. Go to '...' 23 | 2. Click on '....' 24 | 3. Scroll down to '....' 25 | 4. See error 26 | 27 | **Expected behavior** 28 | A clear and concise description of what you expected to happen. 29 | 30 | **Screenshots** 31 | If applicable, add screenshots to help explain your problem. 32 | 33 | **Desktop (please complete the following information):** 34 | - OS: [e.g. iOS] 35 | - Browser [e.g. chrome, safari] 36 | - Version [e.g. 22] 37 | 38 | **Smartphone (please complete the following information):** 39 | - Device: [e.g. iPhone6] 40 | - OS: [e.g. iOS8.1] 41 | - Browser [e.g. stock browser, safari] 42 | - Version [e.g. 22] 43 | 44 | **Additional context** 45 | Add any other context about the problem here. 46 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/custom.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Custom issue template 3 | about: Describe this issue template's purpose here. 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | 11 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/workflows/codeql.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ "main" ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ "main" ] 20 | schedule: 21 | - cron: '22 12 * * 3' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: [ 'go' ] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 37 | # Use only 'java' to analyze code written in Java, Kotlin or both 38 | # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both 39 | # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support 40 | 41 | steps: 42 | - name: Checkout repository 43 | uses: actions/checkout@v3 44 | 45 | # Initializes the CodeQL tools for scanning. 46 | - name: Initialize CodeQL 47 | uses: github/codeql-action/init@v2 48 | with: 49 | languages: ${{ matrix.language }} 50 | # If you wish to specify custom queries, you can do so here or in a config file. 51 | # By default, queries listed here will override any specified in a config file. 52 | # Prefix the list here with "+" to use these queries and those in the config file. 53 | 54 | # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs 55 | # queries: security-extended,security-and-quality 56 | 57 | 58 | # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). 59 | # If this step fails, then you should remove it and run the build manually (see below) 60 | - name: Autobuild 61 | uses: github/codeql-action/autobuild@v2 62 | 63 | # ℹ️ Command-line programs to run using the OS shell. 64 | # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun 65 | 66 | # If the Autobuild fails above, remove it and uncomment the following three lines. 67 | # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. 68 | 69 | # - run: | 70 | # echo "Run, Build Application using script" 71 | # ./location_of_script_within_repo/buildscript.sh 72 | 73 | - name: Perform CodeQL Analysis 74 | uses: github/codeql-action/analyze@v2 75 | with: 76 | category: "/language:${{matrix.language}}" 77 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | libp2p.key 3 | .whypfs 4 | datastore 5 | wallet 6 | wallet_BU 7 | .env 8 | scripts 9 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "extern/filecoin-ffi"] 2 | path = extern/filecoin-ffi 3 | url = https://github.com/filecoin-project/filecoin-ffi.git 4 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, religion, or sexual identity 10 | and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | * Demonstrating empathy and kindness toward other people 21 | * Being respectful of differing opinions, viewpoints, and experiences 22 | * Giving and gracefully accepting constructive feedback 23 | * Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | * Focusing on what is best not just for us as individuals, but for the 26 | overall community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | * The use of sexualized language or imagery, and sexual attention or 31 | advances of any kind 32 | * Trolling, insulting or derogatory comments, and personal or political attacks 33 | * Public or private harassment 34 | * Publishing others' private information, such as a physical or email 35 | address, without their explicit permission 36 | * Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement at 63 | outercore@protocol.ai. 64 | All complaints will be reviewed and investigated promptly and fairly. 65 | 66 | All community leaders are obligated to respect the privacy and security of the 67 | reporter of any incident. 68 | 69 | ## Enforcement Guidelines 70 | 71 | Community leaders will follow these Community Impact Guidelines in determining 72 | the consequences for any action they deem in violation of this Code of Conduct: 73 | 74 | ### 1. Correction 75 | 76 | **Community Impact**: Use of inappropriate language or other behavior deemed 77 | unprofessional or unwelcome in the community. 78 | 79 | **Consequence**: A private, written warning from community leaders, providing 80 | clarity around the nature of the violation and an explanation of why the 81 | behavior was inappropriate. A public apology may be requested. 82 | 83 | ### 2. Warning 84 | 85 | **Community Impact**: A violation through a single incident or series 86 | of actions. 87 | 88 | **Consequence**: A warning with consequences for continued behavior. No 89 | interaction with the people involved, including unsolicited interaction with 90 | those enforcing the Code of Conduct, for a specified period of time. This 91 | includes avoiding interactions in community spaces as well as external channels 92 | like social media. Violating these terms may lead to a temporary or 93 | permanent ban. 94 | 95 | ### 3. Temporary Ban 96 | 97 | **Community Impact**: A serious violation of community standards, including 98 | sustained inappropriate behavior. 99 | 100 | **Consequence**: A temporary ban from any sort of interaction or public 101 | communication with the community for a specified period of time. No public or 102 | private interaction with the people involved, including unsolicited interaction 103 | with those enforcing the Code of Conduct, is allowed during this period. 104 | Violating these terms may lead to a permanent ban. 105 | 106 | ### 4. Permanent Ban 107 | 108 | **Community Impact**: Demonstrating a pattern of violation of community 109 | standards, including sustained inappropriate behavior, harassment of an 110 | individual, or aggression toward or disparagement of classes of individuals. 111 | 112 | **Consequence**: A permanent ban from any sort of public interaction within 113 | the community. 114 | 115 | ## Attribution 116 | 117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 118 | version 2.0, available at 119 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. 120 | 121 | Community Impact Guidelines were inspired by [Mozilla's code of conduct 122 | enforcement ladder](https://github.com/mozilla/diversity). 123 | 124 | [homepage]: https://www.contributor-covenant.org 125 | 126 | For answers to common questions about this code of conduct, see the FAQ at 127 | https://www.contributor-covenant.org/faq. Translations are available at 128 | https://www.contributor-covenant.org/translations. 129 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Delta 2 | 3 | Outercore Engineering Team welcomes contributions to our [open source projects on Github](https://github.com/application-research). 4 | 5 | Please refer to each project's style and contribution guidelines for submitting patches and additions. In general, we follow the "fork-and-pull" Git workflow. 6 | 7 | 1. **Fork** the repo on GitHub 8 | 2. **Clone** the project to your own machine 9 | 3. **Commit** changes to your own branch 10 | 4. **Push** your work back up to your fork 11 | 5. Submit a **Pull request** so that we can review your changes 12 | 13 | NOTE: Be sure to merge the latest from "upstream" before making a pull request! 14 | 15 | ## Issues 16 | 17 | Feel free to use the defined templates submit issues and enhancement requests. 18 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Delta Dockerfile 2 | # Description: Dockerfile for delta 3 | # This is the multi-stage docker image to build and run delta 4 | # Author: Outercore Engineering 5 | # Name: delta 6 | # Email: 7 | # Url: https://delta.store 8 | 9 | FROM golang:1.19 AS builder 10 | 11 | RUN apt-get update && \ 12 | apt-get install -y wget jq hwloc ocl-icd-opencl-dev git libhwloc-dev pkg-config make && \ 13 | apt-get install -y cargo 14 | 15 | WORKDIR /app/ 16 | ADD . /app 17 | 18 | RUN curl https://sh.rustup.rs -sSf | bash -s -- -y 19 | ENV PATH="/root/.cargo/bin:${PATH}" 20 | RUN RUSTFLAGS="-C target-cpu=native -g" FFI_BUILD_FROM_SOURCE=1 FFI_USE_BLST_PORTABLE=1 make 21 | 22 | FROM golang:1.19 23 | 24 | ARG WALLET_DIR="" 25 | ARG REPO="/root/config/.whypfs" 26 | 27 | RUN echo "Building docker image for delta-dm" 28 | RUN echo "WALLET_DIR: ${WALLET_DIR}" 29 | 30 | RUN apt-get update && \ 31 | apt-get install -y hwloc libhwloc-dev ocl-icd-opencl-dev 32 | WORKDIR /root/ 33 | 34 | COPY --from=builder /app/delta ./ 35 | COPY ${WALLET_DIR} /root/config/wallet 36 | CMD ./delta daemon --repo=${REPO} --wallet-dir=${WALLET_DIR} 37 | EXPOSE 1414 -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SHELL=/usr/bin/env bash 2 | GO_BUILD_IMAGE?=golang:1.19 3 | VERSION=$(shell git describe --always --tag --dirty) 4 | COMMIT=$(shell git rev-parse --short HEAD) 5 | DOCKER_COMPOSE_FILE=docker-compose.yml 6 | DOCKER_ORG=0utercore 7 | 8 | .PHONY: all 9 | all: build 10 | 11 | .PHONY: build 12 | build: 13 | git submodule update --init --recursive 14 | make -C extern/filecoin-ffi 15 | go generate 16 | go build -tags netgo -ldflags="-s -w -X main.Commit=$(COMMIT) -X main.Version=$(VERSION)" -o delta 17 | 18 | .PHONY: clean 19 | clean: 20 | rm -f delta 21 | git submodule deinit --all -f 22 | 23 | install: 24 | install -C -m 0755 delta /usr/local/bin 25 | 26 | .PHONY: generate-swagger 27 | generate-swagger: 28 | scripts/swagger/swag.sh 29 | 30 | .PHONY: docker-compose-build 31 | docker-compose-build: 32 | BUILD_DATE=$(shell date -u +"%Y-%m-%dT%H:%M:%SZ") \ 33 | COMMIT=$(shell git rev-parse HEAD) \ 34 | VERSION=$(shell git describe --always --tag --dirty) \ 35 | WALLET_DIR=$(WALLET_DIR) \ 36 | DESCRIPTION=$(DESCRIPTION) \ 37 | TAG=$(TAG) \ 38 | docker-compose -f $(DOCKER_COMPOSE_FILE) build --build-arg WALLET_DIR=$(WALLET_DIR) --build-arg REPO=$(REPO) 39 | 40 | .PHONY: docker-compose-up 41 | docker-compose-up: 42 | docker-compose -f $(DOCKER_COMPOSE_FILE) up 43 | .PHONY: docker-compose-run 44 | docker-compose-run: docker-compose-build docker-compose-up 45 | 46 | .PHONY: docker-compose-down 47 | docker-compose-down: 48 | docker-compose -f $(DOCKER_COMPOSE_FILE) down 49 | 50 | .PHONY: prepare-spec docker-push 51 | docker-push: 52 | docker build -t delta:$(VERSION) . 53 | docker tag delta:$(VERSION) $(DOCKER_ORG)/delta:$(VERSION) 54 | docker push $(DOCKER_ORG)/delta:$(VERSION) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![CodeQL](https://img.shields.io/github/actions/workflow/status/application-research/delta/codeql.yml?label=CodeQL&style=for-the-badge)](https://github.com/application-research/delta/actions/workflows/codeql.yml) 2 | [![Release](https://img.shields.io/github/v/release/application-research/delta?display_name=release&style=for-the-badge)](https://github.com/application-research/delta/releases) 3 | 4 | # Δ Delta 5 | Filecoin Storage Deal Making Service 6 | 7 | Delta is a deal-making service that enables users to make deals with Storage Providers. It is an application that allows users to upload files to the Filecoin network and get them stored by Storage Providers. 8 | 9 | --- 10 | 11 | ![image](https://user-images.githubusercontent.com/4479171/226853191-e19e8fa4-abc1-4652-970f-d3d6cea0df13.png) 12 | 13 | For more information, check out the [docs](docs) 14 | 15 | --- 16 | 17 | # Build and Run Delta 18 | 19 | ## Prepare to `build` 20 | Copy the `.env.example` file to `.env` and update the values as needed. 21 | 22 | ``` 23 | # Node info 24 | NODE_NAME=stg-deal-maker 25 | NODE_DESCRIPTION=Experimental Deal Maker 26 | NODE_TYPE=delta-main 27 | 28 | # Database configuration 29 | MODE=standalone 30 | DB_DSN=delta.db 31 | #REPO=/mnt/.whypfs # shared mounted repo 32 | 33 | # Frequencies 34 | MAX_CLEANUP_WORKERS=1500 35 | ``` 36 | 37 | ## Install and run `Delta` using `docker` 38 | 39 | Running this the first time will generate a wallet. Make sure to get FIL/DataCap from the [faucet](https://verify.glif.io/) and fund the wallet 40 | 41 | Make sure you have docker installed on your machine. 42 | 43 | ### Run the current delta clone using docker-compose 44 | If you already have a wallet with datacap, you can pass it to the command below. This copies over the wallet directory to the containerized app and sets it as the default wallet. 45 | ``` 46 | make docker-compose-run WALLET_DIR= 47 | ``` 48 | 49 | **Check localhost** 50 | 51 | You can check the localhost to see if the delta app is running 52 | ``` 53 | curl --location --request GET 'http://localhost:1414/open/node/info' 54 | ``` 55 | 56 | **Next** 57 | 58 | Now that you can access a live Delta node, you are now ready to make a deal. You can now go to the following guides: 59 | 60 | - [Make an e2e deal](docs/make-e2e-deal.md) 61 | - [Make an import deal](docs/make-import-deal.md) 62 | 63 | 64 | ### Run a specific docker image tag from the docker hub artifactory 65 | We have a few pre-built images on the docker hub artifactory. You can run the following command to run a specific image tag 66 | ``` 67 | cd docker 68 | ./run.sh 69 | ``` 70 | Note: no tag means it'll just get the latest image. 71 | 72 | ## Install, build from source and run `Delta` 73 | ### Install the following pre-req 74 | - go 1.19 75 | - [jq](https://stedolan.github.io/jq/) 76 | - [hwloc](https://www.open-mpi.org/projects/hwloc/) 77 | - opencl 78 | - [rustup](https://rustup.rs/) 79 | - postgresql 80 | 81 | Alternatively, if using Ubuntu, you can run the following commands to install the pre-reqs 82 | ``` 83 | apt-get update && \ 84 | apt-get install -y wget jq hwloc ocl-icd-opencl-dev git libhwloc-dev pkg-config make && \ 85 | apt-get install -y cargo 86 | ``` 87 | 88 | ### Using `make` 89 | ``` 90 | make all 91 | ./delta daemon --repo=.whypfs --wallet-dir= 92 | ``` 93 | 94 | ### Using `go build` 95 | ``` 96 | go build -tags netgo -ldflags '-s -w' -o delta 97 | ./delta daemon --repo=.whypfs --wallet-dir= 98 | ``` 99 | 100 | ## Run `Delta` 101 | ``` 102 | ./delta daemon --mode=standalone 103 | ``` 104 | 105 | ## Test the API server 106 | Try the following endpoints to test the API server 107 | ``` 108 | curl --location --request GET 'http://localhost:1414/open/node/info' 109 | curl --location --request GET 'http://localhost:1414/open/node/peers' 110 | curl --location --request GET 'http://localhost:1414/open/node/host' 111 | ``` 112 | 113 | If it return the following, then the API server is working 114 | ``` 115 | {"name":"stg-deal-maker","description":"Experimental Deal Maker","type":"delta-main"} 116 | ``` 117 | 118 | 119 | # Getting Started with `Delta` 120 | To get started with `Delta`, you can follow the following guides [here](docs) 121 | 122 | ## Author 123 | Protocol Labs Outercore Engineering. 124 | -------------------------------------------------------------------------------- /api/debug_profiler.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "delta/core" 5 | httpprof "net/http/pprof" 6 | "runtime/pprof" 7 | "time" 8 | 9 | "github.com/labstack/echo/v4" 10 | ) 11 | 12 | // ConfigureDebugRouter It configures the router to handle requests for node profiling 13 | func ConfigureDebugProfileRouter(profilerGroup *echo.Group, node *core.DeltaNode) { 14 | 15 | profilerGroup.GET("/pprof/:prof", func(c echo.Context) error { 16 | httpprof.Handler(c.Param("prof")).ServeHTTP(c.Response().Writer, c.Request()) 17 | return nil 18 | }) 19 | 20 | profilerGroup.GET("/cpuprofile", func(c echo.Context) error { 21 | if err := pprof.StartCPUProfile(c.Response()); err != nil { 22 | return err 23 | } 24 | 25 | defer pprof.StopCPUProfile() 26 | 27 | select { 28 | case <-c.Request().Context().Done(): 29 | return c.Request().Context().Err() 30 | case <-time.After(time.Second * 30): 31 | } 32 | return nil 33 | }) 34 | } 35 | -------------------------------------------------------------------------------- /api/health_check.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "delta/core" 5 | "github.com/labstack/echo/v4" 6 | "net/http" 7 | ) 8 | 9 | // ConfigureHealthCheckRouter > ConfigureHealthCheckRouter is a function that takes a pointer to an echo.Group and a pointer to a DeltaNode and 10 | // returns nothing 11 | func ConfigureHealthCheckRouter(healthCheckApiGroup *echo.Group, node *core.DeltaNode) { 12 | 13 | healthCheckAuthApiGroup := healthCheckApiGroup.Group("/check/auth") 14 | 15 | // health check api withouth auth 16 | healthCheckApiGroup.GET("/ping", func(c echo.Context) error { 17 | return c.String(http.StatusOK, "pong") 18 | }) 19 | 20 | // health check api 21 | healthCheckAuthApiGroup.Use(Authenticate(*DeltaNodeConfig)) 22 | healthCheckAuthApiGroup.GET("/ping", func(c echo.Context) error { 23 | return c.String(http.StatusOK, "pong") 24 | }) 25 | } 26 | -------------------------------------------------------------------------------- /api/metrics.go: -------------------------------------------------------------------------------- 1 | // It configures the metrics router 2 | package api 3 | 4 | import ( 5 | "delta/metrics" 6 | "github.com/labstack/echo/v4" 7 | "github.com/prometheus/client_golang/prometheus/promhttp" 8 | "net/http" 9 | ) 10 | 11 | // ConfigMetricsRouter Configuring the metrics router. 12 | // > ConfigMetricsRouter is a function that takes a pointer to an echo.Group and returns nothing 13 | func ConfigMetricsRouter(e *echo.Group) { 14 | // metrics 15 | phandle := promhttp.Handler() 16 | e.GET("/debug/metrics/prometheus", func(e echo.Context) error { 17 | phandle.ServeHTTP(e.Response().Writer, e.Request()) 18 | 19 | return nil 20 | }) 21 | 22 | e.GET("/debug/metrics", func(e echo.Context) error { 23 | return e.JSON(http.StatusOK, "Ok") 24 | //return nil 25 | }) 26 | 27 | e.GET("/debug/metrics", func(e echo.Context) error { 28 | metrics.Exporter().ServeHTTP(e.Response().Writer, e.Request()) 29 | return nil 30 | }) 31 | e.GET("/debug/stack", func(e echo.Context) error { 32 | err := metrics.WriteAllGoroutineStacks(e.Response().Writer) 33 | if err != nil { 34 | log.Error(err) 35 | } 36 | return err 37 | }) 38 | } 39 | -------------------------------------------------------------------------------- /api/node.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "context" 5 | "delta/core" 6 | "delta/models" 7 | "github.com/labstack/echo/v4" 8 | "go.opentelemetry.io/otel" 9 | "go.opentelemetry.io/otel/attribute" 10 | ) 11 | 12 | // ConfigureNodeInfoRouter It configures the router to handle requests for node information 13 | func ConfigureNodeInfoRouter(e *echo.Group, node *core.DeltaNode) { 14 | 15 | nodeGroup := e.Group("/node") 16 | nodeGroup.GET("/info", handleNodeInfo(node)) 17 | nodeGroup.GET("/uuids", handleNodeUuidInfo(node)) 18 | nodeGroup.GET("/addr", handleNodeAddr(node)) 19 | nodeGroup.GET("/peers", handleNodePeers(node)) 20 | nodeGroup.GET("/host", handleNodeHost(node)) 21 | nodeGroup.GET("/api-key", handleNodeHostApiKey(node)) 22 | } 23 | 24 | // If the node is in standalone mode, return the API key 25 | func handleNodeHostApiKey(node *core.DeltaNode) func(c echo.Context) error { 26 | return func(c echo.Context) error { 27 | _, span := otel.Tracer("handleNodeHostApiKey").Start(context.Background(), "handleNodeHostApiKey") 28 | defer span.End() 29 | 30 | span.SetName("ConfigureNodeInfoRouter") 31 | span.SetAttributes(attribute.String("user-agent", c.Request().UserAgent())) 32 | span.SetAttributes(attribute.String("path", c.Path())) 33 | span.SetAttributes(attribute.String("method", c.Request().Method)) 34 | 35 | if node.Config.Common.Mode != "standalone" { 36 | return c.JSON(200, "This is not a standalone node") 37 | } 38 | return c.JSON(200, map[string]string{"standalone_api_key": node.Config.Standalone.APIKey}) 39 | } 40 | 41 | } 42 | 43 | // It returns a function that takes a `DeltaNode` and returns a function that takes an `echo.Context` and returns an 44 | // `error` 45 | func handleNodeHost(node *core.DeltaNode) func(c echo.Context) error { 46 | return func(c echo.Context) error { 47 | _, span := otel.Tracer("handleNodeHost").Start(context.Background(), "handleNodeHostApiKey") 48 | defer span.End() 49 | 50 | span.SetName("ConfigureNodeInfoRouter") 51 | span.SetAttributes(attribute.String("user-agent", c.Request().UserAgent())) 52 | span.SetAttributes(attribute.String("path", c.Path())) 53 | span.SetAttributes(attribute.String("method", c.Request().Method)) 54 | 55 | return c.JSON(200, node.Node.Host.ID()) 56 | } 57 | } 58 | 59 | // It returns a function that takes a `DeltaNode` and returns a function that takes a `Context` and returns an `error` 60 | func handleNodePeers(node *core.DeltaNode) func(c echo.Context) error { 61 | return func(c echo.Context) error { 62 | _, span := otel.Tracer("handleNodePeers").Start(context.Background(), "handleNodeHostApiKey") 63 | defer span.End() 64 | 65 | span.SetName("ConfigureNodeInfoRouter") 66 | span.SetAttributes(attribute.String("user-agent", c.Request().UserAgent())) 67 | span.SetAttributes(attribute.String("path", c.Path())) 68 | span.SetAttributes(attribute.String("method", c.Request().Method)) 69 | return c.JSON(200, node.Node.Host.Network().Peers()) 70 | } 71 | } 72 | 73 | // It returns a function that takes a `DeltaNode` and returns a function that takes a `Context` and returns an `error` 74 | func handleNodeAddr(node *core.DeltaNode) func(c echo.Context) error { 75 | return func(c echo.Context) error { 76 | _, span := otel.Tracer("handleNodePeers").Start(context.Background(), "handleNodeHostApiKey") 77 | defer span.End() 78 | 79 | span.SetName("handleNodeAddr") 80 | span.SetAttributes(attribute.String("user-agent", c.Request().UserAgent())) 81 | span.SetAttributes(attribute.String("path", c.Path())) 82 | span.SetAttributes(attribute.String("method", c.Request().Method)) 83 | return c.JSON(200, node.Node.Host.Addrs()) 84 | } 85 | } 86 | 87 | type Uuids struct { 88 | ID uint `json:"id"` 89 | InstanceUuid string `json:"instance_uuid"` 90 | CreatedAt string `json:"created_at"` 91 | } 92 | 93 | // It returns a function that returns a JSON response with the node's name, description, and type 94 | func handleNodeUuidInfo(node *core.DeltaNode) func(c echo.Context) error { 95 | return func(c echo.Context) error { 96 | _, span := otel.Tracer("handleNodeUuidInfo").Start(context.Background(), "handleNodeUuidInfo") 97 | defer span.End() 98 | 99 | span.SetName("handleNodeUuidInfo") 100 | span.SetAttributes(attribute.String("user-agent", c.Request().UserAgent())) 101 | span.SetAttributes(attribute.String("path", c.Path())) 102 | span.SetAttributes(attribute.String("method", c.Request().Method)) 103 | 104 | // instance meta 105 | var uuids []Uuids 106 | node.DB.Model(&db_models.InstanceMeta{}).Scan(&uuids) 107 | 108 | return c.JSON(200, uuids) 109 | } 110 | } 111 | 112 | // It returns a function that returns a JSON response with the node's name, description, and type 113 | func handleNodeInfo(node *core.DeltaNode) func(c echo.Context) error { 114 | return func(c echo.Context) error { 115 | _, span := otel.Tracer("handleNodePeers").Start(context.Background(), "handleNodeHostApiKey") 116 | defer span.End() 117 | 118 | span.SetName("handleNodeInfo") 119 | span.SetAttributes(attribute.String("user-agent", c.Request().UserAgent())) 120 | span.SetAttributes(attribute.String("path", c.Path())) 121 | span.SetAttributes(attribute.String("method", c.Request().Method)) 122 | 123 | nodeName := node.Config.Node.Name 124 | nodeDescription := node.Config.Node.Description 125 | nodeType := node.Config.Node.Type 126 | commit := node.Config.Common.Commit 127 | version := node.Config.Common.Version 128 | 129 | return c.JSON(200, map[string]string{ 130 | "name": nodeName, 131 | "description": nodeDescription, 132 | "type": nodeType, 133 | "commit": commit, 134 | "version": version, 135 | }) 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /api/open_info.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "context" 5 | "delta/core" 6 | model "delta/models" 7 | "github.com/filecoin-project/go-address" 8 | "github.com/filecoin-project/lotus/chain/types" 9 | "github.com/labstack/echo/v4" 10 | ) 11 | 12 | // TODO: OPTIMIZE!! 13 | func ConfigureOpenInfoCheckRouter(e *echo.Group, node *core.DeltaNode) { 14 | 15 | e.GET("/info/wallet/balance/:address", func(c echo.Context) error { 16 | return handleOpenGetBalance(c, node) 17 | }) 18 | 19 | } 20 | 21 | // It gets the balance of a wallet 22 | func handleOpenGetBalance(c echo.Context, node *core.DeltaNode) error { 23 | var wallet model.Wallet 24 | node.DB.Model(&model.Wallet{}).Where("addr = ?", c.Param("address")).First(&wallet) 25 | 26 | if wallet.ID == 0 { 27 | return c.JSON(400, map[string]interface{}{ 28 | "message": "wallet not found, register the wallet first", 29 | }) 30 | } 31 | 32 | addressFromParam := c.Param("address") 33 | address, err := address.NewFromString(addressFromParam) 34 | if err != nil { 35 | return c.JSON(400, map[string]interface{}{ 36 | "message": "invalid address", 37 | }) 38 | } 39 | bigIntBalance, err := node.LotusApiNode.WalletBalance(context.Background(), address) 40 | if err != nil { 41 | return c.JSON(500, map[string]interface{}{ 42 | "message": "failed to get balance", 43 | }) 44 | } 45 | 46 | act, err := node.LotusApiNode.StateGetActor(context.Background(), address, types.EmptyTSK) 47 | if err != nil { 48 | return c.JSON(500, map[string]interface{}{ 49 | "message": "failed to get actor", 50 | }) 51 | } 52 | 53 | market, err := node.LotusApiNode.StateMarketBalance(context.Background(), address, types.EmptyTSK) 54 | if err != nil { 55 | return c.JSON(500, map[string]interface{}{ 56 | "message": "failed to get market balance", 57 | }) 58 | } 59 | 60 | vcstatus, err := node.LotusApiNode.StateVerifiedClientStatus(context.Background(), address, types.EmptyTSK) 61 | if err != nil { 62 | return c.JSON(500, map[string]interface{}{ 63 | "message": "failed to get verified client status", 64 | }) 65 | } 66 | 67 | avail := types.BigSub(market.Escrow, market.Locked) 68 | 69 | return c.JSON(200, map[string]interface{}{ 70 | "message": "success", 71 | "balance": map[string]interface{}{ 72 | "account": address.String(), 73 | "wallet_balance": types.FIL(bigIntBalance), 74 | "balance": types.FIL(act.Balance), 75 | "market_escrow": types.FIL(market.Escrow), 76 | "market_locked": types.FIL(market.Locked), 77 | "market_available": types.FIL(avail), 78 | "verified_client_balance": vcstatus.Int64(), 79 | }, 80 | }) 81 | } 82 | -------------------------------------------------------------------------------- /api/websocket.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "delta/core" 5 | "fmt" 6 | model "delta/models" 7 | "github.com/gorilla/websocket" 8 | "github.com/labstack/echo/v4" 9 | "sync" 10 | ) 11 | 12 | var ( 13 | upgrader = websocket.Upgrader{} 14 | mutex = &sync.Mutex{} 15 | ) 16 | 17 | // ConfigureWebsocketRouter It creates a new websocket handler that will send the content of the node to the client 18 | func ConfigureWebsocketRouter(e *echo.Group, ln *core.DeltaNode) { 19 | 20 | wsGroup := e.Group("/ws") 21 | // initiate the websocket broadcast 22 | contentChannel := core.ContentChannel{ 23 | Clients: make(map[*core.ClientChannel]bool), 24 | Channel: make(chan model.Content), 25 | } 26 | 27 | pieceCommitmentChannel := core.PieceCommitmentChannel{ 28 | Clients: make(map[*websocket.Conn]bool), 29 | Channel: make(chan model.PieceCommitment), 30 | } 31 | 32 | contentDealChannel := core.ContentDealChannel{ 33 | Clients: make(map[*websocket.Conn]bool), 34 | Channel: make(chan model.ContentDeal), 35 | } 36 | 37 | ln.DeltaEventEmitter.WebsocketBroadcast.ContentChannel = contentChannel 38 | ln.DeltaEventEmitter.WebsocketBroadcast.PieceCommitmentChannel = pieceCommitmentChannel 39 | ln.DeltaEventEmitter.WebsocketBroadcast.ContentDealChannel = contentDealChannel 40 | 41 | wsGroup.GET("/contents/:contentId", handleWebsocketContent(ln)) 42 | wsGroup.GET("/piece-commitments/:pieceCommitmentId", handleWebsocketPieceCommitment(ln)) 43 | wsGroup.GET("/deals/by-uuid/:dealUuid", handleWebsocketContentDeal(ln)) 44 | wsGroup.GET("/deals/by-cid/:cid", handleWebsocketContentDeal(ln)) 45 | 46 | } 47 | 48 | // It upgrades the HTTP connection to a WebSocket connection, registers the new client, and then reads messages from the 49 | // client and sends them to the broadcast channel 50 | func handleWebsocketContent(ln *core.DeltaNode) func(c echo.Context) error { 51 | return func(c echo.Context) error { 52 | // Upgrade HTTP connection to WebSocket connection 53 | conn, err := upgrader.Upgrade(c.Response(), c.Request(), nil) 54 | if err != nil { 55 | fmt.Println("WebSocket upgrade error:", err) 56 | return nil 57 | } 58 | 59 | contentId := c.Param("contentId") 60 | 61 | // Register new client 62 | mutex.Lock() 63 | clientChannelForContent := &core.ClientChannel{ 64 | Conn: conn, 65 | Id: contentId, 66 | } 67 | ln.DeltaEventEmitter.WebsocketBroadcast.ContentChannel.Clients[clientChannelForContent] = true 68 | mutex.Unlock() 69 | 70 | // Close WebSocket connection when client disconnects 71 | defer func() { 72 | mutex.Lock() 73 | delete(ln.DeltaEventEmitter.WebsocketBroadcast.ContentChannel.Clients, clientChannelForContent) 74 | mutex.Unlock() 75 | conn.Close() 76 | }() 77 | 78 | for { 79 | var msg model.Content 80 | err := conn.ReadJSON(&msg) 81 | if err != nil { 82 | fmt.Println("WebSocket read error:", err) 83 | break 84 | } 85 | // Send message to broadcast channel 86 | ln.DeltaEventEmitter.WebsocketBroadcast.ContentChannel.Channel <- msg 87 | } 88 | 89 | return nil 90 | } 91 | } 92 | 93 | // It upgrades the HTTP connection to a WebSocket connection, registers the new client, and then reads messages from the 94 | // client and sends them to the broadcast channel 95 | func handleWebsocketPieceCommitment(ln *core.DeltaNode) func(c echo.Context) error { 96 | return func(c echo.Context) error { 97 | // Upgrade HTTP connection to WebSocket connection 98 | conn, err := upgrader.Upgrade(c.Response(), c.Request(), nil) 99 | if err != nil { 100 | fmt.Println("WebSocket upgrade error:", err) 101 | return nil 102 | } 103 | 104 | // Register new client 105 | mutex.Lock() 106 | ln.DeltaEventEmitter.WebsocketBroadcast.PieceCommitmentChannel.Clients[conn] = true 107 | mutex.Unlock() 108 | 109 | // Close WebSocket connection when client disconnects 110 | defer func() { 111 | mutex.Lock() 112 | delete(ln.DeltaEventEmitter.WebsocketBroadcast.PieceCommitmentChannel.Clients, conn) 113 | mutex.Unlock() 114 | conn.Close() 115 | }() 116 | 117 | for { 118 | var msg model.PieceCommitment 119 | err := conn.ReadJSON(&msg) 120 | if err != nil { 121 | fmt.Println("WebSocket read error:", err) 122 | break 123 | } 124 | // Send message to broadcast channel 125 | ln.DeltaEventEmitter.WebsocketBroadcast.PieceCommitmentChannel.Channel <- msg 126 | } 127 | 128 | return nil 129 | } 130 | } 131 | 132 | // It upgrades the HTTP connection to a WebSocket connection, registers the new client, and then reads messages from the 133 | // client and sends them to the broadcast channel 134 | func handleWebsocketContentDeal(ln *core.DeltaNode) func(c echo.Context) error { 135 | return func(c echo.Context) error { 136 | // Upgrade HTTP connection to WebSocket connection 137 | conn, err := upgrader.Upgrade(c.Response(), c.Request(), nil) 138 | if err != nil { 139 | fmt.Println("WebSocket upgrade error:", err) 140 | return nil 141 | } 142 | 143 | // Register new client 144 | mutex.Lock() 145 | ln.DeltaEventEmitter.WebsocketBroadcast.ContentDealChannel.Clients[conn] = true 146 | mutex.Unlock() 147 | 148 | // Close WebSocket connection when client disconnects 149 | defer func() { 150 | mutex.Lock() 151 | delete(ln.DeltaEventEmitter.WebsocketBroadcast.ContentDealChannel.Clients, conn) 152 | mutex.Unlock() 153 | conn.Close() 154 | }() 155 | 156 | for { 157 | var msg model.ContentDeal 158 | err := conn.ReadJSON(&msg) 159 | if err != nil { 160 | fmt.Println("WebSocket read error:", err) 161 | break 162 | } 163 | // Send message to broadcast channel 164 | ln.DeltaEventEmitter.WebsocketBroadcast.ContentDealChannel.Channel <- msg 165 | } 166 | 167 | return nil 168 | } 169 | } 170 | -------------------------------------------------------------------------------- /cmd/admin.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | c "delta/config" 5 | "github.com/urfave/cli/v2" 6 | ) 7 | 8 | // TODO: add a command to administer the node 9 | func AdminCmd(cfg *c.DeltaConfig) []*cli.Command { 10 | // add a command to run API node 11 | var adminCommands []*cli.Command 12 | 13 | adminCmd := &cli.Command{ 14 | Name: "wallet", 15 | Usage: "Make a delta storage deal", 16 | Flags: []cli.Flag{ 17 | &cli.StringFlag{ 18 | Name: "register-hex", 19 | Usage: "content to store", 20 | }, 21 | &cli.StringFlag{ 22 | Name: "api-key", 23 | Usage: "The API key to use for the request", 24 | }, 25 | }, 26 | } 27 | adminCommands = append(adminCommands, adminCmd) 28 | 29 | return adminCommands 30 | } 31 | -------------------------------------------------------------------------------- /cmd/delta_cmd_node.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "net/http" 7 | "os" 8 | 9 | "github.com/urfave/cli/v2" 10 | ) 11 | 12 | var CLIConnectFlags = []cli.Flag{ 13 | &cli.StringFlag{ 14 | Name: "delta-api", 15 | Usage: "API connection info", 16 | EnvVars: []string{"DELTA_API"}, 17 | Hidden: true, 18 | }, 19 | &cli.StringFlag{ 20 | Name: "delta-auth", 21 | Usage: "delta auth token", 22 | EnvVars: []string{"DELTA_AUTH"}, 23 | Hidden: true, 24 | }, 25 | } 26 | 27 | type DeltaCmdNode struct { 28 | DeltaApi string 29 | DeltaAuth string 30 | } 31 | 32 | // NewDeltaCmdNode It creates a new DeltaCmdNode struct, which is a struct that contains the Delta API URL and the Delta API auth token 33 | func NewDeltaCmdNode(c *cli.Context) (*DeltaCmdNode, error) { 34 | deltaApi := getFlagOrEnvVar(c, "delta-api", "DELTA_API", "http://localhost:1414") 35 | deltaAuth := getFlagOrEnvVar(c, "delta-auth", "DELTA_AUTH", "") 36 | 37 | if deltaAuth == "" { 38 | return nil, fmt.Errorf("DELTA_AUTH env variable or --delta-auth flag is required") 39 | } 40 | 41 | if err := healthCheck(deltaApi, deltaAuth); err != nil { 42 | return nil, fmt.Errorf("unable to communicate with delta daemon: %s", err) 43 | } 44 | 45 | return &DeltaCmdNode{ 46 | DeltaApi: deltaApi, 47 | DeltaAuth: deltaAuth, 48 | }, nil 49 | } 50 | 51 | // If the flag is set, use it. If not, check the environment variable. If that's not set, use the default value 52 | func getFlagOrEnvVar(c *cli.Context, flagName, envVarName, defaultValue string) string { 53 | value := c.String(flagName) 54 | if value == "" { 55 | value = os.Getenv(envVarName) 56 | if value == "" { 57 | value = defaultValue 58 | } 59 | } 60 | return value 61 | } 62 | 63 | // It constructs an HTTP request to the `/open/node/info` endpoint, sets the `Authorization` header to the value of the 64 | // `authKey` parameter, and then makes the request. If the response status code is not 200, it returns an error 65 | func healthCheck(url string, authKey string) error { 66 | req, err := http.NewRequest("GET", url+"/health/check/auth/ping", nil) 67 | if err != nil { 68 | return fmt.Errorf("could not construct http request %v", err) 69 | } 70 | 71 | req.Header.Set("Authorization", "Bearer "+authKey) 72 | 73 | client := &http.Client{} 74 | resp, err := client.Do(req) 75 | if err != nil { 76 | return fmt.Errorf("could not make http request %s", err) 77 | } 78 | 79 | if resp.StatusCode != 200 { 80 | defer resp.Body.Close() 81 | body, err := ioutil.ReadAll(resp.Body) 82 | 83 | if err != nil { 84 | return err 85 | } 86 | 87 | return fmt.Errorf(string(body)) 88 | } 89 | 90 | return err 91 | } 92 | -------------------------------------------------------------------------------- /cmd/sp.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "bytes" 5 | c "delta/config" 6 | "delta/utils" 7 | "encoding/json" 8 | "fmt" 9 | "github.com/urfave/cli/v2" 10 | "io" 11 | "math/rand" 12 | "net/http" 13 | "strconv" 14 | "time" 15 | ) 16 | 17 | type StorageProvider struct { 18 | Providers []Provider `json:"storageProviders"` 19 | } 20 | 21 | type ProviderSelect struct { 22 | Provider Provider `json:"0"` 23 | } 24 | type Provider struct { 25 | ID string `json:"id"` 26 | Address string `json:"address"` 27 | AddressOfOwner string `json:"address_of_owner"` 28 | AddressOfWorker string `json:"address_of_worker"` 29 | AddressOfBeneficiary string `json:"address_of_beneficiary"` 30 | SectorSizeBytes string `json:"sector_size_bytes"` 31 | MaxPieceSizeBytes string `json:"max_piece_size_bytes"` 32 | MinPieceSizeBytes string `json:"min_piece_size_bytes"` 33 | PriceAttofil string `json:"price_attofil"` 34 | PriceVerifiedAttofil string `json:"price_verified_attofil"` 35 | BalanceAttofil string `json:"balance_attofil"` 36 | LockedFundsAttofil string `json:"locked_funds_attofil"` 37 | InitialPledgeAttofil string `json:"initial_pledge_attofil"` 38 | RawPowerBytes string `json:"raw_power_bytes"` 39 | QualityAdjustedPowerBytes string `json:"quality_adjusted_power_bytes"` 40 | TotalRawPowerBytes string `json:"total_raw_power_bytes"` 41 | TotalQualityAdjustedPowerBytes string `json:"total_quality_adjusted_power_bytes"` 42 | TotalStorageDealCount string `json:"total_storage_deal_count"` 43 | TotalSectorsSealedByPostCount string `json:"total_sectors_sealed_by_post_count"` 44 | PeerID string `json:"peer_id"` 45 | Height string `json:"height"` 46 | LotusVersion string `json:"lotus_version"` 47 | Multiaddrs struct { 48 | Addresses []string `json:"addresses"` 49 | } `json:"multiaddrs"` 50 | Metadata interface{} `json:"metadata"` 51 | AddressOfControllers struct { 52 | Addresses []string `json:"addresses"` 53 | } `json:"address_of_controllers"` 54 | Tipset struct { 55 | Cids []struct { 56 | NAMING_FAILED string `json:"/"` 57 | } `json:"cids"` 58 | } `json:"tipset"` 59 | CreatedAt time.Time `json:"created_at"` 60 | UpdatedAt time.Time `json:"updated_at"` 61 | } 62 | 63 | type Pricing struct { 64 | StoragePrice string `json:"storagePrice"` 65 | } 66 | 67 | func SpCmd(cfg *c.DeltaConfig) []*cli.Command { 68 | var spCommands []*cli.Command 69 | spCmd := &cli.Command{ 70 | Name: "sp", 71 | Usage: "SP CLI using data.storage.market API", 72 | Subcommands: []*cli.Command{ 73 | { 74 | Name: "info", 75 | Usage: "Get storage provider info", 76 | Flags: []cli.Flag{ 77 | &cli.StringFlag{ 78 | Name: "addr", 79 | Usage: "The address of the storage provider", 80 | }, 81 | }, Action: func(context *cli.Context) error { 82 | 83 | addr := context.String("addr") 84 | if addr == "" { 85 | return fmt.Errorf("addr is required") 86 | } 87 | 88 | provider, err := fetchProviderByAddr(addr) 89 | if err != nil { 90 | fmt.Println(err) 91 | return err 92 | } 93 | var buffer bytes.Buffer 94 | err = utils.PrettyEncode(provider, &buffer) 95 | if err != nil { 96 | fmt.Println(err) 97 | } 98 | fmt.Println(buffer.String()) 99 | return nil 100 | }, 101 | }, 102 | { 103 | Name: "selection", 104 | Usage: "Get a random storage provider", 105 | Flags: []cli.Flag{ 106 | &cli.Int64Flag{ 107 | Name: "size-in-bytes", 108 | Usage: "The size of the data in bytes", 109 | }, 110 | }, Action: func(context *cli.Context) error { 111 | // Parse query parameters 112 | sizeInBytes := context.Int64("size-in-bytes") 113 | 114 | providers, err := fetchProviders(sizeInBytes) 115 | if err != nil { 116 | fmt.Println(err) 117 | return err 118 | } 119 | 120 | // Select one random provider 121 | rand.Seed(time.Now().UnixNano()) 122 | randomIndex := rand.Intn(len(providers)) 123 | randomProvider := providers[randomIndex] 124 | 125 | if err != nil { 126 | panic(err) 127 | } 128 | var buffer bytes.Buffer 129 | err = utils.PrettyEncode(randomProvider, &buffer) 130 | if err != nil { 131 | fmt.Println(err) 132 | } 133 | fmt.Println(buffer.String()) 134 | 135 | return nil 136 | }, 137 | }, 138 | }, 139 | } 140 | spCommands = append(spCommands, spCmd) 141 | 142 | return spCommands 143 | } 144 | 145 | func fetchProviderByAddr(addr string) (ProviderSelect, error) { 146 | response, err := http.Get("https://data.storage.market/api/providers/" + addr) 147 | if err != nil { 148 | return ProviderSelect{}, err 149 | } 150 | defer response.Body.Close() 151 | 152 | body, err := io.ReadAll(response.Body) 153 | if err != nil { 154 | return ProviderSelect{}, err 155 | } 156 | 157 | var providers ProviderSelect 158 | err = json.Unmarshal(body, &providers) 159 | 160 | if err != nil { 161 | return ProviderSelect{}, err 162 | } 163 | 164 | return providers, nil 165 | } 166 | 167 | func fetchProviders(sizeInBytes int64) ([]Provider, error) { 168 | response, err := http.Get("https://data.storage.market/api/providers") 169 | if err != nil { 170 | return nil, err 171 | } 172 | defer response.Body.Close() 173 | 174 | body, err := io.ReadAll(response.Body) 175 | if err != nil { 176 | return nil, err 177 | } 178 | 179 | var providers StorageProvider 180 | err = json.Unmarshal(body, &providers) 181 | if err != nil { 182 | return nil, err 183 | } 184 | 185 | // Filter providers by piece size 186 | var filteredProviders []Provider 187 | for _, provider := range providers.Providers { 188 | 189 | pvMinPieceSize, _ := strconv.ParseInt(provider.MinPieceSizeBytes, 10, 64) 190 | pvMaxPieceSize, _ := strconv.ParseInt(provider.MaxPieceSizeBytes, 10, 64) 191 | if pvMinPieceSize <= sizeInBytes && pvMaxPieceSize >= sizeInBytes { 192 | filteredProviders = append(filteredProviders, provider) 193 | } 194 | } 195 | 196 | return filteredProviders, nil 197 | } 198 | -------------------------------------------------------------------------------- /cmd/status.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "bytes" 5 | c "delta/config" 6 | "delta/utils" 7 | "encoding/json" 8 | "fmt" 9 | "net/http" 10 | "time" 11 | 12 | "github.com/urfave/cli/v2" 13 | ) 14 | 15 | type StatusResponse struct { 16 | Content struct { 17 | ID int `json:"ID"` 18 | Name string `json:"name"` 19 | Size int `json:"size"` 20 | Cid string `json:"cid"` 21 | PieceCommitmentID int `json:"piece_commitment_id"` 22 | Status string `json:"status"` 23 | RequestType string `json:"request_type"` 24 | ConnectionMode string `json:"connection_mode"` 25 | LastMessage string `json:"last_message"` 26 | CreatedAt time.Time `json:"created_at"` 27 | UpdatedAt time.Time `json:"updated_at"` 28 | } `json:"content"` 29 | DealProposalParameters []struct { 30 | ID int `json:"ID"` 31 | Content int `json:"content"` 32 | Label string `json:"label"` 33 | Duration int `json:"duration"` 34 | CreatedAt time.Time `json:"created_at"` 35 | UpdatedAt time.Time `json:"updated_at"` 36 | } `json:"deal_proposal_parameters"` 37 | DealProposals interface{} `json:"deal_proposals"` 38 | Deals interface{} `json:"deals"` 39 | PieceCommitments []struct { 40 | ID int `json:"ID"` 41 | Cid string `json:"cid"` 42 | Piece string `json:"piece"` 43 | Size int `json:"size"` 44 | PaddedPieceSize int `json:"padded_piece_size"` 45 | UnnpaddedPieceSize int `json:"unnpadded_piece_size"` 46 | Status string `json:"status"` 47 | LastMessage string `json:"last_message"` 48 | CreatedAt time.Time `json:"created_at"` 49 | UpdatedAt time.Time `json:"updated_at"` 50 | } `json:"piece_commitments"` 51 | } 52 | 53 | func StatusCmd(cfg *c.DeltaConfig) []*cli.Command { 54 | // add a command to run API node 55 | var statusCommands []*cli.Command 56 | 57 | statusCmd := &cli.Command{ 58 | Name: "status", 59 | Usage: "Get the status of a content, deal, or piece commitment", 60 | Description: "Get the status of a content, deal, or piece commitment. The type of status can be either content, deal, or piece-commitment.", 61 | Flags: []cli.Flag{ 62 | &cli.StringFlag{ 63 | Name: "type", 64 | Usage: "content, deal, or piece-commitment", 65 | }, 66 | &cli.StringFlag{ 67 | Name: "id", 68 | Usage: "the id of the content, deal, or piece-commitment", 69 | }, 70 | }, 71 | Action: func(context *cli.Context) error { 72 | cmd, err := NewDeltaCmdNode(context) 73 | if err != nil { 74 | return err 75 | } 76 | 77 | typeParam := context.String("type") 78 | idParam := context.String("id") 79 | 80 | fmt.Println(cmd.DeltaApi) 81 | fmt.Println(typeParam) 82 | fmt.Println(cmd.DeltaAuth) 83 | fmt.Println(idParam) 84 | var dealStatusResponse StatusResponse 85 | url := cmd.DeltaApi + "/open/stats/" + typeParam + "/" + idParam 86 | if typeParam == "content" { 87 | // Create a new HTTP request with the desired method and URL. 88 | req, err := http.NewRequest("GET", url, nil) 89 | if err != nil { 90 | panic(err) 91 | } 92 | 93 | // Set the Authorization header. 94 | req.Header.Set("Authorization", "Bearer "+cmd.DeltaAuth) 95 | 96 | // Send the HTTP request and print the response. 97 | resp, err := http.DefaultClient.Do(req) 98 | if err != nil { 99 | panic(err) 100 | } 101 | defer resp.Body.Close() 102 | 103 | // Print the response status code. 104 | fmt.Println(resp.Status) 105 | err = json.NewDecoder(resp.Body).Decode(&dealStatusResponse) 106 | if err != nil { 107 | panic(err) 108 | } 109 | var buffer bytes.Buffer 110 | err = utils.PrettyEncode(dealStatusResponse, &buffer) 111 | if err != nil { 112 | fmt.Println(err) 113 | } 114 | fmt.Println(buffer.String()) 115 | } 116 | 117 | if typeParam == "deal" { 118 | // TODO: implement 119 | fmt.Println("Not implemented yet") 120 | } 121 | 122 | if typeParam == "piece-commitment" { 123 | // TODO: implement 124 | fmt.Println("Not implemented yet") 125 | } 126 | 127 | return nil 128 | }, 129 | } 130 | statusCommands = append(statusCommands, statusCmd) 131 | 132 | return statusCommands 133 | } 134 | -------------------------------------------------------------------------------- /cmd/wallet.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "bytes" 5 | c "delta/config" 6 | "delta/core" 7 | "delta/utils" 8 | "encoding/base64" 9 | "encoding/hex" 10 | "encoding/json" 11 | "fmt" 12 | "net/http" 13 | "time" 14 | 15 | "github.com/urfave/cli/v2" 16 | ) 17 | 18 | type WalletRegisterResponse struct { 19 | Message string `json:"message"` 20 | WalletAddr string `json:"wallet_addr"` 21 | WalletUUID string `json:"wallet_uuid"` 22 | } 23 | 24 | type WalletListResponse struct { 25 | Wallets []struct { 26 | ID int `json:"ID"` 27 | UUID string `json:"uuid"` 28 | Addr string `json:"addr"` 29 | Owner string `json:"owner"` 30 | KeyType string `json:"key_type"` 31 | PrivateKey string `json:"private_key"` 32 | CreatedAt time.Time `json:"created_at"` 33 | UpdatedAt time.Time `json:"updated_at"` 34 | } `json:"wallets"` 35 | } 36 | 37 | type WalletResponse struct { 38 | PublicKey string `json:"public_key,omitempty"` 39 | PrivateKey string `json:"private_key,omitempty"` 40 | KeyType string `json:"key_type,omitempty"` 41 | } 42 | 43 | // TODO: add a command to manage wallet via CLI 44 | func WalletCmd(cfg *c.DeltaConfig) []*cli.Command { 45 | // add a command to run API node 46 | var walletCommands []*cli.Command 47 | 48 | walletCmd := &cli.Command{ 49 | Name: "wallet", 50 | Usage: "Run Delta wallet commands", 51 | Subcommands: []*cli.Command{ 52 | { 53 | Name: "generate", 54 | Usage: "Generate a new wallet", 55 | Flags: []cli.Flag{ 56 | &cli.StringFlag{ 57 | Name: "dir", 58 | Usage: "Wallet directory where the wallet file will be stored", 59 | }, 60 | &cli.BoolFlag{ 61 | Name: "show-private-key", 62 | Usage: "Show private key", 63 | Value: false, 64 | }, 65 | }, 66 | Action: func(context *cli.Context) error { 67 | walletDir := context.String("dir") 68 | showPrivateKey := context.Bool("show-private-key") 69 | 70 | wallet, err := core.SetupWallet(walletDir) 71 | if err != nil { 72 | return err 73 | } 74 | walletAddr, err := wallet.GetDefault() 75 | 76 | walletResponse := WalletResponse{ 77 | PublicKey: walletAddr.String(), 78 | } 79 | 80 | if showPrivateKey { 81 | // import the new wallet 82 | hexedKey := hex.EncodeToString(walletAddr.Payload()) 83 | decodeKey := base64.StdEncoding.EncodeToString([]byte(hexedKey)) 84 | walletResponse.PrivateKey = decodeKey 85 | } 86 | 87 | if err != nil { 88 | panic(err) 89 | } 90 | var buffer bytes.Buffer 91 | err = utils.PrettyEncode(walletResponse, &buffer) 92 | if err != nil { 93 | fmt.Println(err) 94 | } 95 | fmt.Println(buffer.String()) 96 | fmt.Println(utils.Purple + "Wallet generated successfully. Make sure to backup your wallet file." + utils.Purple) 97 | 98 | return nil 99 | }, 100 | }, 101 | { 102 | Name: "register", 103 | Usage: "Register a new wallet", 104 | Flags: []cli.Flag{ 105 | &cli.StringFlag{ 106 | Name: "hex", 107 | Usage: "Hexed wallet from LOTUS/BOOSTD export", 108 | }, 109 | }, 110 | Action: func(context *cli.Context) error { 111 | cmd, err := NewDeltaCmdNode(context) 112 | if err != nil { 113 | return err 114 | } 115 | 116 | hexParam := context.String("hex") 117 | url := cmd.DeltaApi + "/admin/wallet/register-hex" 118 | payload := map[string]string{ 119 | "hex_key": hexParam, 120 | } 121 | data, err := json.Marshal(payload) 122 | if err != nil { 123 | panic(err) 124 | } 125 | 126 | req, err := http.NewRequest("POST", url, bytes.NewBuffer(data)) 127 | req.Header.Set("Authorization", "Bearer "+cmd.DeltaAuth) 128 | req.Header.Set("Content-Type", "application/json") 129 | 130 | client := &http.Client{} 131 | resp, err := client.Do(req) 132 | if err != nil { 133 | panic(err) 134 | } 135 | defer resp.Body.Close() 136 | var response WalletRegisterResponse 137 | err = json.NewDecoder(resp.Body).Decode(&response) 138 | if err != nil { 139 | panic(err) 140 | } 141 | var buffer bytes.Buffer 142 | err = utils.PrettyEncode(response, &buffer) 143 | if err != nil { 144 | fmt.Println(err) 145 | } 146 | fmt.Println(buffer.String()) 147 | return nil 148 | }, 149 | }, 150 | { 151 | Name: "list", 152 | Usage: "List all wallets associated with the API key", 153 | Action: func(context *cli.Context) error { 154 | cmd, err := NewDeltaCmdNode(context) 155 | if err != nil { 156 | return err 157 | } 158 | 159 | url := cmd.DeltaApi + "/admin/wallet/list" 160 | req, err := http.NewRequest("GET", url, nil) 161 | if err != nil { 162 | panic(err) 163 | } 164 | req.Header.Set("Authorization", "Bearer "+cmd.DeltaAuth) 165 | req.Header.Set("Content-Type", "application/json") 166 | 167 | var walletListResponse WalletListResponse 168 | client := &http.Client{} 169 | resp, err := client.Do(req) 170 | if err != nil { 171 | panic(err) 172 | } 173 | defer resp.Body.Close() 174 | err = json.NewDecoder(resp.Body).Decode(&walletListResponse) 175 | if err != nil { 176 | panic(err) 177 | } 178 | var buffer bytes.Buffer 179 | err = utils.PrettyEncode(walletListResponse, &buffer) 180 | if err != nil { 181 | fmt.Println(err) 182 | } 183 | fmt.Println(buffer.String()) 184 | return nil 185 | }, 186 | }, 187 | }, 188 | } 189 | 190 | walletCommands = append(walletCommands, walletCmd) 191 | 192 | return walletCommands 193 | } 194 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "github.com/caarlos0/env/v6" 5 | logging "github.com/ipfs/go-log/v2" 6 | "github.com/joho/godotenv" 7 | "github.com/libp2p/go-libp2p/core/peer" 8 | "github.com/multiformats/go-multiaddr" 9 | ) 10 | 11 | var ( 12 | log = logging.Logger("config") 13 | defaultTestBootstrapPeers []multiaddr.Multiaddr 14 | ) 15 | 16 | type DeltaConfig struct { 17 | Node struct { 18 | Name string `env:"NODE_NAME" envDefault:"delta-deal-maker"` 19 | Description string `env:"NODE_DESCRIPTION" envDefault:"delta-deal-maker"` 20 | Type string `env:"NODE_TYPE" envDefault:"delta-deal-maker"` 21 | InstanceUuid string `env:"INSTANCE_UUID"` 22 | KeepCopies bool `env:"KEEP_COPIES" envDefault:"false"` 23 | AnnounceAddrIP string `env:"ANNOUNCE_ADDR_IP"` 24 | } 25 | 26 | Dispatcher struct { 27 | MaxCleanupWorkers int `env:"MAX_CLEANUP_WORKERS" envDefault:"1500"` 28 | } 29 | 30 | Common struct { 31 | Mode string `env:"MODE" envDefault:"standalone"` 32 | DBDSN string `env:"DB_DSN" envDefault:"delta.db"` 33 | EnableWebsocket bool `env:"ENABLE_WEBSOCKET" envDefault:"false"` 34 | CommpMode string `env:"COMMP_MODE" envDefault:"fast"` // option "filboost" 35 | StatsCollection bool `env:"STATS_COLLECTION" envDefault:"true"` 36 | Commit string `env:"COMMIT"` 37 | Version string `env:"VERSION"` 38 | MaxReplicationFactor int `env:"MAX_REPLICATION_FACTOR" envDefault:"6"` 39 | MaxAutoRetry int `env:"MAX_AUTO_RETRY" envDefault:"3"` 40 | MinE2EFileSize int64 `env:"MIN_E2E_FILE_SIZE" envDefault:"1048576000"` // 1GB 41 | Network string `env:"NETWORK" envDefault:"main"` 42 | } 43 | 44 | // configurable via env vars 45 | ExternalApis struct { 46 | LotusApi string `env:"LOTUS_API" envDefault:"http://api.chain.love"` 47 | AuthSvcApi string `env:"AUTH_SVC_API" envDefault:"https://auth.estuary.tech"` 48 | SpThrottlerApi string `env:"SP_THROTTLER_SVC_API" envDefault:"https://sp-throttler.delta.store"` 49 | SpSelectionApi string `env:"SP_SELECTION_SVC_API" envDefault:"https://sp-select.delta.store/api/providers"` 50 | DealStatusApi string `env:"DEAL_STATUS_API" envDefault:"https://deal-status.estuary.tech"` 51 | } 52 | 53 | Standalone struct { 54 | APIKey string `env:"DELTA_AUTH" envDefault:""` 55 | } 56 | } 57 | 58 | func InitConfig() DeltaConfig { 59 | godotenv.Load() // load from environment OR .env file if it exists 60 | var cfg DeltaConfig 61 | 62 | if err := env.Parse(&cfg); err != nil { 63 | log.Fatal("error parsing config: %+v\n", err) 64 | } 65 | 66 | log.Debug("config parsed successfully") 67 | 68 | return cfg 69 | } 70 | 71 | // BootstrapEstuaryPeers Creating a list of multiaddresses that are used to bootstrap the network. 72 | func BootstrapEstuaryPeers() []peer.AddrInfo { 73 | 74 | for _, s := range []string{ 75 | "/ip4/145.40.90.135/tcp/6746/p2p/12D3KooWNTiHg8eQsTRx8XV7TiJbq3379EgwG6Mo3V3MdwAfThsx", 76 | "/ip4/139.178.68.217/tcp/6744/p2p/12D3KooWCVXs8P7iq6ao4XhfAmKWrEeuKFWCJgqe9jGDMTqHYBjw", 77 | "/ip4/147.75.49.71/tcp/6745/p2p/12D3KooWGBWx9gyUFTVQcKMTenQMSyE2ad9m7c9fpjS4NMjoDien", 78 | "/ip4/147.75.86.255/tcp/6745/p2p/12D3KooWFrnuj5o3tx4fGD2ZVJRyDqTdzGnU3XYXmBbWbc8Hs8Nd", 79 | "/ip4/3.134.223.177/tcp/6745/p2p/12D3KooWN8vAoGd6eurUSidcpLYguQiGZwt4eVgDvbgaS7kiGTup", 80 | "/ip4/35.74.45.12/udp/6746/quic/p2p/12D3KooWLV128pddyvoG6NBvoZw7sSrgpMTPtjnpu3mSmENqhtL7", 81 | "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", 82 | "/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa", 83 | "/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb", 84 | "/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt", 85 | } { 86 | ma, err := multiaddr.NewMultiaddr(s) 87 | if err != nil { 88 | panic(err) 89 | } 90 | defaultTestBootstrapPeers = append(defaultTestBootstrapPeers, ma) 91 | } 92 | 93 | peers, _ := peer.AddrInfosFromP2pAddrs(defaultTestBootstrapPeers...) 94 | return peers 95 | } 96 | -------------------------------------------------------------------------------- /core/commp.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "io" 8 | 9 | "github.com/application-research/filclient" 10 | "github.com/filecoin-project/go-commp-utils/writer" 11 | "github.com/filecoin-project/go-state-types/abi" 12 | "github.com/ipfs/go-cid" 13 | blockstore "github.com/ipfs/go-ipfs-blockstore" 14 | carv2 "github.com/ipld/go-car/v2" 15 | "github.com/labstack/gommon/log" 16 | ) 17 | 18 | type CommpService struct { 19 | DeltaNode *DeltaNode 20 | } 21 | 22 | // GenerateCommPFile Generating a CommP file from a payload file. 23 | // Generating a CommP file from a payload file. 24 | func (c CommpService) GenerateCommPFile(context context.Context, payloadCid cid.Cid, blockstore blockstore.Blockstore) (pieceCid cid.Cid, payloadSize uint64, unPaddedPieceSize abi.UnpaddedPieceSize, err error) { 25 | return filclient.GeneratePieceCommitment(context, payloadCid, blockstore) 26 | } 27 | 28 | // GenerateCommPCarV2 Generating a CommP file from a CARv2 file. 29 | // Generating a CommP file from a CARv2 file. 30 | func (c CommpService) GenerateCommPCarV2(readerFromFile io.Reader) (*abi.PieceInfo, error) { 31 | bytesFromCar, err := io.ReadAll(readerFromFile) 32 | if err != nil { 33 | return nil, fmt.Errorf("error reading car stream: %w", err) 34 | } 35 | rd, err := carv2.NewReader(bytes.NewReader(bytesFromCar)) 36 | if err != nil { 37 | return nil, fmt.Errorf("failed to get CARv2 reader: %w", err) 38 | } 39 | 40 | defer func() { 41 | if err := rd.Close(); err != nil { 42 | log.Warnf("failed to close CARv2 reader: %w", err) 43 | } 44 | }() 45 | 46 | // dump the CARv1 payload of the CARv2 file to the Commp Writer and get back the CommP. 47 | w := &writer.Writer{} 48 | r, err := rd.DataReader() 49 | if err != nil { 50 | return nil, fmt.Errorf("getting data reader for CAR v1 from CAR v2: %w", err) 51 | } 52 | 53 | written, err := io.Copy(w, r) 54 | if err != nil { 55 | return nil, fmt.Errorf("writing to commp writer: %w", err) 56 | } 57 | size, err := c.GetCarSize(readerFromFile, rd) 58 | if err != nil { 59 | return nil, err 60 | } 61 | if size == 0 { 62 | size = int64(len(bytesFromCar)) 63 | } 64 | 65 | if written != size { 66 | return nil, fmt.Errorf("number of bytes written to CommP writer %d not equal to the CARv1 payload size %d", written, rd.Header.DataSize) 67 | } 68 | 69 | pi, err := w.Sum() 70 | if err != nil { 71 | return nil, fmt.Errorf("failed to calculate CommP: %w", err) 72 | } 73 | 74 | return &abi.PieceInfo{ 75 | Size: pi.PieceSize, 76 | PieceCID: pi.PieceCID, 77 | }, nil 78 | } 79 | 80 | // Generate a commP from a reader 81 | func (c CommpService) GenerateCommp(readerFromFile io.ReadSeekCloser) (writer.DataCIDSize, error) { 82 | return fastCommp(readerFromFile) 83 | } 84 | 85 | // GetSize Getting the size of the file. 86 | // Getting the size of the file. 87 | func (c CommpService) GetSize(stream io.Reader) int { 88 | buf := new(bytes.Buffer) 89 | buf.ReadFrom(stream) 90 | return buf.Len() 91 | } 92 | 93 | // GetCarSize Getting the size of the CARv2 file. 94 | // Getting the size of the CARv2 file. 95 | func (c CommpService) GetCarSize(stream io.Reader, rd *carv2.Reader) (int64, error) { 96 | var size int64 97 | switch rd.Version { 98 | case 2: 99 | size = int64(rd.Header.DataSize) 100 | case 1: 101 | bytes, err := io.ReadAll(stream) 102 | if err != nil { 103 | return 0, err 104 | } 105 | size = int64(len(bytes)) 106 | } 107 | return size, nil 108 | } 109 | -------------------------------------------------------------------------------- /core/commp_test.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "reflect" 7 | "testing" 8 | 9 | "github.com/filecoin-project/go-commp-utils/writer" 10 | "github.com/filecoin-project/go-state-types/abi" 11 | "github.com/ipfs/go-cid" 12 | blockstore "github.com/ipfs/go-ipfs-blockstore" 13 | carv2 "github.com/ipld/go-car/v2" 14 | ) 15 | 16 | func TestCommpService_GenerateCommPCarV2(t *testing.T) { 17 | type fields struct { 18 | DeltaNode *DeltaNode 19 | } 20 | type args struct { 21 | readerFromFile io.Reader 22 | } 23 | tests := []struct { 24 | name string 25 | fields fields 26 | args args 27 | want *abi.PieceInfo 28 | wantErr bool 29 | }{ 30 | // TODO: Add test cases. 31 | } 32 | for _, tt := range tests { 33 | t.Run(tt.name, func(t *testing.T) { 34 | c := CommpService{ 35 | DeltaNode: tt.fields.DeltaNode, 36 | } 37 | got, err := c.GenerateCommPCarV2(tt.args.readerFromFile) 38 | if (err != nil) != tt.wantErr { 39 | t.Errorf("GenerateCommPCarV2() error = %v, wantErr %v", err, tt.wantErr) 40 | return 41 | } 42 | if !reflect.DeepEqual(got, tt.want) { 43 | t.Errorf("GenerateCommPCarV2() got = %v, want %v", got, tt.want) 44 | } 45 | }) 46 | } 47 | } 48 | 49 | func TestCommpService_GenerateCommPFile(t *testing.T) { 50 | type fields struct { 51 | DeltaNode *DeltaNode 52 | } 53 | type args struct { 54 | context context.Context 55 | payloadCid cid.Cid 56 | blockstore blockstore.Blockstore 57 | } 58 | tests := []struct { 59 | name string 60 | fields fields 61 | args args 62 | wantPieceCid cid.Cid 63 | wantPayloadSize uint64 64 | wantUnPaddedPieceSize abi.UnpaddedPieceSize 65 | wantErr bool 66 | }{ 67 | // TODO: Add test cases. 68 | } 69 | for _, tt := range tests { 70 | t.Run(tt.name, func(t *testing.T) { 71 | c := CommpService{ 72 | DeltaNode: tt.fields.DeltaNode, 73 | } 74 | gotPieceCid, gotPayloadSize, gotUnPaddedPieceSize, err := c.GenerateCommPFile(tt.args.context, tt.args.payloadCid, tt.args.blockstore) 75 | if (err != nil) != tt.wantErr { 76 | t.Errorf("GenerateCommPFile() error = %v, wantErr %v", err, tt.wantErr) 77 | return 78 | } 79 | if !reflect.DeepEqual(gotPieceCid, tt.wantPieceCid) { 80 | t.Errorf("GenerateCommPFile() gotPieceCid = %v, want %v", gotPieceCid, tt.wantPieceCid) 81 | } 82 | if gotPayloadSize != tt.wantPayloadSize { 83 | t.Errorf("GenerateCommPFile() gotPayloadSize = %v, want %v", gotPayloadSize, tt.wantPayloadSize) 84 | } 85 | if gotUnPaddedPieceSize != tt.wantUnPaddedPieceSize { 86 | t.Errorf("GenerateCommPFile() gotUnPaddedPieceSize = %v, want %v", gotUnPaddedPieceSize, tt.wantUnPaddedPieceSize) 87 | } 88 | }) 89 | } 90 | } 91 | 92 | func TestCommpService_GenerateParallelCommp(t *testing.T) { 93 | type fields struct { 94 | DeltaNode *DeltaNode 95 | } 96 | type args struct { 97 | readerFromFile io.ReadSeekCloser 98 | } 99 | tests := []struct { 100 | name string 101 | fields fields 102 | args args 103 | want writer.DataCIDSize 104 | wantErr bool 105 | }{ 106 | // TODO: Add test cases. 107 | } 108 | for _, tt := range tests { 109 | t.Run(tt.name, func(t *testing.T) { 110 | c := CommpService{ 111 | DeltaNode: tt.fields.DeltaNode, 112 | } 113 | got, err := c.GenerateCommp(tt.args.readerFromFile) 114 | if (err != nil) != tt.wantErr { 115 | t.Errorf("GenerateCommp() error = %v, wantErr %v", err, tt.wantErr) 116 | return 117 | } 118 | if !reflect.DeepEqual(got, tt.want) { 119 | t.Errorf("GenerateCommp() got = %v, want %v", got, tt.want) 120 | } 121 | }) 122 | } 123 | } 124 | 125 | func TestCommpService_GetCarSize(t *testing.T) { 126 | type fields struct { 127 | DeltaNode *DeltaNode 128 | } 129 | type args struct { 130 | stream io.Reader 131 | rd *carv2.Reader 132 | } 133 | tests := []struct { 134 | name string 135 | fields fields 136 | args args 137 | want int64 138 | wantErr bool 139 | }{ 140 | // TODO: Add test cases. 141 | } 142 | for _, tt := range tests { 143 | t.Run(tt.name, func(t *testing.T) { 144 | c := CommpService{ 145 | DeltaNode: tt.fields.DeltaNode, 146 | } 147 | got, err := c.GetCarSize(tt.args.stream, tt.args.rd) 148 | if (err != nil) != tt.wantErr { 149 | t.Errorf("GetCarSize() error = %v, wantErr %v", err, tt.wantErr) 150 | return 151 | } 152 | if got != tt.want { 153 | t.Errorf("GetCarSize() got = %v, want %v", got, tt.want) 154 | } 155 | }) 156 | } 157 | } 158 | 159 | func TestCommpService_GetSize(t *testing.T) { 160 | type fields struct { 161 | DeltaNode *DeltaNode 162 | } 163 | type args struct { 164 | stream io.Reader 165 | } 166 | tests := []struct { 167 | name string 168 | fields fields 169 | args args 170 | want int 171 | }{ 172 | // TODO: Add test cases. 173 | } 174 | for _, tt := range tests { 175 | t.Run(tt.name, func(t *testing.T) { 176 | c := CommpService{ 177 | DeltaNode: tt.fields.DeltaNode, 178 | } 179 | if got := c.GetSize(tt.args.stream); got != tt.want { 180 | t.Errorf("GetSize() = %v, want %v", got, tt.want) 181 | } 182 | }) 183 | } 184 | } 185 | -------------------------------------------------------------------------------- /core/deal_status.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | type DealStatusParam struct { 4 | DealUuid string 5 | MinerAddr string 6 | } 7 | 8 | type DealStatusResult struct { 9 | } 10 | 11 | type DealStatusService struct { 12 | DeltaNode DeltaNode 13 | BoostApi string 14 | } 15 | 16 | func NewDealStatusService(dn DeltaNode) *DealStatusService { 17 | return &DealStatusService{ 18 | DeltaNode: dn, 19 | } 20 | } 21 | 22 | func (d DealStatusService) GetDealStatus(param DealStatusParam) (DealStatusResult, error) { 23 | return DealStatusResult{}, nil 24 | } 25 | 26 | // get dealid 27 | // check status 28 | 29 | //func (d DealStatusService) GetDealID(param DealStatusParam) (DealStatusResult, error) { 30 | // ctx := context.Background() 31 | // d.DeltaNode.LotusApiNode.StateSearchMsg(ctx, types.EmptyTSK, pubcid, 1000, false) 32 | // return DealStatusResult{}, nil 33 | //} 34 | 35 | //func (m *manager) GetProviderDealStatus(ctx context.Context, d *model.ContentDeal, maddr address.Address, dealUUID *uuid.UUID) (*storagemarket.ProviderDealState, bool, error) { 36 | // isPushTransfer := false 37 | // providerDealState, err := m.fc.DealStatus(ctx, maddr, d.PropCid.CID, dealUUID) 38 | // if err != nil && providerDealState == nil { 39 | // isPushTransfer = true 40 | // providerDealState, err = m.fc.DealStatus(ctx, maddr, d.PropCid.CID, nil) 41 | // } 42 | // return providerDealState, isPushTransfer, err 43 | //} 44 | 45 | //func (s *apiV1) handleGetDealInfo(c echo.Context) error { 46 | // dealid, err := strconv.ParseInt(c.Param("dealid"), 10, 64) 47 | // if err != nil { 48 | // return err 49 | // } 50 | // 51 | // deal, err := s.api.StateMarketStorageDeal(c.Request().Context(), abi.DealID(dealid), types.EmptyTSK) 52 | // if err != nil { 53 | // return err 54 | // } 55 | // 56 | // return c.JSON(http.StatusOK, deal) 57 | //} 58 | -------------------------------------------------------------------------------- /core/job_test.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | ) 7 | 8 | type TestProcessor struct { 9 | ID int 10 | Name string 11 | } 12 | 13 | func (tp *TestProcessor) Run() error { 14 | if tp.ID == 0 { 15 | return errors.New("Invalid Test Processor ID") 16 | } 17 | return nil 18 | } 19 | 20 | func TestDispatcher(t *testing.T) { 21 | // create a new dispatcher 22 | dispatcher := CreateNewDispatcher() 23 | 24 | // add a job to the job queue 25 | testProcessor := &TestProcessor{ID: 1, Name: "Test Processor 1"} 26 | dispatcher.AddJob(testProcessor) 27 | 28 | // start the dispatcher with one worker 29 | dispatcher.Start(1) 30 | 31 | // wait until the dispatcher has finished processing all jobs 32 | for !dispatcher.Finished() { 33 | } 34 | 35 | // check if the job was processed successfully 36 | if testProcessor.ID != 1 { 37 | t.Errorf("Test Processor did not run successfully. Expected ID: 1, Actual ID: %d", testProcessor.ID) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /core/libp2p.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "context" 5 | "delta/utils" 6 | "fmt" 7 | model "delta/models" 8 | fc "github.com/application-research/filclient" 9 | datatransfer "github.com/filecoin-project/go-data-transfer" 10 | "github.com/ipfs/go-cid" 11 | "time" 12 | ) 13 | 14 | // SetDataTransferEventsSubscribe `filc.SubscribeToDataTransferEvents(func(event datatransfer.Event, channelState datatransfer.ChannelState) {` 15 | // 16 | // The above function is a callback function that is called whenever a data transfer event occurs. The callback function 17 | // takes two arguments: `event` and `channelState`. The `event` argument is of type `datatransfer.Event` and the 18 | // `channelState` argument is of type `datatransfer.ChannelState` 19 | func SetDataTransferEventsSubscribe(i *DeltaNode) { 20 | fmt.Println(utils.Purple + "Subscribing to transfer channel events..." + utils.Reset) 21 | i.FilClient.SubscribeToDataTransferEvents(func(event datatransfer.Event, channelState datatransfer.ChannelState) { 22 | switch event.Code { 23 | case datatransfer.DataQueued: 24 | fmt.Println("Data Transfer Queued event: ", event, " for transfer id: ", channelState.TransferID(), " for db id: ", channelState.BaseCID()) 25 | case datatransfer.Complete: 26 | fmt.Println("Data Transfer Complete event: ", event, " for transfer id: ", channelState.TransferID(), " for db id: ", channelState.BaseCID()) 27 | case datatransfer.Error, datatransfer.Disconnected, datatransfer.ReceiveDataError, datatransfer.Cancel, datatransfer.RequestTimedOut, datatransfer.SendDataError: 28 | fmt.Println("Data Transfer Error event: ", event, " for transfer id: ", channelState.TransferID(), " for db id: ", channelState.BaseCID()) 29 | } 30 | }) 31 | } 32 | 33 | // SetLibp2pManagerSubscribe It subscribes to the libp2p transfer manager and updates the database with the status of the transfer 34 | func SetLibp2pManagerSubscribe(i *DeltaNode) { 35 | 36 | fmt.Println(utils.Purple + "Subscribing to transfer channel states..." + utils.Reset) 37 | i.FilClient.Libp2pTransferMgr.Subscribe(func(dbid uint, fst fc.ChannelState) { 38 | //fmt.Println("Transfer status: ", fst.Status, " for transfer id: ", fst.TransferID, " for db id: ", dbid) 39 | switch fst.Status { 40 | case datatransfer.Requested: 41 | fmt.Println("Transfer status: ", fst.Status, " for transfer id: ", fst.TransferID, " for db id: ", dbid) 42 | var contentDeal model.ContentDeal 43 | i.DB.Model(&model.ContentDeal{}).Where("id = ?", dbid).Find(&contentDeal) 44 | // save the content deal 45 | contentDeal.TransferStarted = time.Now() 46 | contentDeal.UpdatedAt = time.Now() 47 | contentDeal.LastMessage = utils.DEAL_STATUS_TRANSFER_STARTED 48 | i.DB.Save(&contentDeal) 49 | 50 | case datatransfer.TransferFinished, datatransfer.Completed: 51 | fmt.Println("Transfer status: ", fst.Status, " for transfer id: ", fst.TransferID, " for db id: ", dbid) 52 | //transferId, err := strconv.Atoi(fst.TransferID) 53 | //if err != nil { 54 | // fmt.Println(err) 55 | //} 56 | 57 | // save the content deal 58 | var contentDeal model.ContentDeal 59 | i.DB.Model(&model.ContentDeal{}).Where("id = ?", dbid).Find(&contentDeal) 60 | contentDeal.TransferFinished = time.Now() 61 | contentDeal.SealedAt = time.Now() 62 | contentDeal.UpdatedAt = time.Now() 63 | contentDeal.OnChainAt = time.Now() 64 | contentDeal.LastMessage = utils.DEAL_STATUS_TRANSFER_FINISHED 65 | i.DB.Save(&contentDeal) 66 | 67 | // save the content status 68 | var content model.Content 69 | i.DB.Model(&model.Content{}).Where("id in (select cd.content from content_deals cd where cd.id = ?)", dbid).Find(&content) 70 | content.Status = utils.DEAL_STATUS_TRANSFER_FINISHED 71 | content.LastMessage = utils.DEAL_STATUS_TRANSFER_FINISHED 72 | content.UpdatedAt = time.Now() 73 | i.DB.Save(&content) 74 | 75 | // remove from the blockstore 76 | cidToDelete, err := cid.Decode(content.Cid) 77 | if err != nil { 78 | fmt.Println(err) 79 | } 80 | if i.Config.Node.KeepCopies { 81 | fmt.Println("Keeping a copy of the content - not removing from the blockstore - CID: ", cidToDelete, "") 82 | } else { 83 | go i.Node.Blockservice.DeleteBlock(context.Background(), cidToDelete) 84 | } 85 | case datatransfer.Failed, datatransfer.Failing, datatransfer.Cancelled, datatransfer.InitiatorPaused, datatransfer.ResponderPaused, datatransfer.ChannelNotFoundError: 86 | fmt.Println("Transfer status: ", fst.Status, " for transfer id: ", fst.TransferID, " for db id: ", dbid) 87 | var contentDeal model.ContentDeal 88 | i.DB.Model(&model.ContentDeal{}).Where("id = ?", dbid).Find(&contentDeal) 89 | contentDeal.LastMessage = fst.Message 90 | contentDeal.UpdatedAt = time.Now() 91 | contentDeal.FailedAt = time.Now() 92 | i.DB.Save(&contentDeal) 93 | 94 | var content model.Content 95 | i.DB.Model(&model.Content{}).Where("id in (select cd.content from content_deals cd where cd.id = ?)", dbid).Find(&content) 96 | content.Status = utils.DEAL_STATUS_TRANSFER_FAILED 97 | content.LastMessage = fst.Message 98 | content.UpdatedAt = time.Now() 99 | i.DB.Save(&content) 100 | 101 | // remove from the blockstore 102 | cidToDelete, err := cid.Decode(content.Cid) 103 | if err != nil { 104 | fmt.Println(err) 105 | } 106 | if i.Config.Node.KeepCopies { 107 | fmt.Println("Keeping a copy of the content - not removing from the blockstore - CID: ", cidToDelete, "") 108 | } else { 109 | go i.Node.Blockservice.DeleteBlock(context.Background(), cidToDelete) 110 | } 111 | default: 112 | } 113 | }) 114 | } 115 | -------------------------------------------------------------------------------- /core/libp2p_test.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import "testing" 4 | 5 | func TestSetDataTransferEventsSubscribe(t *testing.T) { 6 | type args struct { 7 | i *DeltaNode 8 | } 9 | tests := []struct { 10 | name string 11 | args args 12 | }{ 13 | // TODO: Add test cases. 14 | } 15 | for _, tt := range tests { 16 | t.Run(tt.name, func(t *testing.T) { 17 | SetDataTransferEventsSubscribe(tt.args.i) 18 | }) 19 | } 20 | } 21 | 22 | func TestSetLibp2pManagerSubscribe(t *testing.T) { 23 | type args struct { 24 | i *DeltaNode 25 | } 26 | tests := []struct { 27 | name string 28 | args args 29 | }{ 30 | // TODO: Add test cases. 31 | } 32 | for _, tt := range tests { 33 | t.Run(tt.name, func(t *testing.T) { 34 | SetLibp2pManagerSubscribe(tt.args.i) 35 | }) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /core/miner_assignment.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "net/http" 7 | "time" 8 | ) 9 | 10 | type Provider struct { 11 | ID string `json:"id"` 12 | Address string `json:"address"` 13 | AddressOfOwner string `json:"address_of_owner"` 14 | AddressOfWorker string `json:"address_of_worker"` 15 | AddressOfBeneficiary string `json:"address_of_beneficiary"` 16 | SectorSizeBytes string `json:"sector_size_bytes"` 17 | MaxPieceSizeBytes string `json:"max_piece_size_bytes"` 18 | MinPieceSizeBytes string `json:"min_piece_size_bytes"` 19 | PriceAttofil string `json:"price_attofil"` 20 | PriceVerifiedAttofil string `json:"price_verified_attofil"` 21 | BalanceAttofil string `json:"balance_attofil"` 22 | LockedFundsAttofil string `json:"locked_funds_attofil"` 23 | InitialPledgeAttofil string `json:"initial_pledge_attofil"` 24 | RawPowerBytes string `json:"raw_power_bytes"` 25 | QualityAdjustedPowerBytes string `json:"quality_adjusted_power_bytes"` 26 | TotalRawPowerBytes string `json:"total_raw_power_bytes"` 27 | TotalQualityAdjustedPowerBytes string `json:"total_quality_adjusted_power_bytes"` 28 | TotalStorageDealCount string `json:"total_storage_deal_count"` 29 | TotalSectorsSealedByPostCount string `json:"total_sectors_sealed_by_post_count"` 30 | PeerID string `json:"peer_id"` 31 | Height string `json:"height"` 32 | LotusVersion string `json:"lotus_version"` 33 | Multiaddrs struct { 34 | Addresses []string `json:"addresses"` 35 | } `json:"multiaddrs"` 36 | Metadata interface{} `json:"metadata"` 37 | AddressOfControllers struct { 38 | Addresses []string `json:"addresses"` 39 | } `json:"address_of_controllers"` 40 | Tipset struct { 41 | Cids []struct { 42 | NAMING_FAILED string `json:"/"` 43 | } `json:"cids"` 44 | } `json:"tipset"` 45 | CreatedAt time.Time `json:"created_at"` 46 | UpdatedAt time.Time `json:"updated_at"` 47 | } 48 | 49 | type MinerAssignmentService struct { 50 | DeltaNode DeltaNode 51 | } 52 | 53 | // GetSPInfo 54 | func NewMinerAssignmentService(node DeltaNode) *MinerAssignmentService { 55 | return &MinerAssignmentService{ 56 | DeltaNode: node, 57 | } 58 | } 59 | 60 | // A function that takes in a parameter, byteSize, and returns a Provider and an error. 61 | func (m MinerAssignmentService) GetSPWithGivenBytes(byteSize int64) (Provider, error) { 62 | bytSizeStr := fmt.Sprintf("%d", byteSize) 63 | fmt.Println("Getting SP with given bytes: ", m.DeltaNode.Config.ExternalApis.SpSelectionApi+"?size_bytes="+bytSizeStr) 64 | resp, err := http.Get(m.DeltaNode.Config.ExternalApis.SpSelectionApi + "?size_bytes=" + bytSizeStr) 65 | if err != nil { 66 | // handle error 67 | fmt.Println("Error making HTTP request:", err) 68 | return Provider{}, err 69 | } 70 | defer resp.Body.Close() 71 | var provider Provider 72 | err = json.NewDecoder(resp.Body).Decode(&provider) 73 | if err != nil { 74 | // handle error 75 | fmt.Println("Error decoding JSON:", err) 76 | return Provider{}, err 77 | } 78 | 79 | return provider, nil 80 | 81 | } 82 | 83 | // A function that takes in two parameters, byteSize and sourceIp, and returns a Provider and an error. 84 | func (m MinerAssignmentService) GetSPWithGivenBytesAndIp(byteSize string, sourceIp string) (Provider, error) { 85 | resp, err := http.Get(m.DeltaNode.Config.ExternalApis.SpSelectionApi + "?size_bytes=" + byteSize + "&source_ip=" + sourceIp) 86 | if err != nil { 87 | // handle error 88 | fmt.Println("Error making HTTP request:", err) 89 | return Provider{}, err 90 | } 91 | defer resp.Body.Close() 92 | var provider Provider 93 | err = json.NewDecoder(resp.Body).Decode(&provider) 94 | if err != nil { 95 | // handle error 96 | fmt.Println("Error decoding JSON:", err) 97 | return Provider{}, err 98 | } 99 | 100 | return provider, nil 101 | } 102 | -------------------------------------------------------------------------------- /core/repair.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | type RepairParam struct { 4 | } 5 | 6 | type RepairResult struct { 7 | } 8 | 9 | type RepairService struct { 10 | DeltaNode DeltaNode 11 | } 12 | 13 | func NewRepairService(dn DeltaNode) *RepairService { 14 | return &RepairService{ 15 | DeltaNode: dn, 16 | } 17 | } 18 | 19 | // RecreateDeal A method of RepairService. 20 | // A method of RepairService. 21 | func (r RepairService) RecreateDeal(param RepairParam) (RepairResult, error) { 22 | return RepairResult{}, nil 23 | } 24 | 25 | // RestartDataTransfer A method of RepairService. 26 | func (r RepairService) RestartDataTransfer(param RepairParam) (RepairResult, error) { 27 | return RepairResult{}, nil 28 | } 29 | 30 | // RecreateCommp A method of RepairService. 31 | func (r RepairService) RecreateCommp(param RepairParam) (RepairResult, error) { 32 | return RepairResult{}, nil 33 | } 34 | 35 | // RecreateContent A method of RepairService. 36 | func (r RepairService) RecreateContent(param RepairParam) (RepairResult, error) { 37 | return RepairResult{}, nil 38 | } 39 | -------------------------------------------------------------------------------- /core/repair_test.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | ) 7 | 8 | func TestNewRepairService(t *testing.T) { 9 | type args struct { 10 | dn DeltaNode 11 | } 12 | tests := []struct { 13 | name string 14 | args args 15 | want *RepairService 16 | }{ 17 | // TODO: Add test cases. 18 | } 19 | for _, tt := range tests { 20 | t.Run(tt.name, func(t *testing.T) { 21 | if got := NewRepairService(tt.args.dn); !reflect.DeepEqual(got, tt.want) { 22 | t.Errorf("NewRepairService() = %v, want %v", got, tt.want) 23 | } 24 | }) 25 | } 26 | } 27 | 28 | func TestRepairService_RecreateCommp(t *testing.T) { 29 | type fields struct { 30 | DeltaNode DeltaNode 31 | } 32 | type args struct { 33 | param RepairParam 34 | } 35 | tests := []struct { 36 | name string 37 | fields fields 38 | args args 39 | want RepairResult 40 | wantErr bool 41 | }{ 42 | // TODO: Add test cases. 43 | } 44 | for _, tt := range tests { 45 | t.Run(tt.name, func(t *testing.T) { 46 | r := RepairService{ 47 | DeltaNode: tt.fields.DeltaNode, 48 | } 49 | got, err := r.RecreateCommp(tt.args.param) 50 | if (err != nil) != tt.wantErr { 51 | t.Errorf("RecreateCommp() error = %v, wantErr %v", err, tt.wantErr) 52 | return 53 | } 54 | if !reflect.DeepEqual(got, tt.want) { 55 | t.Errorf("RecreateCommp() got = %v, want %v", got, tt.want) 56 | } 57 | }) 58 | } 59 | } 60 | 61 | func TestRepairService_RecreateContent(t *testing.T) { 62 | type fields struct { 63 | DeltaNode DeltaNode 64 | } 65 | type args struct { 66 | param RepairParam 67 | } 68 | tests := []struct { 69 | name string 70 | fields fields 71 | args args 72 | want RepairResult 73 | wantErr bool 74 | }{ 75 | // TODO: Add test cases. 76 | } 77 | for _, tt := range tests { 78 | t.Run(tt.name, func(t *testing.T) { 79 | r := RepairService{ 80 | DeltaNode: tt.fields.DeltaNode, 81 | } 82 | got, err := r.RecreateContent(tt.args.param) 83 | if (err != nil) != tt.wantErr { 84 | t.Errorf("RecreateContent() error = %v, wantErr %v", err, tt.wantErr) 85 | return 86 | } 87 | if !reflect.DeepEqual(got, tt.want) { 88 | t.Errorf("RecreateContent() got = %v, want %v", got, tt.want) 89 | } 90 | }) 91 | } 92 | } 93 | 94 | func TestRepairService_RecreateDeal(t *testing.T) { 95 | type fields struct { 96 | DeltaNode DeltaNode 97 | } 98 | type args struct { 99 | param RepairParam 100 | } 101 | tests := []struct { 102 | name string 103 | fields fields 104 | args args 105 | want RepairResult 106 | wantErr bool 107 | }{ 108 | // TODO: Add test cases. 109 | } 110 | for _, tt := range tests { 111 | t.Run(tt.name, func(t *testing.T) { 112 | r := RepairService{ 113 | DeltaNode: tt.fields.DeltaNode, 114 | } 115 | got, err := r.RecreateDeal(tt.args.param) 116 | if (err != nil) != tt.wantErr { 117 | t.Errorf("RecreateDeal() error = %v, wantErr %v", err, tt.wantErr) 118 | return 119 | } 120 | if !reflect.DeepEqual(got, tt.want) { 121 | t.Errorf("RecreateDeal() got = %v, want %v", got, tt.want) 122 | } 123 | }) 124 | } 125 | } 126 | 127 | func TestRepairService_RestartDataTransfer(t *testing.T) { 128 | type fields struct { 129 | DeltaNode DeltaNode 130 | } 131 | type args struct { 132 | param RepairParam 133 | } 134 | tests := []struct { 135 | name string 136 | fields fields 137 | args args 138 | want RepairResult 139 | wantErr bool 140 | }{ 141 | // TODO: Add test cases. 142 | } 143 | for _, tt := range tests { 144 | t.Run(tt.name, func(t *testing.T) { 145 | r := RepairService{ 146 | DeltaNode: tt.fields.DeltaNode, 147 | } 148 | got, err := r.RestartDataTransfer(tt.args.param) 149 | if (err != nil) != tt.wantErr { 150 | t.Errorf("RestartDataTransfer() error = %v, wantErr %v", err, tt.wantErr) 151 | return 152 | } 153 | if !reflect.DeepEqual(got, tt.want) { 154 | t.Errorf("RestartDataTransfer() got = %v, want %v", got, tt.want) 155 | } 156 | }) 157 | } 158 | } 159 | -------------------------------------------------------------------------------- /core/replication.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | // 4 | //import ( 5 | // "fmt" 6 | // model "delta/models" 7 | // "gorm.io/gorm" 8 | // "time" 9 | //) 10 | // 11 | //type ReplicationService struct { 12 | // // `NewReplicationService` is a function that returns a `ReplicationService` struct 13 | // LightNode *DeltaNode 14 | //} 15 | // 16 | //// NewReplicationService Creating a new `ReplicationService` struct. 17 | //func NewReplicationService(ln *DeltaNode) *ReplicationService { 18 | // return &ReplicationService{ 19 | // LightNode: ln, 20 | // } 21 | //} 22 | // 23 | //type DealReplication struct { 24 | // Content model.Content `json:"content"` 25 | // ContentDealProposalParameter model.ContentDealProposalParameters `json:"deal_proposal_parameter"` 26 | //} 27 | // 28 | //func (r ReplicationService) ReplicateContent(contentSource DealReplication, numberOfReplication int, txn *gorm.DB) []model.Content { 29 | // var replicatedContents []model.Content 30 | // for i := 0; i < numberOfReplication; i++ { 31 | // var newContent model.Content 32 | // var newContentDealProposalParameter model.ContentDealProposalParameters 33 | // newContent = contentSource.Content 34 | // newContentDealProposalParameter = contentSource.ContentDealProposalParameter 35 | // newContent.ID = 0 36 | // 37 | // err := txn.Create(&newContent).Error 38 | // if err != nil { 39 | // fmt.Println(err) 40 | // return nil 41 | // } 42 | // 43 | // newContentDealProposalParameter.ID = 0 44 | // newContentDealProposalParameter.Content = newContent.ID 45 | // err = txn.Create(&newContentDealProposalParameter).Error 46 | // if err != nil { 47 | // //tx.Rollback() 48 | // fmt.Println(err) 49 | // return nil 50 | // } 51 | // // assign a miner 52 | // minerAssignService := NewMinerAssignmentService() 53 | // provider, errOnPv := minerAssignService.GetSPWithGivenBytes(newContent.Size) 54 | // if errOnPv != nil { 55 | // fmt.Println(errOnPv) 56 | // return nil 57 | // } 58 | // 59 | // contentMinerAssignment := model.ContentMiner{ 60 | // Miner: provider.Address, 61 | // Content: newContent.ID, 62 | // CreatedAt: time.Now(), 63 | // UpdatedAt: time.Now(), 64 | // } 65 | // err = txn.Create(&contentMinerAssignment).Error 66 | // if err != nil { 67 | // //tx.Rollback() 68 | // fmt.Println(err) 69 | // return nil 70 | // } 71 | // 72 | // replicatedContents = append(replicatedContents, newContent) 73 | // 74 | // } 75 | // return replicatedContents 76 | //} 77 | -------------------------------------------------------------------------------- /core/stats.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import model "delta/models" 4 | 5 | type StatsService struct { 6 | DeltaNode *DeltaNode 7 | } 8 | 9 | type StatsParam struct { 10 | RequestingApiKey string `json:"requesting_api_key"` 11 | } 12 | type PieceCommitmentStatsParam struct { 13 | StatsParam 14 | PieceCommpId int64 `json:"piece_commp_id"` 15 | } 16 | type ContentStatsParam struct { 17 | StatsParam 18 | ContentId int64 `json:"content_id"` 19 | } 20 | type DealStatsParam struct { 21 | StatsParam 22 | DealId int64 `json:"deal_id"` 23 | } 24 | 25 | type StatsResult struct { 26 | Content []model.Content `json:"content"` 27 | Deals []model.ContentDeal `json:"deals"` 28 | PieceCommitments []model.PieceCommitment `json:"piece_commitments"` 29 | } 30 | 31 | type StatsContentResult struct { 32 | Content model.Content `json:"content"` 33 | } 34 | 35 | type StatsDealResult struct { 36 | Deals model.ContentDeal `json:"deals"` 37 | } 38 | 39 | type StatsPieceCommitmentResult struct { 40 | PieceCommitments model.PieceCommitment `json:"piece_commitments"` 41 | } 42 | 43 | func NewStatsStatsService(deltaNode *DeltaNode) *StatsService { 44 | return &StatsService{ 45 | DeltaNode: deltaNode, 46 | } 47 | } 48 | 49 | // Status A function that returns a StatsResult and an error. 50 | func (s *StatsService) Status(param StatsParam) (StatsResult, error) { 51 | var content []model.Content 52 | s.DeltaNode.DB.Raw("select c.* from content_deals cd, contents c where cd.content = c.id and c.requesting_api_key = ?", param.RequestingApiKey).Scan(&content) 53 | 54 | var contentDeal []model.ContentDeal 55 | s.DeltaNode.DB.Raw("select cd.* from content_deals cd, contents c where cd.content = c.id and c.requesting_api_key = ?", param.RequestingApiKey).Scan(&contentDeal) 56 | 57 | // select * from piece_commitments pc, content c where c.piece_commitment_id = pc.id and c.requesting_api_key = ?; 58 | var pieceCommitments []model.PieceCommitment 59 | s.DeltaNode.DB.Raw("select pc.* from piece_commitments pc, contents c where c.piece_commitment_id = pc.id and c.requesting_api_key = ?", param.RequestingApiKey).Scan(&pieceCommitments) 60 | 61 | return StatsResult{ 62 | Content: content, 63 | Deals: contentDeal, 64 | PieceCommitments: pieceCommitments}, nil 65 | } 66 | 67 | // PieceCommitmentStatus A function that returns a StatsPieceCommitmentResult and an error. 68 | func (s *StatsService) PieceCommitmentStatus(param PieceCommitmentStatsParam) (StatsPieceCommitmentResult, error) { 69 | // select * from piece_commitments pc, content c where c.piece_commitment_id = pc.id and c.requesting_api_key = ?; 70 | var pieceCommitment model.PieceCommitment 71 | s.DeltaNode.DB.Raw("select pc.* from piece_commitments pc, contents c where c.piece_commitment_id = pc.id and c.requesting_api_key = ? and pc.id = ?", param.RequestingApiKey, param.PieceCommpId).Scan(&pieceCommitment) 72 | 73 | return StatsPieceCommitmentResult{ 74 | PieceCommitments: pieceCommitment}, nil 75 | } 76 | 77 | // ContentStatus A function that returns a StatsContentResult and an error. 78 | func (s *StatsService) ContentStatus(param ContentStatsParam) (StatsContentResult, error) { 79 | var content model.Content 80 | s.DeltaNode.DB.Raw("select c.* from content_deals cd, contents c where cd.content = c.id and c.requesting_api_key = ? and c.id = ?", param.RequestingApiKey, param.ContentId).Scan(&content) 81 | 82 | return StatsContentResult{Content: content}, nil 83 | } 84 | 85 | // DealStatus A function that returns a StatsDealResult and an error. 86 | func (s *StatsService) DealStatus(param DealStatsParam) (StatsDealResult, error) { 87 | var contentDeal model.ContentDeal 88 | s.DeltaNode.DB.Raw("select cd.* from content_deals cd, contents c where cd.content = c.id and c.requesting_api_key = ? and cd.id = ?", param.RequestingApiKey, param.DealId).Scan(&contentDeal) 89 | 90 | return StatsDealResult{Deals: contentDeal}, nil 91 | } 92 | -------------------------------------------------------------------------------- /core/stats_test.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | ) 7 | 8 | func TestNewStatsStatsService(t *testing.T) { 9 | type args struct { 10 | deltaNode *DeltaNode 11 | } 12 | tests := []struct { 13 | name string 14 | args args 15 | want *StatsService 16 | }{ 17 | // TODO: Add test cases. 18 | } 19 | for _, tt := range tests { 20 | t.Run(tt.name, func(t *testing.T) { 21 | if got := NewStatsStatsService(tt.args.deltaNode); !reflect.DeepEqual(got, tt.want) { 22 | t.Errorf("NewStatsStatsService() = %v, want %v", got, tt.want) 23 | } 24 | }) 25 | } 26 | } 27 | 28 | func TestStatsService_ContentStatus(t *testing.T) { 29 | type fields struct { 30 | DeltaNode *DeltaNode 31 | } 32 | type args struct { 33 | param ContentStatsParam 34 | } 35 | tests := []struct { 36 | name string 37 | fields fields 38 | args args 39 | want StatsContentResult 40 | wantErr bool 41 | }{ 42 | // TODO: Add test cases. 43 | } 44 | for _, tt := range tests { 45 | t.Run(tt.name, func(t *testing.T) { 46 | s := &StatsService{ 47 | DeltaNode: tt.fields.DeltaNode, 48 | } 49 | got, err := s.ContentStatus(tt.args.param) 50 | if (err != nil) != tt.wantErr { 51 | t.Errorf("ContentStatus() error = %v, wantErr %v", err, tt.wantErr) 52 | return 53 | } 54 | if !reflect.DeepEqual(got, tt.want) { 55 | t.Errorf("ContentStatus() got = %v, want %v", got, tt.want) 56 | } 57 | }) 58 | } 59 | } 60 | 61 | func TestStatsService_DealStatus(t *testing.T) { 62 | type fields struct { 63 | DeltaNode *DeltaNode 64 | } 65 | type args struct { 66 | param DealStatsParam 67 | } 68 | tests := []struct { 69 | name string 70 | fields fields 71 | args args 72 | want StatsDealResult 73 | wantErr bool 74 | }{ 75 | // TODO: Add test cases. 76 | } 77 | for _, tt := range tests { 78 | t.Run(tt.name, func(t *testing.T) { 79 | s := &StatsService{ 80 | DeltaNode: tt.fields.DeltaNode, 81 | } 82 | got, err := s.DealStatus(tt.args.param) 83 | if (err != nil) != tt.wantErr { 84 | t.Errorf("DealStatus() error = %v, wantErr %v", err, tt.wantErr) 85 | return 86 | } 87 | if !reflect.DeepEqual(got, tt.want) { 88 | t.Errorf("DealStatus() got = %v, want %v", got, tt.want) 89 | } 90 | }) 91 | } 92 | } 93 | 94 | func TestStatsService_PieceCommitmentStatus(t *testing.T) { 95 | type fields struct { 96 | DeltaNode *DeltaNode 97 | } 98 | type args struct { 99 | param PieceCommitmentStatsParam 100 | } 101 | tests := []struct { 102 | name string 103 | fields fields 104 | args args 105 | want StatsPieceCommitmentResult 106 | wantErr bool 107 | }{ 108 | // TODO: Add test cases. 109 | } 110 | for _, tt := range tests { 111 | t.Run(tt.name, func(t *testing.T) { 112 | s := &StatsService{ 113 | DeltaNode: tt.fields.DeltaNode, 114 | } 115 | got, err := s.PieceCommitmentStatus(tt.args.param) 116 | if (err != nil) != tt.wantErr { 117 | t.Errorf("PieceCommitmentStatus() error = %v, wantErr %v", err, tt.wantErr) 118 | return 119 | } 120 | if !reflect.DeepEqual(got, tt.want) { 121 | t.Errorf("PieceCommitmentStatus() got = %v, want %v", got, tt.want) 122 | } 123 | }) 124 | } 125 | } 126 | 127 | func TestStatsService_Status(t *testing.T) { 128 | type fields struct { 129 | DeltaNode *DeltaNode 130 | } 131 | type args struct { 132 | param StatsParam 133 | } 134 | tests := []struct { 135 | name string 136 | fields fields 137 | args args 138 | want StatsResult 139 | wantErr bool 140 | }{ 141 | // TODO: Add test cases. 142 | } 143 | for _, tt := range tests { 144 | t.Run(tt.name, func(t *testing.T) { 145 | s := &StatsService{ 146 | DeltaNode: tt.fields.DeltaNode, 147 | } 148 | got, err := s.Status(tt.args.param) 149 | if (err != nil) != tt.wantErr { 150 | t.Errorf("Status() error = %v, wantErr %v", err, tt.wantErr) 151 | return 152 | } 153 | if !reflect.DeepEqual(got, tt.want) { 154 | t.Errorf("Status() got = %v, want %v", got, tt.want) 155 | } 156 | }) 157 | } 158 | } 159 | -------------------------------------------------------------------------------- /core/status_logger.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import model "delta/models" 4 | 5 | // StatusLogger is used to change the status of each system objects in a async manner. 6 | // Each state content, commp and transfers needs to be updated and logged in the database. 7 | 8 | type StatusLogger struct { 9 | LightNode DeltaNode 10 | } 11 | 12 | func NewStatusLogger(node DeltaNode) *StatusLogger { 13 | return &StatusLogger{ 14 | LightNode: node, 15 | } 16 | } 17 | 18 | // UpdateContentStatus updates the status of a content object. 19 | func (s *StatusLogger) UpdateContentStatus(content model.Content, status string) error { 20 | tx := s.LightNode.DB.Model(&content).Update("status", status) 21 | if tx.Error != nil { 22 | return tx.Error 23 | } 24 | return nil 25 | } 26 | 27 | // UpdatePieceCommStatus Updating the status of a piece commitment object. 28 | func (s *StatusLogger) UpdatePieceCommStatus(pieceCommp model.PieceCommitment, status string) error { 29 | tx := s.LightNode.DB.Model(&pieceCommp).Update("status", status) 30 | if tx.Error != nil { 31 | return tx.Error 32 | } 33 | return nil 34 | } 35 | 36 | // UpdateContentDealStatus Updating the status of a content deal object. 37 | func (s *StatusLogger) UpdateContentDealStatus(pieceCommp model.ContentDeal, status string) error { 38 | tx := s.LightNode.DB.Model(&pieceCommp).Update("status", status) 39 | if tx.Error != nil { 40 | return tx.Error 41 | } 42 | return nil 43 | } 44 | -------------------------------------------------------------------------------- /core/wallet.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "context" 5 | "crypto/sha256" 6 | "encoding/base64" 7 | "encoding/hex" 8 | "encoding/json" 9 | "fmt" 10 | model "delta/models" 11 | "github.com/filecoin-project/go-address" 12 | "github.com/filecoin-project/lotus/chain/types" 13 | "github.com/filecoin-project/lotus/chain/wallet" 14 | "github.com/google/uuid" 15 | "time" 16 | ) 17 | 18 | type WalletParam struct { 19 | RequestingApiKey string 20 | } 21 | 22 | type GetWalletParam struct { 23 | WalletParam 24 | Address string 25 | } 26 | 27 | type RemoveWalletParam struct { 28 | WalletParam 29 | Address string 30 | } 31 | 32 | type CreateWalletParam struct { 33 | WalletParam 34 | KeyType types.KeyType 35 | } 36 | 37 | type ImportWithHexKey struct { 38 | WalletParam 39 | KeyType types.KeyType `json:"Type"` 40 | PrivateKey string `json:"PrivateKey"` 41 | } 42 | 43 | type ImportWalletParam struct { 44 | WalletParam 45 | KeyType types.KeyType `json:"key_type"` 46 | PrivateKey []byte 47 | } 48 | 49 | type AddWalletResult struct { 50 | Wallet model.Wallet 51 | WalletAddress address.Address 52 | } 53 | 54 | type DeleteWalletResult struct { 55 | WalletAddress string 56 | Message string 57 | } 58 | 59 | type WalletResult struct { 60 | Wallet model.Wallet 61 | } 62 | 63 | type ImportWalletResult struct { 64 | Wallet model.Wallet 65 | WalletAddress address.Address 66 | } 67 | 68 | type WalletService struct { 69 | Context context.Context 70 | DeltaNode *DeltaNode 71 | } 72 | 73 | // NewWalletService Creating a new wallet service. 74 | func NewWalletService(dn *DeltaNode) *WalletService { 75 | return &WalletService{ 76 | DeltaNode: dn, 77 | } 78 | } 79 | 80 | // Create Creating a new wallet and saving it to the database. 81 | func (w WalletService) Create(param CreateWalletParam) (AddWalletResult, error) { 82 | newWallet, err := wallet.NewWallet(wallet.NewMemKeyStore()) 83 | if err != nil { 84 | return AddWalletResult{}, err 85 | } 86 | address, err := newWallet.WalletNew(w.Context, param.KeyType) 87 | 88 | if err != nil { 89 | return AddWalletResult{}, err 90 | } 91 | 92 | // save it on the DB 93 | hexedKey := hex.EncodeToString(address.Payload()) 94 | walletToDb := &model.Wallet{ 95 | Addr: address.String(), 96 | Owner: param.RequestingApiKey, 97 | KeyType: string(param.KeyType), 98 | PrivateKey: hexedKey, 99 | CreatedAt: time.Time{}, 100 | UpdatedAt: time.Time{}, 101 | } 102 | w.DeltaNode.DB.Create(walletToDb) 103 | 104 | return AddWalletResult{ 105 | Wallet: *walletToDb, 106 | WalletAddress: address, 107 | }, nil 108 | } 109 | 110 | func (w WalletService) ImportWithHex(hexKey string, auth string) (ImportWalletResult, error) { 111 | hexString, err := hex.DecodeString(hexKey) 112 | if err != nil { 113 | panic(err) 114 | } 115 | var importWithHexKey ImportWithHexKey 116 | json.Unmarshal(hexString, &importWithHexKey) 117 | fmt.Println(importWithHexKey.KeyType) 118 | bKey, err := base64.StdEncoding.DecodeString(importWithHexKey.PrivateKey) 119 | if err != nil { 120 | panic(err) 121 | 122 | } 123 | result, err := w.Import(ImportWalletParam{ 124 | WalletParam: WalletParam{ 125 | RequestingApiKey: auth, 126 | }, 127 | KeyType: importWithHexKey.KeyType, 128 | PrivateKey: bKey, 129 | }) 130 | 131 | return result, err 132 | 133 | } 134 | 135 | // Import Importing a wallet. 136 | func (w WalletService) Import(param ImportWalletParam) (ImportWalletResult, error) { 137 | newWallet, err := wallet.NewWallet(wallet.NewMemKeyStore()) 138 | if err != nil { 139 | return ImportWalletResult{}, err 140 | } 141 | 142 | hexedWallet := base64.StdEncoding.EncodeToString(param.PrivateKey) 143 | 144 | address, err := newWallet.WalletImport(w.Context, &types.KeyInfo{ 145 | Type: param.KeyType, 146 | PrivateKey: param.PrivateKey, 147 | }) 148 | if err != nil { 149 | return ImportWalletResult{}, err 150 | } 151 | 152 | // save it on the DB 153 | walletUuid, err := uuid.NewUUID() 154 | if err != nil { 155 | return ImportWalletResult{}, err 156 | } 157 | walletToDb := &model.Wallet{ 158 | UuId: walletUuid.String(), 159 | Addr: address.String(), 160 | Owner: param.RequestingApiKey, 161 | KeyType: string(param.KeyType), 162 | PrivateKey: hexedWallet, 163 | CreatedAt: time.Time{}, 164 | UpdatedAt: time.Time{}, 165 | } 166 | w.DeltaNode.DB.Create(walletToDb) 167 | 168 | return ImportWalletResult{ 169 | Wallet: *walletToDb, 170 | WalletAddress: address, 171 | }, nil 172 | } 173 | 174 | // Remove Deleting the wallet from the database. 175 | func (w WalletService) Remove(param RemoveWalletParam) (DeleteWalletResult, error) { 176 | err := w.DeltaNode.DB.Delete(&model.Wallet{}).Where("owner = ? and addr = ?", param.RequestingApiKey, param.Address).Error 177 | if err != nil { 178 | return DeleteWalletResult{ 179 | Message: "Wallet not found", 180 | WalletAddress: param.Address, 181 | }, err 182 | } 183 | return DeleteWalletResult{ 184 | Message: "Wallet deleted", 185 | WalletAddress: param.Address, 186 | }, nil 187 | } 188 | 189 | // List A function that takes a WalletParam and returns a list of model.Wallet and an error. 190 | func (w WalletService) List(param WalletParam) ([]model.Wallet, error) { 191 | var wallets []model.Wallet 192 | w.DeltaNode.DB.Model(&model.Wallet{}).Where("owner = ?", param.RequestingApiKey).Find(&wallets) 193 | return wallets, nil 194 | // Getting the wallet from the database. 195 | } 196 | 197 | // Getting the wallet from the database. 198 | func (w WalletService) Get(param GetWalletParam) (model.Wallet, error) { 199 | var wallet model.Wallet 200 | w.DeltaNode.DB.Model(&model.Wallet{}).Where("owner = ? and addr = ?", param.RequestingApiKey, param.Address).Find(&wallet) 201 | return wallet, nil 202 | } 203 | 204 | func (w WalletService) GetTokenHash(key string) string { 205 | tokenHashBytes := sha256.Sum256([]byte(key)) 206 | // needs to be URL-encodable to send revoke token requests by hash 207 | return base64.RawURLEncoding.EncodeToString(tokenHashBytes[:]) 208 | } 209 | -------------------------------------------------------------------------------- /core/websocket.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | type WebsocketService struct { 4 | DeltaNode *DeltaNode 5 | } 6 | 7 | // `NewWebsocketService` creates a new `WebsocketService` struct and returns a pointer to it 8 | func NewWebsocketService(dn *DeltaNode) *WebsocketService { 9 | return &WebsocketService{ 10 | DeltaNode: dn, 11 | } 12 | } 13 | 14 | // HandlePieceCommitmentMessages A function that is listening to the channel `ws.DeltaNode.WebsocketBroadcast.PieceCommitmentChannel.Channel` and when it 15 | // receives a message, it broadcasts it to all clients. 16 | func (ws *WebsocketService) HandlePieceCommitmentMessages() error { 17 | for { 18 | message := <-ws.DeltaNode.DeltaEventEmitter.WebsocketBroadcast.PieceCommitmentChannel.Channel 19 | // Broadcast to all clients 20 | for client := range ws.DeltaNode.DeltaEventEmitter.WebsocketBroadcast.PieceCommitmentChannel.Clients { 21 | err := client.WriteJSON(message) 22 | if err != nil { 23 | client.Close() 24 | delete(ws.DeltaNode.DeltaEventEmitter.WebsocketBroadcast.PieceCommitmentChannel.Clients, client) 25 | } 26 | } 27 | } 28 | return nil 29 | } 30 | 31 | // HandleContentDealMessages Listening to the channel `ws.DeltaNode.WebsocketBroadcast.ContentDealChannel.Channel` and when it 32 | // // receives a message, it broadcasts it to all clients. 33 | func (ws *WebsocketService) HandleContentDealMessages() error { 34 | for { 35 | message := <-ws.DeltaNode.DeltaEventEmitter.WebsocketBroadcast.ContentDealChannel.Channel 36 | 37 | // Broadcast to all clients 38 | for client := range ws.DeltaNode.DeltaEventEmitter.WebsocketBroadcast.ContentDealChannel.Clients { 39 | err := client.WriteJSON(message) 40 | if err != nil { 41 | client.Close() 42 | delete(ws.DeltaNode.DeltaEventEmitter.WebsocketBroadcast.ContentDealChannel.Clients, client) 43 | } 44 | } 45 | } 46 | return nil 47 | } 48 | 49 | // HandleContentMessages Listening to the channel `ws.DeltaNode.WebsocketBroadcast.ContentChannel.Channel` and when it 50 | // // receives a message, it broadcasts it to all clients. 51 | func (ws *WebsocketService) HandleContentMessages() error { 52 | for { 53 | message := <-ws.DeltaNode.DeltaEventEmitter.WebsocketBroadcast.ContentChannel.Channel 54 | // Broadcast to all clients 55 | for client := range ws.DeltaNode.DeltaEventEmitter.WebsocketBroadcast.ContentChannel.Clients { 56 | err := client.Conn.WriteJSON(message) 57 | if err != nil { 58 | client.Conn.Close() 59 | delete(ws.DeltaNode.DeltaEventEmitter.WebsocketBroadcast.ContentChannel.Clients, client) 60 | } 61 | } 62 | } 63 | return nil 64 | } 65 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.9" 2 | services: 3 | app: 4 | build: . 5 | labels: 6 | - "build-date=${BUILD_DATE}" 7 | - "commit=${COMMIT}" 8 | - "version=${VERSION}" 9 | - "wallet=${WALLET_DIR}" 10 | - "description=${DESCRIPTION}" 11 | ports: 12 | - "${APP_PORT:-1414}:1414" 13 | - "${ANNOUNCE_ADDR_PORT:-6745}:6745/tcp" 14 | environment: 15 | - NODE_NAME=${NODE_NAME:-"delta-node"} 16 | - NODE_DESCRIPTION=${NODE_DESCRIPTION:-"Delta Node"} 17 | - NODE_TYPE=${NODE_TYPE:-"delta"} 18 | - MODE=${MODE:-"cluster"} 19 | - DB_DSN=${DB_DSN:-"delta.db"} 20 | - DELTA_AUTH=${DELTA_AUTH:-""} -------------------------------------------------------------------------------- /docker/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.9" 2 | services: 3 | app: 4 | image: 0utercore/delta:${TAG:-"latest"} 5 | labels: 6 | - "build-date=${BUILD_DATE}" 7 | - "commit=${COMMIT}" 8 | - "version=${VERSION}" 9 | - "wallet=${WALLET_DIR}" 10 | - "description=${DESCRIPTION}" 11 | ports: 12 | - "${APP_PORT:-1414}:1414" 13 | environment: 14 | - NODE_NAME=${NODE_NAME:-"delta-node"} 15 | - NODE_DESCRIPTION=${NODE_DESCRIPTION:-"Delta Node"} 16 | - NODE_TYPE=${NODE_TYPE:-"delta"} 17 | - MODE=${MODE:-"cluster"} 18 | - DB_DSN=${DB_DSN:-"delta.db"} 19 | - DELTA_AUTH=${DELTA_AUTH:-""} 20 | - MAX_CLEANUP_WORKERS=${MAX_CLEANUP_WORKERS:-1500} -------------------------------------------------------------------------------- /docker/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script is used to run docker container pulling from an existing image 4 | ## how to run from root: ./docker/run.sh [tag] 5 | args=("$@") 6 | TAG_ARG=${args[0]} 7 | TAG=${TAG_ARG:-latest} docker-compose -f docker-compose.yml up 8 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # Δ Delta 2 | Generic DealMaking MicroService using whypfs + filclient + estuary_auth 3 | 4 | ![image](https://user-images.githubusercontent.com/4479171/218267752-9a7af133-4e36-4f4c-95da-16b3c7bd73ae.png) 5 | 6 | ## Features 7 | - Make e2e / online and import / offline storage deals. 8 | - Compute piece_commitments using variety of methods 9 | - boost piece commitment computation 10 | - fast piece commitment computation 11 | - filclient piece commitment computation 12 | - Assign deals to specific miners 13 | - Assign deals to specific wallets 14 | - Shows all the deals made for specific user 15 | - Extensive status information about the deals made 16 | - Support for multiple wallets 17 | - Lightweight. Uses a light ipfs node `whypfs-core` to stage the files. 18 | - Cleans up itself after a deal is made. 19 | - Monitors deal progress 20 | - Monitors transfer progress 21 | - Containerized deployment 22 | 23 | ## Getting Started 24 | - To get started on running delta, go to the [getting started to run delta](getting-started-run-delta.md) 25 | - To get started on running delta on calibnet, go to the [getting started to run delta on calibnet](getting-started-run-delta-on-calibnet.md) 26 | - To get started on using a live delta instance, go to the [getting started to use delta](getting-started-use-delta.md) 27 | - To learn more about `delta cli` go to the [delta cli](cli.md) 28 | - To learn more about running delta using docker, go to the [run delta using docker](running-delta-docker.md) 29 | - To learn more about deployment modes, go to the [deployment modes](deployment-modes.md) 30 | - To learn about the metadata request, go to the [deal metadata](deal-metadata.md) 31 | - To get/request an API_KEY, go to the [getting an API_KEY](getting-estuary-api-key.md) 32 | - To make an end-to-end deal, go to the [make e2e deals](make-e2e-deal.md) 33 | - To make an import deal, go to the [make import deals](make-import-deal.md) 34 | - To manage wallets, go to the [managing wallets](manage-wallets.md) 35 | - To learn how to repair a deal, go to the [repairing and retrying deals](repair-retry.md) 36 | - To learn how to access the open statistics and information, go to the [open statistics and information](open-stats-info.md) 37 | - To learn about the content lifecycle and check status of the deals, go to the [content lifecycle and deal status](content-deal-status.md) **[WIP]** 38 | - To learn about the piece commitment computation process flow, go to the [piece commitment computation process flow](process-flow-piece-commitment-compute.md) 39 | - To learn about the storage deal process flow, go to the [storage deal process flow](process-flow-storage-deal.md) 40 | - To generate new swagger documentation for the API, go to [generate swagger documentation](generate-swagger.md) 41 | 42 | # Author 43 | Protocol Labs Outercore Engineering. 44 | -------------------------------------------------------------------------------- /docs/deal-metadata.md: -------------------------------------------------------------------------------- 1 | # Metadata Request 2 | 3 | The metadata request is used to create a deal. It contains the information required to create a deal. 4 | 5 | # Properties of a metadata request 6 | ## cid 7 | The `cid` field is the cid of the content to be stored. This is only required if the `connection_mode` is `import`. If the `connection_mode` is `e2e`, then the `cid` field is not required. 8 | # miner 9 | The `miner` field is the miner to store the content. This is required. 10 | # source 11 | This is the source mulitaddress on where to retrieve the given cid. This is only required when the "cid" field is given. 12 | # unverified_deal_max_price 13 | The `unverified_deal_max_price` field is the maximum price to pay for an unverified deal. This is required only if the `deal_verify_state` is "unverified". If no `unverified_deal_max_price` is specified, Delta will use the default value of 0.000000000000000000 FIL. 14 | # wallet 15 | The `wallet` field is the wallet to use to make the deal. This is optional. If no wallet is specified, Delta will use the default wallet that it generated when it was started. 16 | ## address 17 | The `address` field is the address of the wallet to use to make the deal. This is only required if the `wallet` field is specified. 18 | # piece_commitment 19 | The `piece_commitment` field is the piece commitment of the content to be stored. This is only required if the `connection_mode` is `import`. If the `connection_mode` is `e2e`, then the `piece_commitment` field is not required. 20 | ## piece_cid 21 | The `piece_cid` field is the piece cid of the content to be stored. This is only required if the `connection_mode` is `import`. If the `connection_mode` is `e2e`, then the `piece_cid` field is not required. 22 | ## padded_piece_size 23 | The `padded_piece_size` field is the padded piece size of the content to be stored. This is only required if the `connection_mode` is `import`. If the `connection_mode` is `e2e`, then the `padded_piece_size` field is not required. 24 | ## unpadded_piece_size 25 | The `unpadded_piece_size` field is the unpadded piece size of the content to be stored. This is only required if the `connection_mode` is `import`. If the `connection_mode` is `e2e`, then the `unpadded_piece_size` field is not required. 26 | # transfer_parameters 27 | The `transfer_parameters` field is the transfer parameters set when preparing a deal. It allows deal clients to pull data using the transfer parameter specified. This is only required if the `connection_mode` is `import`. If the `connection_mode` is `e2e`, then the `transfer_parameters` field is not required. 28 | ## url 29 | The `url` field is the url of the content to be stored. This will allow deal clients to pull data from a remote url source. This is only required if the `connection_mode` is `import`. If the `connection_mode` is `e2e`, then the `url` field is not required. 30 | # connection_mode 31 | The `connection_mode` field is the connection mode to use to make the deal. This is either `e2e` or `import`. This is required. 32 | # size 33 | The `size` field is the size of the content to be stored. This is only required if the `connection_mode` is `import`. If the `connection_mode` is `e2e`, then the `size` field is not required. 34 | # remove_unsealed_copy 35 | The `remove_unsealed_copy` field is a boolean field that indicates whether to remove unsealed copies of the content after the deal is made. This is optional. 36 | # skip_ipni_announce 37 | The `skip_ipni_announce` field is a boolean field that indicates whether to skip announcing the deal to interplanetary indexer. This is optional. 38 | # duration_in_days 39 | The `duration_in_days` field is the duration of the deal in days. This is optional. 40 | # start_epoch_in_days 41 | The `start_epoch_at_days` field is the epoch to start the deal. This is optional. 42 | # deal_verify_state 43 | The `deal_verify_state` field is the state of the deal verification. This is to indicate if the deal is from verified FIL or not. This is optional. 44 | valid values are: `verified`, `unverified`. Default: `verified`. 45 | # label 46 | The `label` field is a label for the deal. It has a limit of less than 100 characters. This is optional. 47 | # auto_retry 48 | The `auto_retry` field is a boolean field that indicates whether to automatically retry the deal if it fails. This is optional. 49 | 50 | When set to true, the deal will be retried if the failure falls under the "acceptable" failures. Note that we only have a list of acceptable failures at the moment. 51 | 52 | The deal will be retried with a randomly selected miner based on the file and location of delta instance. 53 | 54 | # Here's the complete structure of the `metadata` request. 55 | ``` 56 | { 57 | "cid": "bafybeidty2dovweduzsne3kkeeg3tllvxd6nc2ifh6ztexvy4krc5pe7om", 58 | "miner":"f01963614", 59 | "wallet": { 60 | "address":"f1mmb3lx7lnzkwsvhridvpugnuzo4mq2xjmawvnfi" 61 | }, 62 | "piece_commitment": { 63 | "piece_cid": "baga6ea4seaqhfvwbdypebhffobtxjyp4gunwgwy2ydanlvbe6uizm5hlccxqmeq", 64 | "padded_piece_size": 4294967296 65 | "unpadded_piece_size": 2500366291 66 | }, 67 | "connection_mode": "import", 68 | "size": 2500366291, 69 | "label": "my deal", 70 | "deal_verify_state": "verified", 71 | "remove_unsealed_copy":true, 72 | "skip_ipni_announce": true, 73 | "duration_in_days": 540, 74 | "auto_retry": false, 75 | "start_epoch_in_days": 14, // days to delay before the deal starts 76 | } 77 | ``` 78 | 79 | # Next 80 | Now that you know how to create a metadata request, you can: 81 | - [Make an import deal](./make-import-deal.md) 82 | - [Check the status of your deal](content-deal-status.md) 83 | - [Learn how to repair a deal](repair-retry.md) 84 | -------------------------------------------------------------------------------- /docs/deployment-modes.md: -------------------------------------------------------------------------------- 1 | # Deployment modes 2 | By default, delta will run on cluster mode but users can standup a standalonemode. standalone mode primarily for those who want to run delta in an isolated environment. This mode will create a local database and a local static API key for all requests. 3 | 4 | ## Prepare the .env file. 5 | Copy the `.env.example` file to `.env` and update the values as needed. 6 | ``` 7 | # Node info 8 | NODE_NAME=delta-node 9 | NODE_DESCRIPTION=Experimental Deal Maker 10 | NODE_TYPE=delta-main 11 | 12 | # Database configuration 13 | MODE=standalone 14 | DB_DSN=delta.db 15 | DELTA_AUTH=[NODE_API_KEY_HERE] 16 | 17 | # Frequencies 18 | MAX_CLEANUP_WORKERS=1500 19 | ``` 20 | Here are the fields in the `.env` file: 21 | 22 | - `NODE_NAME` is the name of the node. 23 | - `NODE_DESCRIPTION` is the description of the node. 24 | - `NODE_TYPE` is the type of the node. 25 | - `MODE` can be `standalone` or `cluster`. If `standalone`, the node will run as a single node. If `cluster`, the node will run as a cluster node. 26 | - `standalone` mode is primarily for those who want to run delta in an isolated environment. This mode will create a local database and a local static API key for all requests. To get a static key, run `https://auth.estuary.tech/register-new-token`. Copy the generated key and paste it in the `DELTA_AUTH` field. 27 | - `cluster` mode is for those who want to run delta as a cluster. This mode will connect to a remote database. In this mode, you don't need to specify the `DELTA_AUTH` field. The node will use the API key provided by `Estuary`. 28 | - `DB_DSN` is the database connection string. If `standalone`, the node will create a local database. If `cluster`, the node will connect to a remote database. 29 | - `DELTA_AUTH` is the API key used to authenticate requests to the node. 30 | - `MAX_CLEANUP_WORKERS` is the maximum number of workers that can be used to clean up the blockstore. This is an optional field. If not specified, the default value is `1500`. 31 | 32 | Put the `.env` file in the same location as the binary/executable. 33 | 34 | 35 | ## MODE=Standalone 36 | With `.env` file in place, run the following command to start the node in standalone mode. 37 | ``` 38 | ./delta daemon --mode=standalone 39 | ``` 40 | 41 | ## MODE=Cluster 42 | By default, delta will run on cluster mode. This mode will create a local database that can be reconfigured to use a remote HA database and estuary-auth as the authentication and authorizatio component. 43 | 44 | When running in cluster mode, users need to register for an API_KEY using the following command. 45 | 46 | ## Request 47 | ``` 48 | curl --location --request GET 'https://auth.estuary.tech/register-new-token' 49 | ``` 50 | 51 | ## Response 52 | ``` 53 | { 54 | "expires": "2123-02-03T21:12:15.632368998Z", 55 | "token": "" 56 | } 57 | ``` 58 | 59 | Place the API_KEY in the .env file. 60 | 61 | ### Run in cluster mode 62 | With `.env` file in place, run the following command to start the node in standalone mode. 63 | ``` 64 | ./delta daemon --mode=cluster 65 | ``` 66 | -------------------------------------------------------------------------------- /docs/generate-swagger.md: -------------------------------------------------------------------------------- 1 | # Delta Swagger Generator 2 | 3 | Delta Swagger Generator is a tool that generates swagger documentation for the Delta APIs. 4 | ## How to run the generator 5 | 6 | ``` 7 | make generate-swagger 8 | ``` 9 | 10 | The above command will do the following: 11 | - scan the `./api` directory for all the `*.go` files starting with router.go 12 | - scan all the annotations on the handlers 13 | - generate the swagger documentation in `./docs/swagger/swagger.yaml` 14 | 15 | ## Check swagger docs using `/swagger` endpoint. 16 | 17 | - To check the generate swagger doc (json), go to [http://localhost:1414/swagger/doc.json](http://localhost:1414/swagger/doc.json) 18 | - To check the swagger ui, go to [http://localhost:1414/swagger/index.html](http://localhost:1414/swagger/index.html) -------------------------------------------------------------------------------- /docs/getting-estuary-api-key.md: -------------------------------------------------------------------------------- 1 | # Getting API Token/Key 2 | 3 | Delta requires an API key to make deals. To make deals, you need to get an API token. You can get an API token from [here](https://estuary.tech/). 4 | 5 | Alternatively, you can also get an API token by running the following request: 6 | 7 | ## Request 8 | ``` 9 | curl --location --request GET 'https://auth.estuary.tech/register-new-token' 10 | ``` 11 | 12 | ## Response 13 | ``` 14 | { 15 | "expires": "2123-02-03T21:12:15.632368998Z", 16 | "token": "" 17 | } 18 | ``` -------------------------------------------------------------------------------- /docs/manage-wallets.md: -------------------------------------------------------------------------------- 1 | # Managing Filecoin Wallets in Delta 2 | Filecoin wallets are essential for making deals with miners. Delta can manage multiple Filecoin wallets. This section will walk you through the steps to manage Filecoin wallets in Delta. 3 | 4 | ## Register a Filecoin wallet 5 | In order to make deals with miners, you need to register a Filecoin wallet with Delta. You can register a Filecoin wallet by sending a `POST` request to the `/api/v1/wallet/register` endpoint. The `metadata` request is the information required to register the wallet. 6 | 7 | To register a wallet to a live Delta node, we can use the `/admin/wallet/register-hex` endpoint. This endpoint is only available on the admin port. 8 | ## Request 9 | ``` 10 | curl --location --request POST 'http://localhost:1414/admin/wallet/register-hex' \ 11 | --header 'Authorization: Bearer [API_KEY]' \ 12 | --header 'Content-Type: application/json' \ 13 | --data-raw '{"hex_key":""}' 14 | ``` 15 | 16 | ## Response 17 | The response will contain the `wallet_addr` and `wallet_uuid` of the registered wallet. 18 | ``` 19 | { 20 | "message": "Successfully imported a wallet address. Please take note of the following information.", 21 | "wallet_addr": "f1mmb3lx7lnzkwsvhridvpugnuzo4mq2xjmawvnfi", 22 | "wallet_uuid": "4d4589d0-c7a2-11ed-b245-9e0bf0c70138" 23 | } 24 | ``` 25 | 26 | We can now use the `wallet_addr` value to make a deal. 27 | 28 | ## Use wallet to prepare the `metadata` request 29 | Once a wallet is registered, we can add a `wallet` field to the `metadata` request to make a deal using that wallet. 30 | ``` 31 | { 32 | "cid": "bafybeidty2dovweduzsne3kkeeg3tllvxd6nc2ifh6ztexvy4krc5pe7om", 33 | "miner":"f01963614", 34 | "wallet": { 35 | "address":"f1mmb3lx7lnzkwsvhridvpugnuzo4mq2xjmawvnfi" 36 | }, 37 | "piece_commitment": { 38 | "piece_cid": "baga6ea4seaqhfvwbdypebhffobtxjyp4gunwgwy2ydanlvbe6uizm5hlccxqmeq", 39 | "padded_piece_size": 4294967296 40 | }, 41 | "connection_mode": "import", 42 | "size": 2500366291, 43 | "remove_unsealed_copy":true, 44 | "skip_ipni_announce": true 45 | } 46 | ``` 47 | 48 | 49 | ## List all registered wallets 50 | ### Request 51 | ``` 52 | curl --location --request GET 'http://localhost:1414/admin/wallet/list' \ 53 | --header 'Authorization: Bearer [API_KEY]' \ 54 | ``` 55 | ### Response 56 | ``` 57 | { 58 | "wallets": [ 59 | { 60 | "ID": 1, 61 | "uuid": "4d4589d0-c7a2-11ed-b245-9e0bf0c70138", 62 | "addr": "f1mmb3lx7lnzkwsvhridvpugnuzo4mq2xjmawvnfi", 63 | "owner": "ESTc904e6ee-8dfe-44b8-864f-37280e1117f9ARY", 64 | "key_type": "secp256k1", 65 | "private_key": "", 66 | "created_at": "2023-03-21T00:39:01.339102-04:00", 67 | "updated_at": "2023-03-21T00:39:01.339102-04:00" 68 | } 69 | ] 70 | } 71 | ``` 72 | -------------------------------------------------------------------------------- /docs/metrics-collection-telemetry.md: -------------------------------------------------------------------------------- 1 | # Delta metrics collection 2 | 3 | Delta collects metric on all nodes. It does this in two ways: 4 | - open telemetry api using opencensus 5 | - a message queue reporting system using [delta-events-consumer](https://github.com/application-research/delta-events-consumer) 6 | 7 | ## How does it work? 8 | Every request that goes thru Delta API is logged. This logged message is then sent to a `nsq` where a background consumer 9 | process collects it and persist is on a timescaleDB/Postgres. 10 | 11 | ### Delta Events Consumer and OTEL API 12 | 13 | Delta uses [delta-events-consumer](https://github.com/application-research/delta-events-consumer/blob/main/README.md) to collect metrics. It is a background process that listens to the `nsq` and persist the data on a timescaleDB/Postgres. 14 | ![image](https://user-images.githubusercontent.com/4479171/226726850-59828c4a-dba8-4082-877a-12efd9474641.png) 15 | 16 | Delta also uses OpenTelemetry API to collect metrics. This is done by using the [opencensus](https://opencensus.io/) library. 17 | 18 | ## Data collected 19 | - `api` - all api calls without the request body 20 | - `api_error` - all api calls that return an error 21 | - `content` - all content related events 22 | - `deal` - all deal related events 23 | - `miner` - all miner related events 24 | - `wallet` - all wallet related events without the wallet seed 25 | - `wallet_error` - all wallet related events that return an error 26 | - `deal proposal` - all deal proposal related events 27 | - `deal proposal error` - all deal proposal related events that return an error 28 | - `deal proposal parameters` - all deal proposal parameters related events 29 | - `deal proposal parameters error` - all deal proposal parameters related events that return an error 30 | - `piece commitment` - all piece commitment related events 31 | - `piece commitment error` - all piece commitment related events that return an error 32 | - `miner assignment` - all miner assignment related events 33 | - `miner assignment error` - all miner assignment related events that return an error 34 | - `wallet assignment` - all wallet assignment related events 35 | - `wallet assignment error` - all wallet assignment related events that return an error 36 | - `transfer` - all transfer related events 37 | - `transfer error` - all transfer related events that return an error 38 | - `status` - all status related events 39 | 40 | Note: We don't collect any request body or wallet seed. The collected data is only used for debugging and monitoring purposes only to improve the product. 41 | Grafana dashboard can be found [here](https://protocollabs.grafana.net/d/xCXVv8-4k/global-delta-dashboard?orgId=1&refresh=10s) -------------------------------------------------------------------------------- /docs/node-information.md: -------------------------------------------------------------------------------- 1 | # Getting Delta Node Information 2 | 3 | Delta is a deal-making service that enables users to make deals with Storage Providers. It is an application that allows users to upload files to the Filecoin network and get them stored by Storage Providers. 4 | 5 | In this section, we will walk you through the steps to use a Delta node to get the status of a deal. 6 | 7 | # Node Information 8 | To get `Delta` node information, we can use the `/open/node/info` endpoint. 9 | ``` 10 | curl --location --request GET 'http://localhost:1414/open/node/info' 11 | ``` 12 | 13 | # Connected Peers 14 | To get `Delta` node connected peers information, we can use the `/open/node/peers` endpoint. 15 | ``` 16 | curl --location --request GET 'http://localhost:1414/open/node/peers' 17 | ``` 18 | 19 | # Node Uuid Information 20 | To get `Delta` node uuids information, we can use the `/open/node/uuids` endpoint. 21 | ``` 22 | curl --location --request GET 'http://localhost:1414/open/node/uuids' 23 | ``` -------------------------------------------------------------------------------- /docs/offline-signing.md: -------------------------------------------------------------------------------- 1 | # Offline Signing 2 | 3 | (*This is a work in progress*) -------------------------------------------------------------------------------- /docs/open-stats-info.md: -------------------------------------------------------------------------------- 1 | # Open statistics and information 2 | 3 | There are several statistics and information endpoints that are open to the public. These endpoints are useful for monitoring the health of the delta service. 4 | 5 | Open endpoints doesn't require any authentication (API_KEY). 6 | 7 | ## Information 8 | ### Get node information 9 | To get `Delta` node information, we can use the `/open/node/info` endpoint. 10 | ``` 11 | curl --location 'http://localhost:1414/open/node/info' 12 | ``` 13 | ### Get node connected peers 14 | To get `Delta` node connected peers information, we can use the `/open/node/peers` endpoint. 15 | ``` 16 | curl --location 'http://localhost:1414/open/node/peers' 17 | ``` 18 | ### Get node uuids 19 | To get `Delta` node uuids information, we can use the `/open/node/uuids` endpoint. 20 | ``` 21 | curl --location 'http://localhost:1414/open/node/uuids' 22 | ``` 23 | Note: UUIDs are delta node identifiers. We use this to identify a delta node in the network. 24 | 25 | ## Stats 26 | ### Get totals info 27 | To get `Delta` node totals information, we can use the `/open/stats/totals/info` endpoint. This includes total e2e and import deals made by the delta node. 28 | ``` 29 | curl --location 'http://localhost:1414/open/stats/totals/info' 30 | ``` 31 | ### Get deal by cid 32 | To get `Delta` node deal information by cid, we can use the `/open/stats/deals/by-cid/:cid` endpoint. 33 | ``` 34 | curl --location 'http://localhost:1414/open/stats/deals/by-cid/' 35 | ``` 36 | ### Get deal by uuid 37 | To get `Delta` node deal information by deal uuid, we can use the `/open/stats/deals/by-uuid/:uuid` endpoint. 38 | ``` 39 | curl --location 'http://localhost:1414/open/stats/deals/by-uuid/' 40 | ``` 41 | ### Get deal by deal id 42 | To get `Delta` node deal information by deal id, we can use the `/open/stats/deals/by-dealid/:dealid` endpoint. 43 | ``` 44 | curl --location 'http://localhost:1414/open/stats/deals/by-dealid/' 45 | ``` 46 | -------------------------------------------------------------------------------- /docs/process-flow-piece-commitment-compute.md: -------------------------------------------------------------------------------- 1 | # Piece Commitment Computation process 2 | 3 | (*This is a work in progress*) 4 | Delta is a deal-making service that enables users to make deals with Storage Providers. It is an application that allows users to upload files to the Filecoin network and get them stored by Storage Providers. 5 | 6 | A content goes thru different stages in delta. One of the stages is the piece commitment computation process. 7 | 8 | ## Content Preparation 9 | - To start, the user will initiate the process by uploading the desired content onto Delta's platform. Once uploaded, Delta will assign the content to a light node and create a corresponding record in its database. To ensure the security and reliability of the content, Delta will assign a miner and wallet to the content record, and create a piece commitment record in its database. This involves computing the piece commitment of the content, which essentially verifies that the content has not been altered in any way. 10 | - Delta will create a deal proposal parameters record in its database, which outlines the terms and conditions of the proposed deal. Based on this record, Delta will then create a deal proposal record, which will serve as a formal agreement between the user and Delta regarding the storage and access of the content on Delta's platform. 11 | 12 | ## Piece Commitment computation 13 | - The piece commitment dispatched job processes piece commitments for Delta. The purpose of this function is to generate a piece commitment record in Delta's database for the uploaded content, which is later used in creating storage deals with miners. 14 | - It begins by updating the status of the content to "CONTENT_PIECE_COMPUTING" in the database, indicating that the content is being processed. Then, it decodes the CID (content identifier) of the uploaded content and checks for any errors. 15 | - It then prepares to compute the piece commitment of the content by retrieving the content's data using the CID. If Delta's configuration is set to fast mode, the piece commitment is generated using a CommpService. Otherwise, if the connection mode is import, the piece commitment is generated using the filclient library. If neither of these conditions is true, the function generates the piece commitment using the CommpService. 16 | - Once the piece commitment is generated, it is saved to the database as a piece commitment record along with its CID, size, and status. The status of the content in the database is updated to "CONTENT_PIECE_ASSIGNED" to indicate that a piece commitment has been generated, and the ID of the newly created piece commitment record is associated with the content. 17 | - Finally, a new StorageDealMakerProcessor is created with the LightNode, Content, and PieceCommitment record, and it is added to the job queue to [create storage deals](process-flow-storage-deal.md) with miners. -------------------------------------------------------------------------------- /docs/process-flow-storage-deal.md: -------------------------------------------------------------------------------- 1 | # Storage Deal Making Process flow 2 | 3 | (*This is a work in progress*) 4 | 5 | Delta is a deal-making service that enables users to make deals with Storage Providers. It is an application that allows users to upload files to the Filecoin network and get them stored by Storage Providers. 6 | 7 | A content goes thru different stages in delta. 8 | 9 | ## Piece Commitment and Deal Proposal Preparation 10 | - To start, the user will initiate the process by uploading the desired content onto Delta's platform. Once uploaded, Delta will assign the content to a light node and create a corresponding record in its database. To ensure the security and reliability of the content, Delta will assign a miner and wallet to the content record, and create a piece commitment record in its database. This involves computing the piece commitment of the content, which essentially verifies that the content has not been altered in any way. 11 | - Delta will create a deal proposal parameters record in its database, which outlines the terms and conditions of the proposed deal. Based on this record, Delta will then create a deal proposal record, which will serve as a formal agreement between the user and Delta regarding the storage and access of the content on Delta's platform. 12 | - The content status is updated to "CONTENT_PIECE_ASSIGNED" to indicate that a piece commitment has been generated, and the ID of the newly [created piece commitment record](process-flow-piece-commitment-compute.md) is associated with the content. 13 | - Once all the meta is available, Delta will then dispatch a job that will call a method called `makeStorageDeal` to make a storage deal proposal for the content. 14 | 15 | ## Making a Storage Deal 16 | - Once all the meta is available, Delta will then dispatch a job that will call a method called `makeStorageDeal` to make a storage deal proposal for the content. 17 | - The purpose of this method is to make a storage deal proposal for a given Content object and PieceCommitment object. 18 | - The first thing this process does is to update the status of the Content object to indicate that a deal proposal is being made. If there is any error during this process, the content status is updated to indicate that the proposal has failed and the error is returned. 19 | - Next, it retrieves the miner address and Filecoin client associated with the content, as well as the deal proposal for the content. If there is any error, the content status is updated to indicate that the proposal has failed and the error is returned. 20 | - It then sets the deal duration, and attempts to decode the payload CID and piece CID from the provided PieceCommitment object. If there is an error in the decoding process, the content status is updated to indicate that the proposal has failed and the error is returned. 21 | - The code then creates a label for the deal proposal, and uses the Filecoin client to make the deal proposal. If there is any error during this process, the content status is updated to indicate that the proposal has failed and the error is returned. 22 | - Assuming the proposal is successful, the code creates a new ContentDeal object and stores it in the database. It also updates the content status to indicate that the proposal is being sent. The proposal is then sent using a separate function, and any errors that occur during this process are handled and the content status is updated accordingly. 23 | - If the proposal is successful, the code updates the content status to indicate that the proposal has been sent. If this is an end-to-end (E2E) content, the data transfer is initiated. -------------------------------------------------------------------------------- /docs/repair-retry.md: -------------------------------------------------------------------------------- 1 | # Repair and Retry storage deals 2 | 3 | Delta has built-in repair and retry functionality. This is useful for when a storage deal fails for some reason. The repair and retry functionality is built into the daemon and can be accessed thru HTTP API. 4 | 5 | Definition of terms: 6 | - retry - retry is a process of dispatching the same content to the deal processor. The content will go thru the same process ie, piece commitment computation to deal making for e2e deal and preparing the deal proposal to sending the deal proposal for import deals. 7 | - repair - repair is a process of redefining some of the metadata of a deal and re-dispatching the deal to the deal processor. 8 | 9 | ## Manual Repair and Retry 10 | Users can also manually repair and retry deals via HTTP API. 11 | 12 | ### Retry a deal for an e2e content 13 | ``` 14 | curl --location --request GET 'http://localhost:1414/api/v1/retry/deal/end-to-end/:contentId' \ 15 | --header 'Authorization: Bearer [API_KEY]' \ 16 | --header 'Content-Type: application/json' 17 | ``` 18 | 19 | ### To retry an e2e deal 20 | ``` 21 | curl --location --request GET 'http://localhost:1414/api/v1/retry/deal/end-to-end/:contentId' \ 22 | --header 'Authorization: Bearer [API_KEY]' \ 23 | --header 'Content-Type: application/json' 24 | ``` 25 | 26 | ### To retry an import deal 27 | ``` 28 | curl --location --request GET 'http://localhost:1414/api/v1/retry/deal/import/:contentId' \ 29 | --header 'Authorization: Bearer [API_KEY]' \ 30 | --header 'Content-Type: application/json' 31 | ``` 32 | 33 | 34 | ## Auto Retry 35 | Users who wants to retry deals can set this up via metadata request. 36 | 37 | The `metadata` parameter is a JSON object that can contain the following fields: 38 | 39 | - `auto_retry` - boolean value that indicates whether the daemon should automatically retry a storage deal if it fails. Default is `false`. 40 | When an auto_retry field is set to true, the deal will run retries on several miners using https://sp-select.delta.store/api/providers until a miner accepts the deal. 41 | 42 | - if miner is set and the auto-retry is set to true, delta will use the given miner. If the miner rejects or faults the deal, Delta will re-try the deal with other miners. 43 | - if the miner is not set and the auto-retry is set to true, delta will look into https://sp-select.delta.store/api/providers to check miners who can accept the deal. If the miner rejects or faults the deal, Delta will re-try the deal with other miners. 44 | - if the miner is set but auto-retry is set to false, delta will use the given miner. It will not retry the deal if the miner rejects the proposal. 45 | - if the miner is not set and auto-retry is set to false, delta will look into https://sp-select.delta.store/api/providers to check miners who can accept the deal. It will not retry the deal if the miner rejects the proposal. 46 | 47 | The [status check](content-deal-status.md) API will return the list of deals that have been retried. 48 | -------------------------------------------------------------------------------- /docs/running-delta-docker.md: -------------------------------------------------------------------------------- 1 | # Running Delta using Docker 2 | 3 | ## Install and run `Delta` using `docker` 4 | Make sure you have docker installed on your machine. You check the following link to install docker on your machine: https://docs.docker.com/engine/install/ 5 | 6 | Once you have docker installed, you're good to proceed to the next steps. 7 | 8 | ### Build a delta image with a specific wallet address 9 | If you already have a wallet with datacap, you can pass it to the command below. This copies over the wallet directory to the containerized app and sets it as the default wallet. 10 | ``` 11 | make docker-compose-build WALLET_DIR= DESCRIPTION=MY_OWN_DELTA_WITH_A_SPECIFIC_WALLET 12 | ``` 13 | You can then run the containerized app using the command below 14 | ``` 15 | make docker-compose-up 16 | ``` 17 | 18 | ### Build and run the current delta clone using docker-compose 19 | Alternatively, you can build and run the current delta clone using docker-compose. 20 | ``` 21 | make docker-compose-run WALLET_DIR= DESCRIPTION=MY_OWN_DELTA_WITH_A_SPECIFIC_WALLET 22 | ``` 23 | 24 | ### Check localhost 25 | You can check the localhost to see if the delta app is running 26 | ``` 27 | curl --location --request GET 'http://localhost:1414/open/node/info' 28 | ``` 29 | 30 | # Next 31 | Now that you can access a live Delta node, you are now ready to make a deal. You can now go to the following guides: 32 | 33 | - [Make an e2e deal](make-e2e-deal.md) 34 | - [Make an import deal](make-import-deal.md) 35 | 36 | If you to run your own `Delta` node, go to [getting started running a delta node](getting-started-run-delta.md). -------------------------------------------------------------------------------- /docs/swagger/swagger.json: -------------------------------------------------------------------------------- 1 | { 2 | "swagger": "2.0", 3 | "info": { 4 | "description": "This is the API for the Delta application.", 5 | "title": "Delta API", 6 | "termsOfService": "http://delta.store", 7 | "contact": { 8 | "name": "API Support" 9 | }, 10 | "license": { 11 | "name": "Apache 2.0 Apache-2.0 OR MIT" 12 | } 13 | }, 14 | "host": "localhost:1414", 15 | "basePath": "/", 16 | "paths": { 17 | "/admin/wallet/balance/:address": { 18 | "post": { 19 | "description": "It creates a new wallet and saves it to the database", 20 | "consumes": [ 21 | "application/json" 22 | ], 23 | "produces": [ 24 | "application/json" 25 | ], 26 | "tags": [ 27 | "Admin" 28 | ], 29 | "summary": "It creates a new wallet and saves it to the database", 30 | "parameters": [ 31 | { 32 | "type": "string", 33 | "description": "address", 34 | "name": "address", 35 | "in": "path", 36 | "required": true 37 | } 38 | ], 39 | "responses": { 40 | "200": { 41 | "description": "OK", 42 | "schema": { 43 | "type": "object", 44 | "additionalProperties": true 45 | } 46 | }, 47 | "400": { 48 | "description": "Bad Request", 49 | "schema": { 50 | "type": "object", 51 | "additionalProperties": true 52 | } 53 | }, 54 | "500": { 55 | "description": "Internal Server Error", 56 | "schema": { 57 | "type": "object", 58 | "additionalProperties": true 59 | } 60 | } 61 | } 62 | } 63 | }, 64 | "/admin/wallet/register": { 65 | "post": { 66 | "description": "It creates a new wallet and saves it to the database", 67 | "consumes": [ 68 | "application/json" 69 | ], 70 | "produces": [ 71 | "application/json" 72 | ], 73 | "tags": [ 74 | "Admin" 75 | ], 76 | "summary": "It creates a new wallet and saves it to the database", 77 | "parameters": [ 78 | { 79 | "type": "string", 80 | "description": "address", 81 | "name": "address", 82 | "in": "path", 83 | "required": true 84 | }, 85 | { 86 | "type": "string", 87 | "description": "key_type", 88 | "name": "key_type", 89 | "in": "path", 90 | "required": true 91 | }, 92 | { 93 | "type": "string", 94 | "description": "private_key", 95 | "name": "private_key", 96 | "in": "path", 97 | "required": true 98 | } 99 | ], 100 | "responses": { 101 | "200": { 102 | "description": "OK", 103 | "schema": { 104 | "$ref": "#/definitions/api.AddWalletRequest" 105 | } 106 | }, 107 | "400": { 108 | "description": "Bad Request", 109 | "schema": { 110 | "type": "object", 111 | "additionalProperties": true 112 | } 113 | }, 114 | "500": { 115 | "description": "Internal Server Error", 116 | "schema": { 117 | "type": "object", 118 | "additionalProperties": true 119 | } 120 | } 121 | } 122 | } 123 | }, 124 | "/admin/wallet/register-hex": { 125 | "post": { 126 | "description": "It creates a new wallet and saves it to the database", 127 | "consumes": [ 128 | "application/json" 129 | ], 130 | "produces": [ 131 | "application/json" 132 | ], 133 | "tags": [ 134 | "Admin" 135 | ], 136 | "summary": "It creates a new wallet and saves it to the database", 137 | "parameters": [ 138 | { 139 | "type": "string", 140 | "description": "address", 141 | "name": "address", 142 | "in": "path", 143 | "required": true 144 | }, 145 | { 146 | "type": "string", 147 | "description": "key_type", 148 | "name": "key_type", 149 | "in": "path", 150 | "required": true 151 | }, 152 | { 153 | "type": "string", 154 | "description": "private_key", 155 | "name": "private_key", 156 | "in": "path", 157 | "required": true 158 | } 159 | ], 160 | "responses": { 161 | "200": { 162 | "description": "OK", 163 | "schema": { 164 | "$ref": "#/definitions/api.AddWalletRequest" 165 | } 166 | }, 167 | "400": { 168 | "description": "Bad Request", 169 | "schema": { 170 | "type": "object", 171 | "additionalProperties": true 172 | } 173 | }, 174 | "500": { 175 | "description": "Internal Server Error", 176 | "schema": { 177 | "type": "object", 178 | "additionalProperties": true 179 | } 180 | } 181 | } 182 | } 183 | } 184 | }, 185 | "definitions": { 186 | "api.AddWalletRequest": { 187 | "type": "object", 188 | "properties": { 189 | "address": { 190 | "type": "string" 191 | }, 192 | "key_type": { 193 | "type": "string" 194 | }, 195 | "private_key": { 196 | "type": "string" 197 | } 198 | } 199 | } 200 | }, 201 | "securityDefinitions": { 202 | "bearerAuth": { 203 | "type": "apiKey", 204 | "name": "Authorization", 205 | "in": "header" 206 | } 207 | }, 208 | "security": [ 209 | { 210 | "bearerAuth": [] 211 | } 212 | ] 213 | } 214 | -------------------------------------------------------------------------------- /docs/swagger/swagger.yaml: -------------------------------------------------------------------------------- 1 | basePath: / 2 | definitions: 3 | api.AddWalletRequest: 4 | properties: 5 | address: 6 | type: string 7 | key_type: 8 | type: string 9 | private_key: 10 | type: string 11 | type: object 12 | host: localhost:1414 13 | info: 14 | contact: 15 | name: API Support 16 | description: This is the API for the Delta application. 17 | license: 18 | name: Apache 2.0 Apache-2.0 OR MIT 19 | termsOfService: http://delta.store 20 | title: Delta API 21 | paths: 22 | /admin/wallet/balance/:address: 23 | post: 24 | consumes: 25 | - application/json 26 | description: It creates a new wallet and saves it to the database 27 | parameters: 28 | - description: address 29 | in: path 30 | name: address 31 | required: true 32 | type: string 33 | produces: 34 | - application/json 35 | responses: 36 | "200": 37 | description: OK 38 | schema: 39 | additionalProperties: true 40 | type: object 41 | "400": 42 | description: Bad Request 43 | schema: 44 | additionalProperties: true 45 | type: object 46 | "500": 47 | description: Internal Server Error 48 | schema: 49 | additionalProperties: true 50 | type: object 51 | summary: It creates a new wallet and saves it to the database 52 | tags: 53 | - Admin 54 | /admin/wallet/register: 55 | post: 56 | consumes: 57 | - application/json 58 | description: It creates a new wallet and saves it to the database 59 | parameters: 60 | - description: address 61 | in: path 62 | name: address 63 | required: true 64 | type: string 65 | - description: key_type 66 | in: path 67 | name: key_type 68 | required: true 69 | type: string 70 | - description: private_key 71 | in: path 72 | name: private_key 73 | required: true 74 | type: string 75 | produces: 76 | - application/json 77 | responses: 78 | "200": 79 | description: OK 80 | schema: 81 | $ref: '#/definitions/api.AddWalletRequest' 82 | "400": 83 | description: Bad Request 84 | schema: 85 | additionalProperties: true 86 | type: object 87 | "500": 88 | description: Internal Server Error 89 | schema: 90 | additionalProperties: true 91 | type: object 92 | summary: It creates a new wallet and saves it to the database 93 | tags: 94 | - Admin 95 | /admin/wallet/register-hex: 96 | post: 97 | consumes: 98 | - application/json 99 | description: It creates a new wallet and saves it to the database 100 | parameters: 101 | - description: address 102 | in: path 103 | name: address 104 | required: true 105 | type: string 106 | - description: key_type 107 | in: path 108 | name: key_type 109 | required: true 110 | type: string 111 | - description: private_key 112 | in: path 113 | name: private_key 114 | required: true 115 | type: string 116 | produces: 117 | - application/json 118 | responses: 119 | "200": 120 | description: OK 121 | schema: 122 | $ref: '#/definitions/api.AddWalletRequest' 123 | "400": 124 | description: Bad Request 125 | schema: 126 | additionalProperties: true 127 | type: object 128 | "500": 129 | description: Internal Server Error 130 | schema: 131 | additionalProperties: true 132 | type: object 133 | summary: It creates a new wallet and saves it to the database 134 | tags: 135 | - Admin 136 | swagger: "2.0" 137 | securityDefinitions: 138 | bearerAuth: 139 | type: apiKey 140 | name: Authorization 141 | in: header 142 | security: 143 | - bearerAuth: [] 144 | -------------------------------------------------------------------------------- /jobs/clean_up.go: -------------------------------------------------------------------------------- 1 | package jobs 2 | 3 | import ( 4 | "context" 5 | "delta/core" 6 | "delta/utils" 7 | "fmt" 8 | model "delta/models" 9 | "github.com/ipfs/go-cid" 10 | "time" 11 | ) 12 | 13 | // ContentCleanUpProcessor `ContentCleanUpProcessor` is a struct that has a `Context` and a `LightNode` field. 14 | // @property Context - The context of the current request. 15 | // @property LightNode - This is the node that we want to clean up. 16 | type ContentCleanUpProcessor struct { 17 | Context context.Context 18 | LightNode *core.DeltaNode 19 | } 20 | 21 | // NewItemContentCleanUpProcessor `NewItemContentCleanUpProcessor` creates a new `ContentCleanUpProcessor` instance 22 | func NewContentCleanUpProcessor(ln *core.DeltaNode) IProcessor { 23 | return &ContentCleanUpProcessor{ 24 | Context: context.Background(), 25 | LightNode: ln, 26 | } 27 | } 28 | 29 | // Run Cleaning up the database. 30 | func (i ContentCleanUpProcessor) Run() error { 31 | 32 | // clear up finished CID deals. 33 | var contentsOnline []model.Content 34 | i.LightNode.DB.Model(&model.Content{}).Where("status = ? and connection_mode = ?", "transfer-finished", "e2e").Find(&contentsOnline) 35 | 36 | for _, content := range contentsOnline { 37 | cidD, err := cid.Decode(content.Cid) 38 | if err != nil { 39 | fmt.Println("error in decoding cid", err) 40 | continue 41 | } 42 | err = i.LightNode.Node.Blockservice.DeleteBlock(context.Background(), cidD) 43 | if err != nil { 44 | fmt.Println("error in deleting block", err) 45 | continue 46 | } 47 | } 48 | 49 | var contentsOffline []model.Content 50 | i.LightNode.DB.Model(&model.Content{}).Where("status = ? and connection_mode = ?", "deal-proposal-sent", "import").Find(&contentsOffline) 51 | 52 | for _, content := range contentsOffline { 53 | cidD, err := cid.Decode(content.Cid) 54 | if err != nil { 55 | fmt.Println("error in decoding cid", err) 56 | continue 57 | } 58 | err = i.LightNode.Node.Blockservice.DeleteBlock(context.Background(), cidD) 59 | if err != nil { 60 | fmt.Println("error in deleting block", err) 61 | continue 62 | } 63 | } 64 | 65 | // clear up failed CID deals. 66 | var contentDeals []model.ContentDeal 67 | i.LightNode.DB.Model(&model.ContentDeal{}).Where("failed = ?", true).Find(&contentDeals) 68 | 69 | for _, contentDeal := range contentDeals { 70 | var content model.Content 71 | i.LightNode.DB.Model(&model.Content{}).Where("id = ?", contentDeal.Content).Find(&content) 72 | cidD, err := cid.Decode(content.Cid) 73 | if err != nil { 74 | fmt.Println("error in decoding cid", err) 75 | continue 76 | } 77 | err = i.LightNode.Node.Blockservice.DeleteBlock(context.Background(), cidD) 78 | if err != nil { 79 | fmt.Println("error in deleting block", err) 80 | continue 81 | } 82 | } 83 | 84 | // clear up cids that are older than 3 days. 85 | var oldContents []model.Content 86 | i.LightNode.DB.Model(&model.Content{}).Where("status not in(?,?,?,?) and created_at < ?", "transfer-failed", "deal-proposal-failed", "transfer-finished", "deal-proposal-sent", time.Now().AddDate(0, 0, -3)).Find(&oldContents) 87 | 88 | for _, content := range oldContents { 89 | 90 | cidD, err := cid.Decode(content.Cid) 91 | if err != nil { 92 | fmt.Println("error in decoding cid", err) 93 | continue 94 | } 95 | err = i.LightNode.Node.Blockservice.DeleteBlock(context.Background(), cidD) 96 | if err != nil { 97 | fmt.Println("error in deleting block", err) 98 | continue 99 | } 100 | 101 | content.Status = utils.DEAL_STATUS_TRANSFER_FAILED 102 | content.LastMessage = "Transfer failed. Record is older than 3 days." 103 | i.LightNode.DB.Save(&content) 104 | 105 | } 106 | return nil 107 | } 108 | -------------------------------------------------------------------------------- /jobs/clean_up_content.go: -------------------------------------------------------------------------------- 1 | package jobs 2 | 3 | import ( 4 | "context" 5 | "delta/core" 6 | "fmt" 7 | model "delta/models" 8 | "github.com/ipfs/go-cid" 9 | ) 10 | 11 | // ItemContentCleanUpProcessor ContentCleanUpProcessor `ContentCleanUpProcessor` is a struct that has a `Context` and a `LightNode` field. 12 | // @property Context - The context of the current request. 13 | // @property LightNode - This is the node that we want to clean up. 14 | type ItemContentCleanUpProcessor struct { 15 | Content model.Content 16 | LightNode *core.DeltaNode 17 | } 18 | 19 | // NewItemContentCleanUpProcessor `NewItemContentCleanUpProcessor` creates a new `ContentCleanUpProcessor` instance 20 | func NewItemContentCleanUpProcessor(ln *core.DeltaNode, content model.Content) IProcessor { 21 | return &ItemContentCleanUpProcessor{ 22 | Content: content, 23 | LightNode: ln, 24 | } 25 | } 26 | 27 | // Run Cleaning up the database. 28 | func (i ItemContentCleanUpProcessor) Run() error { 29 | 30 | // clear up finished CID deals. 31 | var contentsOnline []model.Content 32 | i.LightNode.DB.Model(&model.Content{}).Where("status = ? and connection_mode = ? and id = ?", "transfer-finished", "e2e", i.Content.ID).Find(&contentsOnline) 33 | 34 | for _, content := range contentsOnline { 35 | cidD, err := cid.Decode(content.Cid) 36 | if err != nil { 37 | fmt.Println("error in decoding cid", err) 38 | continue 39 | } 40 | err = i.LightNode.Node.Blockservice.DeleteBlock(context.Background(), cidD) 41 | if err != nil { 42 | fmt.Println("error in deleting block", err) 43 | continue 44 | } 45 | } 46 | 47 | // clear up failed CID deals. 48 | var contentDeals []model.ContentDeal 49 | i.LightNode.DB.Model(&model.ContentDeal{}).Where("failed = ? and content = ?", true, i.Content.ID).Find(&contentDeals) 50 | 51 | for _, contentDeal := range contentDeals { 52 | var content model.Content 53 | i.LightNode.DB.Model(&model.Content{}).Where("id = ?", contentDeal.Content).Find(&content) 54 | cidD, err := cid.Decode(content.Cid) 55 | if err != nil { 56 | fmt.Println("error in decoding cid", err) 57 | continue 58 | } 59 | err = i.LightNode.Node.Blockservice.DeleteBlock(context.Background(), cidD) 60 | if err != nil { 61 | fmt.Println("error in deleting block", err) 62 | continue 63 | } 64 | } 65 | return nil 66 | } 67 | -------------------------------------------------------------------------------- /jobs/data_transfer_restart.go: -------------------------------------------------------------------------------- 1 | package jobs 2 | 3 | import ( 4 | "context" 5 | "delta/core" 6 | "delta/utils" 7 | "fmt" 8 | model "delta/models" 9 | "github.com/application-research/filclient" 10 | ) 11 | 12 | // DataTransferRestartListenerProcessor `DataTransferRestartListenerProcessor` is a struct that contains a `LightNode` and a `ContentDeal`. 13 | // @property LightNode - The light node that is used to send the data transfer restart message. 14 | // @property ContentDeal - The content deal that is being processed. 15 | type DataTransferRestartListenerProcessor struct { 16 | LightNode *core.DeltaNode 17 | // `NewDataTransferRestartProcessor` is a function that returns a `DataTransferRestartListenerProcessor` struct 18 | ContentDeal model.ContentDeal 19 | } 20 | 21 | // NewDataTransferRestartProcessor Creating a new `DataTransferRestartListenerProcessor` struct. 22 | func NewDataTransferRestartProcessor(ln *core.DeltaNode, contentDeal model.ContentDeal) IProcessor { 23 | return &DataTransferRestartListenerProcessor{ 24 | LightNode: ln, 25 | ContentDeal: contentDeal, 26 | } 27 | } 28 | 29 | // Run Restarting the data transfer. 30 | func (d DataTransferRestartListenerProcessor) Run() error { 31 | // get the deal data transfer state pull deals 32 | dtChan, err := utils.GetChannelID(d.ContentDeal.DTChan) 33 | if err != nil { 34 | fmt.Println(err) 35 | return err 36 | } 37 | channelId := dtChan 38 | st, err := d.LightNode.FilClient.TransferStatus(context.Background(), &channelId) 39 | if err != nil && err != filclient.ErrNoTransferFound { 40 | fmt.Println(err) 41 | return err 42 | } 43 | 44 | if st == nil { 45 | return fmt.Errorf("no data transfer state was found") 46 | } 47 | 48 | err = d.LightNode.FilClient.RestartTransfer(context.Background(), &channelId) 49 | if err != nil { 50 | return err 51 | } 52 | return nil 53 | } 54 | -------------------------------------------------------------------------------- /jobs/data_transfer_status.go: -------------------------------------------------------------------------------- 1 | package jobs 2 | 3 | import ( 4 | "delta/core" 5 | "delta/utils" 6 | "fmt" 7 | model "delta/models" 8 | "github.com/application-research/filclient" 9 | datatransfer "github.com/filecoin-project/go-data-transfer" 10 | "strconv" 11 | "time" 12 | ) 13 | 14 | // DataTransferStatusListenerProcessor It's a struct that contains a pointer to a DeltaNode. 15 | // @property LightNode - This is the DeltaNode that will be used to process the data transfer. 16 | type DataTransferStatusListenerProcessor struct { 17 | LightNode *core.DeltaNode 18 | } 19 | 20 | // NewDataTransferStatusListenerProcessor It creates a new instance of the `DataTransferStatusListenerProcessor` struct, and returns a pointer to it 21 | func NewDataTransferStatusListenerProcessor(ln *core.DeltaNode) IProcessor { 22 | return &DataTransferStatusListenerProcessor{ 23 | LightNode: ln, 24 | } 25 | } 26 | 27 | // Run It's a function that is called when the data transfer status changes. 28 | func (d DataTransferStatusListenerProcessor) Run() error { 29 | d.LightNode.FilClient.Libp2pTransferMgr.Subscribe(func(dbid uint, fst filclient.ChannelState) { 30 | fmt.Println("Data Transfer Status Listener: ", fst.Status) 31 | switch fst.Status { 32 | case datatransfer.Requested: 33 | d.LightNode.DB.Model(&model.ContentDeal{}).Where("id = ?", dbid).Updates(model.ContentDeal{ 34 | TransferStarted: time.Now(), 35 | }) 36 | case datatransfer.TransferFinished, datatransfer.Completed: 37 | transferId, err := strconv.Atoi(fst.TransferID) 38 | if err != nil { 39 | fmt.Println(err) 40 | } 41 | d.LightNode.DB.Model(&model.ContentDeal{}).Where("id = ?", dbid).Updates(model.ContentDeal{ 42 | DealID: int64(transferId), 43 | TransferFinished: time.Now(), 44 | SealedAt: time.Now(), 45 | LastMessage: utils.DEAL_STATUS_TRANSFER_FINISHED, 46 | }) 47 | d.LightNode.DB.Model(&model.Content{}).Where("id = (select content from content_deals cd where cd.id = ?)", dbid).Updates(model.Content{ 48 | Status: utils.DEAL_STATUS_TRANSFER_FINISHED, 49 | }) 50 | case datatransfer.Failed: 51 | var contentDeal model.ContentDeal 52 | d.LightNode.DB.Model(&model.ContentDeal{}).Where("id = ?", dbid).Updates(model.ContentDeal{ 53 | FailedAt: time.Now(), 54 | }).Find(&contentDeal) 55 | d.LightNode.DB.Model(&model.Content{}).Joins("left join content_deals as cd on cd.content = c.id").Where("cd.id = ?", dbid).Updates(model.Content{ 56 | Status: utils.DEAL_STATUS_TRANSFER_FAILED, 57 | }) 58 | 59 | d.LightNode.Dispatcher.AddJobAndDispatch(NewDataTransferRestartProcessor(d.LightNode, contentDeal), 1) 60 | default: 61 | } 62 | }) 63 | fmt.Println("Data Transfer Status Listener Ended") 64 | 65 | return nil 66 | } 67 | -------------------------------------------------------------------------------- /jobs/deal_status_check.go: -------------------------------------------------------------------------------- 1 | package jobs 2 | 3 | import ( 4 | "context" 5 | "delta/core" 6 | model "delta/models" 7 | "encoding/base64" 8 | "fmt" 9 | fc "github.com/application-research/filclient" 10 | "github.com/filecoin-project/go-address" 11 | "github.com/filecoin-project/go-fil-markets/storagemarket" 12 | "github.com/filecoin-project/lotus/chain/types" 13 | "github.com/filecoin-project/lotus/chain/wallet" 14 | "github.com/google/uuid" 15 | "github.com/ipfs/go-cid" 16 | ) 17 | 18 | type DealStatusCheck struct { 19 | LightNode *core.DeltaNode 20 | Content *model.Content 21 | } 22 | 23 | func (d DealStatusCheck) Run() error { 24 | var contentDeals []model.ContentDeal 25 | // get the latest content deal of the content 26 | d.LightNode.DB.Where("content = ?", d.Content.ID).Order("created_at desc").Find(&contentDeals) 27 | 28 | filcOfContent, errFilc := d.GetAssignedFilclientForContent(*d.Content) 29 | if errFilc != nil { 30 | return errFilc 31 | } 32 | 33 | for _, contentDeal := range contentDeals { 34 | 35 | if contentDeal.DealUUID == "" { 36 | return nil 37 | } 38 | 39 | miner, err := address.NewFromString(contentDeal.Miner) 40 | if err != nil { 41 | return err 42 | } 43 | 44 | cidProp, err := cid.Decode(contentDeal.PropCid) 45 | if err != nil { 46 | return err 47 | 48 | } 49 | dealUuid, err := uuid.Parse(contentDeal.DealUUID) 50 | if err != nil { 51 | return err 52 | } 53 | 54 | // get the status 55 | status, err := filcOfContent.DealStatus(context.Background(), miner, cidProp, &dealUuid) 56 | if err != nil { 57 | return err 58 | } 59 | contentDeal.DealID = int64(status.DealID) 60 | 61 | if status.State != storagemarket.StorageDealUnknown { 62 | d.Content.Status = storagemarket.DealStates[status.State] 63 | d.Content.LastMessage = storagemarket.DealStatesDescriptions[status.State] 64 | contentDeal.LastMessage = storagemarket.DealStatesDescriptions[status.State] 65 | contentDeal.DealID = int64(status.DealID) 66 | } 67 | d.LightNode.DB.Save(&contentDeal) 68 | d.LightNode.DB.Save(&d.Content) 69 | } 70 | return nil 71 | } 72 | 73 | func NewDealStatusCheck(ln *core.DeltaNode, content *model.Content) IProcessor { 74 | return &DealStatusCheck{ 75 | LightNode: ln, 76 | Content: content, 77 | } 78 | } 79 | 80 | func (d DealStatusCheck) GetAssignedFilclientForContent(content model.Content) (*fc.FilClient, error) { 81 | api := d.LightNode.LotusApiNode 82 | var storageWalletAssignment model.ContentWallet 83 | d.LightNode.DB.Model(&model.ContentWallet{}).Where("content = ?", content.ID).Find(&storageWalletAssignment) 84 | 85 | if storageWalletAssignment.ID != 0 { 86 | newWallet, err := wallet.NewWallet(wallet.NewMemKeyStore()) 87 | if err != nil { 88 | fmt.Println("error on new wallet", err) 89 | return nil, err 90 | } 91 | 92 | // get the wallet entry 93 | var wallet model.Wallet 94 | d.LightNode.DB.Model(&model.Wallet{}).Where("id = ?", storageWalletAssignment.WalletId).Find(&wallet) 95 | decodedPkey, err := base64.StdEncoding.DecodeString(wallet.PrivateKey) 96 | if err != nil { 97 | fmt.Println("error on base64 decode", err) 98 | return nil, err 99 | } 100 | 101 | newWalletAddr, err := newWallet.WalletImport(context.Background(), &types.KeyInfo{ 102 | Type: types.KeyType(wallet.KeyType), 103 | PrivateKey: decodedPkey, 104 | }) 105 | 106 | if err != nil { 107 | fmt.Println("error on wallet_estuary import", err) 108 | return nil, err 109 | } 110 | // new filclient just for this request 111 | filclient, err := fc.NewClient(d.LightNode.Node.Host, api, newWallet, newWalletAddr, d.LightNode.Node.Blockstore, d.LightNode.Node.Datastore, d.LightNode.Node.Config.DatastoreDir.Directory) 112 | if err != nil { 113 | fmt.Println("error on filclient", err) 114 | return nil, err 115 | } 116 | core.SetLibp2pManagerSubscribe(d.LightNode) 117 | return filclient, err 118 | } 119 | 120 | return d.LightNode.FilClient, nil 121 | } 122 | -------------------------------------------------------------------------------- /jobs/instance_meta.go: -------------------------------------------------------------------------------- 1 | package jobs 2 | 3 | import ( 4 | "delta/core" 5 | model "delta/models" 6 | "runtime" 7 | "syscall" 8 | ) 9 | 10 | // InstanceMetaProcessor It's a struct that contains a pointer to a DeltaNode. 11 | // @property LightNode - This is the node that is being processed. 12 | type InstanceMetaProcessor struct { 13 | LightNode *core.DeltaNode 14 | } 15 | 16 | // NewInstanceMetaProcessor > This function creates a new instance of the `InstanceMetaProcessor` struct and returns it as an `IProcessor` interface 17 | func NewInstanceMetaProcessor(ln *core.DeltaNode, contentDeal model.ContentDeal) IProcessor { 18 | return &InstanceMetaProcessor{ 19 | LightNode: ln, 20 | } 21 | } 22 | 23 | // Run It's checking if the CPU or Mem is above the meta set. 24 | func (d InstanceMetaProcessor) Run() error { 25 | // check if CPU or Mem is above the meta set 26 | memStats := &runtime.MemStats{} 27 | runtime.ReadMemStats(memStats) 28 | totalMemory := memStats.Sys 29 | 30 | // Get the number of available CPUs 31 | //numCPUs := runtime.NumCPU() 32 | 33 | // Get the total amount of storage space in bytes 34 | var stat syscall.Statfs_t 35 | syscall.Statfs("/", &stat) 36 | totalStorage := stat.Blocks * uint64(stat.Bsize) 37 | 38 | // if it's above, set the api to read only 39 | if d.LightNode.MetaInfo.StorageLimit < totalStorage || d.LightNode.MetaInfo.MemoryLimit < totalMemory { 40 | d.LightNode.MetaInfo.DisableRequest = true 41 | d.LightNode.MetaInfo.DisableCommitmentPieceGeneration = true 42 | d.LightNode.MetaInfo.DisableStorageDeal = true 43 | d.LightNode.MetaInfo.DisableOnlineDeals = true 44 | d.LightNode.MetaInfo.DisableOfflineDeals = true 45 | } 46 | 47 | return nil 48 | } 49 | -------------------------------------------------------------------------------- /jobs/log_event.go: -------------------------------------------------------------------------------- 1 | package jobs 2 | 3 | //import ( 4 | // "delta/core" 5 | // "github.com/application-research/delta-db/messaging" 6 | //) 7 | // 8 | //// LogEventProcessor `LogEventProcessor` is a struct that contains a `LightNode` and a `LogEvent`. 9 | //// @property LightNode - The node that the event is being processed for. 10 | //// @property LogEvent - This is the event that we want to process. 11 | //type LogEventProcessor struct { 12 | // LightNode *core.DeltaNode 13 | // LogEvent messaging.LogEvent 14 | //} 15 | // 16 | //// NewLogEvent > This function creates a new LogEventProcessor object and returns it 17 | //func NewLogEvent(ln *core.DeltaNode, logEvent messaging.LogEvent) IProcessor { 18 | // return &LogEventProcessor{ 19 | // LightNode: ln, 20 | // LogEvent: logEvent, 21 | // } 22 | //} 23 | // 24 | //// Run Saving the log event to the database. 25 | //func (l LogEventProcessor) Run() error { 26 | // // save log event 27 | // l.LightNode.DB.Create(&l.LogEvent) 28 | // return nil 29 | //} 30 | -------------------------------------------------------------------------------- /jobs/miner_check.go: -------------------------------------------------------------------------------- 1 | package jobs 2 | 3 | import ( 4 | "delta/core" 5 | "encoding/json" 6 | "fmt" 7 | model "delta/models" 8 | "gorm.io/gorm" 9 | "net/http" 10 | ) 11 | 12 | type MinerCheckProcessor struct { 13 | Processor 14 | } 15 | 16 | // NewMinerCheckProcessor job run to get updated miner list and their piece sizes 17 | // `NewMinerCheckProcessor` creates a new `MinerCheckProcessor` object and returns it 18 | func NewMinerCheckProcessor(ln *core.DeltaNode) IProcessor { 19 | return &MinerCheckProcessor{ 20 | Processor{ 21 | LightNode: ln, 22 | }, 23 | } 24 | } 25 | 26 | // Run Getting the list of miners and their prices from the API and storing them in the database. 27 | func (m MinerCheckProcessor) Run() error { 28 | 29 | // remove any record of the miner on the list 30 | m.LightNode.DB.Transaction(func(tx *gorm.DB) error { 31 | // delete all 32 | tx.Delete(model.MinerInfo{}).Where("id > 0") 33 | return nil 34 | }) 35 | 36 | // refresh list 37 | // get the miner list 38 | var minerInfos []model.MinerInfo 39 | req, err := http.Get("https://api.estuary.tech/public/miners/") 40 | if err != nil { 41 | fmt.Println(err) 42 | return err 43 | } 44 | defer req.Body.Close() 45 | json.NewDecoder(req.Body).Decode(&minerInfos) 46 | fmt.Println(minerInfos) 47 | 48 | // for each miner info, get miner price 49 | var minerPrices []model.MinerPrice 50 | for _, minerInfo := range minerInfos { 51 | fmt.Println(minerInfo.Addr) 52 | var minerPrice model.MinerPrice 53 | reqMinerPrice, err := http.Get("https://api.estuary.tech/public/miners/storage/query/" + minerInfo.Addr) 54 | if err != nil { 55 | fmt.Println(err) 56 | return err 57 | } 58 | defer reqMinerPrice.Body.Close() 59 | json.NewDecoder(reqMinerPrice.Body).Decode(&minerPrice) 60 | minerPrices = append(minerPrices, minerPrice) 61 | } 62 | 63 | fmt.Println(minerPrices) 64 | 65 | m.LightNode.DB.Transaction(func(tx *gorm.DB) error { 66 | // insert all 67 | tx.Create(&minerInfos) 68 | tx.Create(&minerPrices) 69 | return nil 70 | }) 71 | return nil 72 | } 73 | -------------------------------------------------------------------------------- /jobs/piece_commp_compute.go: -------------------------------------------------------------------------------- 1 | package jobs 2 | 3 | import ( 4 | "context" 5 | "delta/core" 6 | "delta/utils" 7 | "io" 8 | "time" 9 | 10 | model "delta/models" 11 | "github.com/application-research/filclient" 12 | "github.com/filecoin-project/go-state-types/abi" 13 | "github.com/ipfs/go-cid" 14 | "github.com/labstack/gommon/log" 15 | ) 16 | 17 | // PieceCommpProcessor `PieceCommpProcessor` is a struct that contains a `context.Context`, a `*core.DeltaNode`, a `model.Content`, a 18 | // `filclient.DealConfig`, and a `*core.CommpService`. 19 | // @property Context - The context of the current request 20 | // @property LightNode - The light node that is currently being processed 21 | // @property Content - The content of the piece of data 22 | // @property DealPieceConfig - The configuration of the piece of information, including the piece of information, the piece 23 | // of information, the piece of information, the piece of information, the piece of information, the piece of information, 24 | // the piece of information, the piece of information, the piece of information, the piece of information, the 25 | // @property CommpService - The CommpService object is used to communicate with the Commp protocol. 26 | type PieceCommpProcessor struct { 27 | Context context.Context 28 | LightNode *core.DeltaNode 29 | Content model.Content 30 | DealPieceConfig filclient.DealConfig 31 | CommpService *core.CommpService 32 | } 33 | 34 | // NewPieceCommpProcessor `NewPieceCommpProcessor` is a function that returns a `PieceCommpProcessor` struct 35 | func NewPieceCommpProcessor(ln *core.DeltaNode, content model.Content) IProcessor { 36 | commpService := new(core.CommpService) 37 | return &PieceCommpProcessor{ 38 | LightNode: ln, 39 | Content: content, 40 | Context: context.Background(), 41 | CommpService: commpService, 42 | } 43 | } 44 | 45 | // Run The process of generating the commp. 46 | func (i PieceCommpProcessor) Run() error { 47 | 48 | // if you already have the piece entry for the CID, let's just create a new record with the same commp 49 | var content model.Content 50 | var existingCommp model.PieceCommitment 51 | 52 | i.LightNode.DB.Model(&i.Content).Where("id = ?", i.Content.ID).Find(&content) 53 | i.LightNode.DB.Model(&model.PieceCommitment{}).Where("cid = ?", i.Content.Cid).Find(&existingCommp) 54 | if existingCommp.ID != 0 { 55 | // just assign it if it's already there. 56 | i.Content.Status = utils.CONTENT_PIECE_ASSIGNED 57 | i.Content.PieceCommitmentId = existingCommp.ID 58 | i.Content.UpdatedAt = time.Now() 59 | i.LightNode.DB.Save(&i.Content) 60 | 61 | // then launch the deal maker with the content and the existing commp 62 | item := NewStorageDealMakerProcessor(i.LightNode, i.Content, existingCommp) 63 | i.LightNode.Dispatcher.AddJobAndDispatch(item, 1) 64 | return nil 65 | } 66 | 67 | i.Content.Status = utils.CONTENT_PIECE_COMPUTING 68 | i.Content.UpdatedAt = time.Now() 69 | i.LightNode.DB.Save(&content) 70 | 71 | payloadCid, err := cid.Decode(i.Content.Cid) 72 | if err != nil { 73 | i.Content.Status = utils.CONTENT_PIECE_COMPUTING_FAILED 74 | i.Content.LastMessage = err.Error() 75 | i.Content.UpdatedAt = time.Now() 76 | i.LightNode.DB.Save(&i.Content) 77 | } 78 | 79 | // prepare the commp 80 | node, err := i.LightNode.Node.GetFile(context.Background(), payloadCid) 81 | nodeCopy := node 82 | 83 | bytesFromCar, err := io.ReadAll(nodeCopy) 84 | if err != nil { 85 | log.Error(err) 86 | } 87 | var pieceCid cid.Cid 88 | var payloadSize uint64 89 | var unPaddedPieceSize abi.UnpaddedPieceSize 90 | var paddedPieceSize abi.PaddedPieceSize 91 | 92 | if i.LightNode.Config.Common.CommpMode == utils.COMMP_MODE_FAST { 93 | 94 | pieceInfo, err := i.CommpService.GenerateCommp(node) 95 | if err != nil { 96 | i.LightNode.DB.Model(&i.Content).Where("id = ?", i.Content.ID).Updates(model.Content{ 97 | Status: utils.CONTENT_FAILED_TO_PROCESS, 98 | LastMessage: err.Error(), 99 | UpdatedAt: time.Now(), 100 | }) 101 | return err 102 | } 103 | pieceCid = pieceInfo.PieceCID 104 | paddedPieceSize = abi.PaddedPieceSize(pieceInfo.PayloadSize) 105 | unPaddedPieceSize = pieceInfo.PieceSize.Unpadded() 106 | 107 | payloadSize = uint64(len(bytesFromCar)) 108 | 109 | } else { 110 | 111 | if i.Content.ConnectionMode == utils.CONNECTION_MODE_IMPORT { 112 | pieceCid, payloadSize, unPaddedPieceSize, err = filclient.GeneratePieceCommitment(context.Background(), payloadCid, i.LightNode.Node.Blockstore) 113 | if err != nil { 114 | i.LightNode.DB.Model(&i.Content).Where("id = ?", i.Content.ID).Updates(model.Content{ 115 | Status: utils.CONTENT_FAILED_TO_PROCESS, 116 | LastMessage: err.Error(), 117 | UpdatedAt: time.Now(), 118 | }) 119 | return err 120 | } 121 | paddedPieceSize = abi.PaddedPieceSize(payloadSize) 122 | 123 | } else { 124 | pieceCid, payloadSize, unPaddedPieceSize, err = i.CommpService.GenerateCommPFile(i.Context, payloadCid, i.LightNode.Node.Blockstore) 125 | paddedPieceSize = unPaddedPieceSize.Padded() 126 | if err != nil { 127 | i.LightNode.DB.Model(&i.Content).Where("id = ?", i.Content.ID).Updates(model.Content{ 128 | Status: utils.CONTENT_FAILED_TO_PROCESS, 129 | LastMessage: err.Error(), 130 | UpdatedAt: time.Now(), 131 | }) 132 | return err 133 | } 134 | } 135 | } 136 | 137 | if err != nil { 138 | // put this back to the queue 139 | i.LightNode.Dispatcher.AddJobAndDispatch(NewPieceCommpProcessor(i.LightNode, i.Content), 1) 140 | return err 141 | } 142 | 143 | // save the commp to the database 144 | commpRec := &model.PieceCommitment{ 145 | Cid: payloadCid.String(), 146 | Piece: pieceCid.String(), 147 | Size: int64(payloadSize), 148 | PaddedPieceSize: uint64(paddedPieceSize), 149 | UnPaddedPieceSize: uint64(unPaddedPieceSize), 150 | Status: "open", 151 | CreatedAt: time.Now(), 152 | UpdatedAt: time.Now(), 153 | } 154 | 155 | i.LightNode.DB.Create(commpRec) 156 | 157 | // update the content record 158 | i.Content.Status = utils.CONTENT_PIECE_ASSIGNED 159 | i.Content.PieceCommitmentId = commpRec.ID 160 | i.Content.UpdatedAt = time.Now() 161 | i.LightNode.DB.Save(&i.Content) 162 | 163 | // add this to the job queue 164 | item := NewStorageDealMakerProcessor(i.LightNode, i.Content, *commpRec) 165 | i.LightNode.Dispatcher.AddJobAndDispatch(item, 1) 166 | 167 | return nil 168 | } 169 | -------------------------------------------------------------------------------- /jobs/processor.go: -------------------------------------------------------------------------------- 1 | package jobs 2 | 3 | import ( 4 | "context" 5 | "delta/core" 6 | ) 7 | 8 | type JobExecutable func() error 9 | 10 | // IProcessor is an interface that has a Run method that returns an error. 11 | // @property {error} Run - This is the main function of the processor. It will be called by the processor manager. 12 | type IProcessor interface { 13 | Run() error 14 | } 15 | 16 | // Processor `Processor` is a struct that contains a `context.Context` and a `*core.DeltaNode`. 17 | // @property Context - The context of the processor. 18 | // @property LightNode - This is the node that is being processed. 19 | type Processor struct { 20 | Context context.Context 21 | LightNode *core.DeltaNode 22 | } 23 | -------------------------------------------------------------------------------- /jobs/repair.go: -------------------------------------------------------------------------------- 1 | package jobs 2 | 3 | import ( 4 | "delta/core" 5 | ) 6 | 7 | type RepairProcessor struct { 8 | LightNode *core.DeltaNode 9 | } 10 | 11 | // NewRepairProcessor `NewRepairProcessor` creates a new `RepairProcessor` struct and returns it 12 | func NewRepairProcessor(ln *core.DeltaNode) IProcessor { 13 | return &RepairProcessor{ 14 | LightNode: ln, 15 | } 16 | } 17 | 18 | // Run DB heavy process. We need to check the status of the content and requeue the job if needed. 19 | func (i RepairProcessor) Run() error { 20 | return nil 21 | } 22 | -------------------------------------------------------------------------------- /jobs/retry.go: -------------------------------------------------------------------------------- 1 | package jobs 2 | 3 | import ( 4 | "context" 5 | "delta/core" 6 | "delta/utils" 7 | "fmt" 8 | model "delta/models" 9 | "github.com/ipfs/go-cid" 10 | "time" 11 | ) 12 | 13 | type RetryProcessor struct { 14 | LightNode *core.DeltaNode 15 | } 16 | 17 | // NewRetryProcessor `NewRetryProcessor` creates a new `RetryProcessor` instance 18 | func NewRetryProcessor(ln *core.DeltaNode) IProcessor { 19 | return &RetryProcessor{ 20 | LightNode: ln, 21 | } 22 | } 23 | 24 | // Run DB heavy process. We need to check the status of the content and requeue the job if needed. 25 | // Checking the status of the content and requeue the job if needed. 26 | func (i RetryProcessor) Run() error { 27 | 28 | // collect all cids 29 | var cidsToDelete []cid.Cid 30 | 31 | // if the content is hanging in the middle of the process after a day, let's retry it. 32 | var contents []model.Content 33 | i.LightNode.DB.Model(&model.Content{}).Where("status not in(?,?,?,?) and created_at > ?", "transfer-failed", "deal-proposal-failed", "transfer-finished", "deal-proposal-sent", time.Now().Add(-24*time.Hour)).Find(&contents) 34 | 35 | // Checking the status of the content and requeue the job if needed. 36 | for _, content := range contents { 37 | // get the piece 38 | // This is the retry logic. 39 | if content.Status == utils.CONTENT_PINNED || content.Status == utils.CONTENT_PIECE_COMPUTING { 40 | 41 | // record the retry as a piece-commp 42 | i.LightNode.DB.Model(&model.RetryDealCount{}).Create(&model.RetryDealCount{ 43 | Type: "piece-commitment", 44 | OldId: content.ID, 45 | CreatedAt: time.Now(), 46 | UpdatedAt: time.Now(), 47 | }) 48 | i.LightNode.Dispatcher.AddJobAndDispatch(NewPieceCommpProcessor(i.LightNode, content), 1) 49 | } else if content.Status == utils.CONTENT_PIECE_COMPUTED || content.Status == utils.CONTENT_DEAL_SENDING_PROPOSAL || content.Status == utils.CONTENT_DEAL_MAKING_PROPOSAL { 50 | var pieceCommp model.PieceCommitment 51 | i.LightNode.DB.Model(&model.PieceCommitment{}).Where("id = (select piece_commitment_id from contents c where c.id = ?)", content.ID).Find(&pieceCommp) 52 | i.LightNode.Dispatcher.AddJobAndDispatch(NewStorageDealMakerProcessor(i.LightNode, content, pieceCommp), 1) 53 | } else if content.Status == utils.CONTENT_FAILED_TO_PIN || content.Status == utils.DEAL_STATUS_TRANSFER_FAILED || content.Status == utils.CONTENT_DEAL_PROPOSAL_FAILED || content.Status == utils.CONTENT_PIECE_COMPUTING_FAILED { 54 | // delete/ignore 55 | cidToDelete, err := cid.Decode(content.Cid) 56 | if err != nil { 57 | fmt.Println("error in decoding cid", err) 58 | continue 59 | } 60 | cidsToDelete = append(cidsToDelete, cidToDelete) 61 | } else { 62 | // fail it entirely 63 | content.Status = utils.CONTENT_FAILED_TO_PROCESS 64 | content.LastMessage = "failed to process even after retrying." 65 | i.LightNode.DB.Model(&content).Where("id = ?", content.ID).Updates(content) 66 | cidToDelete, err := cid.Decode(content.Cid) 67 | if err != nil { 68 | fmt.Println("error in decoding cid", err) 69 | continue 70 | } 71 | cidsToDelete = append(cidsToDelete, cidToDelete) 72 | } 73 | 74 | } 75 | 76 | // delete the cids 77 | err := i.LightNode.Node.DAGService.RemoveMany(context.Background(), cidsToDelete) 78 | if err != nil { 79 | fmt.Println("error in unpinning cid", err) 80 | } 81 | 82 | return nil 83 | } 84 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | // It creates a new Echo instance, adds some middleware, creates a new WhyPFS node, creates a new GatewayHandler, and then 2 | // adds a route to the Echo instance 3 | package main 4 | 5 | import ( 6 | "delta/cmd" 7 | c "delta/config" 8 | _ "embed" 9 | "fmt" 10 | _ "net/http" 11 | "os" 12 | 13 | logging "github.com/ipfs/go-log/v2" 14 | "github.com/urfave/cli/v2" 15 | ) 16 | 17 | var ( 18 | log = logging.Logger("delta") 19 | ) 20 | 21 | var Commit string 22 | var Version string 23 | 24 | // It initializes the config, gets all the commands, and runs the app. 25 | func main() { 26 | 27 | // get the config 28 | cfg := c.InitConfig() 29 | cfg.Common.Commit = Commit 30 | cfg.Common.Version = Version 31 | 32 | // get all the commands 33 | var commands []*cli.Command 34 | 35 | // commands 36 | commands = append(commands, cmd.DaemonCmd(&cfg)...) 37 | 38 | // cli 39 | commands = append(commands, cmd.CarCmd(&cfg)...) 40 | commands = append(commands, cmd.CommpCmd(&cfg)...) 41 | commands = append(commands, cmd.DealCmd(&cfg)...) 42 | commands = append(commands, cmd.SpCmd(&cfg)...) 43 | commands = append(commands, cmd.StatusCmd(&cfg)...) 44 | commands = append(commands, cmd.WalletCmd(&cfg)...) 45 | 46 | app := &cli.App{ 47 | Commands: commands, 48 | Name: "delta", 49 | Description: "A deal making engine microservice for the filecoin network", 50 | Version: fmt.Sprintf("%s+git.%s\n", cfg.Common.Version, cfg.Common.Commit), 51 | Flags: cmd.CLIConnectFlags, 52 | Usage: "delta [command] [arguments]", 53 | } 54 | 55 | if err := app.Run(os.Args); err != nil { 56 | log.Fatal(err) 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /metrics/metrics.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "io" 5 | "net/http" 6 | "runtime" 7 | 8 | "github.com/labstack/gommon/log" 9 | 10 | //#nosec G108 - exposing the profiling endpoint is expected 11 | _ "net/http/pprof" 12 | 13 | "contrib.go.opencensus.io/exporter/prometheus" 14 | promclient "github.com/prometheus/client_golang/prometheus" 15 | ) 16 | 17 | // It creates a Prometheus exporter that exports the metrics from the default Prometheus registry 18 | func Exporter() http.Handler { 19 | // Prometheus globals are exposed as interfaces, but the prometheus 20 | // OpenCensus exporter expects a concrete *Registry. The concrete type of 21 | // the globals are actually *Registry, so we downcast them, staying 22 | // defensive in case things change under the hood. 23 | registry, ok := promclient.DefaultRegisterer.(*promclient.Registry) 24 | if !ok { 25 | log.Warnf("failed to export default prometheus registry; some metrics will be unavailable; unexpected type: %T", promclient.DefaultRegisterer) 26 | } 27 | exporter, err := prometheus.NewExporter(prometheus.Options{ 28 | Registry: registry, 29 | Namespace: "delta", 30 | }) 31 | if err != nil { 32 | log.Errorf("could not create the prometheus stats exporter: %v", err) 33 | } 34 | 35 | return exporter 36 | } 37 | 38 | // It writes the stack traces of all goroutines to the given writer 39 | func WriteAllGoroutineStacks(w io.Writer) error { 40 | buf := make([]byte, 64<<20) 41 | for i := 0; ; i++ { 42 | n := runtime.Stack(buf, true) 43 | if n < len(buf) { 44 | buf = buf[:n] 45 | break 46 | } 47 | if len(buf) >= 1<<30 { 48 | // Filled 1 GB - stop there. 49 | break 50 | } 51 | buf = make([]byte, 2*len(buf)) 52 | } 53 | _, err := w.Write(buf) 54 | return err 55 | } 56 | -------------------------------------------------------------------------------- /models/base.go: -------------------------------------------------------------------------------- 1 | package db_models 2 | 3 | import ( 4 | "io/ioutil" 5 | "net/http" 6 | "os" 7 | ) 8 | 9 | func GetPublicIP() (string, error) { 10 | resp, err := http.Get("https://ifconfig.me") // important to get the public ip if possible. 11 | if err != nil { 12 | return "", err 13 | } 14 | defer resp.Body.Close() 15 | body, err := ioutil.ReadAll(resp.Body) 16 | if err != nil { 17 | return "", err 18 | } 19 | return string(body), nil 20 | } 21 | 22 | func GetHostname() string { 23 | hostname, err := os.Hostname() 24 | if err != nil { 25 | return "unknown" 26 | } 27 | return hostname 28 | } 29 | -------------------------------------------------------------------------------- /models/batch_import_content.go: -------------------------------------------------------------------------------- 1 | package db_models 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // BatchImport create an entry first 8 | type BatchImport struct { 9 | ID int64 `gorm:"primaryKey"` 10 | Uuid string `json:"uuid" gorm:"index:,option:CONCURRENTLY"` 11 | Status string `json:"status"` 12 | CreatedAt time.Time `json:"created_at"` 13 | UpdatedAt time.Time `json:"updated_at"` 14 | } 15 | 16 | //func (u *BatchImport) AfterSave(tx *gorm.DB) (err error) { 17 | // 18 | // var instanceFromDb InstanceMeta 19 | // // get the latest instance uuid 20 | // tx.Raw("SELECT * FROM instance_meta ORDER BY id DESC LIMIT 1").Scan(&instanceFromDb) 21 | // 22 | // tx.Model(&InstanceMeta{}).Where("id > 0").First(&instanceFromDb) 23 | // 24 | // if instanceFromDb.ID == 0 { 25 | // return 26 | // } 27 | // 28 | // // get instance info 29 | // ip, err := GetPublicIP() 30 | // if err != nil { 31 | // return 32 | // } 33 | // 34 | // log := BatchImportLog{ 35 | // Uuid: u.Uuid, 36 | // Status: u.Status, 37 | // NodeInfo: GetHostname(), 38 | // RequesterInfo: ip, 39 | // DeltaNodeUuid: instanceFromDb.InstanceUuid, 40 | // SystemBatchImportId: u.ID, 41 | // CreatedAt: time.Now(), 42 | // UpdatedAt: time.Now(), 43 | // } 44 | // 45 | // deltaMetricsBaseMessage := DeltaMetricsBaseMessage{ 46 | // ObjectType: "BatchImportLog", 47 | // Object: log, 48 | // } 49 | // 50 | // messageBytes, err := json.Marshal(deltaMetricsBaseMessage) 51 | // if err != nil { 52 | // return err 53 | // } 54 | // producer.Publish(messageBytes) 55 | // return 56 | //} 57 | 58 | // BatchContent associate the content to a batch 59 | type BatchImportContent struct { 60 | ID int64 `gorm:"primaryKey"` 61 | BatchImportID int64 `json:"batch_import_id" gorm:"index:,option:CONCURRENTLY"` 62 | ContentID int64 `json:"content_id" gorm:"index:,option:CONCURRENTLY"` // check status of the content 63 | CreatedAt time.Time `json:"created_at"` 64 | UpdatedAt time.Time `json:"updated_at"` 65 | } 66 | 67 | //func (u *BatchImportContent) AfterSave(tx *gorm.DB) (err error) { 68 | // 69 | // var instanceFromDb InstanceMeta 70 | // // get the latest instance uuid 71 | // tx.Raw("SELECT * FROM instance_meta ORDER BY id DESC LIMIT 1").Scan(&instanceFromDb) 72 | // 73 | // tx.Model(&InstanceMeta{}).Where("id > 0").First(&instanceFromDb) 74 | // 75 | // if instanceFromDb.ID == 0 { 76 | // return 77 | // } 78 | // 79 | // // get instance info 80 | // ip, err := GetPublicIP() 81 | // if err != nil { 82 | // return 83 | // } 84 | // 85 | // log := BatchImportContentLog{ 86 | // BatchImportID: u.BatchImportID, 87 | // ContentID: u.ContentID, 88 | // NodeInfo: GetHostname(), 89 | // RequesterInfo: ip, 90 | // DeltaNodeUuid: instanceFromDb.InstanceUuid, 91 | // SystemBatchContentId: u.ID, 92 | // CreatedAt: time.Now(), 93 | // UpdatedAt: time.Now(), 94 | // } 95 | // 96 | // deltaMetricsBaseMessage := DeltaMetricsBaseMessage{ 97 | // ObjectType: "BatchImportContentLog", 98 | // Object: log, 99 | // } 100 | // 101 | // messageBytes, err := json.Marshal(deltaMetricsBaseMessage) 102 | // if err != nil { 103 | // return err 104 | // } 105 | // producer.Publish(messageBytes) 106 | // return 107 | //} 108 | -------------------------------------------------------------------------------- /models/content.go: -------------------------------------------------------------------------------- 1 | package db_models 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type Content struct { 8 | ID int64 `gorm:"primaryKey"` 9 | Name string `json:"name"` 10 | Size int64 `json:"size"` 11 | Cid string `json:"cid"` 12 | RequestingApiKey string `json:"requesting_api_key,omitempty"` 13 | PieceCommitmentId int64 `json:"piece_commitment_id,omitempty"` 14 | Status string `json:"status"` 15 | RequestType string `json:"request_type"` // default signed, or unsigned 16 | ConnectionMode string `json:"connection_mode"` // offline or online 17 | AutoRetry bool `json:"auto_retry"` 18 | LastMessage string `json:"last_message"` 19 | CreatedAt time.Time `json:"created_at"` 20 | UpdatedAt time.Time `json:"updated_at"` 21 | } 22 | 23 | //func (u *Content) AfterSave(tx *gorm.DB) (err error) { 24 | // 25 | // var instanceFromDb InstanceMeta 26 | // // get the latest instance info based on created_at 27 | // tx.Raw("SELECT * FROM instance_meta ORDER BY id DESC LIMIT 1").Scan(&instanceFromDb) 28 | // 29 | // if instanceFromDb.ID == 0 { 30 | // return 31 | // } 32 | // 33 | // var contentFromDb Content 34 | // tx.Model(&Content{}).Where("id = ?", u.ID).First(&contentFromDb) 35 | // 36 | // if contentFromDb.ID == 0 { 37 | // return 38 | // } 39 | // // get instance info 40 | // ip, err := GetPublicIP() 41 | // if err != nil { 42 | // return 43 | // } 44 | // 45 | // log := ContentLog{ 46 | // Name: u.Name, 47 | // Size: u.Size, 48 | // Cid: u.Cid, 49 | // RequestingApiKey: u.RequestingApiKey, 50 | // PieceCommitmentId: u.PieceCommitmentId, 51 | // Status: u.Status, 52 | // ConnectionMode: u.ConnectionMode, 53 | // LastMessage: u.LastMessage, 54 | // AutoRetry: u.AutoRetry, 55 | // NodeInfo: GetHostname(), 56 | // RequesterInfo: ip, 57 | // DeltaNodeUuid: instanceFromDb.InstanceUuid, 58 | // SystemContentId: u.ID, 59 | // CreatedAt: time.Now(), 60 | // UpdatedAt: time.Now(), 61 | // } 62 | // 63 | // deltaMetricsBaseMessage := DeltaMetricsBaseMessage{ 64 | // ObjectType: "ContentLog", 65 | // Object: log, 66 | // } 67 | // 68 | // messageBytes, err := json.Marshal(deltaMetricsBaseMessage) 69 | // if err != nil { 70 | // return err 71 | // } 72 | // producer.Publish(messageBytes) 73 | // 74 | // return 75 | //} 76 | -------------------------------------------------------------------------------- /models/content_deal.go: -------------------------------------------------------------------------------- 1 | package db_models 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type ContentDeal struct { 8 | ID int64 `gorm:"primaryKey"` 9 | Content int64 `json:"content" gorm:"index:,option:CONCURRENTLY"` 10 | //Content Content `gorm:"references:ID"` 11 | PropCid string `json:"propCid"` 12 | DealUUID string `json:"dealUuid"` 13 | Miner string `json:"miner"` 14 | DealID int64 `json:"dealId"` 15 | Failed bool `json:"failed"` 16 | Verified bool `json:"verified"` 17 | Slashed bool `json:"slashed"` 18 | FailedAt time.Time `json:"failedAt,omitempty"` 19 | DTChan string `json:"dtChan" gorm:"index"` 20 | TransferStarted time.Time `json:"transferStarted"` 21 | TransferFinished time.Time `json:"transferFinished"` 22 | OnChainAt time.Time `json:"onChainAt"` 23 | SealedAt time.Time `json:"sealedAt"` 24 | LastMessage string `json:"lastMessage"` 25 | DealProtocolVersion string `json:"deal_protocol_version"` 26 | MinerVersion string `json:"miner_version,omitempty"` 27 | CreatedAt time.Time `json:"created_at"` 28 | UpdatedAt time.Time `json:"updated_at"` 29 | } 30 | 31 | //func (u *ContentDeal) AfterSave(tx *gorm.DB) (err error) { 32 | // 33 | // var instanceFromDb InstanceMeta 34 | // // get the latest instance uuid 35 | // tx.Raw("SELECT * FROM instance_meta ORDER BY id DESC LIMIT 1").Scan(&instanceFromDb) 36 | // 37 | // tx.Model(&InstanceMeta{}).Where("id > 0").First(&instanceFromDb) 38 | // 39 | // if instanceFromDb.ID == 0 { 40 | // return 41 | // } 42 | // 43 | // var contentDealLog ContentDeal 44 | // tx.Model(&ContentDeal{}).Where("id = ?", u.ID).First(&contentDealLog) 45 | // 46 | // if contentDealLog.ID == 0 { 47 | // return 48 | // } 49 | // // get instance info 50 | // ip, err := GetPublicIP() 51 | // if err != nil { 52 | // return 53 | // } 54 | // 55 | // log := ContentDealLog{ 56 | // Content: u.Content, 57 | // PropCid: u.PropCid, 58 | // DealUUID: u.DealUUID, 59 | // Miner: u.Miner, 60 | // DealID: u.DealID, 61 | // Failed: u.Failed, 62 | // Verified: u.Verified, 63 | // Slashed: u.Slashed, 64 | // FailedAt: u.FailedAt, 65 | // DTChan: u.DTChan, 66 | // TransferStarted: u.TransferStarted, 67 | // TransferFinished: u.TransferFinished, 68 | // OnChainAt: u.OnChainAt, 69 | // SealedAt: u.SealedAt, 70 | // LastMessage: u.LastMessage, 71 | // DealProtocolVersion: u.DealProtocolVersion, 72 | // MinerVersion: u.MinerVersion, 73 | // DeltaNodeUuid: instanceFromDb.InstanceUuid, 74 | // NodeInfo: GetHostname(), 75 | // RequesterInfo: ip, 76 | // SystemContentDealId: u.ID, 77 | // CreatedAt: time.Now(), 78 | // UpdatedAt: time.Now(), 79 | // } 80 | // 81 | // deltaMetricsBaseMessage := DeltaMetricsBaseMessage{ 82 | // ObjectType: "ContentDealLog", 83 | // Object: log, 84 | // } 85 | // 86 | // messageBytes, err := json.Marshal(deltaMetricsBaseMessage) 87 | // if err != nil { 88 | // return err 89 | // } 90 | // producer.Publish(messageBytes) 91 | // return 92 | //} 93 | -------------------------------------------------------------------------------- /models/content_deal_proposal.go: -------------------------------------------------------------------------------- 1 | package db_models 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type ContentDealProposal struct { 8 | ID int64 `gorm:"primaryKey"` 9 | Content int64 `json:"content" gorm:"index:,option:CONCURRENTLY"` 10 | Unsigned string `json:"unsigned"` 11 | Signed string `json:"signed"` 12 | Meta string `json:"meta"` 13 | CreatedAt time.Time `json:"created_at"` 14 | UpdatedAt time.Time `json:"updated_at"` 15 | } 16 | 17 | //func (u *ContentDealProposal) AfterSave(tx *gorm.DB) (err error) { 18 | // 19 | // var instanceFromDb InstanceMeta 20 | // tx.Raw("SELECT * FROM instance_meta ORDER BY id DESC LIMIT 1").Scan(&instanceFromDb) 21 | // 22 | // if instanceFromDb.ID == 0 { 23 | // return 24 | // } 25 | // 26 | // var contentDealProposal ContentDealProposal 27 | // tx.Model(&ContentDealProposal{}).Where("id = ?", u.ID).First(&contentDealProposal) 28 | // 29 | // if contentDealProposal.ID == 0 { 30 | // return 31 | // } 32 | // 33 | // // get instance info 34 | // ip, err := GetPublicIP() 35 | // if err != nil { 36 | // return 37 | // } 38 | // 39 | // log := ContentDealProposalLog{ 40 | // Content: u.Content, 41 | // Unsigned: u.Unsigned, 42 | // Signed: u.Signed, 43 | // Meta: u.Meta, 44 | // NodeInfo: GetHostname(), 45 | // RequesterInfo: ip, 46 | // DeltaNodeUuid: instanceFromDb.InstanceUuid, 47 | // SystemContentDealProposalId: u.ID, 48 | // CreatedAt: time.Now(), 49 | // UpdatedAt: time.Now(), 50 | // } 51 | // 52 | // deltaMetricsBaseMessage := DeltaMetricsBaseMessage{ 53 | // ObjectType: "ContentDealProposalLog", 54 | // Object: log, 55 | // } 56 | // 57 | // messageBytes, err := json.Marshal(deltaMetricsBaseMessage) 58 | // producer.Publish(messageBytes) 59 | // 60 | // return 61 | //} 62 | -------------------------------------------------------------------------------- /models/content_deal_proposal_parameters.go: -------------------------------------------------------------------------------- 1 | package db_models 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type ContentDealProposalParameters struct { 8 | ID int64 `gorm:"primaryKey"` 9 | Content int64 `json:"content" gorm:"index:,option:CONCURRENTLY"` 10 | Label string `json:"label,omitempty"` 11 | Duration int64 `json:"duration,omitempty"` 12 | StartEpoch int64 `json:"start_epoch,omitempty"` 13 | EndEpoch int64 `json:"end_epoch,omitempty"` 14 | TransferParams string `json:"transfer_parameters,omitempty"` 15 | RemoveUnsealedCopy bool `json:"remove_unsealed_copy"` 16 | SkipIPNIAnnounce bool `json:"skip_ipni_announce"` 17 | VerifiedDeal bool `json:"verified_deal"` 18 | UnverifiedDealMaxPrice string `json:"unverified_deal_max_price"` 19 | CreatedAt time.Time `json:"created_at" json:"created-at"` 20 | UpdatedAt time.Time `json:"updated_at" json:"updated-at"` 21 | } 22 | 23 | //func (u *ContentDealProposalParameters) AfterSave(tx *gorm.DB) (err error) { 24 | // 25 | // var instanceFromDb InstanceMeta 26 | // tx.Raw("SELECT * FROM instance_meta ORDER BY id DESC LIMIT 1").Scan(&instanceFromDb) 27 | // 28 | // if instanceFromDb.ID == 0 { 29 | // return 30 | // } 31 | // 32 | // var contentDealProposalParams ContentDealProposalParameters 33 | // tx.Model(&ContentDealProposalParameters{}).Where("id = ?", u.ID).First(&contentDealProposalParams) 34 | // 35 | // if contentDealProposalParams.ID == 0 { 36 | // return 37 | // } 38 | // 39 | // // get instance info 40 | // ip, err := GetPublicIP() 41 | // if err != nil { 42 | // return 43 | // } 44 | // log := ContentDealProposalParametersLog{ 45 | // Content: u.Content, 46 | // Label: u.Label, 47 | // Duration: u.Duration, 48 | // StartEpoch: u.StartEpoch, 49 | // EndEpoch: u.EndEpoch, 50 | // TransferParams: u.TransferParams, 51 | // RemoveUnsealedCopy: u.RemoveUnsealedCopy, 52 | // SkipIPNIAnnounce: u.SkipIPNIAnnounce, 53 | // VerifiedDeal: u.VerifiedDeal, 54 | // UnverifiedDealMaxPrice: u.UnverifiedDealMaxPrice, 55 | // NodeInfo: GetHostname(), 56 | // RequesterInfo: ip, 57 | // DeltaNodeUuid: instanceFromDb.InstanceUuid, 58 | // SystemContentDealProposalParametersId: u.ID, 59 | // CreatedAt: time.Now(), 60 | // UpdatedAt: time.Now(), 61 | // } 62 | // 63 | // deltaMetricsBaseMessage := DeltaMetricsBaseMessage{ 64 | // ObjectType: "ContentDealProposalParametersLog", 65 | // Object: log, 66 | // } 67 | // 68 | // messageBytes, err := json.Marshal(deltaMetricsBaseMessage) 69 | // producer.Publish(messageBytes) 70 | // 71 | // return 72 | //} 73 | -------------------------------------------------------------------------------- /models/content_miner.go: -------------------------------------------------------------------------------- 1 | package db_models 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type ContentMiner struct { 8 | ID int64 `gorm:"primaryKey"` 9 | Content int64 `json:"content" gorm:"index:,option:CONCURRENTLY"` 10 | Miner string `json:"miner"` 11 | CreatedAt time.Time `json:"created_at"` 12 | UpdatedAt time.Time `json:"updated_at"` 13 | } 14 | 15 | //func (u *ContentMiner) AfterSave(tx *gorm.DB) (err error) { 16 | // 17 | // var instanceFromDb InstanceMeta 18 | // tx.Raw("SELECT * FROM instance_meta ORDER BY id DESC LIMIT 1").Scan(&instanceFromDb) 19 | // 20 | // if instanceFromDb.ID == 0 { 21 | // return 22 | // } 23 | // 24 | // var contentMiner ContentMiner 25 | // tx.Model(&ContentMiner{}).Where("id = ?", u.ID).First(&contentMiner) 26 | // 27 | // if contentMiner.ID == 0 { 28 | // return 29 | // } 30 | // 31 | // // get instance info 32 | // ip, err := GetPublicIP() 33 | // if err != nil { 34 | // return 35 | // } 36 | // 37 | // log := ContentMinerLog{ 38 | // Content: u.Content, 39 | // Miner: u.Miner, 40 | // NodeInfo: GetHostname(), 41 | // RequesterInfo: ip, 42 | // SystemContentMinerId: u.ID, 43 | // DeltaNodeUuid: instanceFromDb.InstanceUuid, 44 | // CreatedAt: time.Now(), 45 | // UpdatedAt: time.Now(), 46 | // } 47 | // 48 | // deltaMetricsBaseMessage := DeltaMetricsBaseMessage{ 49 | // ObjectType: "ContentMinerLog", 50 | // Object: log, 51 | // } 52 | // 53 | // messageBytes, err := json.Marshal(deltaMetricsBaseMessage) 54 | // producer.Publish(messageBytes) 55 | // 56 | // return 57 | //} 58 | -------------------------------------------------------------------------------- /models/content_wallet.go: -------------------------------------------------------------------------------- 1 | package db_models 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type ContentWallet struct { 8 | ID int64 `gorm:"primaryKey"` 9 | Content int64 `json:"content" gorm:"index:,option:CONCURRENTLY"` 10 | WalletId int64 `json:"wallet_id" gorm:"index:,option:CONCURRENTLY"` 11 | CreatedAt time.Time `json:"created_at"` 12 | UpdatedAt time.Time `json:"updated_at"` 13 | } 14 | 15 | //func (u *ContentWallet) AfterSave(tx *gorm.DB) (err error) { 16 | // 17 | // var instanceFromDb InstanceMeta 18 | // tx.Raw("SELECT * FROM instance_meta ORDER BY id DESC LIMIT 1").Scan(&instanceFromDb) 19 | // 20 | // if instanceFromDb.ID == 0 { 21 | // return 22 | // } 23 | // 24 | // var contentWallet ContentWallet 25 | // tx.Model(&ContentWallet{}).Where("id = ?", u.ID).First(&contentWallet) 26 | // 27 | // if contentWallet.ID == 0 { 28 | // return 29 | // } 30 | // 31 | // // get instance info 32 | // ip, err := GetPublicIP() 33 | // if err != nil { 34 | // return 35 | // } 36 | // log := ContentWalletLog{ 37 | // Content: u.Content, 38 | // WalletId: u.WalletId, 39 | // NodeInfo: GetHostname(), 40 | // RequesterInfo: ip, 41 | // SystemContentWalletId: u.ID, 42 | // DeltaNodeUuid: instanceFromDb.InstanceUuid, 43 | // CreatedAt: time.Now(), 44 | // UpdatedAt: time.Now(), 45 | // } 46 | // 47 | // deltaMetricsBaseMessage := DeltaMetricsBaseMessage{ 48 | // ObjectType: "ContentWalletLog", 49 | // Object: log, 50 | // } 51 | // 52 | // messageBytes, err := json.Marshal(deltaMetricsBaseMessage) 53 | // producer.Publish(messageBytes) 54 | // 55 | // return 56 | //} 57 | -------------------------------------------------------------------------------- /models/database.go: -------------------------------------------------------------------------------- 1 | package db_models 2 | 3 | import ( 4 | "fmt" 5 | "github.com/application-research/delta-db/messaging" 6 | "gorm.io/gorm/logger" 7 | "time" 8 | 9 | "gorm.io/driver/postgres" 10 | "gorm.io/driver/sqlite" 11 | "gorm.io/gorm" 12 | ) 13 | 14 | var producer *messaging.DeltaMetricsMessageProducer 15 | 16 | type DeltaMetricsBaseMessage struct { 17 | ObjectType string `json:"object_type"` 18 | Object interface{} `json:"object"` 19 | } 20 | 21 | func OpenDatabase(dbDsn string) (*gorm.DB, error) { 22 | // use postgres 23 | var DB *gorm.DB 24 | var err error 25 | 26 | if dbDsn[:8] == "postgres" { 27 | DB, err = gorm.Open(postgres.Open(dbDsn), &gorm.Config{ 28 | Logger: logger.Default.LogMode(logger.Silent), 29 | }) 30 | } else { 31 | DB, err = gorm.Open(sqlite.Open(dbDsn), &gorm.Config{ 32 | Logger: logger.Default.LogMode(logger.Silent), 33 | }) 34 | } 35 | 36 | // generate new models. 37 | ConfigureModels(DB) // create models. 38 | 39 | if err != nil { 40 | return nil, err 41 | } 42 | return DB, nil 43 | } 44 | 45 | func ConfigureModels(db *gorm.DB) { 46 | db.AutoMigrate(&Content{}, &ContentDeal{}, &PieceCommitment{}, &MinerInfo{}, &MinerPrice{}, &messaging.LogEvent{}, &ContentMiner{}, &ProcessContentCounter{}, &ContentWallet{}, &ContentDealProposalParameters{}, &Wallet{}, &ContentDealProposal{}, &InstanceMeta{}, &RetryDealCount{}, &BatchImport{}, &BatchImportContent{}) 47 | } 48 | 49 | type ProcessContentCounter struct { 50 | ID int64 `gorm:"primaryKey"` 51 | Content int64 `json:"content" gorm:"index:,option:CONCURRENTLY"` 52 | Counter int64 `json:"counter"` 53 | CreatedAt time.Time `json:"created_at"` 54 | UpdatedAt time.Time `json:"updated_at"` 55 | } 56 | 57 | type MinerInfo struct { 58 | ID int64 `gorm:"primaryKey"` 59 | Addr string `json:"addr"` // same as Miner from MinerPrice 60 | Name string `json:"name"` 61 | Suspended bool `json:"suspended"` 62 | Version string `json:"version"` 63 | ChainInfo string `json:"chain_info"` 64 | SuspendedReason string `json:"suspendedReason,omitempty"` 65 | CreatedAt time.Time `json:"created_at"` 66 | UpdatedAt time.Time `json:"updated_at"` 67 | } 68 | 69 | type MinerPrice struct { 70 | ID int64 `gorm:"primaryKey"` 71 | Miner string `json:"miner"` 72 | Price string `json:"price"` 73 | VerifiedPrice string `json:"verifiedPrice"` 74 | MinPieceSize int64 `json:"minPieceSize"` 75 | MaxPieceSize int64 `json:"maxPieceSize"` 76 | MinerVersion string `json:"miner_version"` 77 | CreatedAt time.Time `json:"created_at"` 78 | UpdatedAt time.Time `json:"updated_at"` 79 | } 80 | 81 | type AdminUser struct { 82 | ID int64 `gorm:"primaryKey"` 83 | Username string `json:"username"` 84 | Password string `json:"password"` 85 | CreatedAt time.Time `json:"created_at"` 86 | UpdatedAt time.Time `json:"updated_at"` 87 | } 88 | 89 | type RetryDealCount struct { 90 | ID int64 `gorm:"primaryKey"` 91 | Type string `json:"type"` 92 | OldId int64 `json:"old_id"` 93 | NewId int64 `json:"new_id"` 94 | CreatedAt time.Time `json:"created_at"` 95 | UpdatedAt time.Time `json:"updated_at"` 96 | } 97 | 98 | var ErrNoChannelID = fmt.Errorf("no data transfer channel id in deal") 99 | -------------------------------------------------------------------------------- /models/instance_meta.go: -------------------------------------------------------------------------------- 1 | package db_models 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type InstanceMeta struct { 8 | // gorm id 9 | ID int64 `gorm:"primary_key" json:"id"` 10 | InstanceUuid string `json:"instance_uuid"` 11 | InstanceHostName string `json:"instance_host_name"` 12 | InstanceNodeName string `json:"instance_node_name"` 13 | OSDetails string `json:"os_details"` 14 | PublicIp string `json:"public_ip"` 15 | MemoryLimit uint64 `json:"memory_limit"` 16 | CpuLimit uint64 `json:"cpu_limit"` 17 | StorageLimit uint64 `json:"storage_limit"` 18 | DisableRequest bool `json:"disable_requests"` 19 | DisableCommitmentPieceGeneration bool `json:"disable_commitment_piece_generation"` 20 | DisableStorageDeal bool `json:"disable_storage_deal"` 21 | DisableOnlineDeals bool `json:"disable_online_deals"` 22 | DisableOfflineDeals bool `json:"disable_offline_deals"` 23 | NumberOfCpus uint64 `json:"number_of_cpus"` 24 | StorageInBytes uint64 `json:"storage_in_bytes"` 25 | SystemMemory uint64 `json:"system_memory"` 26 | HeapMemory uint64 `json:"heap_memory"` 27 | HeapInUse uint64 `json:"heap_in_use"` 28 | StackInUse uint64 `json:"stack_in_use"` 29 | InstanceStart time.Time `json:"instance_start"` 30 | BytesPerCpu uint64 `json:"bytes_per_cpu"` 31 | CreatedAt time.Time `json:"created_at"` 32 | UpdatedAt time.Time `json:"updated_at"` 33 | } 34 | 35 | //func (u *InstanceMeta) AfterSave(tx *gorm.DB) (err error) { 36 | // 37 | // var contentFromDb Content 38 | // tx.Model(&Content{}).Where("id = ?", u.ID).First(&contentFromDb) 39 | // 40 | // if contentFromDb.ID == 0 { 41 | // return 42 | // } 43 | // // get instance info 44 | // ip, err := GetPublicIP() 45 | // if err != nil { 46 | // return 47 | // } 48 | // 49 | // log := InstanceMetaLog{ 50 | // InstanceUuid: u.InstanceUuid, 51 | // InstanceHostName: u.InstanceHostName, 52 | // InstanceNodeName: u.InstanceNodeName, 53 | // OSDetails: u.OSDetails, 54 | // PublicIp: u.PublicIp, 55 | // MemoryLimit: u.MemoryLimit, 56 | // CpuLimit: u.CpuLimit, 57 | // StorageLimit: u.StorageLimit, 58 | // DisableRequest: u.DisableRequest, 59 | // DisableCommitmentPieceGeneration: u.DisableCommitmentPieceGeneration, 60 | // DisableStorageDeal: u.DisableStorageDeal, 61 | // DisableOnlineDeals: u.DisableOnlineDeals, 62 | // DisableOfflineDeals: u.DisableOfflineDeals, 63 | // NumberOfCpus: u.NumberOfCpus, 64 | // StorageInBytes: u.StorageInBytes, 65 | // SystemMemory: u.SystemMemory, 66 | // HeapMemory: u.HeapMemory, 67 | // HeapInUse: u.HeapInUse, 68 | // StackInUse: u.StackInUse, 69 | // InstanceStart: u.InstanceStart, 70 | // BytesPerCpu: u.BytesPerCpu, 71 | // NodeInfo: GetHostname(), 72 | // RequesterInfo: ip, 73 | // DeltaNodeUuid: u.InstanceUuid, 74 | // SystemInstanceMetaId: u.ID, 75 | // CreatedAt: time.Now(), 76 | // UpdatedAt: time.Now(), 77 | // } 78 | // 79 | // deltaMetricsBaseMessage := DeltaMetricsBaseMessage{ 80 | // ObjectType: "InstanceMetaLog", 81 | // Object: log, 82 | // } 83 | // 84 | // messageBytes, err := json.Marshal(deltaMetricsBaseMessage) 85 | // if err != nil { 86 | // return err 87 | // } 88 | // producer.Publish(messageBytes) 89 | // 90 | // return 91 | //} 92 | -------------------------------------------------------------------------------- /models/log_event.go: -------------------------------------------------------------------------------- 1 | package db_models 2 | -------------------------------------------------------------------------------- /models/piece_commitment.go: -------------------------------------------------------------------------------- 1 | package db_models 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type PieceCommitment struct { 8 | ID int64 `gorm:"primaryKey"` 9 | Cid string `json:"cid"` 10 | Piece string `json:"piece"` 11 | Size int64 `json:"size"` 12 | PaddedPieceSize uint64 `json:"padded_piece_size"` 13 | UnPaddedPieceSize uint64 `json:"unnpadded_piece_size"` 14 | Status string `json:"status"` // open, in-progress, completed (closed). 15 | LastMessage string `json:"last_message"` 16 | CreatedAt time.Time `json:"created_at"` 17 | UpdatedAt time.Time `json:"updated_at"` 18 | } 19 | 20 | //func (u *PieceCommitment) AfterSave(tx *gorm.DB) (err error) { 21 | // 22 | // var instanceFromDb InstanceMeta 23 | // tx.Raw("SELECT * FROM instance_meta ORDER BY id DESC LIMIT 1").Scan(&instanceFromDb) 24 | // 25 | // if instanceFromDb.ID == 0 { 26 | // return 27 | // } 28 | // 29 | // var pieceComm PieceCommitment 30 | // tx.Model(&PieceCommitment{}).Where("id = ?", u.ID).First(&pieceComm) 31 | // 32 | // if pieceComm.ID == 0 { 33 | // return 34 | // } 35 | // 36 | // // get instance info 37 | // ip, err := GetPublicIP() 38 | // if err != nil { 39 | // return 40 | // } 41 | // log := PieceCommitmentLog{ 42 | // Cid: u.Cid, 43 | // Piece: u.Piece, 44 | // Size: u.Size, 45 | // PaddedPieceSize: u.PaddedPieceSize, 46 | // UnPaddedPieceSize: u.UnPaddedPieceSize, 47 | // Status: u.Status, 48 | // LastMessage: u.LastMessage, 49 | // NodeInfo: GetHostname(), 50 | // RequesterInfo: ip, 51 | // SystemContentPieceCommitmentId: u.ID, 52 | // DeltaNodeUuid: instanceFromDb.InstanceUuid, 53 | // CreatedAt: time.Now(), 54 | // UpdatedAt: time.Now(), 55 | // } 56 | // 57 | // deltaMetricsBaseMessage := DeltaMetricsBaseMessage{ 58 | // ObjectType: "PieceCommitmentLog", 59 | // Object: log, 60 | // } 61 | // 62 | // messageBytes, err := json.Marshal(deltaMetricsBaseMessage) 63 | // producer.Publish(messageBytes) 64 | // 65 | // return 66 | //} 67 | -------------------------------------------------------------------------------- /models/repair_request.go: -------------------------------------------------------------------------------- 1 | package db_models 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type RepairRequest struct { 8 | ID int64 `gorm:"primaryKey"` 9 | ObjectId int64 `json:"object_id"` 10 | Type string `json:"type"` 11 | CreatedAt time.Time `json:"created_at"` 12 | UpdatedAt time.Time `json:"updated_at"` 13 | } 14 | -------------------------------------------------------------------------------- /models/wallet.go: -------------------------------------------------------------------------------- 1 | package db_models 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // Wallet time series log events 8 | type Wallet struct { 9 | ID int64 `gorm:"primaryKey"` 10 | UuId string `json:"uuid"` 11 | Addr string `json:"addr"` 12 | Owner string `json:"owner"` 13 | KeyType string `json:"key_type"` 14 | PrivateKey string `json:"private_key"` 15 | CreatedAt time.Time `json:"created_at"` 16 | UpdatedAt time.Time `json:"updated_at"` 17 | } 18 | 19 | //func (7u *Wallet) AfterSave(tx *gorm.DB) (err error) { 20 | // 21 | // var instanceFromDb InstanceMeta 22 | // tx.Raw("SELECT * FROM instance_meta ORDER BY id DESC LIMIT 1").Scan(&instanceFromDb) 23 | // 24 | // if instanceFromDb.ID == 0 { 25 | // return 26 | // } 27 | // 28 | // var walletFromDb Wallet 29 | // tx.Model(&Wallet{}).Where("id = ?", u.ID).First(&walletFromDb) 30 | // 31 | // if walletFromDb.ID == 0 { 32 | // return 33 | // } 34 | // // get instance info 35 | // ip, err := GetPublicIP() 36 | // if err != nil { 37 | // return 38 | // } 39 | // 40 | // log := WalletLog{ 41 | // UuId: u.UuId, 42 | // Addr: u.Addr, 43 | // Owner: u.Owner, 44 | // KeyType: "REDACTED", 45 | // PrivateKey: "REDACTED", 46 | // NodeInfo: GetHostname(), 47 | // RequesterInfo: ip, 48 | // SystemWalletId: u.ID, 49 | // DeltaNodeUuid: instanceFromDb.InstanceUuid, 50 | // CreatedAt: time.Now(), 51 | // UpdatedAt: time.Now(), 52 | // } 53 | // 54 | // deltaMetricsBaseMessage := DeltaMetricsBaseMessage{ 55 | // ObjectType: "WalletLog", 56 | // Object: log, 57 | // } 58 | // 59 | // messageBytes, err := json.Marshal(deltaMetricsBaseMessage) 60 | // if err != nil { 61 | // return err 62 | // } 63 | // producer.Publish(messageBytes) 64 | // 65 | // return 66 | //} 67 | -------------------------------------------------------------------------------- /plugins/interface.go: -------------------------------------------------------------------------------- 1 | package plugins 2 | 3 | import ( 4 | "net/http" 5 | ) 6 | 7 | // simple plugin interface 8 | type PluginInOut interface { 9 | API() // URL https://auth.estuary.tech/ 10 | 11 | // structure of the input must be defined 12 | Input(interface{}) error 13 | 14 | // structure of the output must be defined 15 | Output() interface{} // can be an error 16 | } 17 | 18 | type Plugin interface { 19 | Name() string 20 | Initialize() error 21 | Middleware() func(http.Handler) http.Handler 22 | } 23 | 24 | type PluginRegistry struct { 25 | plugins map[string]Plugin 26 | } 27 | 28 | // flow 29 | // 1. load all the plugins in this directory 30 | // 2. override pre-existing plugin with the plugins from this directory 31 | // 3. run the plugins 32 | -------------------------------------------------------------------------------- /plugins/registry.go: -------------------------------------------------------------------------------- 1 | package plugins 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "plugin" 7 | ) 8 | 9 | func NewPluginRegistry() *PluginRegistry { 10 | return &PluginRegistry{ 11 | plugins: make(map[string]Plugin), 12 | } 13 | } 14 | 15 | func (r *PluginRegistry) Register(plugin Plugin) { 16 | r.plugins[plugin.Name()] = plugin 17 | } 18 | 19 | func (r *PluginRegistry) Middleware(name string) (func(http.Handler) http.Handler, bool) { 20 | plugin, ok := r.plugins[name] 21 | if !ok { 22 | return nil, false 23 | } 24 | return plugin.Middleware(), true 25 | } 26 | 27 | func LoadPlugin(path string, registry *PluginRegistry) error { 28 | plug, err := plugin.Open(path) 29 | if err != nil { 30 | return err 31 | } 32 | 33 | symPlugin, err := plug.Lookup("PluginInstance") 34 | if err != nil { 35 | return err 36 | } 37 | 38 | pluginInstance, ok := symPlugin.(Plugin) 39 | if !ok { 40 | return fmt.Errorf("unexpected plugin type: %T", symPlugin) 41 | } 42 | 43 | registry.Register(pluginInstance) 44 | return pluginInstance.Initialize() 45 | } 46 | -------------------------------------------------------------------------------- /utils/constants.go: -------------------------------------------------------------------------------- 1 | // A package that is used to define the constants used in the project. 2 | package utils 3 | 4 | var Reset = "\033[0m" 5 | var Red = "\033[31m" 6 | var Green = "\033[32m" 7 | var Yellow = "\033[33m" 8 | var Blue = "\033[34m" 9 | var Purple = "\033[35m" 10 | var Cyan = "\033[36m" 11 | var Gray = "\033[37m" 12 | var White = "\033[97m" 13 | 14 | const ( 15 | DELTA_LABEL string = "seal-the-delta-deal" 16 | CONTENT_PINNED string = "pinned" 17 | CONTENT_FAILED_TO_PIN string = "failed-to-pin" 18 | CONTENT_FAILED_TO_PROCESS string = "failed-to-process" 19 | 20 | CONTENT_PIECE_COMPUTING = "piece-computing" 21 | CONTENT_PIECE_COMPUTED = "piece-computed" 22 | CONTENT_PIECE_COMPUTING_FAILED = "piece-computing-failed" 23 | CONTENT_PIECE_ASSIGNED = "piece-assigned" 24 | 25 | CONTENT_DEAL_MAKING_PROPOSAL = "making-deal-proposal" 26 | CONTENT_DEAL_SENDING_PROPOSAL = "sending-deal-proposal" 27 | CONTENT_DEAL_PROPOSAL_SENT = "deal-proposal-sent" 28 | CONTENT_DEAL_PROPOSAL_FAILED = "deal-proposal-failed" 29 | 30 | DEAL_STATUS_TRANSFER_STARTED = "transfer-started" 31 | DEAL_STATUS_TRANSFER_FINISHED = "transfer-finished" 32 | DEAL_STATUS_TRANSFER_FAILED = "transfer-failed" 33 | 34 | BATCH_IMPORT_STATUS_COMPLETED = "completed" 35 | BATCH_IMPORT_STATUS_FAILED = "failed" 36 | BATCH_IMPORT_STATUS_STARTED = "started" 37 | 38 | COMMP_STATUS_OPEN = "open" 39 | COMMP_STATUS_COMITTED = "committed" 40 | 41 | CONNECTION_MODE_E2E = "e2e" 42 | CONNECTION_MODE_IMPORT = "import" 43 | 44 | DEAL_VERIFIED = "verified" 45 | DEAL_UNVERIFIED = "unverified" 46 | 47 | EPOCH_540_DAYS = 1555200 48 | EPOCH_PER_DAY = 2880 49 | EPOCH_PER_HOUR = 60 * 2 50 | FILECOIN_GENESIS_UNIX_EPOCH = 1598306400 51 | DEFAULT_DURATION = EPOCH_540_DAYS - (EPOCH_PER_DAY * 21) 52 | 53 | COMMP_MODE_FAST = "fast" 54 | COMMP_MODE_STREAM = "stream" 55 | COMPP_MODE_FILBOOST = "filboost" 56 | 57 | MAX_DEAL_RETRY = 10 58 | ) 59 | -------------------------------------------------------------------------------- /utils/encode.go: -------------------------------------------------------------------------------- 1 | // It takes a data structure and an output stream, encodes the data structure into JSON format, and writes it to the output 2 | // stream 3 | package utils 4 | 5 | import ( 6 | "encoding/json" 7 | "io" 8 | ) 9 | 10 | // PrettyEncode Encoding the data into JSON format and writing it to the output stream. 11 | func PrettyEncode(data interface{}, out io.Writer) error { 12 | enc := json.NewEncoder(out) 13 | enc.SetIndent("", " ") 14 | if err := enc.Encode(data); err != nil { 15 | return err 16 | } 17 | return nil 18 | } 19 | -------------------------------------------------------------------------------- /utils/epoch_height.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | "time" 7 | ) 8 | 9 | func DateToHeight(time time.Time) int64 { 10 | // time to string date only 11 | dateStr := time.Format("2006-01-02") 12 | return UnixToHeight(DateToUnixEpoch(dateStr, "00:00:00")) 13 | } 14 | 15 | func HeightToDate(height int64) time.Time { 16 | return time.Unix(HeightToUnix(height)*1000, 0) 17 | } 18 | 19 | func DateToUnixEpoch(dateStr string, timeStr string) int64 { 20 | dtStr := fmt.Sprintf("%sT%sZ", dateStr, timeStr) 21 | timeT, err := time.Parse(time.RFC3339, dtStr) 22 | if err != nil { 23 | fmt.Println(err) 24 | panic(err) 25 | } 26 | 27 | unixTimeSeconds := float64(timeT.Unix()) 28 | return int64(unixTimeSeconds) 29 | } 30 | 31 | func UnixToHeight(unixEpoch int64) int64 { 32 | return int64(math.Floor(float64(unixEpoch-FILECOIN_GENESIS_UNIX_EPOCH) / 30)) 33 | } 34 | 35 | func HeightToUnix(height int64) int64 { 36 | return (height * 30) + FILECOIN_GENESIS_UNIX_EPOCH 37 | } 38 | -------------------------------------------------------------------------------- /utils/file.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "github.com/pkg/errors" 5 | "io" 6 | ) 7 | 8 | // GetFileSize returns the size of a file in bytes. 9 | func GetFileSize(file io.Reader) (int64, error) { 10 | // Check if the reader also implements the Seeker interface 11 | seeker, ok := file.(io.Seeker) 12 | if !ok { 13 | return 0, errors.New("file size retrieval not supported") 14 | } 15 | 16 | // Get the current offset position 17 | currentPos, err := seeker.Seek(0, io.SeekCurrent) 18 | if err != nil { 19 | return 0, err 20 | } 21 | 22 | // Seek to the end to get the file size 23 | fileSize, err := seeker.Seek(0, io.SeekEnd) 24 | if err != nil { 25 | return 0, err 26 | } 27 | 28 | // Restore the original offset position 29 | _, err = seeker.Seek(currentPos, io.SeekStart) 30 | if err != nil { 31 | return 0, err 32 | } 33 | 34 | // Return the file size 35 | return fileSize, nil 36 | } 37 | -------------------------------------------------------------------------------- /utils/model_utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "delta/models" 6 | "github.com/application-research/filclient" 7 | datatransfer "github.com/filecoin-project/go-data-transfer" 8 | ) 9 | 10 | // GetChannelID `GetChannelID` takes a string and returns a `datatransfer.ChannelID` and an error 11 | func GetChannelID(dtChannel string) (datatransfer.ChannelID, error) { 12 | if dtChannel == "" { 13 | return datatransfer.ChannelID{}, db_models.ErrNoChannelID 14 | } 15 | 16 | chid, err := filclient.ChannelIDFromString(dtChannel) 17 | if err != nil { 18 | err = fmt.Errorf("incorrectly formatted data transfer channel ID in contentDeal record: %w", err) 19 | return datatransfer.ChannelID{}, err 20 | } 21 | return *chid, nil 22 | } 23 | -------------------------------------------------------------------------------- /utils/status_change.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | // StatusChange `StatusChange` is a struct with two fields, `To` and `Model`. The `To` field is a string, and the `Model` field is an 4 | // interface{}. 5 | // @property {string} To - The status to change to. 6 | // @property Model - The model that is being changed. 7 | type StatusChange struct { 8 | To string 9 | Model interface{} 10 | } 11 | --------------------------------------------------------------------------------