├── .env.example
├── .gitignore
├── LICENSE-APACHE
├── LICENSE-MIT
├── README.md
├── box-img-lg-template.png
├── box-img-sm-template.png
├── client.js
├── contracts
├── FPS.sol
├── Lighthouse.sol
├── LighthouseV2.sol
├── Migrations.sol
└── abi
│ ├── Lighthouse.json
│ └── LighthouseV2.json
├── migrations
└── 1_initial_migration.js
├── package-lock.json
├── package.json
├── res
└── lighthouse.png
├── sampleInput
├── src
├── @types
│ └── cid.d.ts
├── index.js
├── index.ts
├── parsers
│ ├── index.js
│ ├── index.ts
│ ├── infura
│ │ ├── contract.js
│ │ ├── contract.ts
│ │ ├── index.js
│ │ └── index.ts
│ └── vulcanize
│ │ ├── index.js
│ │ └── index.ts
└── storage-adapter
│ ├── index.js
│ ├── index.ts
│ ├── interfaces.js
│ ├── interfaces.ts
│ └── storage-providers
│ ├── index.js
│ ├── index.ts
│ ├── powergate.js
│ ├── powergate.ts
│ ├── provider.js
│ └── provider.ts
├── test
└── .gitkeep
├── truffle-box.json
├── truffle-config.js
└── tsconfig.json
/.env.example:
--------------------------------------------------------------------------------
1 | INFURA_URL="wss://rinkeby.infura.io/ws/v3/2cf3ec9bd42d4099b8620c2a6ee8c51a"
2 |
3 | VULCANIZE_URL="https://lighthouse.vdb.to/graphql"
4 |
5 | POWERGATE_URL=
6 | POW_ID=
7 | POW_TOKEN=
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | node_modules/
2 | .env
3 | dist
4 |
--------------------------------------------------------------------------------
/LICENSE-APACHE:
--------------------------------------------------------------------------------
1 | Copyright (c) 2020 Filecoin Project
2 |
3 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
4 |
5 | http://www.apache.org/licenses/LICENSE-2.0
6 |
7 | Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
--------------------------------------------------------------------------------
/LICENSE-MIT:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2020 Filecoin Project
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Lighthouse Node
2 | This project provides a way for Ethereum smart contracts to request Filecoin storage of CIDs in IPFS via Powergate. Hence, Ethereum developers can use this to request verifiable storage of data to Filecoin from Ethereum smart contracts.
3 |
4 | ### Website
5 | http://lighthouse.ninja
6 | ### Blog Post
7 | Read more about it on the blog here - https://nanditmehra123.medium.com/lighthouse-filecoin-ethereum-cross-chain-infra-project-66c041a1a1db
8 | ### Demo Video
9 | https://vimeo.com/552468707
10 | https://www.youtube.com/watch?v=E2-cfbdSXtM
11 |
12 | 
13 |
14 | The parser at src/parser:
15 |
16 | 1. Takes data feed from external sources VulcanizeDB.
17 | 2. Parses the data feed into a set of parameters which can be passed to the storage-adapter.
18 |
19 | The Storage adapter at src/storage-adapter:
20 |
21 | 1. Accepts the arguments passed from the parser. (storage adapter is modular enough to be taken out of this repo and used as an dependency. It can be used by any other project.)
22 | 2. Uses the arguments passed to store the data (cid) according to the config.
23 | 3. Publishes storage status via websocket.
24 |
25 | ## How to run
26 |
27 | 1. Clone this repo
28 | 2. Open a terminal window and run `> npm install`
29 | 3. To start the lighthouse node run `> node src/index.js`
30 |
31 | Now the lighthouse node is up and running, feel free to run it locally on your system or host on a server.
32 |
33 | This node also starts a socket.io server at port 3002 with which a socket.io client can interact to get the storage info of a file.
34 |
35 | **Sample code for socket.io client -**
36 | ```js
37 | const io = require("socket.io-client");
38 | const socket = io("ws://localhost:3002");
39 |
40 | // handle the event sent with socket.send()
41 | socket.on("message", data => {
42 | console.log(data);
43 | });
44 |
45 | socket.on("connect", () => {
46 | // or with emit() and custom event names
47 | socket.emit("cid", "QmbazJSQWTwmtAFqDgqxMG5JxGhg3kgYwdapaQDMM88HTP"); // put in your here for which storage-info is requested
48 | });
49 |
50 | // handle the event sent with socket.emit()
51 | socket.on("storageInfo", (storageInfo) => {
52 | console.log('storageInfo for cid:', storageInfo);
53 | });
54 | ```
55 |
56 | ## Use Truffle box to Interact with Lighthouse
57 | Demo Video - https://vimeo.com/552468707
58 | - make a new directory -
59 | mkdir lighthouse
60 | - cd into that -
61 | cd lighthouse
62 | - run the following command -
63 | truffle unbox nandit123/lighthouse
64 | - Install the packages -
65 | npm i
66 | - Compile Smart Contracts -
67 | truffle compile
68 | - Migrate and Deploy -
69 | truffle migrate --network rinkeby
70 | - Now start truffle console using -
71 | truffle console
72 | - Run the following command to get the instance of the deployed smart contract -
73 | let specificInstance = await FPS.at("0xdFEa08D7c2B43498Bfe32778334c9279956057F0");
74 | - Now run this command to store data to filecoin -
75 | let result = specificInstance.store("sampleCid", "SampleConfig");
76 | - You can get the transaction hash of the previous call by typing in -
77 | result
78 |
79 | - To get the storageInfo replace your cid in the client.js file and run -
80 | node client.js
81 |
--------------------------------------------------------------------------------
/box-img-lg-template.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/nandit123/lighthouse/44e4b36276b418ac35ed72bd946d70bab1d4c790/box-img-lg-template.png
--------------------------------------------------------------------------------
/box-img-sm-template.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/nandit123/lighthouse/44e4b36276b418ac35ed72bd946d70bab1d4c790/box-img-sm-template.png
--------------------------------------------------------------------------------
/client.js:
--------------------------------------------------------------------------------
1 | const io = require("socket.io-client");
2 | // const socket = io("ws://localhost:3002"); //local
3 | const socket = io("ws://13.126.82.18:3002"); // hosted
4 |
5 | const fs = require("fs");
6 |
7 | // handle the event sent with socket.send()
8 | socket.on("message", data => {
9 | console.log(data);
10 | });
11 |
12 | socket.on("connect", () => {
13 | // or with emit() and custom event names
14 | socket.emit("cid", "QmTJYDuVWNnRn6g4AjRE5adZ9GwTm7Sgu2Zj75a3xibDRJ");
15 | // socket.emit("retrieveFile", "QmcpabSYeXqNQ3hPyr5vefGbxnqkDmCD5pWm4RRoQoMor4");
16 | });
17 |
18 | // handle the event sent with socket.emit()
19 | socket.on("storageInfo", (storageInfo) => {
20 | console.log('storageInfo for cid:', storageInfo);
21 | });
22 |
23 | // handle the event sent with socket.emit()
24 | socket.on("retrieveFile", (file) => {
25 | console.log('typeof', typeof file)
26 | // console.log('file:', file)
27 | // console.log('file Uint8Array:', file);
28 | fs.appendFile('retrievedFile', Buffer.from(new Uint8Array(file)), function(err) {
29 | if (err) {
30 | console.log('error creating file');
31 | } else {
32 | console.log('file created successfully');
33 | }
34 | })
35 | });
36 |
37 |
--------------------------------------------------------------------------------
/contracts/FPS.sol:
--------------------------------------------------------------------------------
1 | // SPDX-License-Identifier: MIT
2 | pragma solidity >=0.4.22 <0.8.0;
3 |
4 | // https://ethereum.stackexchange.com/questions/17094/how-to-store-ipfs-hash-using-bytes32
5 | // https://medium.com/temporal-cloud/efficient-usable-an`d-cheap-storage-of-ipfs-hashes-in-solidity-smart-contracts-eb3bef129eba
6 |
7 | contract FPS {
8 | struct Content {
9 | string cid;
10 | string config;
11 | }
12 |
13 | event StorageRequest(address uploader, string cid, string config);
14 |
15 | mapping(address => mapping(string => Content)) uploaderToContent;
16 |
17 | function store(string calldata cid, string calldata config)
18 | external
19 | returns (bytes32)
20 | {
21 | uploaderToContent[msg.sender][cid] = Content(cid, config);
22 | emit StorageRequest(msg.sender, cid, config);
23 | // should return a requestId
24 | }
25 | }
26 |
--------------------------------------------------------------------------------
/contracts/Lighthouse.sol:
--------------------------------------------------------------------------------
1 | pragma solidity >=0.4.22 <0.8.0;
2 |
3 | contract Lighthouse {
4 | address owner = msg.sender;
5 |
6 | struct Content {
7 | string cid;
8 | string config;
9 | uint fileCost;
10 | }
11 |
12 | event StorageRequest(address uploader, string cid, string config, uint fileCost);
13 |
14 | mapping(address => mapping(string => Content)) public requests;
15 |
16 | function store(string calldata cid, string calldata config)
17 | external
18 | payable
19 | {
20 | uint fileCost = msg.value;
21 | requests[msg.sender][cid] = Content(cid, config, fileCost);
22 | emit StorageRequest(msg.sender, cid, config, msg.value);
23 | }
24 |
25 | function getPaid(uint amount, address payable recipient)
26 | external
27 | {
28 | require(msg.sender == owner);
29 | recipient.transfer(amount);
30 | }
31 |
32 | fallback () external payable {}
33 | }
34 |
--------------------------------------------------------------------------------
/contracts/LighthouseV2.sol:
--------------------------------------------------------------------------------
1 | // SPDX-License-Identifier: GPL-3.0
2 |
3 | pragma solidity >=0.4.22 <0.8.0;
4 |
5 | contract Lighthouse {
6 | address owner = msg.sender;
7 |
8 | struct Content {
9 | string cid;
10 | string config;
11 | uint fileCost;
12 | }
13 |
14 | struct Status {
15 | string dealIds;
16 | bool active;
17 | // mapping (uint => string) miners;
18 | }
19 |
20 | event StorageRequest(address uploader, string cid, string config, uint fileCost);
21 | event StorageStatusRequest(address requester, string cid);
22 |
23 | mapping(address => mapping(string => Content)) public requests;
24 | mapping(string => Status) public statuses; // address -> cid -> status
25 |
26 | function store(string calldata cid, string calldata config)
27 | external
28 | payable
29 | {
30 | uint fileCost = msg.value;
31 | requests[msg.sender][cid] = Content(cid, config, fileCost);
32 | emit StorageRequest(msg.sender, cid, config, msg.value);
33 | }
34 |
35 | function getPaid(uint amount, address payable recipient)
36 | external
37 | {
38 | require(msg.sender == owner);
39 | recipient.transfer(amount);
40 | }
41 |
42 | function requestStorageStatus(string calldata cid)
43 | external
44 | {
45 | emit StorageStatusRequest(msg.sender, cid);
46 | }
47 |
48 | function publishStorageStatus(string calldata cid, string calldata dealIds, bool active)
49 | external
50 | { // restrict it to only to the user
51 | require(msg.sender == owner);
52 | statuses[cid] = Status(dealIds, active);
53 | }
54 |
55 | fallback () external payable {}
56 | }
57 |
--------------------------------------------------------------------------------
/contracts/Migrations.sol:
--------------------------------------------------------------------------------
1 | // SPDX-License-Identifier: MIT
2 | pragma solidity >=0.4.22 <0.9.0;
3 |
4 | contract Migrations {
5 | address public owner = msg.sender;
6 | uint public last_completed_migration;
7 |
8 | modifier restricted() {
9 | require(
10 | msg.sender == owner,
11 | "This function is restricted to the contract's owner"
12 | );
13 | _;
14 | }
15 |
16 | function setCompleted(uint completed) public restricted {
17 | last_completed_migration = completed;
18 | }
19 | }
20 |
--------------------------------------------------------------------------------
/contracts/abi/Lighthouse.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "anonymous": false,
4 | "inputs": [
5 | {
6 | "indexed": false,
7 | "internalType": "address",
8 | "name": "uploader",
9 | "type": "address"
10 | },
11 | {
12 | "indexed": false,
13 | "internalType": "string",
14 | "name": "cid",
15 | "type": "string"
16 | },
17 | {
18 | "indexed": false,
19 | "internalType": "string",
20 | "name": "config",
21 | "type": "string"
22 | },
23 | {
24 | "indexed": false,
25 | "internalType": "uint256",
26 | "name": "fileCost",
27 | "type": "uint256"
28 | }
29 | ],
30 | "name": "StorageRequest",
31 | "type": "event"
32 | },
33 | {
34 | "stateMutability": "payable",
35 | "type": "fallback"
36 | },
37 | {
38 | "inputs": [
39 | {
40 | "internalType": "uint256",
41 | "name": "amount",
42 | "type": "uint256"
43 | },
44 | {
45 | "internalType": "address payable",
46 | "name": "recipient",
47 | "type": "address"
48 | }
49 | ],
50 | "name": "getPaid",
51 | "outputs": [],
52 | "stateMutability": "nonpayable",
53 | "type": "function"
54 | },
55 | {
56 | "inputs": [
57 | {
58 | "internalType": "address",
59 | "name": "",
60 | "type": "address"
61 | },
62 | {
63 | "internalType": "string",
64 | "name": "",
65 | "type": "string"
66 | }
67 | ],
68 | "name": "requests",
69 | "outputs": [
70 | {
71 | "internalType": "string",
72 | "name": "cid",
73 | "type": "string"
74 | },
75 | {
76 | "internalType": "string",
77 | "name": "config",
78 | "type": "string"
79 | },
80 | {
81 | "internalType": "uint256",
82 | "name": "fileCost",
83 | "type": "uint256"
84 | }
85 | ],
86 | "stateMutability": "view",
87 | "type": "function"
88 | },
89 | {
90 | "inputs": [
91 | {
92 | "internalType": "string",
93 | "name": "cid",
94 | "type": "string"
95 | },
96 | {
97 | "internalType": "string",
98 | "name": "config",
99 | "type": "string"
100 | }
101 | ],
102 | "name": "store",
103 | "outputs": [],
104 | "stateMutability": "payable",
105 | "type": "function"
106 | }
107 | ]
--------------------------------------------------------------------------------
/contracts/abi/LighthouseV2.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "anonymous": false,
4 | "inputs": [
5 | {
6 | "indexed": false,
7 | "internalType": "address",
8 | "name": "uploader",
9 | "type": "address"
10 | },
11 | {
12 | "indexed": false,
13 | "internalType": "string",
14 | "name": "cid",
15 | "type": "string"
16 | },
17 | {
18 | "indexed": false,
19 | "internalType": "string",
20 | "name": "config",
21 | "type": "string"
22 | },
23 | {
24 | "indexed": false,
25 | "internalType": "uint256",
26 | "name": "fileCost",
27 | "type": "uint256"
28 | }
29 | ],
30 | "name": "StorageRequest",
31 | "type": "event"
32 | },
33 | {
34 | "anonymous": false,
35 | "inputs": [
36 | {
37 | "indexed": false,
38 | "internalType": "address",
39 | "name": "requester",
40 | "type": "address"
41 | },
42 | {
43 | "indexed": false,
44 | "internalType": "string",
45 | "name": "cid",
46 | "type": "string"
47 | }
48 | ],
49 | "name": "StorageStatusRequest",
50 | "type": "event"
51 | },
52 | {
53 | "stateMutability": "payable",
54 | "type": "fallback"
55 | },
56 | {
57 | "inputs": [
58 | {
59 | "internalType": "uint256",
60 | "name": "amount",
61 | "type": "uint256"
62 | },
63 | {
64 | "internalType": "address payable",
65 | "name": "recipient",
66 | "type": "address"
67 | }
68 | ],
69 | "name": "getPaid",
70 | "outputs": [],
71 | "stateMutability": "nonpayable",
72 | "type": "function"
73 | },
74 | {
75 | "inputs": [
76 | {
77 | "internalType": "string",
78 | "name": "cid",
79 | "type": "string"
80 | },
81 | {
82 | "internalType": "string",
83 | "name": "dealIds",
84 | "type": "string"
85 | },
86 | {
87 | "internalType": "bool",
88 | "name": "active",
89 | "type": "bool"
90 | }
91 | ],
92 | "name": "publishStorageStatus",
93 | "outputs": [],
94 | "stateMutability": "nonpayable",
95 | "type": "function"
96 | },
97 | {
98 | "inputs": [
99 | {
100 | "internalType": "string",
101 | "name": "cid",
102 | "type": "string"
103 | }
104 | ],
105 | "name": "requestStorageStatus",
106 | "outputs": [],
107 | "stateMutability": "nonpayable",
108 | "type": "function"
109 | },
110 | {
111 | "inputs": [
112 | {
113 | "internalType": "address",
114 | "name": "",
115 | "type": "address"
116 | },
117 | {
118 | "internalType": "string",
119 | "name": "",
120 | "type": "string"
121 | }
122 | ],
123 | "name": "requests",
124 | "outputs": [
125 | {
126 | "internalType": "string",
127 | "name": "cid",
128 | "type": "string"
129 | },
130 | {
131 | "internalType": "string",
132 | "name": "config",
133 | "type": "string"
134 | },
135 | {
136 | "internalType": "uint256",
137 | "name": "fileCost",
138 | "type": "uint256"
139 | }
140 | ],
141 | "stateMutability": "view",
142 | "type": "function"
143 | },
144 | {
145 | "inputs": [
146 | {
147 | "internalType": "string",
148 | "name": "",
149 | "type": "string"
150 | }
151 | ],
152 | "name": "statuses",
153 | "outputs": [
154 | {
155 | "internalType": "string",
156 | "name": "dealIds",
157 | "type": "string"
158 | },
159 | {
160 | "internalType": "bool",
161 | "name": "active",
162 | "type": "bool"
163 | }
164 | ],
165 | "stateMutability": "view",
166 | "type": "function"
167 | },
168 | {
169 | "inputs": [
170 | {
171 | "internalType": "string",
172 | "name": "cid",
173 | "type": "string"
174 | },
175 | {
176 | "internalType": "string",
177 | "name": "config",
178 | "type": "string"
179 | }
180 | ],
181 | "name": "store",
182 | "outputs": [],
183 | "stateMutability": "payable",
184 | "type": "function"
185 | }
186 | ]
--------------------------------------------------------------------------------
/migrations/1_initial_migration.js:
--------------------------------------------------------------------------------
1 | const Migrations = artifacts.require("Migrations");
2 | const FPS = artifacts.require("FPS");
3 |
4 | module.exports = function (deployer) {
5 | deployer.deploy(Migrations);
6 | deployer.deploy(FPS);
7 | };
8 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "lighthouse-node",
3 | "version": "0.0.1",
4 | "description": "Lighthouse client",
5 | "main": "index.js",
6 | "scripts": {
7 | "test": "echo \"Error: no test specified\" && exit 1"
8 | },
9 | "repository": {
10 | "type": "git",
11 | "url": "git+https://github.com/dappkit/lighthouse.git"
12 | },
13 | "keywords": [
14 | "lighthouse",
15 | "powergate",
16 | "ipfs",
17 | "filecoin",
18 | "ethereum",
19 | "infura",
20 | "vulcanizedb"
21 | ],
22 | "author": "nandit123 (nanditmehra123@gmail.com), vasa (vasa.develop@gmail.com)",
23 | "license": "MIT",
24 | "bugs": {
25 | "url": "https://github.com/dappkit/lighthouse/issues"
26 | },
27 | "homepage": "https://github.com/dappkit/lighthouse#readme",
28 | "dependencies": {
29 | "@textile/powergate-client": "^4.1.0",
30 | "@truffle/hdwallet-provider": "^1.4.0",
31 | "dotenv": "^8.2.0",
32 | "express": "^4.17.1",
33 | "fs": "0.0.1-security",
34 | "ipfs-core": "^0.9.1",
35 | "ipfs-http-client": "^51.0.1",
36 | "kikstart-graphql-client": "^1.5.0",
37 | "node-cron": "^3.0.0",
38 | "postgraphile": "^4.11.0",
39 | "rimraf": "^3.0.2",
40 | "socket.io": "^2.4.1",
41 | "socket.io-client": "^2.4.0",
42 | "web3": "^1.2.11"
43 | }
44 | }
45 |
--------------------------------------------------------------------------------
/res/lighthouse.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/nandit123/lighthouse/44e4b36276b418ac35ed72bd946d70bab1d4c790/res/lighthouse.png
--------------------------------------------------------------------------------
/sampleInput:
--------------------------------------------------------------------------------
1 | rinkeby contract - 0xdFEa08D7c2B43498Bfe32778334c9279956057F0
2 | CID - QmcpabSYeXqNQ3hPyr5vefGbxnqkDmCD5pWm4RRoQoMor4
3 | Config - {hot:{enabled:true,allowUnfreeze:true,ipfs:{addTimeout:900},unfreezeMaxPrice:0},cold:{enabled:true,filecoin:{replicationFactor:1,dealMinDuration:518400,excludedMinersList:[],trustedMinersList:[],countryCodesList:[],renew:{enabled:true,threshold:1},address:f3rpbm3bt4muydk3iq5ainss6phht4bjbe5dq6egrx4rwzqjgwc5eruyloozvf6qjunubo467neaqsvbzyxnna,maxPrice:100000000000,fastRetrieval:true,dealStartOffset:8640,verifiedDeal:true}},repairable:false}
4 |
--------------------------------------------------------------------------------
/src/@types/cid.d.ts:
--------------------------------------------------------------------------------
1 | /**
2 | * Class representing a CID ``
3 | * , as defined in [ipld/cid](https://github.com/multiformats/cid).
4 | */
5 | declare class CID {
6 | /**
7 | * Create a new CID.
8 | *
9 | * The algorithm for argument input is roughly:
10 | * ```
11 | * if (cid)
12 | * -> create a copy
13 | * else if (str)
14 | * if (1st char is on multibase table) -> CID String
15 | * else -> bs58 encoded multihash
16 | * else if (Buffer)
17 | * if (1st byte is 0 or 1) -> CID
18 | * else -> multihash
19 | * else if (Number)
20 | * -> construct CID by parts
21 | * ```
22 | *
23 | * @example
24 | * new CID(, , , )
25 | * new CID()
26 | * new CID()
27 | * new CID()
28 | * new CID()
29 | * new CID()
30 | */
31 | constructor(
32 | version: 0 | 1,
33 | codec: string,
34 | multhash: Buffer,
35 | multibaseName?: string
36 | );
37 | constructor(cid: CID);
38 | constructor(str: string);
39 | constructor(buf: Buffer);
40 |
41 | /**
42 | * The version of the CID.
43 | */
44 | version: number;
45 |
46 | /**
47 | * The codec of the CID.
48 | */
49 | codec: string;
50 |
51 | /**
52 | * The multihash of the CID.
53 | */
54 | multihash: Buffer;
55 |
56 | /**
57 | * Multibase name as string.
58 | */
59 | multibaseName: string;
60 |
61 | /**
62 | * The CID as a `Buffer`
63 | */
64 | readonly buffer: Buffer;
65 |
66 | /**
67 | * The prefix of the CID.
68 | */
69 | readonly prefix: Buffer;
70 |
71 | /**
72 | * Convert to a CID of version `0`.
73 | */
74 | toV0(): CID;
75 |
76 | /**
77 | * Convert to a CID of version `1`.
78 | */
79 | toV1(): CID;
80 |
81 | /**
82 | * Encode the CID into a string.
83 | *
84 | * @param base Base encoding to use.
85 | */
86 | toBaseEncodedString(base?: string): string;
87 |
88 | /**
89 | * Encode the CID into a string.
90 | */
91 | toString(base?: string): string;
92 |
93 | /**
94 | * Serialize to a plain object.
95 | */
96 | toJSON(): { codec: string; version: 0 | 1; hash: Buffer };
97 |
98 | /**
99 | * Compare equality with another CID.
100 | *
101 | * @param other The other CID.
102 | */
103 | equals(other: any): boolean;
104 |
105 | /**
106 | * Test if the given input is a valid CID object.
107 | * Throws if it is not.
108 | *
109 | * @param other The other CID.
110 | */
111 | static validateCID(other: any): void;
112 |
113 | static isCID(mixed: any): boolean;
114 |
115 | static codecs: Record;
116 | }
117 |
118 | export = CID;
119 |
--------------------------------------------------------------------------------
/src/index.js:
--------------------------------------------------------------------------------
1 | "use strict";
2 | exports.__esModule = true;
3 | var parsers_1 = require("./parsers");
4 | require("dotenv").config();
5 | var infuraURL = process.env.INFURA_URL;
6 | var vulcanizeURL = process.env.VULCANIZE_URL;
7 | var powergateHost = process.env.POWERGATE_URL;
8 | var config = {
9 | powergate: {
10 | host: powergateHost,
11 | config: {
12 | hot: {
13 | enabled: true,
14 | allowUnfreeze: true,
15 | ipfs: { addTimeout: "900" },
16 | unfreezeMaxPrice: "0"
17 | },
18 | cold: {
19 | enabled: true,
20 | filecoin: {
21 | replicationFactor: "1",
22 | dealMinDuration: "518400",
23 | excludedMiners: [],
24 | trustedMiners: ["f01240", "f0678914", "f022352", "f010446", "f02576"],
25 | countryCodes: [],
26 | renew: { enabled: true, threshold: "1" },
27 | address: "f3rpbm3bt4muydk3iq5ainss6phht4bjbe5dq6egrx4rwzqjgwc5eruyloozvf6qjunubo467neaqsvbzyxnna",
28 | maxPrice: "100000000000",
29 | fastRetrieval: true,
30 | dealStartOffset: "8640",
31 | verifiedDeal: true
32 | }
33 | },
34 | repairable: false
35 | }
36 | }
37 | };
38 | // Start parsers
39 | var parser = new parsers_1["default"](config, infuraURL, vulcanizeURL);
40 | parser.start();
41 | parser.socket();
42 |
--------------------------------------------------------------------------------
/src/index.ts:
--------------------------------------------------------------------------------
1 | import Parser from "./parsers";
2 |
3 | require("dotenv").config();
4 |
5 | const infuraURL = process.env.INFURA_URL;
6 | const vulcanizeURL = process.env.VULCANIZE_URL;
7 | const powergateHost = process.env.POWERGATE_URL;
8 | const config = {
9 | powergate: {
10 | host: powergateHost,
11 | config: {
12 | hot: {
13 | enabled: true,
14 | allowUnfreeze: true,
15 | ipfs: { addTimeout: "900" },
16 | unfreezeMaxPrice: "0",
17 | },
18 | cold: {
19 | enabled: true,
20 | filecoin: {
21 | replicationFactor: "1",
22 | dealMinDuration: "518400",
23 | excludedMiners: [],
24 | trustedMiners: ["f01240", "f0678914", "f022352", "f010446", "f02576"],
25 | countryCodes: [],
26 | renew: { enabled: true, threshold: "1" },
27 | address:
28 | "f3rpbm3bt4muydk3iq5ainss6phht4bjbe5dq6egrx4rwzqjgwc5eruyloozvf6qjunubo467neaqsvbzyxnna",
29 | maxPrice: "100000000000",
30 | fastRetrieval: true,
31 | dealStartOffset: "8640",
32 | verifiedDeal: true
33 | },
34 | },
35 | repairable: false,
36 | },
37 | },
38 | };
39 |
40 | // Start parsers
41 |
42 | const parser = new Parser(config, infuraURL, vulcanizeURL);
43 | parser.start();
44 | parser.socket();
45 |
--------------------------------------------------------------------------------
/src/parsers/index.js:
--------------------------------------------------------------------------------
1 | "use strict";
2 | var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3 | function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4 | return new (P || (P = Promise))(function (resolve, reject) {
5 | function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6 | function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7 | function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8 | step((generator = generator.apply(thisArg, _arguments || [])).next());
9 | });
10 | };
11 | var __generator = (this && this.__generator) || function (thisArg, body) {
12 | var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
13 | return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
14 | function verb(n) { return function (v) { return step([n, v]); }; }
15 | function step(op) {
16 | if (f) throw new TypeError("Generator is already executing.");
17 | while (_) try {
18 | if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
19 | if (y = 0, t) op = [op[0] & 2, t.value];
20 | switch (op[0]) {
21 | case 0: case 1: t = op; break;
22 | case 4: _.label++; return { value: op[1], done: false };
23 | case 5: _.label++; y = op[1]; op = [0]; continue;
24 | case 7: op = _.ops.pop(); _.trys.pop(); continue;
25 | default:
26 | if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
27 | if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
28 | if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
29 | if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
30 | if (t[2]) _.ops.pop();
31 | _.trys.pop(); continue;
32 | }
33 | op = body.call(thisArg, _);
34 | } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
35 | if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
36 | }
37 | };
38 | var __asyncValues = (this && this.__asyncValues) || function (o) {
39 | if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
40 | var m = o[Symbol.asyncIterator], i;
41 | return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
42 | function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
43 | function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
44 | };
45 | exports.__esModule = true;
46 | var infura_1 = require("./infura");
47 | var vulcanize_1 = require("./vulcanize");
48 | var storage_adapter_1 = require("../storage-adapter");
49 | var ipfs_http_client_1 = require("ipfs-http-client");
50 | var io = require("socket.io")(3002, { cors: { origins: ["*"] } });
51 | var fs = require("fs");
52 | var rimraf = require("rimraf");
53 | var IPFS = require('ipfs-core');
54 | var ipfsClient = require('ipfs-http-client');
55 | var express = require('express');
56 | var Files = {};
57 | var Parser = /** @class */ (function () {
58 | function Parser(config, infuraURL, vulcanizeURL) {
59 | this.config = config;
60 | this.storageAdapter = new storage_adapter_1["default"](this.config);
61 | this.infura = new infura_1["default"](infuraURL, this.storageAdapter);
62 | this.vulcanize = new vulcanize_1["default"](vulcanizeURL, this.storageAdapter);
63 | }
64 | Parser.prototype.start = function () {
65 | // Use Infura as a fallback source
66 | // When the Lighthouse node starts, if we get a response from Vulcanize
67 | // before the set Timeout, then we woud use Vulcanize, otherwise, use Infura
68 | this.vulcanize.start();
69 | this.vulcanize.listenEventStorageRequest(this.storageAdapter);
70 | this.vulcanize.cronJob(this.storageAdapter);
71 | this.infura.start();
72 | this.httpServer(this.storageAdapter);
73 | };
74 | Parser.prototype.getStorageInfo = function (cid) {
75 | return this.storageAdapter.getStorageInfo(cid);
76 | };
77 | Parser.prototype.socket = function () {
78 | var _this = this;
79 | console.log('socket started');
80 | io.on("connection", function (socket) {
81 | // either with send()
82 | socket.send("Welcome to Lighthouse!");
83 | // handle the event sent with socket.emit()
84 | socket.on("cid", function (cid) { return __awaiter(_this, void 0, void 0, function () {
85 | var storageInfo, _a, _b, _c, _d, _e, _f, _g, e_1;
86 | return __generator(this, function (_h) {
87 | switch (_h.label) {
88 | case 0:
89 | console.log("cid recieved:", cid);
90 | _h.label = 1;
91 | case 1:
92 | _h.trys.push([1, 4, , 5]);
93 | _b = (_a = console).log;
94 | _c = ['storageInfo is'];
95 | _e = (_d = JSON).stringify;
96 | return [4 /*yield*/, this.storageAdapter.getStorageInfo(cid)];
97 | case 2:
98 | _b.apply(_a, _c.concat([_e.apply(_d, [_h.sent()])]));
99 | _g = (_f = JSON).stringify;
100 | return [4 /*yield*/, this.storageAdapter.getStorageInfo(cid)];
101 | case 3:
102 | storageInfo = _g.apply(_f, [_h.sent()]);
103 | return [3 /*break*/, 5];
104 | case 4:
105 | e_1 = _h.sent();
106 | console.log('entered catch');
107 | storageInfo = { storageInfo: 'No Storage Deal found for this CID' };
108 | return [3 /*break*/, 5];
109 | case 5:
110 | // or with emit() and custom event names
111 | socket.emit("storageInfo", storageInfo);
112 | return [2 /*return*/];
113 | }
114 | });
115 | }); });
116 | // publish the storage status of cid onto the smart contract
117 | socket.on("publishStatus", function (cid) { return __awaiter(_this, void 0, void 0, function () {
118 | var storageInfo, _a, _b, _c, _d, _e, _f, _g, e_2;
119 | return __generator(this, function (_h) {
120 | switch (_h.label) {
121 | case 0:
122 | console.log("cid recieved:", cid);
123 | _h.label = 1;
124 | case 1:
125 | _h.trys.push([1, 4, , 5]);
126 | _b = (_a = console).log;
127 | _c = ['storageInfo is'];
128 | _e = (_d = JSON).stringify;
129 | return [4 /*yield*/, this.storageAdapter.getStorageInfo(cid)];
130 | case 2:
131 | _b.apply(_a, _c.concat([_e.apply(_d, [_h.sent()])]));
132 | _g = (_f = JSON).stringify;
133 | return [4 /*yield*/, this.storageAdapter.getStorageInfo(cid)];
134 | case 3:
135 | storageInfo = _g.apply(_f, [_h.sent()]);
136 | return [3 /*break*/, 5];
137 | case 4:
138 | e_2 = _h.sent();
139 | console.log('entered catch');
140 | storageInfo = { storageInfo: 'No Storage Deal found for this CID' };
141 | return [3 /*break*/, 5];
142 | case 5:
143 | // or with emit() and custom event names
144 | socket.emit("storageInfo", storageInfo);
145 | return [2 /*return*/];
146 | }
147 | });
148 | }); });
149 | socket.on("Start", function (data) { return __awaiter(_this, void 0, void 0, function () {
150 | var Name, Path, Place, Stat;
151 | return __generator(this, function (_a) {
152 | switch (_a.label) {
153 | case 0:
154 | Name = data['Name'];
155 | Path = data['Path'];
156 | Files[Name] = {
157 | FileSize: data['Size'],
158 | Data: "",
159 | Downloaded: 0
160 | };
161 | return [4 /*yield*/, fs.mkdir(Path, function (err) {
162 | if (err) {
163 | return console.error(err);
164 | }
165 | console.log('Directory created: ', Path);
166 | })];
167 | case 1:
168 | _a.sent();
169 | Place = 0;
170 | try {
171 | Stat = fs.statSync(Path + '/' + Name);
172 | if (Stat.isFile()) {
173 | Files[Name]['Downloaded'] = Stat.size;
174 | Place = Stat.size / 54288;
175 | }
176 | }
177 | catch (error) {
178 | console.log('It is a new file');
179 | }
180 | fs.open(Path + "/" + Name, "a", '0755', function (err, fd) {
181 | if (err) {
182 | console.log('file open error', err);
183 | }
184 | else {
185 | Files[Name]['Handler'] = fd; // we store file handler so we can write to it later
186 | socket.emit('MoreData', { 'Place': Place, Percent: 0 });
187 | }
188 | });
189 | return [2 /*return*/];
190 | }
191 | });
192 | }); });
193 | socket.on('Upload', function (data) { return __awaiter(_this, void 0, void 0, function () {
194 | var Name, Path, Place, Percent;
195 | var _this = this;
196 | return __generator(this, function (_a) {
197 | switch (_a.label) {
198 | case 0:
199 | console.log('entered Upload');
200 | Name = data['Name'];
201 | Path = data['Path'];
202 | Files[Name]['Downloaded'] += data['Data'].length;
203 | Files[Name]['Data'] += data['Data'];
204 | if (!(Files[Name]['Downloaded'] == Files[Name]['FileSize'])) return [3 /*break*/, 2];
205 | return [4 /*yield*/, fs.write(Files[Name]['Handler'], Files[Name]['Data'], null, 'Binary', function (err, Writen) { return __awaiter(_this, void 0, void 0, function () {
206 | var path;
207 | return __generator(this, function (_a) {
208 | //Get Thumbnail Here
209 | console.log('File downloaded fully !!', Name);
210 | socket.emit('FileDownloaded', 'Yes');
211 | try {
212 | path = Path + '/' + Name;
213 | // let cidObject: any = await this.storageAdapter.stageFile(path);
214 | // console.log('cid is:', cidObject);
215 | socket.emit('FileInfo', { name: Name, size: Files[Name]['FileSize'] });
216 | // fs.unlink(path, (err) => {
217 | // if (err) throw err;
218 | // console.log(path + ' was deleted')
219 | // });
220 | }
221 | catch (e) {
222 | console.log('stageFile error:', e);
223 | }
224 | return [2 /*return*/];
225 | });
226 | }); })];
227 | case 1:
228 | _a.sent();
229 | return [3 /*break*/, 5];
230 | case 2:
231 | if (!(Files[Name]['Data'].length > 10485760)) return [3 /*break*/, 4];
232 | return [4 /*yield*/, fs.write(Files[Name]['Handler'], Files[Name]['Data'], null, 'Binary', function (err, Writen) {
233 | Files[Name]['Data'] = ""; //Reset The Buffer
234 | var Place = Files[Name]['Downloaded'] / 524288;
235 | var Percent = (Files[Name]['Downloaded'] / Files[Name]['FileSize']) * 100;
236 | socket.emit('MoreData', { 'Place': Place, 'Percent': Percent });
237 | })];
238 | case 3:
239 | _a.sent();
240 | return [3 /*break*/, 5];
241 | case 4:
242 | Place = Files[Name]['Downloaded'] / 524288;
243 | Percent = (Files[Name]['Downloaded'] / Files[Name]['FileSize']) * 100;
244 | socket.emit('MoreData', { 'Place': Place, 'Percent': Percent });
245 | _a.label = 5;
246 | case 5: return [2 /*return*/];
247 | }
248 | });
249 | }); });
250 | socket.on("GetCid", function (path) { return __awaiter(_this, void 0, void 0, function () {
251 | var cid, ipfs, globSourceOptions, _a, _b, file, e_3_1;
252 | var e_3, _c;
253 | return __generator(this, function (_d) {
254 | switch (_d.label) {
255 | case 0:
256 | console.log('GetCid for folder:', path);
257 | return [4 /*yield*/, this.storageAdapter.stageFolder(path)];
258 | case 1:
259 | cid = _d.sent();
260 | console.log('cid is:', cid);
261 | return [4 /*yield*/, ipfsClient.create({ host: 'localhost', port: '5001', protocol: 'http' })];
262 | case 2:
263 | ipfs = _d.sent();
264 | console.log('ipfs instance created');
265 | globSourceOptions = {
266 | recursive: true
267 | };
268 | console.log('path:', path);
269 | _d.label = 3;
270 | case 3:
271 | _d.trys.push([3, 8, 9, 14]);
272 | _a = __asyncValues(ipfs.addAll(ipfs_http_client_1.globSource('./' + path, globSourceOptions)));
273 | _d.label = 4;
274 | case 4: return [4 /*yield*/, _a.next()];
275 | case 5:
276 | if (!(_b = _d.sent(), !_b.done)) return [3 /*break*/, 7];
277 | file = _b.value;
278 | console.log(file);
279 | _d.label = 6;
280 | case 6: return [3 /*break*/, 4];
281 | case 7: return [3 /*break*/, 14];
282 | case 8:
283 | e_3_1 = _d.sent();
284 | e_3 = { error: e_3_1 };
285 | return [3 /*break*/, 14];
286 | case 9:
287 | _d.trys.push([9, , 12, 13]);
288 | if (!(_b && !_b.done && (_c = _a["return"]))) return [3 /*break*/, 11];
289 | return [4 /*yield*/, _c.call(_a)];
290 | case 10:
291 | _d.sent();
292 | _d.label = 11;
293 | case 11: return [3 /*break*/, 13];
294 | case 12:
295 | if (e_3) throw e_3.error;
296 | return [7 /*endfinally*/];
297 | case 13: return [7 /*endfinally*/];
298 | case 14:
299 | rimraf(path, function () { console.log("deleted folder:", path); });
300 | socket.emit('FolderCid', { cid: cid });
301 | return [2 /*return*/];
302 | }
303 | });
304 | }); });
305 | socket.on("GetCidSize", function (cid) { return __awaiter(_this, void 0, void 0, function () {
306 | var ipfs, cidInfo, e_4;
307 | return __generator(this, function (_a) {
308 | switch (_a.label) {
309 | case 0:
310 | console.log("Size for cid:", cid);
311 | return [4 /*yield*/, ipfsClient.create({ host: 'localhost', port: '5001', protocol: 'http' })];
312 | case 1:
313 | ipfs = _a.sent();
314 | console.log('ipfsClient created');
315 | _a.label = 2;
316 | case 2:
317 | _a.trys.push([2, 4, , 5]);
318 | console.log('inside try block');
319 | return [4 /*yield*/, ipfs.files.stat("/ipfs/" + cid)];
320 | case 3:
321 | cidInfo = _a.sent();
322 | console.log('cidinfo:', cidInfo);
323 | socket.emit('CidSize', { size: cidInfo.cumulativeSize });
324 | return [3 /*break*/, 5];
325 | case 4:
326 | e_4 = _a.sent();
327 | console.log("getCIDSize error:", e_4);
328 | socket.emit('CidSize', { size: "Error" });
329 | return [3 /*break*/, 5];
330 | case 5: return [2 /*return*/];
331 | }
332 | });
333 | }); });
334 | socket.on("retrieveFile", function (cid) { return __awaiter(_this, void 0, void 0, function () {
335 | var file, e_5;
336 | return __generator(this, function (_a) {
337 | switch (_a.label) {
338 | case 0:
339 | console.log("cid recieved:", cid);
340 | _a.label = 1;
341 | case 1:
342 | _a.trys.push([1, 3, , 4]);
343 | console.log('entered retrieveFile');
344 | return [4 /*yield*/, this.storageAdapter.retrieveFile(cid)];
345 | case 2:
346 | // console.log('storageInfo is', JSON.stringify(await this.storageAdapter.getStorageInfo(cid)));
347 | file = (_a.sent()).buffer;
348 | return [3 /*break*/, 4];
349 | case 3:
350 | e_5 = _a.sent();
351 | console.log('entered catch');
352 | file = 'error';
353 | return [3 /*break*/, 4];
354 | case 4:
355 | // or with emit() and custom event names
356 | socket.emit("retrieveFile", file);
357 | return [2 /*return*/];
358 | }
359 | });
360 | }); });
361 | });
362 | };
363 | Parser.prototype.httpServer = function (storageAdapter) {
364 | var app = express();
365 | app.get('/storageDealRecords', function (req, res) {
366 | return __awaiter(this, void 0, void 0, function () {
367 | var records;
368 | return __generator(this, function (_a) {
369 | switch (_a.label) {
370 | case 0: return [4 /*yield*/, storageAdapter.storageDealRecords()];
371 | case 1:
372 | records = _a.sent();
373 | res.status(200).send(records);
374 | return [2 /*return*/];
375 | }
376 | });
377 | });
378 | });
379 | app.listen(3000, console.log('http server listening at port 3000'));
380 | };
381 | return Parser;
382 | }());
383 | exports["default"] = Parser;
384 |
--------------------------------------------------------------------------------
/src/parsers/index.ts:
--------------------------------------------------------------------------------
1 | import Infura from "./infura";
2 | import Vulcanize from "./vulcanize";
3 | import StorageAdapter from "../storage-adapter";
4 | import { globSource } from 'ipfs-http-client';
5 |
6 | const io = require("socket.io")(3002, { cors: {origins: ["*"] } });
7 | const fs = require("fs");
8 | const rimraf = require("rimraf");
9 | const IPFS = require('ipfs-core')
10 | const ipfsClient = require('ipfs-http-client');
11 | const express = require('express');
12 |
13 | var Files = {};
14 |
15 | class Parser {
16 | config: any;
17 | infura: Infura;
18 | vulcanize: Vulcanize;
19 | storageAdapter: StorageAdapter;
20 | constructor(config: any, infuraURL: string, vulcanizeURL: string) {
21 | this.config = config;
22 | this.storageAdapter = new StorageAdapter(this.config);
23 | this.infura = new Infura(infuraURL, this.storageAdapter);
24 | this.vulcanize = new Vulcanize(vulcanizeURL, this.storageAdapter);
25 | }
26 |
27 | start() {
28 | // Use Infura as a fallback source
29 | // When the Lighthouse node starts, if we get a response from Vulcanize
30 | // before the set Timeout, then we woud use Vulcanize, otherwise, use Infura
31 | this.vulcanize.start();
32 | this.vulcanize.listenEventStorageRequest(this.storageAdapter);
33 | this.vulcanize.cronJob(this.storageAdapter);
34 | this.infura.start();
35 | this.httpServer(this.storageAdapter);
36 | }
37 |
38 | getStorageInfo(cid) {
39 | return this.storageAdapter.getStorageInfo(cid);
40 | }
41 |
42 | socket() {
43 | console.log('socket started')
44 | io.on("connection", socket => {
45 | // either with send()
46 | socket.send("Welcome to Lighthouse!");
47 |
48 | // handle the event sent with socket.emit()
49 | socket.on("cid", async (cid) => {
50 | console.log("cid recieved:", cid);
51 |
52 | let storageInfo;
53 | try {
54 | console.log('storageInfo is', JSON.stringify(await this.storageAdapter.getStorageInfo(cid)));
55 | storageInfo = JSON.stringify(await this.storageAdapter.getStorageInfo(cid));
56 | } catch(e) {
57 | console.log('entered catch');
58 | storageInfo = { storageInfo: 'No Storage Deal found for this CID' };
59 | }
60 | // or with emit() and custom event names
61 | socket.emit("storageInfo", storageInfo);
62 | });
63 |
64 | // publish the storage status of cid onto the smart contract
65 | socket.on("publishStatus", async (cid) => {
66 | console.log("cid recieved:", cid);
67 |
68 | let storageInfo;
69 | try {
70 | console.log('storageInfo is', JSON.stringify(await this.storageAdapter.getStorageInfo(cid)));
71 | storageInfo = JSON.stringify(await this.storageAdapter.getStorageInfo(cid));
72 | } catch(e) {
73 | console.log('entered catch');
74 | storageInfo = { storageInfo: 'No Storage Deal found for this CID' };
75 | }
76 | // or with emit() and custom event names
77 | socket.emit("storageInfo", storageInfo);
78 | });
79 |
80 | socket.on("Start", async (data) => {
81 | var Name = data['Name'];
82 | var Path = data['Path'];
83 | Files[Name] = { // create a new entry in Files variable
84 | FileSize: data['Size'],
85 | Data: "",
86 | Downloaded: 0
87 | }
88 | await fs.mkdir(Path, (err) => {
89 | if (err) {
90 | return console.error(err);
91 | }
92 | console.log('Directory created: ', Path);
93 | });
94 | var Place = 0;
95 | try {
96 | var Stat = fs.statSync(Path + '/' + Name);
97 | if (Stat.isFile()) {
98 | Files[Name]['Downloaded'] = Stat.size;
99 | Place = Stat.size / 54288;
100 | }
101 | } catch (error) {
102 | console.log('It is a new file');
103 | }
104 | fs.open(Path + "/" + Name, "a", '0755', function (err, fd) {
105 | if (err) {
106 | console.log('file open error', err);
107 | } else {
108 | Files[Name]['Handler'] = fd; // we store file handler so we can write to it later
109 | socket.emit('MoreData', { 'Place': Place, Percent: 0 });
110 | }
111 | })
112 | });
113 |
114 | socket.on('Upload', async (data) => {
115 | console.log('entered Upload');
116 | var Name = data['Name'];
117 | var Path = data['Path'];
118 | Files[Name]['Downloaded'] += data['Data'].length;
119 | Files[Name]['Data'] += data['Data'];
120 | if(Files[Name]['Downloaded'] == Files[Name]['FileSize']) //If File is Fully Uploaded
121 | {
122 | await fs.write(Files[Name]['Handler'], Files[Name]['Data'], null, 'Binary', async (err, Writen) => {
123 | //Get Thumbnail Here
124 | console.log('File downloaded fully !!', Name);
125 | socket.emit('FileDownloaded', 'Yes');
126 |
127 | try {
128 | let path = Path + '/' + Name;
129 | // let cidObject: any = await this.storageAdapter.stageFile(path);
130 | // console.log('cid is:', cidObject);
131 | socket.emit('FileInfo', { name: Name, size: Files[Name]['FileSize']});
132 | // fs.unlink(path, (err) => {
133 | // if (err) throw err;
134 | // console.log(path + ' was deleted')
135 | // });
136 | } catch (e) {
137 | console.log('stageFile error:', e);
138 | }
139 | });
140 | }
141 | else if(Files[Name]['Data'].length > 10485760){ //If the Data Buffer reaches 10MB
142 | await fs.write(Files[Name]['Handler'], Files[Name]['Data'], null, 'Binary', function(err, Writen){
143 | Files[Name]['Data'] = ""; //Reset The Buffer
144 | var Place = Files[Name]['Downloaded'] / 524288;
145 | var Percent = (Files[Name]['Downloaded'] / Files[Name]['FileSize']) * 100;
146 | socket.emit('MoreData', { 'Place' : Place, 'Percent' : Percent});
147 | });
148 | }
149 | else
150 | {
151 | var Place = Files[Name]['Downloaded'] / 524288;
152 | var Percent = (Files[Name]['Downloaded'] / Files[Name]['FileSize']) * 100;
153 | socket.emit('MoreData', { 'Place' : Place, 'Percent' : Percent});
154 | }
155 | });
156 |
157 | socket.on("GetCid", async (path) => {
158 | console.log('GetCid for folder:', path);
159 | let cid: any = await this.storageAdapter.stageFolder(path);
160 | console.log('cid is:', cid);
161 | const ipfs = await ipfsClient.create({ host: 'localhost', port: '5001', protocol: 'http' })
162 | console.log('ipfs instance created');
163 |
164 | //options specific to globSource
165 | const globSourceOptions = {
166 | recursive: true
167 | };
168 | console.log('path:', path);
169 | for await (const file of ipfs.addAll(globSource('./' + path, globSourceOptions))) {
170 | console.log(file)
171 | }
172 |
173 | rimraf(path, function () { console.log("deleted folder:", path); });
174 | socket.emit('FolderCid', {cid: cid});
175 | });
176 |
177 | socket.on("GetCidSize", async(cid) => {
178 | console.log("Size for cid:", cid);
179 | const ipfs = await ipfsClient.create({ host: 'localhost', port: '5001', protocol: 'http' })
180 | console.log('ipfsClient created');
181 | try {
182 | console.log('inside try block');
183 | let cidInfo = await ipfs.files.stat("/ipfs/" + cid);
184 | console.log('cidinfo:', cidInfo);
185 | socket.emit('CidSize', { size: cidInfo.cumulativeSize });
186 | } catch (e) {
187 | console.log("getCIDSize error:", e);
188 | socket.emit('CidSize', { size: "Error" });
189 | }
190 | });
191 |
192 | socket.on("retrieveFile", async (cid) => {
193 | console.log("cid recieved:", cid);
194 |
195 | let file;
196 | try {
197 | console.log('entered retrieveFile');
198 | // console.log('storageInfo is', JSON.stringify(await this.storageAdapter.getStorageInfo(cid)));
199 | file = (await this.storageAdapter.retrieveFile(cid)).buffer;
200 | } catch(e) {
201 | console.log('entered catch');
202 | file = 'error';
203 | }
204 | // or with emit() and custom event names
205 | socket.emit("retrieveFile", file);
206 | });
207 | });
208 | }
209 |
210 | httpServer(storageAdapter: StorageAdapter) {
211 | let app = express();
212 | app.get('/storageDealRecords', async function (req, res) {
213 | let records = await storageAdapter.storageDealRecords();
214 | res.status(200).send(records);
215 | })
216 |
217 | app.listen(3000, console.log('http server listening at port 3000'));
218 | }
219 | }
220 |
221 | export default Parser;
222 |
--------------------------------------------------------------------------------
/src/parsers/infura/contract.js:
--------------------------------------------------------------------------------
1 | "use strict";
2 | exports.__esModule = true;
3 | exports.contractAddress = exports.abi = void 0;
4 | exports.abi = [
5 | {
6 | anonymous: false,
7 | inputs: [
8 | {
9 | indexed: false,
10 | internalType: "address",
11 | name: "uploader",
12 | type: "address"
13 | },
14 | {
15 | indexed: false,
16 | internalType: "string",
17 | name: "cid",
18 | type: "string"
19 | },
20 | {
21 | indexed: false,
22 | internalType: "string",
23 | name: "config",
24 | type: "string"
25 | },
26 | {
27 | indexed: false,
28 | internalType: "enum FPS.storageStatus",
29 | name: "status",
30 | type: "uint8"
31 | },
32 | ],
33 | name: "CidStatusUpdate",
34 | type: "event"
35 | },
36 | {
37 | anonymous: false,
38 | inputs: [
39 | {
40 | indexed: false,
41 | internalType: "address",
42 | name: "uploader",
43 | type: "address"
44 | },
45 | {
46 | indexed: false,
47 | internalType: "string",
48 | name: "cid",
49 | type: "string"
50 | },
51 | {
52 | indexed: false,
53 | internalType: "string",
54 | name: "config",
55 | type: "string"
56 | },
57 | ],
58 | name: "StorageRequest",
59 | type: "event"
60 | },
61 | {
62 | inputs: [
63 | {
64 | internalType: "string",
65 | name: "cid",
66 | type: "string"
67 | },
68 | {
69 | internalType: "string",
70 | name: "config",
71 | type: "string"
72 | },
73 | ],
74 | name: "store",
75 | outputs: [],
76 | stateMutability: "nonpayable",
77 | type: "function"
78 | },
79 | {
80 | inputs: [
81 | {
82 | internalType: "string",
83 | name: "cid",
84 | type: "string"
85 | },
86 | {
87 | internalType: "enum FPS.storageStatus",
88 | name: "status",
89 | type: "uint8"
90 | },
91 | ],
92 | name: "updateStorageStatus",
93 | outputs: [],
94 | stateMutability: "nonpayable",
95 | type: "function"
96 | },
97 | ];
98 | exports.contractAddress = "0xbbeff2b19d8d4b2fcbed78bf4a7a51bc5d912d76";
99 |
--------------------------------------------------------------------------------
/src/parsers/infura/contract.ts:
--------------------------------------------------------------------------------
1 | export const abi = [
2 | {
3 | anonymous: false,
4 | inputs: [
5 | {
6 | indexed: false,
7 | internalType: "address",
8 | name: "uploader",
9 | type: "address",
10 | },
11 | {
12 | indexed: false,
13 | internalType: "string",
14 | name: "cid",
15 | type: "string",
16 | },
17 | {
18 | indexed: false,
19 | internalType: "string",
20 | name: "config",
21 | type: "string",
22 | },
23 | {
24 | indexed: false,
25 | internalType: "enum FPS.storageStatus",
26 | name: "status",
27 | type: "uint8",
28 | },
29 | ],
30 | name: "CidStatusUpdate",
31 | type: "event",
32 | },
33 | {
34 | anonymous: false,
35 | inputs: [
36 | {
37 | indexed: false,
38 | internalType: "address",
39 | name: "uploader",
40 | type: "address",
41 | },
42 | {
43 | indexed: false,
44 | internalType: "string",
45 | name: "cid",
46 | type: "string",
47 | },
48 | {
49 | indexed: false,
50 | internalType: "string",
51 | name: "config",
52 | type: "string",
53 | },
54 | ],
55 | name: "StorageRequest",
56 | type: "event",
57 | },
58 | {
59 | inputs: [
60 | {
61 | internalType: "string",
62 | name: "cid",
63 | type: "string",
64 | },
65 | {
66 | internalType: "string",
67 | name: "config",
68 | type: "string",
69 | },
70 | ],
71 | name: "store",
72 | outputs: [],
73 | stateMutability: "nonpayable",
74 | type: "function",
75 | },
76 | {
77 | inputs: [
78 | {
79 | internalType: "string",
80 | name: "cid",
81 | type: "string",
82 | },
83 | {
84 | internalType: "enum FPS.storageStatus",
85 | name: "status",
86 | type: "uint8",
87 | },
88 | ],
89 | name: "updateStorageStatus",
90 | outputs: [],
91 | stateMutability: "nonpayable",
92 | type: "function",
93 | },
94 | ];
95 |
96 | export const contractAddress = "0xbbeff2b19d8d4b2fcbed78bf4a7a51bc5d912d76";
97 |
--------------------------------------------------------------------------------
/src/parsers/infura/index.js:
--------------------------------------------------------------------------------
1 | "use strict";
2 | exports.__esModule = true;
3 | var Web3 = require("web3");
4 | var contract_1 = require("./contract");
5 | var Infura = /** @class */ (function () {
6 | function Infura(url, storageAdapter) {
7 | this.storageAdapter = storageAdapter;
8 | this.web3 = new Web3(new Web3.providers.WebsocketProvider(url));
9 | this.contract = new this.web3.eth.Contract(contract_1.abi, contract_1.contractAddress, {
10 | from: "0x073Ab1C0CAd3677cDe9BDb0cDEEDC2085c029579",
11 | gasPrice: "20000000000"
12 | });
13 | }
14 | Infura.prototype.start = function () {
15 | this.contract.events.StorageRequest(function (error, event) {
16 | if (error) {
17 | // TODO: Log error
18 | throw Error(error);
19 | }
20 | else {
21 | console.log(event.returnValues);
22 | }
23 | });
24 | };
25 | return Infura;
26 | }());
27 | exports["default"] = Infura;
28 |
--------------------------------------------------------------------------------
/src/parsers/infura/index.ts:
--------------------------------------------------------------------------------
1 | const Web3 = require("web3");
2 | import { abi, contractAddress } from "./contract";
3 | import StorageAdapter from "../../storage-adapter";
4 |
5 | class Infura {
6 | web3: Web3;
7 | contract: any;
8 | storageAdapter: StorageAdapter;
9 |
10 | constructor(url: string, storageAdapter: StorageAdapter) {
11 | this.storageAdapter = storageAdapter;
12 |
13 | this.web3 = new Web3(new Web3.providers.WebsocketProvider(url));
14 | this.contract = new this.web3.eth.Contract(
15 | abi,
16 | contractAddress,
17 | {
18 | from: "0x073Ab1C0CAd3677cDe9BDb0cDEEDC2085c029579",
19 | gasPrice: "20000000000",
20 | }
21 | );
22 | }
23 |
24 | start() {
25 | this.contract.events.StorageRequest((error, event) => {
26 | if (error) {
27 | // TODO: Log error
28 | throw Error(error);
29 | } else {
30 | console.log(event.returnValues);
31 | }
32 | });
33 | }
34 | }
35 |
36 | export default Infura;
37 |
--------------------------------------------------------------------------------
/src/parsers/vulcanize/index.js:
--------------------------------------------------------------------------------
1 | "use strict";
2 | var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3 | function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4 | return new (P || (P = Promise))(function (resolve, reject) {
5 | function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6 | function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7 | function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8 | step((generator = generator.apply(thisArg, _arguments || [])).next());
9 | });
10 | };
11 | var __generator = (this && this.__generator) || function (thisArg, body) {
12 | var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
13 | return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
14 | function verb(n) { return function (v) { return step([n, v]); }; }
15 | function step(op) {
16 | if (f) throw new TypeError("Generator is already executing.");
17 | while (_) try {
18 | if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
19 | if (y = 0, t) op = [op[0] & 2, t.value];
20 | switch (op[0]) {
21 | case 0: case 1: t = op; break;
22 | case 4: _.label++; return { value: op[1], done: false };
23 | case 5: _.label++; y = op[1]; op = [0]; continue;
24 | case 7: op = _.ops.pop(); _.trys.pop(); continue;
25 | default:
26 | if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
27 | if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
28 | if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
29 | if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
30 | if (t[2]) _.ops.pop();
31 | _.trys.pop(); continue;
32 | }
33 | op = body.call(thisArg, _);
34 | } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
35 | if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
36 | }
37 | };
38 | exports.__esModule = true;
39 | var kik = require('kikstart-graphql-client');
40 | var Web3 = require("web3");
41 | var cron = require('node-cron');
42 | var abiEventStorageRequest = {
43 | "anonymous": false,
44 | "inputs": [
45 | {
46 | "indexed": false,
47 | "name": "uploader",
48 | "type": "address"
49 | },
50 | {
51 | "indexed": false,
52 | "name": "cid",
53 | "type": "string"
54 | },
55 | {
56 | "indexed": false,
57 | "name": "config",
58 | "type": "string"
59 | },
60 | {
61 | "indexed": false,
62 | "name": "fileCost",
63 | "type": "uint256"
64 | }
65 | ],
66 | "name": "StorageRequest",
67 | "type": "event"
68 | };
69 | var abiEventStorageStatusRequest = {
70 | "anonymous": false,
71 | "inputs": [
72 | {
73 | "indexed": false,
74 | "name": "requester",
75 | "type": "address"
76 | },
77 | {
78 | "indexed": false,
79 | "name": "cid",
80 | "type": "string"
81 | }
82 | ],
83 | "name": "StorageStatusRequest",
84 | "type": "event"
85 | };
86 | var query = "subscription SubscriptionEvents {\n listen(topic: \"events\") {\n relatedNode {\n ... on ContractId1EventId1 {\n eventId\n mhKey\n dataUploader\n dataCid\n dataConfig\n id\n headerId\n }\n }\n }\n }";
87 | function modifyConfig(config) {
88 | config = config.replace(/ /g, '');
89 | config = config.replace(/([a-zA-Z0-9-]+):([a-zA-Z0-9-]+)/g, "\"$1\":\"$2\"");
90 | config = config.replace(/(\w+:)|(\w+ :)/g, function (matchedStr) {
91 | return '"' + matchedStr.substring(0, matchedStr.length - 1) + '":';
92 | });
93 | return config;
94 | }
95 | function publishStorageStatus(web3, PUBLIC_KEY, PRIVATE_KEY, lighthouseContract, contractAddress, cid, dealIds, active) {
96 | return __awaiter(this, void 0, void 0, function () {
97 | var nonce, gasEstimate, e_1, tx, signPromise;
98 | return __generator(this, function (_a) {
99 | switch (_a.label) {
100 | case 0: return [4 /*yield*/, web3.eth.getTransactionCount(PUBLIC_KEY, 'pending')];
101 | case 1:
102 | nonce = _a.sent();
103 | _a.label = 2;
104 | case 2:
105 | _a.trys.push([2, 4, , 5]);
106 | return [4 /*yield*/, lighthouseContract.methods.publishStorageStatus(cid, dealIds, active).estimateGas({ from: PUBLIC_KEY })];
107 | case 3:
108 | gasEstimate = _a.sent(); // estimate gas
109 | return [3 /*break*/, 5];
110 | case 4:
111 | e_1 = _a.sent();
112 | console.log('error:', e_1);
113 | return [3 /*break*/, 5];
114 | case 5:
115 | console.log('gasEstimate:', gasEstimate);
116 | tx = {
117 | 'from': PUBLIC_KEY,
118 | 'to': contractAddress,
119 | 'nonce': nonce,
120 | 'gas': gasEstimate,
121 | 'maxFeePerGas': 1000000108,
122 | 'data': lighthouseContract.methods.publishStorageStatus(cid, dealIds, active).encodeABI()
123 | };
124 | signPromise = web3.eth.accounts.signTransaction(tx, PRIVATE_KEY);
125 | signPromise.then(function (signedTx) {
126 | web3.eth.sendSignedTransaction(signedTx.rawTransaction, function (err, hash) {
127 | if (!err) {
128 | console.log("The hash of your transaction is: ", hash);
129 | }
130 | else {
131 | console.log("Something went wrong when submitting your transaction:", err);
132 | }
133 | });
134 | })["catch"](function (err) {
135 | console.log("Promise failed:", err);
136 | });
137 | return [2 /*return*/];
138 | }
139 | });
140 | });
141 | }
142 | var client = new kik.GraphQLClient({
143 | uri: 'https://lighthouse.vdb.to/graphql',
144 | wsUri: 'wss://lighthouse.vdb.to/graphql'
145 | });
146 | var Vulcanize = /** @class */ (function () {
147 | function Vulcanize(url, storageAdapter) {
148 | this.storageAdapter = storageAdapter;
149 | }
150 | // make call to vulcanize graphql here to get the event
151 | Vulcanize.prototype.start = function () {
152 | var _this = this;
153 | console.log('entered vulcanize');
154 | client.runSubscription(query)
155 | .subscribe({
156 | next: function (res) {
157 | console.log(JSON.stringify(res.data));
158 | console.log('cid:', res.data.listen.relatedNode.dataCid);
159 | console.log('config:', res.data.listen.relatedNode.dataConfig);
160 | console.log('address:', res.data.listen.relatedNode.dataUploader);
161 | var cid = res.data.listen.relatedNode.dataCid;
162 | var configString = res.data.listen.relatedNode.dataConfig;
163 | var config = JSON.parse(modifyConfig(configString));
164 | var address = '0x' + res.data.listen.relatedNode.dataUploader;
165 | var jobId = _this.storageAdapter.store(address, cid, config);
166 | console.log('cid:', cid); //, '& jobId:', jobId)
167 | },
168 | error: function (error) { return console.error(error); },
169 | complete: function () { return console.log('done'); }
170 | });
171 | };
172 | Vulcanize.prototype.listenEventStorageRequest = function (storageAdapter) {
173 | console.log('Listening to Event Storage Request');
174 | var unindexedEventsStorageRequest = abiEventStorageRequest.inputs.filter(function (event) { return event.indexed === false; });
175 | var unindexedEventsStorageStatusRequest = abiEventStorageStatusRequest.inputs.filter(function (event) { return event.indexed === false; });
176 | var web3 = new Web3(process.env.ALCHEMY_WSS);
177 | var web3Http = new Web3(process.env.ALCHEMY_HTTPS);
178 | console.log('Alchemy_wss:', process.env.ALCHEMY_WSS);
179 | web3.eth.subscribe("logs", { "address": process.env.LIGHTHOUSE_SMART_CONTRACT }, function (error, result) {
180 | return __awaiter(this, void 0, void 0, function () {
181 | var data, event_1, cid, configString, config, address, jobId, _a, PUBLIC_KEY, PRIVATE_KEY, event_2, cid, requester, storageInfo, dealProposals, dealList, active, i, contractAbi, contractAddress, lighthouseContract, _b;
182 | return __generator(this, function (_c) {
183 | switch (_c.label) {
184 | case 0:
185 | if (!!error) return [3 /*break*/, 9];
186 | console.log('result:', result);
187 | data = result.data;
188 | console.log('now going to decode the data:', result.data);
189 | _c.label = 1;
190 | case 1:
191 | _c.trys.push([1, 2, , 8]);
192 | event_1 = web3Http.eth.abi.decodeLog(unindexedEventsStorageRequest, data);
193 | console.log('event:', event_1);
194 | console.log('address:', event_1.uploader);
195 | console.log('cid:', event_1.cid);
196 | console.log('config:', event_1.config);
197 | console.log('fileCost:', event_1.fileCost);
198 | cid = event_1.cid;
199 | configString = event_1.config;
200 | config = JSON.parse(modifyConfig(configString));
201 | console.log('modifiedConfig:', config);
202 | address = event_1.uploader;
203 | jobId = storageAdapter.store(address, cid, config);
204 | console.log('cid:', cid); //, '& jobId:', jobId)
205 | return [3 /*break*/, 8];
206 | case 2:
207 | _a = _c.sent();
208 | PUBLIC_KEY = process.env.PUBLIC_KEY;
209 | PRIVATE_KEY = process.env.PRIVATE_KEY;
210 | event_2 = web3Http.eth.abi.decodeLog(unindexedEventsStorageStatusRequest, data);
211 | console.log('event:', event_2);
212 | console.log('requester:', event_2.requester);
213 | console.log('cid:', event_2.cid);
214 | cid = event_2.cid;
215 | requester = event_2.requester;
216 | return [4 /*yield*/, storageAdapter.getStorageInfo(cid)];
217 | case 3:
218 | storageInfo = _c.sent();
219 | console.log('storageInfo:', storageInfo); // need dealID and active status
220 | _c.label = 4;
221 | case 4:
222 | _c.trys.push([4, 6, , 7]);
223 | console.log('cold filecoin:', storageInfo.cidInfo.currentStorageInfo.cold.filecoin);
224 | dealProposals = storageInfo.cidInfo.currentStorageInfo.cold.filecoin.proposalsList;
225 | console.log('dealProposals:', dealProposals);
226 | dealList = [];
227 | active = void 0;
228 | for (i = 0; i < dealProposals.length; i++) {
229 | console.log('dealID:', dealProposals[i]["dealId"]);
230 | dealList.push(dealProposals[i]["dealId"]);
231 | active = true;
232 | }
233 | console.log('deal list:', dealList);
234 | contractAbi = require("./../../../contracts/abi/LighthouseV2.json");
235 | contractAddress = process.env.LIGHTHOUSE_SMART_CONTRACT;
236 | lighthouseContract = new web3.eth.Contract(contractAbi, contractAddress);
237 | return [4 /*yield*/, publishStorageStatus(web3Http, PUBLIC_KEY, PRIVATE_KEY, lighthouseContract, contractAddress, cid, dealList.toString(), active)];
238 | case 5:
239 | _c.sent();
240 | return [3 /*break*/, 7];
241 | case 6:
242 | _b = _c.sent();
243 | console.log('hot:', storageInfo.cidInfo.currentStorageInfo.hot);
244 | return [3 /*break*/, 7];
245 | case 7: return [3 /*break*/, 8];
246 | case 8: return [3 /*break*/, 10];
247 | case 9:
248 | console.log('Event Storage Request', error);
249 | _c.label = 10;
250 | case 10: return [2 /*return*/];
251 | }
252 | });
253 | });
254 | });
255 | };
256 | Vulcanize.prototype.cronJob = function (storageAdapter) {
257 | var _this = this;
258 | console.log('cron job scheduled');
259 | cron.schedule(process.env.CRON_EXPRESSION, function () { return __awaiter(_this, void 0, void 0, function () {
260 | var records, recordsList, storageInfoList, i, cid, dealId, active, deals, PUBLIC_KEY, PRIVATE_KEY, web3, web3Http, contractAbi, contractAddress, lighthouseContract, _i, _a, _b, cid, status_1, currentDealIds, e_2;
261 | return __generator(this, function (_c) {
262 | switch (_c.label) {
263 | case 0:
264 | console.log('cron expressions', process.env.CRON_EXPRESSION);
265 | return [4 /*yield*/, storageAdapter.storageDealRecords()];
266 | case 1:
267 | records = _c.sent();
268 | recordsList = records.recordsList;
269 | storageInfoList = {};
270 | for (i = 0; i < recordsList.length; i++) {
271 | cid = recordsList[i].rootCid;
272 | dealId = recordsList[i].dealInfo.dealId;
273 | active = !recordsList[i].pending;
274 | if (storageInfoList[cid]) { // multiple deals for same cid
275 | deals = storageInfoList[cid].dealList;
276 | deals.push(dealId);
277 | storageInfoList[cid] = { dealList: deals, active: true }; // @To-do : handle active
278 | }
279 | else { // first deal of cid
280 | storageInfoList[cid] = { dealList: [dealId], active: true };
281 | }
282 | }
283 | PUBLIC_KEY = process.env.PUBLIC_KEY;
284 | PRIVATE_KEY = process.env.PRIVATE_KEY;
285 | web3 = new Web3(process.env.ALCHEMY_WSS);
286 | web3Http = new Web3(process.env.ALCHEMY_HTTPS);
287 | contractAbi = require("./../../../contracts/abi/LighthouseV2.json");
288 | contractAddress = process.env.LIGHTHOUSE_SMART_CONTRACT;
289 | lighthouseContract = new web3.eth.Contract(contractAbi, contractAddress);
290 | _i = 0, _a = Object.entries(storageInfoList);
291 | _c.label = 2;
292 | case 2:
293 | if (!(_i < _a.length)) return [3 /*break*/, 10];
294 | _b = _a[_i], cid = _b[0], status_1 = _b[1];
295 | _c.label = 3;
296 | case 3:
297 | _c.trys.push([3, 8, , 9]);
298 | return [4 /*yield*/, lighthouseContract.methods.statuses(cid).call()];
299 | case 4:
300 | currentDealIds = _c.sent();
301 | currentDealIds = currentDealIds.dealIds;
302 | if (!(currentDealIds !== status_1["dealList"].toString())) return [3 /*break*/, 6];
303 | console.log('need to update deal for cid:', cid);
304 | return [4 /*yield*/, publishStorageStatus(web3Http, PUBLIC_KEY, PRIVATE_KEY, lighthouseContract, contractAddress, cid, status_1["dealList"].toString(), status_1["active"])];
305 | case 5:
306 | _c.sent();
307 | return [3 /*break*/, 7];
308 | case 6:
309 | console.log('no update to this cid:', cid);
310 | _c.label = 7;
311 | case 7: return [3 /*break*/, 9];
312 | case 8:
313 | e_2 = _c.sent();
314 | console.log('error:', e_2);
315 | return [3 /*break*/, 9];
316 | case 9:
317 | _i++;
318 | return [3 /*break*/, 2];
319 | case 10: return [2 /*return*/];
320 | }
321 | });
322 | }); });
323 | };
324 | return Vulcanize;
325 | }());
326 | exports["default"] = Vulcanize;
327 |
--------------------------------------------------------------------------------
/src/parsers/vulcanize/index.ts:
--------------------------------------------------------------------------------
1 | import StorageAdapter from "../../storage-adapter";
2 |
3 | var kik = require('kikstart-graphql-client');
4 | var Web3 = require("web3")
5 | var cron = require('node-cron');
6 |
7 | const abiEventStorageRequest = {
8 | "anonymous": false,
9 | "inputs": [
10 | {
11 | "indexed": false,
12 | "name": "uploader",
13 | "type": "address"
14 | },
15 | {
16 | "indexed": false,
17 | "name": "cid",
18 | "type": "string"
19 | },
20 | {
21 | "indexed": false,
22 | "name": "config",
23 | "type": "string"
24 | },
25 | {
26 | "indexed": false,
27 | "name": "fileCost",
28 | "type": "uint256"
29 | }
30 | ],
31 | "name": "StorageRequest",
32 | "type": "event"
33 | };
34 |
35 | const abiEventStorageStatusRequest = {
36 | "anonymous": false,
37 | "inputs": [
38 | {
39 | "indexed": false,
40 | "name": "requester",
41 | "type": "address"
42 | },
43 | {
44 | "indexed": false,
45 | "name": "cid",
46 | "type": "string"
47 | }
48 | ],
49 | "name": "StorageStatusRequest",
50 | "type": "event"
51 | };
52 |
53 | const query = `subscription SubscriptionEvents {
54 | listen(topic: "events") {
55 | relatedNode {
56 | ... on ContractId1EventId1 {
57 | eventId
58 | mhKey
59 | dataUploader
60 | dataCid
61 | dataConfig
62 | id
63 | headerId
64 | }
65 | }
66 | }
67 | }`
68 |
69 | function modifyConfig(config) {
70 | config = config.replace(/ /g,'');
71 | config = config.replace(/([a-zA-Z0-9-]+):([a-zA-Z0-9-]+)/g, "\"$1\":\"$2\"");
72 | config = config.replace(/(\w+:)|(\w+ :)/g, function(matchedStr) {
73 | return '"' + matchedStr.substring(0, matchedStr.length - 1) + '":';
74 | });
75 | return config;
76 | }
77 |
78 | async function publishStorageStatus(web3, PUBLIC_KEY, PRIVATE_KEY, lighthouseContract, contractAddress, cid, dealIds, active) {
79 | const nonce = await web3.eth.getTransactionCount(PUBLIC_KEY, 'pending'); // get latest nonce
80 |
81 | let gasEstimate;
82 | try {
83 | gasEstimate = await lighthouseContract.methods.publishStorageStatus(cid, dealIds, active).estimateGas({from: PUBLIC_KEY}); // estimate gas
84 | } catch (e) {
85 | console.log('error:', e)
86 | }
87 | console.log('gasEstimate:', gasEstimate);
88 | // Create the transaction
89 | const tx = {
90 | 'from': PUBLIC_KEY,
91 | 'to': contractAddress,
92 | 'nonce': nonce,
93 | 'gas': gasEstimate,
94 | 'maxFeePerGas': 1000000108,
95 | 'data': lighthouseContract.methods.publishStorageStatus(cid, dealIds, active).encodeABI()
96 | };
97 |
98 | // Sign the transaction
99 | const signPromise = web3.eth.accounts.signTransaction(tx, PRIVATE_KEY);
100 | signPromise.then((signedTx) => {
101 | web3.eth.sendSignedTransaction(signedTx.rawTransaction, function(err, hash) {
102 | if (!err) {
103 | console.log("The hash of your transaction is: ", hash);
104 | } else {
105 | console.log("Something went wrong when submitting your transaction:", err)
106 | }
107 | });
108 | }).catch((err) => {
109 | console.log("Promise failed:", err);
110 | });
111 | }
112 |
113 | const client = new kik.GraphQLClient({
114 | uri: 'https://lighthouse.vdb.to/graphql',
115 | wsUri: 'wss://lighthouse.vdb.to/graphql',
116 | });
117 |
118 | class Vulcanize {
119 | storageAdapter: StorageAdapter;
120 |
121 | constructor(url: string, storageAdapter: any) {
122 | this.storageAdapter = storageAdapter;
123 | }
124 |
125 | // make call to vulcanize graphql here to get the event
126 | start() {
127 | console.log('entered vulcanize');
128 | client.runSubscription(query)
129 | .subscribe({
130 | next: res => {
131 | console.log(JSON.stringify(res.data));
132 | console.log('cid:', res.data.listen.relatedNode.dataCid);
133 | console.log('config:', res.data.listen.relatedNode.dataConfig);
134 | console.log('address:', res.data.listen.relatedNode.dataUploader);
135 |
136 | let cid = res.data.listen.relatedNode.dataCid;
137 | let configString = res.data.listen.relatedNode.dataConfig;
138 | let config = JSON.parse(modifyConfig(configString));
139 | let address = '0x' + res.data.listen.relatedNode.dataUploader;
140 |
141 | const jobId = this.storageAdapter.store(address, cid, config);
142 | console.log('cid:', cid); //, '& jobId:', jobId)
143 | },
144 | error: error => console.error(error),
145 | complete: () => console.log('done'),
146 | })
147 | }
148 |
149 | listenEventStorageRequest(storageAdapter: StorageAdapter) {
150 | console.log('Listening to Event Storage Request')
151 | const unindexedEventsStorageRequest = abiEventStorageRequest.inputs.filter(event => event.indexed === false);
152 | const unindexedEventsStorageStatusRequest = abiEventStorageStatusRequest.inputs.filter(event => event.indexed === false);
153 |
154 | const web3 = new Web3(process.env.ALCHEMY_WSS);
155 | const web3Http = new Web3(process.env.ALCHEMY_HTTPS);
156 |
157 | console.log('Alchemy_wss:', process.env.ALCHEMY_WSS);
158 | web3.eth.subscribe("logs", {"address": process.env.LIGHTHOUSE_SMART_CONTRACT}, async function(error, result){
159 | if (!error) {
160 | console.log('result:', result);
161 | let data = result.data;
162 | console.log('now going to decode the data:', result.data);
163 |
164 | try { // storage request event
165 | let event = web3Http.eth.abi.decodeLog(unindexedEventsStorageRequest, data);
166 | console.log('event:', event);
167 | console.log('address:', event.uploader);
168 | console.log('cid:', event.cid);
169 | console.log('config:', event.config);
170 | console.log('fileCost:', event.fileCost);
171 |
172 | let cid = event.cid;
173 | let configString = event.config;
174 | let config = JSON.parse(modifyConfig(configString));
175 | console.log('modifiedConfig:', config);
176 | let address = event.uploader;
177 |
178 | const jobId = storageAdapter.store(address, cid, config);
179 | console.log('cid:', cid); //, '& jobId:', jobId)
180 | } catch { // storage status request event
181 | const PUBLIC_KEY = process.env.PUBLIC_KEY;
182 | const PRIVATE_KEY = process.env.PRIVATE_KEY;
183 |
184 | let event = web3Http.eth.abi.decodeLog(unindexedEventsStorageStatusRequest, data);
185 | console.log('event:', event);
186 | console.log('requester:', event.requester);
187 | console.log('cid:', event.cid);
188 |
189 | let cid = event.cid;
190 | let requester = event.requester;
191 |
192 | let storageInfo: any = await storageAdapter.getStorageInfo(cid);
193 | console.log('storageInfo:', storageInfo); // need dealID and active status
194 | try {
195 | console.log('cold filecoin:', storageInfo.cidInfo.currentStorageInfo.cold.filecoin);
196 | let dealProposals = storageInfo.cidInfo.currentStorageInfo.cold.filecoin.proposalsList;
197 | console.log('dealProposals:', dealProposals);
198 | let dealList = [];
199 | let active;
200 |
201 | for (let i = 0; i < dealProposals.length; i++) {
202 | console.log('dealID:', dealProposals[i]["dealId"]);
203 | dealList.push(dealProposals[i]["dealId"]);
204 | active = true;
205 | }
206 | console.log('deal list:', dealList);
207 | // console.log('hot:', storageInfo.cidInfo.currentStorageInfo.hot);
208 | const contractAbi = require("./../../../contracts/abi/LighthouseV2.json");
209 | const contractAddress = process.env.LIGHTHOUSE_SMART_CONTRACT;
210 | const lighthouseContract = new web3.eth.Contract(contractAbi, contractAddress);
211 |
212 | await publishStorageStatus(web3Http, PUBLIC_KEY, PRIVATE_KEY, lighthouseContract, contractAddress, cid, dealList.toString(), active)
213 | } catch { // no on-chain deal
214 | console.log('hot:', storageInfo.cidInfo.currentStorageInfo.hot);
215 | }
216 | }
217 | } else {
218 | console.log('Event Storage Request', error);
219 | }
220 | });
221 | }
222 |
223 | cronJob(storageAdapter: StorageAdapter) {
224 | console.log('cron job scheduled');
225 | cron.schedule(process.env.CRON_EXPRESSION, async () => {
226 | console.log('cron expressions', process.env.CRON_EXPRESSION);
227 | let records: any = await storageAdapter.storageDealRecords();
228 | let recordsList = records.recordsList;
229 |
230 | let storageInfoList:any = {};
231 |
232 | for (var i = 0; i < recordsList.length; i++) {
233 | let cid = recordsList[i].rootCid;
234 | let dealId = recordsList[i].dealInfo.dealId;
235 | let active = !recordsList[i].pending;
236 |
237 | if (storageInfoList[cid]) { // multiple deals for same cid
238 | let deals = storageInfoList[cid].dealList;
239 | deals.push(dealId);
240 | storageInfoList[cid] = { dealList: deals, active: true }; // @To-do : handle active
241 | } else { // first deal of cid
242 | storageInfoList[cid] = { dealList: [dealId], active: true };
243 | }
244 | }
245 |
246 | const PUBLIC_KEY = process.env.PUBLIC_KEY;
247 | const PRIVATE_KEY = process.env.PRIVATE_KEY;
248 |
249 | const web3 = new Web3(process.env.ALCHEMY_WSS);
250 | const web3Http = new Web3(process.env.ALCHEMY_HTTPS);
251 |
252 | const contractAbi = require("./../../../contracts/abi/LighthouseV2.json");
253 | const contractAddress = process.env.LIGHTHOUSE_SMART_CONTRACT;
254 | const lighthouseContract = new web3.eth.Contract(contractAbi, contractAddress);
255 |
256 | for (const [cid, status] of Object.entries(storageInfoList)) {
257 | try {
258 | let currentDealIds = await lighthouseContract.methods.statuses(cid).call();
259 | currentDealIds = currentDealIds.dealIds;
260 | if (currentDealIds !== status["dealList"].toString()) {
261 | console.log('need to update deal for cid:', cid);
262 | await publishStorageStatus(web3Http, PUBLIC_KEY, PRIVATE_KEY, lighthouseContract, contractAddress, cid, status["dealList"].toString(), status["active"])
263 | } else {
264 | console.log('no update to this cid:', cid);
265 | }
266 | } catch (e) {
267 | console.log('error:', e);
268 | }
269 | }
270 | });
271 | }
272 | }
273 |
274 | export default Vulcanize;
275 |
--------------------------------------------------------------------------------
/src/storage-adapter/index.js:
--------------------------------------------------------------------------------
1 | "use strict";
2 | exports.__esModule = true;
3 | var storage_providers_1 = require("./storage-providers");
4 | var StorageAdapter = /** @class */ (function () {
5 | /**
6 | *
7 | * @param config
8 | * {
9 | * "powergate": {config here}
10 | * }
11 | */
12 | function StorageAdapter(config) {
13 | this.config = config;
14 | // initialize the storage provider
15 | this.storageProvider = new storage_providers_1["default"](this.config.powergate);
16 | }
17 | StorageAdapter.prototype.store = function (address, cid, config) {
18 | if (!config) {
19 | config = this.config;
20 | }
21 | if (!address) {
22 | // TODO: Log error
23 | throw Error("An Ethereum address should be passed as an argument in the store function");
24 | }
25 | return this.storageProvider.store(address, cid, config);
26 | };
27 | StorageAdapter.prototype.getStorageInfo = function (cid) {
28 | console.log('cid in storage-adapter');
29 | return this.storageProvider.getStorageInfo(cid);
30 | };
31 | StorageAdapter.prototype.storageDealRecords = function () {
32 | return this.storageProvider.storageDealRecords();
33 | };
34 | StorageAdapter.prototype.stageFile = function (path) {
35 | console.log('path in storage-adapter');
36 | return this.storageProvider.stageFile(path);
37 | };
38 | StorageAdapter.prototype.stageFolder = function (path) {
39 | console.log('path in storage-adapter');
40 | return this.storageProvider.stageFolder(path);
41 | };
42 | StorageAdapter.prototype.retrieveFile = function (cid) {
43 | console.log('cid in storage-adapter');
44 | return this.storageProvider.retrieveFile(cid);
45 | };
46 | return StorageAdapter;
47 | }());
48 | exports["default"] = StorageAdapter;
49 |
--------------------------------------------------------------------------------
/src/storage-adapter/index.ts:
--------------------------------------------------------------------------------
1 | import StorageProvider from "./storage-providers";
2 |
3 | class StorageAdapter {
4 | config: any;
5 | storageProvider: StorageProvider;
6 | /**
7 | *
8 | * @param config
9 | * {
10 | * "powergate": {config here}
11 | * }
12 | */
13 |
14 | constructor(config: object) {
15 | this.config = config;
16 | // initialize the storage provider
17 | this.storageProvider = new StorageProvider(this.config.powergate);
18 | }
19 |
20 | store(address: string, cid: string, config?: object): Promise {
21 | if (!config) {
22 | config = this.config;
23 | }
24 | if (!address) {
25 | // TODO: Log error
26 | throw Error(
27 | "An Ethereum address should be passed as an argument in the store function"
28 | );
29 | }
30 | return this.storageProvider.store(address, cid, config);
31 | }
32 |
33 | getStorageInfo(cid: string): Promise