├── .eslintrc.json ├── .gitattributes ├── .github ├── .kodiak.toml ├── dependabot.yml └── workflows │ └── ci.yml ├── .gitignore ├── .husky ├── .gitignore └── pre-commit ├── .prettierignore ├── .prettierrc ├── CHANGELOG.md ├── LICENSE ├── README.md ├── index.js ├── index.test.js ├── integration ├── encryptedfile.test.js ├── file.test.js ├── index.js ├── skydb.test.js └── skydb_v2.test.js ├── jest.config.js ├── package.json ├── scripts ├── download.js ├── download_data.js ├── file.js ├── get_metadata.js ├── pin.js ├── skydb.js ├── skydb_v2.js ├── string_to_uint8_array.js ├── upload.js ├── upload_data.js └── upload_progress.js ├── src ├── client.js ├── defaults.js ├── download.js ├── download.test.js ├── pin.test.js ├── skydb.js ├── skydb.test.js ├── skydb_v2.js ├── skydb_v2.test.js ├── skylink_sia.js ├── upload.js ├── upload.test.js ├── utils.js ├── utils.test.js ├── utils_encoding.js ├── utils_registry.js ├── utils_string.js ├── utils_testing.js └── utils_validation.js ├── testdata ├── data.json ├── dir1 │ └── file3.txt ├── file1.txt └── file2.txt └── yarn.lock /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "parserOptions": { 3 | "ecmaVersion": 2020, 4 | "sourceType": "module" 5 | }, 6 | "env": { 7 | "es2020": true, 8 | "jest": true, 9 | "node": true 10 | }, 11 | "extends": "eslint:recommended" 12 | } 13 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | * text=auto eol=lf 2 | -------------------------------------------------------------------------------- /.github/.kodiak.toml: -------------------------------------------------------------------------------- 1 | # Auto-merge Dependabot PRs. Requires also `.github/workflows/dependabot-auto-merge.yml`. 2 | # See https://hackmd.io/@SkynetLabs/ryFfInNXc. 3 | 4 | version = 1 5 | 6 | [approve] 7 | # note: remove the "[bot]" suffix from GitHub Bot usernames. 8 | # Instead of "dependabot[bot]" use "dependabot". 9 | auto_approve_usernames = ["dependabot"] 10 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: npm 4 | directory: "/" 5 | schedule: 6 | interval: monthly 7 | open-pull-requests-limit: 10 8 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: "CI" 2 | 3 | on: 4 | push: 5 | branches: [master] 6 | pull_request: 7 | 8 | # For auto-approving/merging PRs. 9 | permissions: 10 | pull-requests: write 11 | contents: write 12 | 13 | jobs: 14 | check: 15 | name: "Checks and Tests" 16 | runs-on: ${{ matrix.os }} 17 | 18 | strategy: 19 | fail-fast: false 20 | matrix: 21 | os: [ubuntu-latest, macos-latest, windows-latest] 22 | node-version: [14.x, 16.x, 18.x] 23 | 24 | steps: 25 | - name: "Run Yarn Basic Checks" 26 | uses: SkynetLabs/.github/.github/actions/yarn-basic-checks@master 27 | with: 28 | node-version: ${{ matrix.node-version }} 29 | build: false 30 | env: 31 | # Use the pro server if an API key is found, otherwise use siasky.net. 32 | # This is for PRs run from forks, which do not have access to secrets. 33 | # 34 | # Ternary syntax. See https://github.com/actions/runner/issues/409#issuecomment-752775072. 35 | SKYNET_JS_INTEGRATION_TEST_SERVER: ${{ secrets.SKYNET_API_KEY && 'https://skynetpro.net' || 'https://siasky.net' }} 36 | SKYNET_JS_INTEGRATION_TEST_SKYNET_API_KEY: ${{ secrets.SKYNET_API_KEY }} 37 | 38 | # Auto-merge Dependabot PRs. Requires also `.github/.kodiak.toml`. 39 | # See https://hackmd.io/@SkynetLabs/ryFfInNXc. 40 | dependabot: 41 | name: "Approve and Merge Dependabot PRs" 42 | # - Must be a PR. 43 | # - The latest actor must be Dependabot. This prevents other users from 44 | # sneaking in changes into the PR. 45 | if: ${{ github.event_name == 'pull_request' && github.actor == 'dependabot[bot]' }} 46 | runs-on: ubuntu-latest 47 | 48 | steps: 49 | - name: "Run Auto-Approval and Auto-Merge" 50 | uses: SkynetLabs/.github/.github/actions/dependabot-approve-and-merge@master 51 | with: 52 | github-token: ${{ secrets.GITHUB_TOKEN }} 53 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | package-lock.json 3 | .DS_Store 4 | -------------------------------------------------------------------------------- /.husky/.gitignore: -------------------------------------------------------------------------------- 1 | _ 2 | -------------------------------------------------------------------------------- /.husky/pre-commit: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | . "$(dirname "$0")/_/husky.sh" 3 | 4 | yarn lint-staged 5 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | package.json -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "printWidth": 120 3 | } 4 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 6 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 7 | 8 | Types of changes: 9 | 10 | - `Added` for new features. 11 | - `Changed` for changes in existing functionality. 12 | - `Removed` for now removed features. 13 | - `Deprecated` for soon-to-be removed features. 14 | - `Fixed` for any bug fixes. 15 | - `Security` in case of vulnerabilities. 16 | 17 | ## Unreleased 18 | 19 | ## [2.8.0] 20 | 21 | ### Fixed 22 | 23 | - A bug has been fixed in `downloadFileHns` where it was creating a detached promise. 24 | 25 | ### Added 26 | 27 | - `disableDefaultPath` option for `uploadDirectory`. 28 | - `format` option for `downloadFile` and `downloadFileHns`. 29 | 30 | ## [2.7.0] 31 | 32 | ### Added 33 | 34 | - `stringToUint8ArrayUtf8` and `uint8ArrayToStringUtf8` conversion utilities 35 | (for working with registry entries). 36 | - Export more methods from `skynet-js`: 37 | - `downloadFileHns` 38 | - `getHnsUrl`, `getHnsresUrl` 39 | - `getFileContent`, `getFileContentBinary`, `getFileContentHns`, `getFileContentBinaryHns` 40 | - `resolveHns`, 41 | - `pinSkylink` 42 | - `file.getJSON` 43 | 44 | ## [2.6.0] 45 | 46 | ### Added 47 | 48 | - Add `uploadData` and `downloadData`. 49 | - Add SkyDB and SkyDB V2 (accessed with `client.db` and `client.dbV2`). By 50 | @parajbs in https://github.com/SkynetLabs/nodejs-skynet/pull/140 51 | 52 | ## [2.5.1] 53 | 54 | ### Fixed 55 | 56 | - Fix `skynetApiKey` not being passed for certain methods. 57 | 58 | ## [2.5.0] 59 | 60 | ### Added 61 | 62 | - Add `skynetApiKey` option for portal API keys. 63 | 64 | ## [2.4.1] 65 | 66 | ### Fixed 67 | 68 | - Fix Options not being marshaled from the `nodejs` client to the `skynet-js` client. 69 | - Fix an issue with `client.getEntryLink`. 70 | - Fix `skynet-js` client initialization in Node context (portal URL undefined). 71 | 72 | ## [2.4.0] 73 | 74 | ### Added 75 | 76 | - Added `client.getMetadata`. 77 | 78 | ## [2.3.1] 79 | 80 | ### Fixed 81 | 82 | - Fixed bug with paths containing `.` and `..` as inputs to `uploadDirectory`. 83 | 84 | ## [2.3.0] 85 | 86 | ### Added 87 | 88 | - Added `client.getSkylinkUrl`. 89 | - Added `client.file.getEntryData` and `client.file.getEntryLink`. 90 | - Added `client.db.setDataLink`. 91 | - Added `client.registry.getEntry`, `client.registry.getEntryUrl`, 92 | `client.registry.getEntryLink`, `client.registry.setEntry`, and 93 | `client.registry.postSignedEntry`. 94 | - Added `genKeyPairAndSeed`, `genKeyPairFromSeed`, `getEntryLink` function exports. 95 | 96 | ## [2.2.0] 97 | 98 | ### Added 99 | 100 | - Added `errorPages` and `tryFiles` options when uploading directories 101 | 102 | ### Fixed 103 | 104 | - Fixed custom client portal URL being ignored 105 | 106 | ## [2.1.0] 107 | 108 | ### Added 109 | 110 | - Added tus protocol to `uploadFile` for large, resumable file uploads. 111 | - Added ability to set custom cookies. 112 | 113 | ## [2.0.1] 114 | 115 | ## Fixed 116 | 117 | - Fixed length limits for request bodies. 118 | - Fixed upload errors due to missing headers. 119 | 120 | ### Changed 121 | 122 | - Remove leading slash in directory path before uploading an absolute path. 123 | 124 | ## [2.0.0] 125 | 126 | ### Added 127 | 128 | - `customDirname` upload option 129 | 130 | ### Changed 131 | 132 | - This SDK has been updated to match Browser JS and require a client. You will 133 | first need to create a client and then make all API calls from this client. 134 | - Connection options can now be passed to the client, in addition to individual 135 | API calls, to be applied to all API calls. 136 | - The `defaultPortalUrl` string has been renamed to `defaultSkynetPortalUrl` and 137 | `defaultPortalUrl` is now a function. 138 | 139 | ## [1.1.0] 140 | 141 | ### Added 142 | 143 | - Common Options object 144 | - API authentication 145 | - `dryRun` option 146 | 147 | ### Fixed 148 | 149 | - Some upload bugs were fixed. 150 | 151 | ## [1.0.0] 152 | 153 | ### Added 154 | 155 | - Upload and download functionality. 156 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Nebulous 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | **⛔️ Due to [Skynet Labs organisation shut down](https://skynetlabs.com/news/skynet-labs-full-shutdown), this repo has been archived and will no longer be maintained.** 2 | 3 | --- 4 | 5 | # Skynet NodeJS SDK 6 | 7 | [![No Maintenance Intended](http://unmaintained.tech/badge.svg)](http://unmaintained.tech/) 8 | [![Version](https://img.shields.io/github/package-json/v/SkynetLabs/nodejs-skynet)](https://www.npmjs.com/package/@skynetlabs/skynet-nodejs) 9 | [![Build Status](https://img.shields.io/github/workflow/status/SkynetLabs/nodejs-skynet/Pull%20Request)](https://github.com/SkynetLabs/nodejs-skynet/actions) 10 | [![Contributors](https://img.shields.io/github/contributors/SkynetLabs/nodejs-skynet)](https://github.com/SkynetLabs/nodejs-skynet/graphs/contributors) 11 | [![License](https://img.shields.io/github/license/SkynetLabs/nodejs-skynet)](https://github.com/SkynetLabs/nodejs-skynet) 12 | 13 | An SDK for integrating Skynet into Node.js applications. 14 | 15 | ## Installing 16 | 17 | Using `npm`: 18 | 19 | ```sh 20 | npm install @skynetlabs/skynet-nodejs 21 | ``` 22 | 23 | Using `yarn`: 24 | 25 | ```sh 26 | yarn add @skynetlabs/skynet-nodejs 27 | ``` 28 | 29 | ## Documentation 30 | 31 | For documentation complete with examples, please see [the Skynet SDK docs](https://siasky.net/docs/?javascript--node#introduction). 32 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const { SkynetClient } = require("./src/client"); 4 | 5 | const { 6 | defaultOptions, 7 | defaultSkynetPortalUrl, 8 | defaultPortalUrl, 9 | uriSkynetPrefix, 10 | formatSkylink, 11 | } = require("./src/utils"); 12 | 13 | const { 14 | genKeyPairAndSeed, 15 | genKeyPairFromSeed, 16 | getEntryLink, 17 | stringToUint8ArrayUtf8, 18 | uint8ArrayToStringUtf8, 19 | } = require("skynet-js"); 20 | 21 | module.exports = { 22 | SkynetClient, 23 | 24 | defaultOptions, 25 | defaultPortalUrl, 26 | defaultSkynetPortalUrl, 27 | uriSkynetPrefix, 28 | formatSkylink, 29 | 30 | // Re-export utilities from skynet-js. 31 | 32 | genKeyPairAndSeed, 33 | genKeyPairFromSeed, 34 | getEntryLink, 35 | stringToUint8ArrayUtf8, 36 | uint8ArrayToStringUtf8, 37 | }; 38 | -------------------------------------------------------------------------------- /index.test.js: -------------------------------------------------------------------------------- 1 | const { SkynetClient } = require("./index"); 2 | 3 | describe("SkynetClient", () => { 4 | it("should contain all api methods", () => { 5 | const client = new SkynetClient(); 6 | 7 | // Download 8 | expect(client).toHaveProperty("downloadData"); 9 | expect(client).toHaveProperty("downloadFile"); 10 | expect(client).toHaveProperty("downloadFileHns"); 11 | expect(client).toHaveProperty("getMetadata"); 12 | 13 | // Upload 14 | expect(client).toHaveProperty("uploadData"); 15 | expect(client).toHaveProperty("uploadFile"); 16 | expect(client).toHaveProperty("uploadDirectory"); 17 | }); 18 | }); 19 | -------------------------------------------------------------------------------- /integration/encryptedfile.test.js: -------------------------------------------------------------------------------- 1 | const { SkynetClient, defaultPortalUrl } = require("../index"); 2 | const portalUrl = defaultPortalUrl(); 3 | const client = new SkynetClient(portalUrl); 4 | 5 | describe(`Encrypted File API integration tests for portal '${portalUrl}'`, () => { 6 | const userID = "4dfb9ce035e4e44711c1bb0a0901ce3adc2a928b122ee7b45df6ac47548646b0"; 7 | // Path seed for "test.hns/encrypted". 8 | const pathSeed = "fe2c5148646532a442dd117efab3ff2a190336da506e363f80fb949513dab811"; 9 | 10 | it("Should get existing encrypted JSON", async () => { 11 | const expectedJson = { message: "foo" }; 12 | 13 | const { data } = await client.file.getJSONEncrypted(userID, pathSeed); 14 | 15 | expect(data).toEqual(expectedJson); 16 | }); 17 | 18 | it("Should return null for inexistant encrypted JSON", async () => { 19 | const pathSeed = "a".repeat(64); 20 | 21 | const { data } = await client.file.getJSONEncrypted(userID, pathSeed); 22 | 23 | expect(data).toBeNull(); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /integration/file.test.js: -------------------------------------------------------------------------------- 1 | const { SkynetClient, defaultPortalUrl, uriSkynetPrefix } = require("../index"); 2 | const portalUrl = defaultPortalUrl(); 3 | const client = new SkynetClient(portalUrl); 4 | 5 | describe(`File API integration tests for portal '${portalUrl}'`, () => { 6 | const userID = "89e5147864297b80f5ddf29711ba8c093e724213b0dcbefbc3860cc6d598cc35"; 7 | const path = "snew.hns/asdf"; 8 | 9 | it("Should get existing File API JSON data", async () => { 10 | const expected = { name: "testnames" }; 11 | 12 | const { data: received } = await client.file.getJSON(userID, path); 13 | expect(received).toEqual(expect.objectContaining(expected)); 14 | }); 15 | 16 | it("Should get existing File API entry data", async () => { 17 | const expected = new Uint8Array([ 18 | 65, 65, 67, 116, 77, 77, 114, 101, 122, 76, 56, 82, 71, 102, 105, 98, 104, 67, 53, 79, 98, 120, 48, 83, 102, 69, 19 | 106, 48, 77, 87, 108, 106, 95, 112, 55, 97, 95, 77, 107, 90, 85, 81, 45, 77, 57, 65, 20 | ]); 21 | 22 | const { data: received } = await client.file.getEntryData(userID, path); 23 | expect(received).toEqual(expected); 24 | }); 25 | 26 | it("Should get an existing entry link for a user ID and path", async () => { 27 | const expected = `${uriSkynetPrefix}AQAKDRJbfAOOp3Vk8L-cjuY2d34E8OrEOy_PTsD0xCkYOQ`; 28 | 29 | const entryLink = await client.file.getEntryLink(userID, path); 30 | expect(entryLink).toEqual(expected); 31 | }); 32 | }); 33 | -------------------------------------------------------------------------------- /integration/index.js: -------------------------------------------------------------------------------- 1 | const axios = require("axios"); 2 | const axiosRetry = require("axios-retry"); 3 | const { SkynetClient, DEFAULT_SKYNET_PORTAL_URL } = require("../index"); 4 | 5 | // retry if we're getting rate limited 6 | axiosRetry(axios, { 7 | retries: 10, 8 | retryDelay: axiosRetry.exponentialDelay, 9 | retryCondition: (e) => axiosRetry.isNetworkOrIdempotentRequestError(e) || e.response?.status === 429, 10 | }); 11 | 12 | // To test a specific server. 13 | // 14 | // Example: 15 | // 16 | // SKYNET_JS_INTEGRATION_TEST_SERVER=https://eu-fin-1.siasky.net yarn run jest integration 17 | const portal = process.env.SKYNET_JS_INTEGRATION_TEST_SERVER || DEFAULT_SKYNET_PORTAL_URL; 18 | // Allow setting a custom API key for e.g. authentication for running tests on paid portals. 19 | // 20 | // Example: 21 | // 22 | // SKYNET_JS_INTEGRATION_TEST_SKYNET_API_KEY=foo yarn run jest integration 23 | const skynetApiKey = process.env.SKYNET_JS_INTEGRATION_TEST_SKYNET_API_KEY; 24 | 25 | const customOptions = { skynetApiKey }; 26 | const client = new SkynetClient(portal, customOptions); 27 | 28 | module.exports = { 29 | client, 30 | portal, 31 | }; 32 | -------------------------------------------------------------------------------- /integration/skydb.test.js: -------------------------------------------------------------------------------- 1 | const { client, portal } = require("."); 2 | const { genKeyPairAndSeed, formatSkylink } = require("../index"); 3 | 4 | const dataKey = "testdatakey"; 5 | const data = { example: "This is some example JSON data." }; 6 | const { publicKey, privateKey } = genKeyPairAndSeed(); 7 | 8 | const skylink = "AAA_uTgxYiKcqpGMLNe2V52fLc3FivZBZStLVqMSeHnGtQ"; 9 | const dataLink = "sia://AAChv5I6FTTqd8_6mtIOgwd5AhxurcYY9OSc-FacWlurEw"; 10 | const rawEntryData = Uint8Array.from([ 11 | 0, 0, 161, 191, 146, 58, 21, 52, 234, 119, 207, 250, 154, 210, 14, 131, 7, 121, 2, 28, 110, 173, 198, 24, 244, 228, 12 | 156, 248, 86, 156, 90, 91, 171, 19, 13 | ]); 14 | const rawBytesData = 15 | "[123,34,95,100,97,116,97,34,58,123,34,101,120,97,109,112,108,101,34,58,34,84,104,105,115,32,105,115,32,115,111,109,101,32,101,120,97,109,112,108,101,32,74,83,79,78,32,100,97,116,97,32,50,46,34,125,44,34,95,118,34,58,50,125]"; 16 | 17 | describe(`SkyDB V1 end to end integration tests for portal '${portal}'`, () => { 18 | it("should be a dataLink set", async () => { 19 | await client.db.setDataLink(privateKey, dataKey, dataLink); 20 | }); 21 | 22 | it("should set entryData", async () => { 23 | const receivedData = await client.db.setEntryData(privateKey, dataKey, rawEntryData); 24 | 25 | await expect(receivedData["data"]).toEqual(rawEntryData); 26 | }); 27 | 28 | it("should get entryData", async () => { 29 | const receivedData = await client.db.getEntryData(publicKey, dataKey); 30 | 31 | await expect(receivedData["data"]).toEqual(rawEntryData); 32 | }); 33 | 34 | it("should get rawBytes", async () => { 35 | const receivedData = await client.db.getRawBytes(publicKey, dataKey); 36 | 37 | await expect(formatSkylink(receivedData["dataLink"])).toEqual(dataLink); 38 | await expect("[" + receivedData["data"] + "]").toEqual(rawBytesData); 39 | }); 40 | 41 | it("should set jsonData to skydb", async () => { 42 | const receivedData = await client.db.setJSON(privateKey, dataKey, data); 43 | 44 | await expect(formatSkylink(receivedData["dataLink"])).toEqual(`sia://${skylink}`); 45 | }); 46 | 47 | it("should get jsonData from skydb", async () => { 48 | const receivedData = await client.db.getJSON(publicKey, dataKey); 49 | 50 | await expect(receivedData.data).toEqual(data); 51 | }); 52 | 53 | it("should delete jsonData on skydb", async () => { 54 | await client.db.deleteJSON(privateKey, dataKey); 55 | const receivedData = await client.db.getJSON(publicKey, dataKey); 56 | 57 | await expect(receivedData.data).toEqual(null); 58 | await expect(receivedData["dataLink"]).toEqual(null); 59 | }); 60 | }); 61 | -------------------------------------------------------------------------------- /integration/skydb_v2.test.js: -------------------------------------------------------------------------------- 1 | const { client, portal } = require("."); 2 | const { genKeyPairAndSeed, formatSkylink } = require("../index"); 3 | 4 | const dataKey = "testdatakey"; 5 | const data = { example: "This is some example JSON data for SkyDB V2." }; 6 | 7 | const skylink = "AAB1QJWQV0y2ynDXnJvOt0uh-THq-pJj2_layW5fjPXhTQ"; 8 | const dataLink = "sia://AAChv5I6FTTqd8_6mtIOgwd5AhxurcYY9OSc-FacWlurEw"; 9 | const rawEntryData = Uint8Array.from([ 10 | 0, 0, 161, 191, 146, 58, 21, 52, 234, 119, 207, 250, 154, 210, 14, 131, 7, 121, 2, 28, 110, 173, 198, 24, 244, 228, 11 | 156, 248, 86, 156, 90, 91, 171, 19, 12 | ]); 13 | const rawBytesData = 14 | "[123,34,95,100,97,116,97,34,58,123,34,101,120,97,109,112,108,101,34,58,34,84,104,105,115,32,105,115,32,115,111,109,101,32,101,120,97,109,112,108,101,32,74,83,79,78,32,100,97,116,97,32,50,46,34,125,44,34,95,118,34,58,50,125]"; 15 | 16 | describe(`SkyDB V2 end to end integration tests for portal '${portal}'`, () => { 17 | it("should set and get jsonData to skydb", async () => { 18 | const { publicKey, privateKey } = genKeyPairAndSeed(); 19 | 20 | // Should get initial jsonData from skydb. 21 | let receivedData = await client.dbV2.getJSON(publicKey, dataKey); 22 | await expect(receivedData.data).toEqual(null); 23 | 24 | // Set jsonData. 25 | receivedData = await client.dbV2.setJSON(privateKey, dataKey, data); 26 | await expect(formatSkylink(receivedData["dataLink"])).toEqual(`sia://${skylink}`); 27 | 28 | // Should get new jsonData from skydb. 29 | receivedData = await client.dbV2.getJSON(publicKey, dataKey); 30 | await expect(receivedData.data).toEqual(data); 31 | }); 32 | 33 | it("should set and get entry data", async () => { 34 | const { publicKey, privateKey } = genKeyPairAndSeed(); 35 | 36 | await client.dbV2.setDataLink(privateKey, dataKey, dataLink); 37 | 38 | // Should set entryData. 39 | let receivedData = await client.dbV2.setEntryData(privateKey, dataKey, rawEntryData); 40 | await expect(receivedData["data"]).toEqual(rawEntryData); 41 | 42 | // Should get entryData. 43 | receivedData = await client.dbV2.getEntryData(publicKey, dataKey); 44 | await expect(receivedData["data"]).toEqual(rawEntryData); 45 | 46 | // Should get rawBytes. 47 | receivedData = await client.dbV2.getRawBytes(publicKey, dataKey); 48 | await expect(formatSkylink(receivedData["dataLink"])).toEqual(dataLink); 49 | await expect("[" + receivedData["data"] + "]").toEqual(rawBytesData); 50 | 51 | // Should delete jsonData on skydb. 52 | await client.dbV2.deleteJSON(privateKey, dataKey); 53 | receivedData = await client.dbV2.getJSON(publicKey, dataKey); 54 | 55 | await expect(receivedData.data).toEqual(null); 56 | await expect(receivedData.dataLink).toEqual(null); 57 | }); 58 | }); 59 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | /* 2 | * For a detailed explanation regarding each configuration property, visit: 3 | * https://jestjs.io/docs/en/configuration.html 4 | */ 5 | 6 | const config = { 7 | // From old package.json. 8 | testTimeout: 120000, 9 | // Automatically clear mock calls and instances between every test 10 | clearMocks: true, 11 | // An array of glob patterns indicating a set of files for which coverage information should be collected 12 | // collectCoverageFrom: ["**/*.ts"], 13 | // An object that configures minimum threshold enforcement for coverage results 14 | // coverageThreshold: { 15 | // global: { 16 | // branches: 98, 17 | // functions: 98, 18 | // lines: 98, 19 | // statements: 98, 20 | // }, 21 | // }, 22 | // The root directory that Jest should scan for tests and modules within 23 | roots: ["integration", "src"], 24 | 25 | // All imported modules in your tests should be mocked automatically 26 | // automock: false, 27 | 28 | // Stop running tests after `n` failures 29 | // bail: 0, 30 | 31 | // The directory where Jest should store its cached dependency information 32 | // cacheDirectory: "C:\\Users\\dgh\\AppData\\Local\\Temp\\jest", 33 | 34 | // Indicates whether the coverage information should be collected while executing the test 35 | // collectCoverage: false, 36 | 37 | // The directory where Jest should output its coverage files 38 | // coverageDirectory: "../coverage", 39 | 40 | // An array of regexp pattern strings used to skip coverage collection 41 | // coveragePathIgnorePatterns: [ 42 | // "\\\\node_modules\\\\" 43 | // ], 44 | 45 | // Indicates which provider should be used to instrument code for coverage 46 | // coverageProvider: "babel", 47 | 48 | // A list of reporter names that Jest uses when writing coverage reports 49 | // coverageReporters: [ 50 | // "json", 51 | // "text", 52 | // "lcov", 53 | // "clover" 54 | // ], 55 | 56 | // A path to a custom dependency extractor 57 | // dependencyExtractor: undefined, 58 | 59 | // Make calling deprecated APIs throw helpful error messages 60 | // errorOnDeprecated: false, 61 | 62 | // Force coverage collection from ignored files using an array of glob patterns 63 | // forceCoverageMatch: [], 64 | 65 | // A path to a module which exports an async function that is triggered once before all test suites 66 | // globalSetup: undefined, 67 | 68 | // A path to a module which exports an async function that is triggered once after all test suites 69 | // globalTeardown: undefined, 70 | 71 | // A set of global variables that need to be available in all test environments 72 | // globals: {}, 73 | 74 | // The maximum amount of workers used to run your tests. Can be specified as % or a number. E.g. maxWorkers: 10% will use 10% of your CPU amount + 1 as the maximum worker number. maxWorkers: 2 will use a maximum of 2 workers. 75 | // maxWorkers: "50%", 76 | 77 | // An array of directory names to be searched recursively up from the requiring module's location 78 | // moduleDirectories: [ 79 | // "node_modules" 80 | // ], 81 | 82 | // An array of file extensions your modules use 83 | // moduleFileExtensions: [ 84 | // "js", 85 | // "json", 86 | // "jsx", 87 | // "ts", 88 | // "tsx", 89 | // "node" 90 | // ], 91 | 92 | // A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module 93 | // moduleNameMapper: {}, 94 | 95 | // An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader 96 | // modulePathIgnorePatterns: [], 97 | 98 | // Activates notifications for test results 99 | // notify: false, 100 | 101 | // An enum that specifies notification mode. Requires { notify: true } 102 | // notifyMode: "failure-change", 103 | 104 | // A preset that is used as a base for Jest's configuration 105 | // preset: undefined, 106 | 107 | // Run tests from one or more projects 108 | // projects: undefined, 109 | 110 | // Use this configuration option to add custom reporters to Jest 111 | // reporters: undefined, 112 | 113 | // Automatically reset mock state between every test 114 | // resetMocks: false, 115 | 116 | // Reset the module registry before running each individual test 117 | // resetModules: false, 118 | 119 | // A path to a custom resolver 120 | // resolver: undefined, 121 | 122 | // Automatically restore mock state between every test 123 | // restoreMocks: false, 124 | 125 | // A list of paths to directories that Jest should use to search for files in 126 | // roots: [ 127 | // "" 128 | // ], 129 | 130 | // Allows you to use a custom runner instead of Jest's default test runner 131 | // runner: "jest-runner", 132 | 133 | // The paths to modules that run some code to configure or set up the testing environment before each test 134 | // setupFiles: [], 135 | 136 | // A list of paths to modules that run some code to configure or set up the testing framework before each test 137 | // setupFilesAfterEnv: [], 138 | 139 | // The number of seconds after which a test is considered as slow and reported as such in the results. 140 | // slowTestThreshold: 5, 141 | 142 | // A list of paths to snapshot serializer modules Jest should use for snapshot testing 143 | // snapshotSerializers: [], 144 | 145 | // The test environment that will be used for testing 146 | // testEnvironment: "jsdom", 147 | 148 | // Options that will be passed to the testEnvironment 149 | // testEnvironmentOptions: {}, 150 | 151 | // Adds a location field to test results 152 | // testLocationInResults: false, 153 | 154 | // The glob patterns Jest uses to detect test files 155 | // testMatch: [ 156 | // "**/__tests__/**/*.[jt]s?(x)", 157 | // "**/?(*.)+(spec|test).[tj]s?(x)" 158 | // ], 159 | 160 | // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped 161 | // testPathIgnorePatterns: [ 162 | // "\\\\node_modules\\\\" 163 | // ], 164 | 165 | // The regexp pattern or array of patterns that Jest uses to detect test files 166 | // testRegex: [], 167 | 168 | // This option allows the use of a custom results processor 169 | // testResultsProcessor: undefined, 170 | 171 | // This option allows use of a custom test runner 172 | // testRunner: "jasmine2", 173 | 174 | // This option sets the URL for the jsdom environment. It is reflected in properties such as location.href 175 | // testURL: "http://localhost", 176 | 177 | // Setting this value to "fake" allows the use of fake timers for functions such as "setTimeout" 178 | // timers: "real", 179 | 180 | // A map from regular expressions to paths to transformers 181 | // transform: undefined, 182 | 183 | // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation 184 | // transformIgnorePatterns: [ 185 | // "\\\\node_modules\\\\", 186 | // "\\.pnp\\.[^\\\\]+$" 187 | // ], 188 | 189 | // An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them 190 | // unmockedModulePathPatterns: undefined, 191 | 192 | // Indicates whether each individual test should be reported during the run 193 | // verbose: undefined, 194 | 195 | // An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode 196 | // watchPathIgnorePatterns: [], 197 | 198 | // Whether to use watchman for file crawling 199 | // watchman: true, 200 | }; 201 | 202 | module.exports = config; 203 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@skynetlabs/skynet-nodejs", 3 | "version": "2.9.0", 4 | "description": "Skynet SDK", 5 | "repository": "https://github.com/SkynetLabs/nodejs-skynet", 6 | "main": "index.js", 7 | "scripts": { 8 | "test": "jest", 9 | "format": "prettier --write .", 10 | "lint": "yarn lint:eslint", 11 | "lint:eslint": "eslint . --max-warnings 0", 12 | "prepare": "husky install" 13 | }, 14 | "lint-staged": { 15 | "*.{js,jsx,ts,tsx}": [ 16 | "eslint --max-warnings 0", 17 | "prettier --write" 18 | ], 19 | "*.{json,yml,md}": [ 20 | "prettier --write" 21 | ] 22 | }, 23 | "keywords": [ 24 | "Decentralised", 25 | "Cloud Storage", 26 | "Sia" 27 | ], 28 | "author": "Peter-Jan Brone", 29 | "publishConfig": { 30 | "access": "public" 31 | }, 32 | "contributors": [ 33 | { 34 | "name": "Kunal Kamble", 35 | "email": "kunal@kunalkamble.com", 36 | "url": "https://kunalkamble.com" 37 | } 38 | ], 39 | "license": "MIT", 40 | "dependencies": { 41 | "axios": "0.27.2", 42 | "form-data": "4.0.0", 43 | "mime": "^3.0.0", 44 | "skynet-js": "^4.3.0", 45 | "@skynetlabs/tus-js-client": "^3.0.0", 46 | "url-join": "^4.0.1" 47 | }, 48 | "devDependencies": { 49 | "axios-mock-adapter": "^1.20.0", 50 | "axios-retry": "^3.2.5", 51 | "cli-progress": "^3.11.2", 52 | "eslint": "^8.0.0", 53 | "husky": "^8.0.1", 54 | "jest": "^29.0.1", 55 | "lint-staged": "^13.0.0", 56 | "prettier": "^2.0.5", 57 | "tmp": "0.2.1" 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /scripts/download.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Demo script to test all download functions. 3 | * 4 | * Example usage: node scripts/download.js "sia://XABvi7JtJbQSMAcDwnUnmp2FKDPjg8_tTTFP4BwMSxVdEg" "sky-os" 5 | * 6 | * Example with default data: node scripts/download.js 7 | */ 8 | 9 | const fs = require("fs"); 10 | var dir = "./tmp/download/"; 11 | 12 | if (!fs.existsSync(dir)) { 13 | fs.mkdirSync(dir, { recursive: true }); 14 | } 15 | 16 | (async () => { 17 | const { SkynetClient, defaultSkynetPortalUrl } = require(".."); 18 | const portalUrl = defaultSkynetPortalUrl; 19 | const client = new SkynetClient(`${portalUrl}`); 20 | const defaultSkylink = "sia://XABvi7JtJbQSMAcDwnUnmp2FKDPjg8_tTTFP4BwMSxVdEg"; 21 | const defaultHnsLink = "sky-os"; 22 | const defaultDownloadPath = "./tmp/download/"; 23 | let usedSkylink; 24 | let usedHnsLink; 25 | 26 | if (process.argv[2] === null || process.argv[2] === undefined) { 27 | usedSkylink = defaultSkylink; 28 | console.log("\n\n\nusedSkylink = " + usedSkylink); 29 | usedHnsLink = defaultHnsLink; 30 | console.log("usedHnsLink = " + usedHnsLink); 31 | } else { 32 | usedSkylink = process.argv[2]; 33 | console.log("\n\n\nusedSkylink = " + usedSkylink); 34 | usedHnsLink = process.argv[3]; 35 | console.log("usedHnsLink = " + usedHnsLink); 36 | } 37 | 38 | //1. use downloadFile to get a file from skynet. 39 | async function downloadFile(path, skylink) { 40 | await client 41 | .downloadFile(path, skylink) 42 | .then(() => { 43 | console.log("\n\n\n1. use downloadFile to get a file from skynet."); 44 | }) 45 | .catch((err) => { 46 | console.log("\n1. Get Error: ", err); 47 | }); 48 | } 49 | 50 | //2. use downloadFileHns to get a hns file from skynet. 51 | async function downloadFileHns(path, hnsLink) { 52 | await client 53 | .downloadFileHns(path, hnsLink) 54 | .then((res) => { 55 | console.log("\n\n2. use downloadFileHns to get a hns file from skynet."); 56 | console.log("hnsLink: " + res); 57 | }) 58 | .catch((err) => { 59 | console.log("\n2. Get Error: ", err); 60 | }); 61 | } 62 | 63 | //3. use getSkylinkUrl to get the url from a skylink. 64 | async function getSkylinkUrl(skylink) { 65 | await client 66 | .getSkylinkUrl(skylink) 67 | .then((res) => { 68 | console.log("\n\n3. use getSkylinkUrl to get the url from a skylink."); 69 | console.log("skylinkUrl: " + res); 70 | }) 71 | .catch((err) => { 72 | console.log("\n3. Get Error: ", err); 73 | }); 74 | } 75 | 76 | //4. use getHnsUrl to get the hnsUrl from a hnsLink. 77 | async function getHnsUrl(hnsLink) { 78 | await client 79 | .getHnsUrl(hnsLink) 80 | .then((res) => { 81 | console.log("\n\n4. use getHnsUrl to get the hnsUrl from a hnsLink."); 82 | console.log("hnsUrl: " + res); 83 | }) 84 | .catch((err) => { 85 | console.log("\n4. Get Error: ", err); 86 | }); 87 | } 88 | 89 | //5. use getHnsresUrl to get the hnsresUrl from a hnsLink. 90 | async function getHnsresUrl(hnsLink) { 91 | await client 92 | .getHnsresUrl(hnsLink) 93 | .then((res) => { 94 | console.log("\n\n5. use getHnsresUrl to get the hnsresUrl from a hnsLink."); 95 | console.log("hnsresUrl: " + res); 96 | }) 97 | .catch((err) => { 98 | console.log("\n5. Get Error: ", err); 99 | }); 100 | } 101 | 102 | //6. use getMetadata to get the metadata from a skylink. 103 | async function getMetadata(skylink) { 104 | await client 105 | .getMetadata(skylink) 106 | .then((res) => { 107 | console.log("\n\n6. use getMetadata to get the metadata from a skylink."); 108 | console.log("Metadata: " + JSON.stringify(res)); 109 | }) 110 | .catch((err) => { 111 | console.log("\n6. Get Error: ", err); 112 | }); 113 | } 114 | 115 | //7. use getFileContent to get the fileContent from a skylink. 116 | async function getFileContent(entryLink) { 117 | await client 118 | .getFileContent(entryLink) 119 | .then((res) => { 120 | console.log("\n\n7. use getFileContent to get the fileContent from a skylink."); 121 | console.log("contentType: " + res.contentType); 122 | console.log("portalUrl: " + res.portalUrl); 123 | console.log("skylink: " + res.skylink); 124 | }) 125 | .catch((err) => { 126 | console.log("\n7. Get Error: ", err); 127 | }); 128 | } 129 | 130 | //8. use getFileContentBinary to get the fileContentBinary from a skylink. 131 | async function getFileContentBinary(skylink) { 132 | await client 133 | .getFileContentBinary(skylink) 134 | .then((res) => { 135 | console.log("\n\n8. use getFileContentBinary to get the fileContentBinary from a skylink."); 136 | console.log("contentType: " + res.contentType); 137 | console.log("portalUrl: " + res.portalUrl); 138 | console.log("skylink: " + res.skylink); 139 | }) 140 | .catch((err) => { 141 | console.log("\n8. Get Error: ", err); 142 | }); 143 | } 144 | 145 | //9. use getFileContentHns to get the hnsFileContent from a hnsLink. 146 | async function getFileContentHns(hnsLink) { 147 | await client 148 | .getFileContentHns(hnsLink) 149 | .then((res) => { 150 | console.log("\n\n9. use getFileContentHns to get the hns fileContent from a hnsLink."); 151 | console.log("contentType: " + res.contentType); 152 | console.log("portalUrl: " + res.portalUrl); 153 | console.log("skylink: " + res.skylink); 154 | }) 155 | .catch((err) => { 156 | console.log("\n9. Get Error: ", err); 157 | }); 158 | } 159 | 160 | //10. use getFileContentBinaryHns to get the hnsFileContentBinary from a hnsLink. 161 | async function getFileContentBinaryHns(hnsLink) { 162 | await client 163 | .getFileContentBinaryHns(hnsLink) 164 | .then((res) => { 165 | console.log("\n\n10. use getFileContentBinaryHns to get the hnsFileContentBinary from a hnsLink."); 166 | console.log("contentType: " + res.contentType); 167 | console.log("portalUrl: " + res.portalUrl); 168 | console.log("skylink: " + res.skylink); 169 | }) 170 | .catch((err) => { 171 | console.log("\n10. Get Error: ", err); 172 | }); 173 | } 174 | 175 | //11. use resolveHns to get from a hnsLink the data. 176 | async function resolveHns(hnsLink) { 177 | await client 178 | .resolveHns(hnsLink) 179 | .then((res) => { 180 | console.log("\n\n11. use resolveHns to get from a hnsLink the data."); 181 | console.log("data: " + JSON.stringify(res.data)); 182 | console.log("skylink: " + res.skylink); 183 | }) 184 | .catch((err) => { 185 | console.log("\n11. Get Error: ", err); 186 | }); 187 | } 188 | 189 | async function main() { 190 | await downloadFile(defaultDownloadPath + "sia.pdf", usedSkylink); 191 | await downloadFileHns(defaultDownloadPath + usedHnsLink + ".html", usedHnsLink); 192 | await getSkylinkUrl(usedSkylink); 193 | await getHnsUrl(usedHnsLink); 194 | await getHnsresUrl(usedHnsLink); 195 | await getMetadata(usedSkylink); 196 | await getFileContent(usedSkylink); 197 | await getFileContentBinary(usedSkylink); 198 | await getFileContentHns(usedHnsLink); 199 | await getFileContentBinaryHns(usedHnsLink); 200 | await resolveHns(usedHnsLink); 201 | } 202 | main(); 203 | })(); 204 | -------------------------------------------------------------------------------- /scripts/download_data.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Demo script that downloads data for all skylinks passed in as CLI arguments. 3 | * 4 | * Example usage: node scripts/download_data.js 5 | */ 6 | 7 | const process = require("process"); 8 | 9 | const { SkynetClient } = require(".."); 10 | 11 | const client = new SkynetClient(); 12 | 13 | const promises = process.argv 14 | // Ignore the first two arguments. 15 | .slice(2) 16 | .map(async (skylink) => await client.downloadData(skylink)); 17 | 18 | (async () => { 19 | const results = await Promise.allSettled(promises); 20 | results.forEach((result) => { 21 | if (result.status === "fulfilled") { 22 | console.log(result.value); 23 | } else { 24 | console.log(result.reason); 25 | } 26 | }); 27 | })(); 28 | -------------------------------------------------------------------------------- /scripts/file.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Demo script for test the funktion "file.getJSON" and "file.getJSONEncrypted". 3 | * 4 | * Doc for "file.getJSON" and "file.getJSONEncrypted" usage: node scripts/file.js userId pathSeed myskyJsonPath 5 | * 6 | * Example: node scripts/file.js "4dfb9ce035e4e44711c1bb0a0901ce3adc2a928b122ee7b45df6ac47548646b0" "fe2c5148646532a442dd117efab3ff2a190336da506e363f80fb949513dab811" "test.hns/encrypted" 7 | * 8 | * Example with default data: node scripts/file.js 9 | * 10 | */ 11 | 12 | (async () => { 13 | // Fill in default variables here: 14 | const defaultUserId = "4dfb9ce035e4e44711c1bb0a0901ce3adc2a928b122ee7b45df6ac47548646b0"; 15 | const defaultPathSeed = "fe2c5148646532a442dd117efab3ff2a190336da506e363f80fb949513dab811"; 16 | const defaultMyskyJsonPath = "test.hns/encrypted"; 17 | // end default variables 18 | 19 | const { SkynetClient, defaultSkynetPortalUrl } = require(".."); 20 | const portalUrl = defaultSkynetPortalUrl; 21 | const client = new SkynetClient(`${portalUrl}`); 22 | let usedUserId; 23 | let usedPathSeed; 24 | let usedMyskyJsonPath; 25 | 26 | if (process.argv[2] === null || process.argv[2] === undefined) { 27 | usedUserId = defaultUserId; 28 | console.log("\n\n\nusedUserId = " + usedUserId); 29 | usedPathSeed = defaultPathSeed; 30 | console.log("usedPathSeed = " + usedPathSeed); 31 | usedMyskyJsonPath = defaultMyskyJsonPath; 32 | console.log("usedMyskyJsonPath = " + usedMyskyJsonPath); 33 | } else { 34 | usedUserId = process.argv[2]; 35 | console.log("\n\n\nusedUserId = " + usedUserId); 36 | usedPathSeed = process.argv[3]; 37 | console.log("usedPathSeed = " + usedPathSeed); 38 | usedMyskyJsonPath = process.argv[4]; 39 | console.log("usedMyskyJsonPath = " + usedMyskyJsonPath); 40 | } 41 | 42 | // 1. use file.getJSON to get the data. 43 | async function filegetJSON(userId, path) { 44 | await client.file 45 | .getJSON(userId, path) 46 | .then((res) => { 47 | console.log("\n\n1. use file.getJSON to get the data."); 48 | console.log("data: " + JSON.stringify(res.data)); 49 | console.log("dataLink: " + res.dataLink); 50 | }) 51 | .catch((err) => { 52 | console.log("\n1. Get Error: ", err); 53 | }); 54 | } 55 | 56 | // 2. use file.getJSONEncrypted to get the encrypted data. 57 | async function getJSONEncrypted(userId, path) { 58 | await client.file 59 | .getJSONEncrypted(userId, path) 60 | .then((res) => { 61 | console.log("\n\n2. use file.getJSONEncrypted to get the encrypted data."); 62 | console.log("data: " + JSON.stringify(res.data)); 63 | }) 64 | .catch((err) => { 65 | console.log("\n2. Get Error: ", err); 66 | }); 67 | } 68 | 69 | async function main() { 70 | await filegetJSON(usedUserId, usedMyskyJsonPath); 71 | await getJSONEncrypted(usedUserId, usedPathSeed); 72 | } 73 | main(); 74 | })(); 75 | -------------------------------------------------------------------------------- /scripts/get_metadata.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Demo script that gets metadata for all skylinks passed in as CLI arguments. 3 | * 4 | * Example usage: node scripts/get_metadata.js 5 | */ 6 | 7 | const process = require("process"); 8 | 9 | const { SkynetClient } = require(".."); 10 | 11 | const client = new SkynetClient(); 12 | 13 | const promises = process.argv 14 | // Ignore the first two arguments. 15 | .slice(2) 16 | .map(async (skylink) => await client.getMetadata(skylink)); 17 | 18 | (async () => { 19 | const results = await Promise.allSettled(promises); 20 | results.forEach((result) => { 21 | if (result.status === "fulfilled") { 22 | console.log(result.value); 23 | } else { 24 | console.log(result.reason); 25 | } 26 | }); 27 | })(); 28 | -------------------------------------------------------------------------------- /scripts/pin.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Demo script for test funktion "pinSkylink". 3 | * 4 | * Example for "pinSkylink" usage: node scripts/pin.js "sia://XABvi7JtJbQSMAcDwnUnmp2FKDPjg8_tTTFP4BwMSxVdEg" 5 | * 6 | * Example with default data: node scripts/pin.js 7 | * 8 | */ 9 | 10 | (async () => { 11 | const { SkynetClient, defaultSkynetPortalUrl } = require(".."); 12 | 13 | const portalUrl = defaultSkynetPortalUrl; 14 | const client = new SkynetClient(`${portalUrl}`); 15 | const defaultSkylink = "sia://XABvi7JtJbQSMAcDwnUnmp2FKDPjg8_tTTFP4BwMSxVdEg"; 16 | let usedSkylink; 17 | 18 | if (process.argv[2] === null || process.argv[2] === undefined) { 19 | usedSkylink = defaultSkylink; 20 | console.log("\n\nusedSkylink = " + usedSkylink); 21 | } else { 22 | usedSkylink = process.argv[2]; 23 | console.log("usedSkylink = " + usedSkylink); 24 | } 25 | 26 | // 1. use pinSkylink to pin a skylink to a portal. 27 | async function pinSkylink(skylink) { 28 | await client 29 | .pinSkylink(skylink) 30 | .then((res) => { 31 | console.log("\n\n\n1. use pinSkylink to pin a skylink to a portal."); 32 | console.log("skylink: " + res.skylink); 33 | }) 34 | .catch((err) => { 35 | console.log("\n1. Get Error: ", err); 36 | }); 37 | } 38 | 39 | pinSkylink(usedSkylink); 40 | })(); 41 | -------------------------------------------------------------------------------- /scripts/skydb.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Demo script for test all funktions from SkyDB V1. 3 | * 4 | * Example for SkyDB V1 usage: node scripts/skydb.js dataKey "Seed Test Key" ./testdata/data.json 5 | * 6 | * Example with default data: node scripts/skydb.js dataKey "Seed Test Key" 7 | * 8 | */ 9 | 10 | const fs = require("fs"); 11 | const process = require("process"); 12 | 13 | (async () => { 14 | const { SkynetClient, defaultSkynetPortalUrl, genKeyPairFromSeed, formatSkylink } = require(".."); 15 | const portalUrl = defaultSkynetPortalUrl; 16 | const client = new SkynetClient(`${portalUrl}`); 17 | 18 | const dataKey = process.argv[2]; 19 | console.log("\n\ndataKey = " + dataKey); 20 | const seedKey = process.argv[3]; 21 | console.log("seedKey = " + seedKey); 22 | 23 | let data; 24 | const pathJsonData = process.argv[4]; 25 | if (process.argv[4] === undefined) { 26 | console.log("Default dataJson used."); 27 | data = { example: "This is some example JSON data" }; 28 | } else { 29 | console.log("pathJsonData from command line argument"); 30 | const rawJSONdata = fs.readFileSync(pathJsonData); 31 | data = JSON.parse(rawJSONdata); 32 | } 33 | console.log("data: " + JSON.stringify(data)); 34 | 35 | const { publicKey, privateKey } = genKeyPairFromSeed(seedKey); 36 | console.log("\n\npublicKey: " + publicKey); 37 | console.log("privateKey: " + privateKey + "\n"); 38 | 39 | const rawEntryData = Uint8Array.from([ 40 | 0, 0, 161, 191, 146, 58, 21, 52, 234, 119, 207, 250, 154, 210, 14, 131, 7, 121, 2, 28, 110, 173, 198, 24, 244, 228, 41 | 156, 248, 86, 156, 90, 91, 171, 19, 42 | ]); 43 | const dataLink = "sia://AAChv5I6FTTqd8_6mtIOgwd5AhxurcYY9OSc-FacWlurEw"; 44 | 45 | // 1. use db.setJSON to add new data 46 | await client.db 47 | .setJSON(privateKey, dataKey, data) 48 | .then((res) => { 49 | console.log("\n1. use db.setJSON to add new data."); 50 | console.log(`db.setJSON:`); 51 | console.log(`Saved dataLink: ${formatSkylink(res.dataLink)}`); 52 | console.log(`Saved Data: ${JSON.stringify(res.data)}`); 53 | }) 54 | .catch((err) => { 55 | console.log("\n1. Error: ", err); 56 | }); 57 | 58 | // 2. use db.getJSON to get the data. 59 | await client.db 60 | .getJSON(publicKey, dataKey) 61 | .then((data) => { 62 | console.log("\n2. use db.getJSON to get the data."); 63 | console.log(`db.getJSON:`); 64 | console.log("Retrieved dataLink: " + formatSkylink(data["dataLink"])); 65 | console.log("Retrieved Data: " + JSON.stringify(data["data"]) + "\n"); 66 | }) 67 | .catch((err) => { 68 | console.log("\n2. Get Error: ", JSON.stringify(err)); 69 | }); 70 | 71 | // 3. use db.deleteJSON to delete the dataKey from skydb. 72 | await client.db 73 | .deleteJSON(privateKey, dataKey) 74 | .then(() => { 75 | console.log("\n3. use db.deleteJSON to delete the dataKey from skydb."); 76 | }) 77 | .catch((err) => { 78 | console.log("\n3. Get Error: ", JSON.stringify(err)); 79 | }); 80 | 81 | // 4. use db.getJSON to check dataKey and data after deleteJSON. 82 | await client.db 83 | .getJSON(publicKey, dataKey) 84 | .then((data) => { 85 | console.log("\n4. use db.getJSON to check dataKey and data after deleteJSON."); 86 | console.log(`db.getJSON:`); 87 | console.log("Retrieved dataLink: " + data["dataLink"]); 88 | console.log("Retrieved Data: " + JSON.stringify(data["data"]) + "\n"); 89 | }) 90 | .catch((err) => { 91 | console.log("\n4. Get Error: ", JSON.stringify(err)); 92 | }); 93 | 94 | // 5. use db.setDataLink to set a new dataLink. 95 | await client.db 96 | .setDataLink(privateKey, dataKey, dataLink) 97 | .then(() => { 98 | console.log("\n5. use db.setDataLink to set a new dataLink."); 99 | }) 100 | .catch((err) => { 101 | console.log("\n5. Error: ", err); 102 | }); 103 | 104 | // 6. use db.getJSON to check dataKey and data after setDataLink. 105 | await client.db 106 | .getJSON(publicKey, dataKey) 107 | .then((data) => { 108 | console.log("\n6. use db.getJSON to check dataKey and data after setDataLink."); 109 | console.log(`db.getJSON:`); 110 | console.log("Retrieved dataLink: " + formatSkylink(data["dataLink"])); 111 | console.log("Retrieved Data: " + JSON.stringify(data["data"]) + "\n"); 112 | }) 113 | .catch((err) => { 114 | console.log("\n6. Get Error: ", JSON.stringify(err)); 115 | }); 116 | 117 | // 7. use db.setEntryData to set the EntryData. 118 | await client.db 119 | .setEntryData(privateKey, dataKey, rawEntryData) 120 | .then(() => { 121 | console.log("\n7. use db.setEntryData to set the EntryData."); 122 | }) 123 | .catch((err) => { 124 | console.log("\n7. Get Error: ", err); 125 | }); 126 | 127 | // 8. use db.getEntryData . 128 | await client.db 129 | .getEntryData(publicKey, dataKey) 130 | .then((data) => { 131 | console.log("\n8. use db.getEntryData to get the EntryData."); 132 | console.log(`db.getEntryData:`); 133 | console.log("Retrieved EntryData: " + data["data"] + "\n"); 134 | }) 135 | .catch((err) => { 136 | console.log("\n8. Get Error: ", JSON.stringify(err)); 137 | }); 138 | 139 | // 9. use db.getRawBytes to get dataLink and data. 140 | await client.db 141 | .getRawBytes(publicKey, dataKey) 142 | .then((data) => { 143 | console.log("\n9. use db.getRawBytes to get dataLink and data."); 144 | console.log(`db.getRawBytes:`); 145 | console.log("Retrieved dataLink: " + formatSkylink(data["dataLink"])); 146 | console.log("Retrieved data: " + data["data"] + "\n"); 147 | }) 148 | .catch((err) => { 149 | console.log("\n9. Get Error: ", JSON.stringify(err)); 150 | }); 151 | 152 | // 10. use db.deleteEntryData to delete the EntryData from skydb. 153 | await client.db 154 | .deleteEntryData(privateKey, dataKey) 155 | .then(() => { 156 | console.log("\n10. use db.deleteEntryData to delete the EntryData from skydb."); 157 | }) 158 | .catch((err) => { 159 | console.log("\n10. Get Error: ", JSON.stringify(err)); 160 | }); 161 | 162 | // 11. use db.getEntryData to check data after db.deleteEntryData. 163 | await client.db 164 | .getEntryData(publicKey, dataKey) 165 | .then((data) => { 166 | console.log("\n11. use db.getEntryData to check data after db.deleteEntryData."); 167 | console.log(`db.getEntryData:`); 168 | console.log("Retrieved EntryData: " + JSON.stringify(data) + "\n"); 169 | }) 170 | .catch((err) => { 171 | console.log("\n11. Get Error: ", JSON.stringify(err)); 172 | }); 173 | 174 | // 12. use db.getRawBytes to check dataKey and data after db.deleteEntryData. 175 | await client.db 176 | .getRawBytes(publicKey, dataKey) 177 | .then((data) => { 178 | console.log("\n12. use db.getRawBytes to check data after db.deleteEntryData."); 179 | console.log(`db.getRawBytes:`); 180 | console.log("Retrieved dataLink: " + data["dataLink"]); 181 | console.log("Retrieved data: " + data["data"] + "\n"); 182 | }) 183 | .catch((err) => { 184 | console.log("\n12. Get Error: ", JSON.stringify(err)); 185 | }); 186 | })(); 187 | -------------------------------------------------------------------------------- /scripts/skydb_v2.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Demo script for test all funktions from SkyDB V2. 3 | * 4 | * Example for SkyDB V2 usage: node scripts/skydb_v2.js dataKey "Seed Test Key" ./testdata/data.json 5 | * 6 | * Example with default data: node scripts/skydb_v2.js dataKey "Seed Test Key" 7 | * 8 | */ 9 | 10 | const fs = require("fs"); 11 | const process = require("process"); 12 | 13 | (async () => { 14 | const { SkynetClient, defaultSkynetPortalUrl, genKeyPairFromSeed, formatSkylink } = require(".."); 15 | const portalUrl = defaultSkynetPortalUrl; 16 | const client = new SkynetClient(`${portalUrl}`); 17 | 18 | const dataKey = process.argv[2]; 19 | console.log("\n\ndataKey = " + dataKey); 20 | const seedKey = process.argv[3]; 21 | console.log("seedKey = " + seedKey); 22 | 23 | let data; 24 | const pathJsonData = process.argv[4]; 25 | if (process.argv[4] === undefined) { 26 | console.log("Default dataJson used."); 27 | data = { example: "This is some example JSON data for SkyDB V2. " }; 28 | } else { 29 | console.log("pathJsonData from command line argument"); 30 | const rawJSONdata = fs.readFileSync(pathJsonData); 31 | data = JSON.parse(rawJSONdata); 32 | } 33 | console.log("data: " + JSON.stringify(data)); 34 | 35 | const { publicKey, privateKey } = genKeyPairFromSeed(seedKey); 36 | console.log("\n\npublicKey: " + publicKey); 37 | console.log("privateKey: " + privateKey + "\n"); 38 | 39 | const rawEntryData = Uint8Array.from([ 40 | 0, 0, 161, 191, 146, 58, 21, 52, 234, 119, 207, 250, 154, 210, 14, 131, 7, 121, 2, 28, 110, 173, 198, 24, 244, 228, 41 | 156, 248, 86, 156, 90, 91, 171, 19, 42 | ]); 43 | const dataLink = "sia://AAChv5I6FTTqd8_6mtIOgwd5AhxurcYY9OSc-FacWlurEw"; 44 | 45 | // 1. Always call dbV2.getJSON first, because dbV2.setJSON does not make a network request to get the latest revision number. 46 | await client.dbV2 47 | .getJSON(publicKey, dataKey) 48 | .then((data) => { 49 | console.log( 50 | "\n1. Always call dbV2.getJSON first, because dbV2.setJSON does not make a network request to get the latest revision number." 51 | ); 52 | console.log(`dbV2.getJSON:`); 53 | console.log("Retrieved dataLink: " + data["dataLink"]); 54 | console.log("Retrieved Data: " + JSON.stringify(data["data"]) + "\n"); 55 | }) 56 | .catch((err) => { 57 | console.log("\n1. Get Error: ", JSON.stringify(err)); 58 | }); 59 | 60 | // 2. "dbV2.setJSON" can then to set new Data. 61 | await client.dbV2 62 | .setJSON(privateKey, dataKey, data) 63 | .then((res) => { 64 | console.log('\n2. "dbV2.setJSON" can then to set new Data.'); 65 | console.log(`dbV2.setJSON:`); 66 | console.log(`Saved dataLink: ${formatSkylink(res.dataLink)}`); 67 | console.log(`Saved Data: ${JSON.stringify(res.data)}`); 68 | }) 69 | .catch((err) => { 70 | console.log("\n2. Error: ", err); 71 | }); 72 | 73 | // 3. "dbV2.getJSON" called again for current data. 74 | await client.dbV2 75 | .getJSON(publicKey, dataKey) 76 | .then((data) => { 77 | console.log('\n3. "dbV2.getJSON" called again for current data.'); 78 | console.log(`dbV2.getJSON:`); 79 | console.log("Retrieved dataLink: " + formatSkylink(data["dataLink"])); 80 | console.log("Retrieved Data: " + JSON.stringify(data["data"]) + "\n"); 81 | }) 82 | .catch((err) => { 83 | console.log("\n3. Get Error: ", JSON.stringify(err)); 84 | }); 85 | 86 | // 4. use dbV2.deleteJSON to delete the dataKey from skydb. 87 | await client.dbV2 88 | .deleteJSON(privateKey, dataKey) 89 | .then(() => { 90 | console.log("\n4. use dbV2.deleteJSON to delete the dataKey from skydb."); 91 | }) 92 | .catch((err) => { 93 | console.log("\n4. Get Error: ", JSON.stringify(err)); 94 | }); 95 | 96 | // 5. use dbV2.getJSON to check dataKey and data after deleteJSON. 97 | await client.dbV2 98 | .getJSON(publicKey, dataKey) 99 | .then((data) => { 100 | console.log("\n5. use dbV2.getJSON to check dataKey and data after deleteJSON."); 101 | console.log(`dbV2.getJSON:`); 102 | console.log("Retrieved dataLink: " + data["dataLink"]); 103 | console.log("Retrieved Data: " + JSON.stringify(data["data"]) + "\n"); 104 | }) 105 | .catch((err) => { 106 | console.log("\n5. Get Error: ", JSON.stringify(err)); 107 | }); 108 | 109 | // 6. use dbV2.setDataLink to set a new dataLink. 110 | await client.dbV2 111 | .setDataLink(privateKey, dataKey, dataLink) 112 | .then(() => { 113 | console.log("\n6. use dbV2.setDataLink to set a new dataLink."); 114 | }) 115 | .catch((err) => { 116 | console.log("\n6. Error: ", err); 117 | }); 118 | 119 | // 7. use dbV2.getJSON to check dataKey and data after setDataLink. 120 | await client.dbV2 121 | .getJSON(publicKey, dataKey) 122 | .then((data) => { 123 | console.log("\n7. use dbV2.getJSON to check dataKey and data after setDataLink."); 124 | console.log(`dbV2.getJSON:`); 125 | console.log("Retrieved dataLink: " + formatSkylink(data["dataLink"])); 126 | console.log("Retrieved Data: " + JSON.stringify(data["data"]) + "\n"); 127 | }) 128 | .catch((err) => { 129 | console.log("\n7. Get Error: ", JSON.stringify(err)); 130 | }); 131 | 132 | // 8. use dbV2.setEntryData to set the EntryData. 133 | await client.dbV2 134 | .setEntryData(privateKey, dataKey, rawEntryData) 135 | .then(() => { 136 | console.log("\n8. use dbV2.setEntryData to set the EntryData."); 137 | }) 138 | .catch((err) => { 139 | console.log("\n8. Get Error: ", err); 140 | }); 141 | 142 | // 9. use dbV2.getEntryData . 143 | await client.dbV2 144 | .getEntryData(publicKey, dataKey) 145 | .then((data) => { 146 | console.log("\n9. use dbV2.getEntryData to get the EntryData."); 147 | console.log(`dbV2.getEntryData:`); 148 | console.log("Retrieved EntryData: " + data["data"] + "\n"); 149 | }) 150 | .catch((err) => { 151 | console.log("\n9. Get Error: ", JSON.stringify(err)); 152 | }); 153 | 154 | // 10. use dbV2.getRawBytes to get dataLink and data. 155 | await client.dbV2 156 | .getRawBytes(publicKey, dataKey) 157 | .then((data) => { 158 | console.log("\n10. use dbV2.getRawBytes to get dataLink and data."); 159 | console.log(`dbV2.getRawBytes:`); 160 | console.log("Retrieved dataLink: " + formatSkylink(data["dataLink"])); 161 | console.log("Retrieved data: " + data["data"] + "\n"); 162 | }) 163 | .catch((err) => { 164 | console.log("\n10. Get Error: ", JSON.stringify(err)); 165 | }); 166 | 167 | // 11. use dbV2.deleteEntryData to delete the EntryData from skydbV2. 168 | await client.dbV2 169 | .deleteEntryData(privateKey, dataKey) 170 | .then(() => { 171 | console.log("\n11. use dbV2.deleteEntryData to delete the EntryData from skydbV2."); 172 | }) 173 | .catch((err) => { 174 | console.log("\n11. Get Error: ", JSON.stringify(err)); 175 | }); 176 | 177 | // 12. use dbV2.getEntryData to check data after dbV2.deleteEntryData. 178 | await client.dbV2 179 | .getEntryData(publicKey, dataKey) 180 | .then((data) => { 181 | console.log("\n12. use dbV2.getEntryData to check data after dbV2.deleteEntryData."); 182 | console.log(`dbV2.getEntryData:`); 183 | console.log("Retrieved EntryData: " + JSON.stringify(data) + "\n"); 184 | }) 185 | .catch((err) => { 186 | console.log("\n12. Get Error: ", JSON.stringify(err)); 187 | }); 188 | 189 | // 13. use dbV2.getRawBytes to check dataKey and data after dbV2.deleteEntryData. 190 | await client.dbV2 191 | .getRawBytes(publicKey, dataKey) 192 | .then((data) => { 193 | console.log("\n13. use dbV2.getRawBytes to check data after dbV2.deleteEntryData."); 194 | console.log(`dbV2.getRawBytes:`); 195 | console.log("Retrieved dataLink: " + data["dataLink"]); 196 | console.log("Retrieved data: " + data["data"] + "\n"); 197 | }) 198 | .catch((err) => { 199 | console.log("\n13. Get Error: ", JSON.stringify(err)); 200 | }); 201 | })(); 202 | -------------------------------------------------------------------------------- /scripts/string_to_uint8_array.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Demo script to convert strings to Uint8Arrays. Useful for testing or for constructing registry entries. 3 | * 4 | * Example usage: node scripts/string_to_uint8_array.js "bar" 5 | */ 6 | 7 | const { stringToUint8ArrayUtf8 } = require(".."); 8 | 9 | process.argv 10 | // Ignore the first two arguments. 11 | .slice(2) 12 | .map((str) => console.log(stringToUint8ArrayUtf8(str))); 13 | -------------------------------------------------------------------------------- /scripts/upload.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Demo script that uploads all paths passed in as CLI arguments. 3 | * 4 | * Example usage: node scripts/upload.js 5 | */ 6 | 7 | const fs = require("fs"); 8 | const process = require("process"); 9 | 10 | const { SkynetClient } = require(".."); 11 | 12 | const client = new SkynetClient(); 13 | 14 | const promises = process.argv 15 | // Ignore the first two arguments. 16 | .slice(2) 17 | // Use appropriate function for dir or for file. Note that this throws if the 18 | // path doesn't exist; we print an error later. 19 | .map((path) => 20 | fs.promises 21 | .lstat(path) 22 | .then((stat) => (stat.isDirectory() ? client.uploadDirectory(path) : client.uploadFile(path))) 23 | ); 24 | 25 | (async () => { 26 | const results = await Promise.allSettled(promises); 27 | results.forEach((result) => { 28 | if (result.status === "fulfilled") { 29 | console.log(result.value); 30 | } else { 31 | console.log(result.reason); 32 | } 33 | }); 34 | })(); 35 | -------------------------------------------------------------------------------- /scripts/upload_data.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Demo script that uploads data passed in as CLI arguments. 3 | * 4 | * Example usage: node scripts/upload_data.js 5 | */ 6 | 7 | const process = require("process"); 8 | 9 | const { SkynetClient } = require(".."); 10 | 11 | const client = new SkynetClient(); 12 | 13 | (async () => { 14 | const filename = process.argv[2]; 15 | const data = process.argv[3]; 16 | 17 | const skylink = await client.uploadData(data, filename); 18 | 19 | console.log(skylink); 20 | })(); 21 | -------------------------------------------------------------------------------- /scripts/upload_progress.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Demo script for uploading a file with a minimum size of 42mb with options onUploadProgress and numParallelUploads. 3 | * 4 | * Example usage: node scripts/upload_progress.js 5 | */ 6 | 7 | const fs = require("fs"); 8 | const process = require("process"); 9 | 10 | const { SkynetClient } = require(".."); 11 | const { onUploadProgress } = require("../src/utils_testing"); 12 | 13 | const client = new SkynetClient("", { numParallelUploads: 2, onUploadProgress }); 14 | 15 | const promises = process.argv 16 | // Ignore the first two arguments. 17 | .slice(2) 18 | // Use appropriate function for dir or for file. Note that this throws if the 19 | // path doesn't exist; we print an error later. 20 | .map((path) => 21 | fs.promises 22 | .lstat(path) 23 | .then((stat) => (stat.isDirectory() ? client.uploadDirectory(path) : client.uploadFile(path))) 24 | ); 25 | 26 | (async () => { 27 | const results = await Promise.allSettled(promises); 28 | results.forEach((result) => { 29 | if (result.status === "fulfilled") { 30 | console.log(result.value); 31 | } else { 32 | console.log(result.reason); 33 | } 34 | }); 35 | })(); 36 | -------------------------------------------------------------------------------- /src/client.js: -------------------------------------------------------------------------------- 1 | const axios = require("axios"); 2 | const { SkynetClient: BrowserSkynetClient } = require("skynet-js"); 3 | 4 | const { defaultPortalUrl, makeUrl } = require("./utils.js"); 5 | 6 | const { setJSONdbV1 } = require("./skydb.js"); 7 | const { setJSONdbV2 } = require("./skydb_v2.js"); 8 | 9 | class SkynetClient { 10 | /** 11 | * The Skynet Client which can be used to access Skynet. 12 | * @constructor 13 | * @param {string} [portalUrl="https://siasky.net"] - The portal URL to use to access Skynet, if specified. To use the default portal while passing custom options, use "". 14 | * @param {Object} [customOptions={}] - Configuration for the client. 15 | * @param {string} [customOptions.APIKey] - Authentication password to use. 16 | * @param {string} [customOptions.skynetApiKey] - Authentication API key to use for a Skynet portal (sets the "Skynet-Api-Key" header). 17 | * @param {string} [customCookie=""] - Custom cookie header to set. 18 | * @param {string} [customOptions.customUserAgent=""] - Custom user agent header to set. 19 | * @param {Function} [customOptions.onUploadProgress] - Optional callback to track progress. 20 | */ 21 | constructor(portalUrl, customOptions = {}) { 22 | // Check if portal URL provided twice. 23 | 24 | if (portalUrl && customOptions.portalUrl) { 25 | throw new Error( 26 | "Both 'portalUrl' parameter provided and 'customOptions.portalUrl' provided. Please pass only one in order to avoid conflicts." 27 | ); 28 | } 29 | 30 | // Add portal URL to options if given. 31 | 32 | this.customOptions = { ...customOptions }; 33 | // If portal was not given, the default portal URL will be used. 34 | if (portalUrl) { 35 | // Set the portalUrl if given. 36 | this.customOptions.portalUrl = portalUrl; 37 | } 38 | 39 | // Re-export selected client methods from skynet-js. 40 | 41 | // Create the browser client. It requires an explicit portal URL to be passed in Node contexts. We also have to pass valid custom options, so we remove any unexpected ones. 42 | const browserClientOptions = { ...this.customOptions }; 43 | delete browserClientOptions.portalUrl; 44 | const browserClient = new BrowserSkynetClient(portalUrl || defaultPortalUrl(), browserClientOptions); 45 | this.browserClient = browserClient; 46 | 47 | // Download 48 | this.getSkylinkUrl = browserClient.getSkylinkUrl.bind(browserClient); 49 | this.getHnsUrl = browserClient.getHnsUrl.bind(browserClient); 50 | this.getHnsresUrl = browserClient.getHnsresUrl.bind(browserClient); 51 | this.getMetadata = browserClient.getMetadata.bind(browserClient); 52 | this.getFileContent = browserClient.getFileContent.bind(browserClient); 53 | this.getFileContentBinary = browserClient.getFileContentBinary.bind(browserClient); 54 | this.getFileContentHns = browserClient.getFileContentHns.bind(browserClient); 55 | this.getFileContentBinaryHns = browserClient.getFileContentBinaryHns.bind(browserClient); 56 | this.resolveHns = browserClient.resolveHns.bind(browserClient); 57 | 58 | // Pin 59 | this.pinSkylink = browserClient.pinSkylink.bind(browserClient); 60 | 61 | // File API 62 | this.file = { 63 | getJSON: browserClient.file.getJSON.bind(browserClient), 64 | getEntryData: browserClient.file.getEntryData.bind(browserClient), 65 | getEntryLink: browserClient.file.getEntryLink.bind(browserClient), 66 | getJSONEncrypted: browserClient.file.getJSONEncrypted.bind(browserClient), 67 | }; 68 | 69 | // SkyDB v1 (deprecated) 70 | this.db = { 71 | getJSON: browserClient.db.getJSON.bind(browserClient), 72 | // We define `setJSONdbV1` in this SDK, so bind it to the current client. 73 | setJSON: setJSONdbV1.bind(this), 74 | deleteJSON: browserClient.db.deleteJSON.bind(browserClient), 75 | setDataLink: browserClient.db.setDataLink.bind(browserClient), 76 | getEntryData: browserClient.db.getEntryData.bind(browserClient), 77 | setEntryData: browserClient.db.setEntryData.bind(browserClient), 78 | deleteEntryData: browserClient.db.deleteEntryData.bind(browserClient), 79 | getRawBytes: browserClient.db.getRawBytes.bind(browserClient), 80 | }; 81 | 82 | // SkyDB v2 83 | this.dbV2 = { 84 | getJSON: browserClient.dbV2.getJSON.bind(browserClient), 85 | // We define `setJSONdbV1` in this SDK, so bind it to the current client. 86 | setJSON: setJSONdbV2.bind(this), 87 | deleteJSON: browserClient.dbV2.deleteJSON.bind(browserClient), 88 | setDataLink: browserClient.dbV2.setDataLink.bind(browserClient), 89 | getEntryData: browserClient.dbV2.getEntryData.bind(browserClient), 90 | setEntryData: browserClient.dbV2.setEntryData.bind(browserClient), 91 | deleteEntryData: browserClient.dbV2.deleteEntryData.bind(browserClient), 92 | getRawBytes: browserClient.dbV2.getRawBytes.bind(browserClient), 93 | 94 | // Holds the cached revision numbers, protected by mutexes to prevent 95 | // concurrent access. 96 | revisionNumberCache: browserClient.dbV2.revisionNumberCache, 97 | }; 98 | 99 | // Registry 100 | this.registry = { 101 | getEntry: browserClient.registry.getEntry.bind(browserClient), 102 | getEntryUrl: browserClient.registry.getEntryUrl.bind(browserClient), 103 | // Don't bind the client since this method doesn't take the client. 104 | getEntryLink: browserClient.registry.getEntryLink, 105 | setEntry: browserClient.registry.setEntry.bind(browserClient), 106 | postSignedEntry: browserClient.registry.postSignedEntry.bind(browserClient), 107 | }; 108 | } 109 | 110 | /** 111 | * Creates and executes a request. 112 | * @param {Object} config - Configuration for the request. See docs for constructor for the full list of options. 113 | */ 114 | executeRequest(config) { 115 | let url = config.url; 116 | if (!url) { 117 | url = makeUrl(config.portalUrl, config.endpointPath, config.extraPath ? config.extraPath : ""); 118 | } 119 | 120 | // Build headers. 121 | const headers = buildRequestHeaders( 122 | config.headers, 123 | config.customUserAgent, 124 | config.customCookie, 125 | config.skynetApiKey 126 | ); 127 | 128 | return axios({ 129 | url, 130 | method: config.method, 131 | data: config.data, 132 | params: config.params, 133 | headers, 134 | auth: config.APIKey && { username: "", password: config.APIKey }, 135 | responseType: config.responseType, 136 | maxContentLength: Infinity, 137 | maxBodyLength: Infinity, 138 | }); 139 | } 140 | } 141 | 142 | /** 143 | * Helper function that builds the request headers. 144 | * 145 | * @param [baseHeaders] - Any base headers. 146 | * @param [customUserAgent] - A custom user agent to set. 147 | * @param [customCookie] - A custom cookie. 148 | * @param [skynetApiKey] - Authentication key to use for a Skynet portal. 149 | * @returns - The built headers. 150 | */ 151 | function buildRequestHeaders(baseHeaders, customUserAgent, customCookie, skynetApiKey) { 152 | const returnHeaders = { ...baseHeaders }; 153 | // Set some headers from common options. 154 | if (customUserAgent) { 155 | returnHeaders["User-Agent"] = customUserAgent; 156 | } 157 | if (customCookie) { 158 | returnHeaders["Cookie"] = customCookie; 159 | } 160 | if (skynetApiKey) { 161 | returnHeaders["Skynet-Api-Key"] = skynetApiKey; 162 | } 163 | return returnHeaders; 164 | } 165 | 166 | // Export the client. 167 | 168 | module.exports = { SkynetClient, buildRequestHeaders }; 169 | 170 | // Get the following files to run or the client's methods won't be defined. 171 | require("./download.js"); 172 | require("./upload.js"); 173 | -------------------------------------------------------------------------------- /src/defaults.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const { defaultOptions } = require("./utils"); 4 | 5 | /** 6 | * The tus chunk size is (4MiB - encryptionOverhead) * dataPieces, set in skyd. 7 | */ 8 | const TUS_CHUNK_SIZE = (1 << 22) * 10; 9 | 10 | /** 11 | * Indicates what the default chunk size multiplier is. 12 | */ 13 | const DEFAULT_TUS_CHUNK_SIZE_MULTIPLIER = 3; 14 | 15 | /** 16 | * Indicates how many parts should be uploaded in parallel, by default. 17 | */ 18 | const DEFAULT_TUS_PARALLEL_UPLOADS = 2; 19 | 20 | /** 21 | * The retry delays, in ms. Data is stored in skyd for up to 20 minutes, so the 22 | * total delays should not exceed that length of time. 23 | */ 24 | const DEFAULT_TUS_RETRY_DELAYS = [0, 5000, 15000, 60000, 300000, 600000]; 25 | 26 | /** 27 | * Indicates the default stagger percent between chunk uploads. 28 | */ 29 | const DEFAULT_TUS_STAGGER_PERCENT = 50; 30 | 31 | /** 32 | * The portal file field name. 33 | */ 34 | const PORTAL_FILE_FIELD_NAME = "file"; 35 | /** 36 | * The portal directory file field name. 37 | */ 38 | const PORTAL_DIRECTORY_FILE_FIELD_NAME = "files[]"; 39 | 40 | const DEFAULT_BASE_OPTIONS = { 41 | APIKey: "", 42 | skynetApiKey: "", 43 | customUserAgent: "", 44 | customCookie: "", 45 | loginFn: undefined, 46 | }; 47 | 48 | const DEFAULT_DOWNLOAD_OPTIONS = { 49 | ...defaultOptions("/"), 50 | format: "", 51 | }; 52 | 53 | const DEFAULT_DOWNLOAD_HNS_OPTIONS = { 54 | ...DEFAULT_DOWNLOAD_OPTIONS, 55 | endpointDownloadHns: "hns", 56 | hnsSubdomain: "hns", 57 | }; 58 | 59 | const DEFAULT_GET_METADATA_OPTIONS = { 60 | ...defaultOptions("/"), 61 | }; 62 | 63 | const DEFAULT_UPLOAD_OPTIONS = { 64 | ...defaultOptions("/skynet/skyfile"), 65 | endpointLargeUpload: "/skynet/tus", 66 | 67 | portalFileFieldname: PORTAL_FILE_FIELD_NAME, 68 | portalDirectoryFileFieldname: PORTAL_DIRECTORY_FILE_FIELD_NAME, 69 | customFilename: "", 70 | customDirname: "", 71 | disableDefaultPath: false, 72 | dryRun: false, 73 | errorPages: undefined, 74 | tryFiles: undefined, 75 | 76 | // Large files. 77 | chunkSizeMultiplier: DEFAULT_TUS_CHUNK_SIZE_MULTIPLIER, 78 | largeFileSize: TUS_CHUNK_SIZE, 79 | numParallelUploads: DEFAULT_TUS_PARALLEL_UPLOADS, 80 | staggerPercent: DEFAULT_TUS_STAGGER_PERCENT, 81 | retryDelays: DEFAULT_TUS_RETRY_DELAYS, 82 | }; 83 | 84 | const DEFAULT_GET_ENTRY_OPTIONS = { 85 | ...DEFAULT_BASE_OPTIONS, 86 | endpointGetEntry: "/skynet/registry", 87 | }; 88 | 89 | const DEFAULT_SET_ENTRY_OPTIONS = { 90 | ...DEFAULT_BASE_OPTIONS, 91 | endpointSetEntry: "/skynet/registry", 92 | }; 93 | 94 | /** 95 | * The default options for get JSON. Includes the default get entry and download 96 | * options. 97 | */ 98 | const DEFAULT_GET_JSON_OPTIONS = { 99 | ...DEFAULT_BASE_OPTIONS, 100 | ...DEFAULT_GET_ENTRY_OPTIONS, 101 | ...DEFAULT_DOWNLOAD_OPTIONS, 102 | endpointPath: "/skynet/skyfile", 103 | cachedDataLink: undefined, 104 | }; 105 | 106 | /** 107 | * The default options for set JSON. Includes the default upload, get JSON, and 108 | * set entry options. 109 | */ 110 | const DEFAULT_SET_JSON_OPTIONS = { 111 | ...DEFAULT_BASE_OPTIONS, 112 | ...DEFAULT_UPLOAD_OPTIONS, 113 | ...DEFAULT_GET_JSON_OPTIONS, 114 | ...DEFAULT_SET_ENTRY_OPTIONS, 115 | endpointPath: "/skynet/skyfile", 116 | }; 117 | 118 | module.exports = { 119 | TUS_CHUNK_SIZE, 120 | DEFAULT_BASE_OPTIONS, 121 | DEFAULT_DOWNLOAD_OPTIONS, 122 | DEFAULT_DOWNLOAD_HNS_OPTIONS, 123 | DEFAULT_GET_METADATA_OPTIONS, 124 | DEFAULT_UPLOAD_OPTIONS, 125 | DEFAULT_GET_ENTRY_OPTIONS, 126 | DEFAULT_SET_ENTRY_OPTIONS, 127 | DEFAULT_GET_JSON_OPTIONS, 128 | DEFAULT_SET_JSON_OPTIONS, 129 | }; 130 | -------------------------------------------------------------------------------- /src/download.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-unused-vars */ 2 | 3 | "use strict"; 4 | 5 | const fs = require("fs"); 6 | 7 | const { SkynetClient } = require("./client"); 8 | const { DEFAULT_DOWNLOAD_OPTIONS, DEFAULT_DOWNLOAD_HNS_OPTIONS } = require("./defaults"); 9 | const { trimSiaPrefix } = require("./utils_string"); 10 | 11 | /** 12 | * Downloads in-memory data from a skylink. 13 | * 14 | * @param {string} skylink - The skylink. 15 | * @param {Object} [customOptions={}] - Configuration options. 16 | * @returns - The data. 17 | */ 18 | SkynetClient.prototype.downloadData = async function (skylink, customOptions = {}) { 19 | const opts = { ...DEFAULT_DOWNLOAD_OPTIONS, ...this.customOptions, ...customOptions }; 20 | 21 | skylink = trimSiaPrefix(skylink); 22 | 23 | const response = await this.executeRequest({ 24 | ...opts, 25 | method: "get", 26 | extraPath: skylink, 27 | responseType: "arraybuffer", 28 | }); 29 | return response.data; 30 | }; 31 | 32 | /** 33 | * Downloads a file from the given skylink. 34 | * 35 | * @param {string} path - The path to download the file to. 36 | * @param {Object} [customOptions] - Configuration options. 37 | * @param {Object} [customOptions.format] - The format (tar or zip) to download the file as. 38 | * @returns - The skylink. 39 | */ 40 | SkynetClient.prototype.downloadFile = function (path, skylink, customOptions = {}) { 41 | const opts = { ...DEFAULT_DOWNLOAD_OPTIONS, ...this.customOptions, ...customOptions }; 42 | 43 | skylink = trimSiaPrefix(skylink); 44 | 45 | const writer = fs.createWriteStream(path); 46 | 47 | const params = buildDownloadParams(opts.format); 48 | 49 | return new Promise((resolve, reject) => { 50 | this.executeRequest({ 51 | ...opts, 52 | method: "get", 53 | extraPath: skylink, 54 | responseType: "stream", 55 | params, 56 | }) 57 | .then((response) => { 58 | response.data.pipe(writer); 59 | writer.on("finish", resolve); 60 | writer.on("error", reject); 61 | }) 62 | .catch((error) => { 63 | reject(error); 64 | }); 65 | }); 66 | }; 67 | 68 | /** 69 | * Downloads a file from the given HNS domain. 70 | * 71 | * @param {string} path - The path to download the file to. 72 | * @param {Object} [customOptions] - Configuration options. 73 | * @param {Object} [customOptions.format] - The format (tar or zip) to download the file as. 74 | * @returns - The skylink. 75 | */ 76 | SkynetClient.prototype.downloadFileHns = async function (path, domain, customOptions = {}) { 77 | const opts = { ...DEFAULT_DOWNLOAD_HNS_OPTIONS, ...this.customOptions, ...customOptions }; 78 | 79 | const url = await this.getHnsUrl(domain); 80 | const params = buildDownloadParams(opts.format); 81 | 82 | const writer = fs.createWriteStream(path); 83 | 84 | await new Promise((resolve, reject) => { 85 | this.executeRequest({ 86 | ...opts, 87 | method: "get", 88 | url: url, 89 | responseType: "stream", 90 | params, 91 | }) 92 | .then((response) => { 93 | response.data.pipe(writer); 94 | writer.on("finish", resolve); 95 | writer.on("error", reject); 96 | }) 97 | .catch((error) => { 98 | reject(error); 99 | }); 100 | }); 101 | 102 | return url; 103 | }; 104 | 105 | function buildDownloadParams(format) { 106 | const params = {}; 107 | if (format) { 108 | params.format = format; 109 | } 110 | return params; 111 | } 112 | -------------------------------------------------------------------------------- /src/download.test.js: -------------------------------------------------------------------------------- 1 | const axios = require("axios"); 2 | const { PassThrough } = require("stream"); 3 | const tmp = require("tmp"); 4 | 5 | const { SkynetClient, defaultPortalUrl } = require("../index"); 6 | const { trimForwardSlash } = require("./utils_string"); 7 | const { extractNonSkylinkPath } = require("./utils_testing"); 8 | 9 | jest.mock("axios"); 10 | 11 | const attachment = "?attachment=true"; 12 | const portalUrl = defaultPortalUrl(); 13 | const hnsLink = "sky-os"; 14 | const skylink = "XABvi7JtJbQSMAcDwnUnmp2FKDPjg8_tTTFP4BwMSxVdEg"; 15 | const skylinkBase32 = "bg06v2tidkir84hg0s1s4t97jaeoaa1jse1svrad657u070c9calq4g"; 16 | const client = new SkynetClient(); 17 | const expectedHnsUrl = `https://${hnsLink}.hns.siasky.net/`; 18 | const expectedHnsUrlNoSubdomain = `${portalUrl}/hns/${hnsLink}`; 19 | const expectedHnsresUrl = `${portalUrl}/hnsres/${hnsLink}`; 20 | const sialink = `sia://${skylink}`; 21 | const entryLink = "AQDwh1jnoZas9LaLHC_D4-2yP9XYDdZzNtz62H4Dww1jDA"; 22 | 23 | const validSkylinkVariations = [`sia://${skylink}`, `https://siasky.net/${skylink}`, skylink]; 24 | const validHnsLinkVariations = [hnsLink, `hns:${hnsLink}`, `hns://${hnsLink}`]; 25 | 26 | describe("downloadFile", () => { 27 | const body = "asdf"; 28 | const mockResponse = `{ "data": ${body} }`; 29 | 30 | beforeEach(() => { 31 | const mockStream = new PassThrough(); 32 | mockStream.push(mockResponse); 33 | mockStream.end(); // Mark that we pushed all the data. 34 | 35 | axios.mockResolvedValue({ 36 | data: { 37 | body, 38 | pipe: function (writer) { 39 | mockStream.pipe(writer); 40 | }, 41 | }, 42 | }); 43 | }); 44 | 45 | it("should send get request to default portal", async () => { 46 | const tmpFile = tmp.fileSync(); 47 | 48 | await client.downloadFile(tmpFile.name, skylink); 49 | 50 | expect(axios).toHaveBeenCalledWith( 51 | expect.objectContaining({ 52 | url: `${portalUrl}/${skylink}`, 53 | method: "get", 54 | responseType: "stream", 55 | }) 56 | ); 57 | 58 | tmpFile.removeCallback(); 59 | }); 60 | 61 | it("should use custom options if defined on the API call", async () => { 62 | const tmpFile = tmp.fileSync(); 63 | 64 | await client.downloadFile(tmpFile.name, skylink, { 65 | portalUrl: "http://localhost", 66 | customCookie: "skynet-jwt=foo", 67 | }); 68 | 69 | expect(axios).toHaveBeenCalledWith( 70 | expect.objectContaining({ 71 | url: `http://localhost/${skylink}`, 72 | method: "get", 73 | responseType: "stream", 74 | headers: expect.objectContaining({ Cookie: "skynet-jwt=foo" }), 75 | }) 76 | ); 77 | 78 | tmpFile.removeCallback(); 79 | }); 80 | 81 | it("should use custom connection options if defined on the client", async () => { 82 | const tmpFile = tmp.fileSync(); 83 | const client = new SkynetClient("", { 84 | APIKey: "foobar", 85 | skynetApiKey: "api-key-1", 86 | customUserAgent: "Sia-Agent", 87 | customCookie: "skynet-jwt=foo", 88 | }); 89 | 90 | await client.downloadFile(tmpFile.name, skylink, { 91 | APIKey: "barfoo", 92 | skynetApiKey: "api-key-2", 93 | customUserAgent: "Sia-Agent-2", 94 | customCookie: "skynet-jwt=bar", 95 | }); 96 | 97 | expect(axios).toHaveBeenCalledWith( 98 | expect.objectContaining({ 99 | url: `${portalUrl}/${skylink}`, 100 | auth: { username: "", password: "barfoo" }, 101 | headers: expect.objectContaining({ 102 | "User-Agent": "Sia-Agent-2", 103 | Cookie: "skynet-jwt=bar", 104 | "Skynet-Api-Key": "api-key-2", 105 | }), 106 | }) 107 | ); 108 | 109 | tmpFile.removeCallback(); 110 | }); 111 | }); 112 | 113 | describe("downloadData", () => { 114 | const body = "asdf"; 115 | 116 | beforeEach(() => { 117 | axios.mockResolvedValue({ data: body }); 118 | }); 119 | 120 | it("should send get request to default portal", async () => { 121 | const data = await client.downloadData(skylink); 122 | 123 | expect(axios).toHaveBeenCalledWith( 124 | expect.objectContaining({ 125 | url: `${portalUrl}/${skylink}`, 126 | method: "get", 127 | responseType: "arraybuffer", 128 | }) 129 | ); 130 | 131 | expect(data).toEqual(body); 132 | }); 133 | }); 134 | 135 | describe("downloadFileHns", () => { 136 | const body = "asdf"; 137 | const mockResponse = `{ "data": ${body} }`; 138 | 139 | beforeEach(() => { 140 | const mockStream = new PassThrough(); 141 | mockStream.push(mockResponse); 142 | mockStream.end(); // Mark that we pushed all the data. 143 | 144 | axios.mockResolvedValue({ 145 | data: { 146 | body, 147 | pipe: function (writer) { 148 | mockStream.pipe(writer); 149 | }, 150 | }, 151 | }); 152 | }); 153 | 154 | it.each(validHnsLinkVariations)("should download with the correct link using hns link %s", async (input) => { 155 | const tmpFile = tmp.fileSync(); 156 | const url = await client.downloadFileHns(tmpFile.name, input, { download: true, subdomain: true }); 157 | 158 | expect(url).toEqual(`${expectedHnsUrl}`); 159 | await tmpFile.removeCallback(); 160 | }); 161 | 162 | it("should set format query parameter if passed as an option", async () => { 163 | const tmpFile = tmp.fileSync(); 164 | await client.downloadFileHns(tmpFile.name, hnsLink, { format: "zip" }); 165 | 166 | expect(axios).toHaveBeenCalledWith( 167 | expect.objectContaining({ 168 | url: expectedHnsUrl, 169 | method: "get", 170 | responseType: "stream", 171 | params: { format: "zip" }, 172 | }) 173 | ); 174 | 175 | await tmpFile.removeCallback(); 176 | }); 177 | }); 178 | 179 | describe("getSkylinkUrl", () => { 180 | const expectedUrl = `${portalUrl}/${skylink}`; 181 | 182 | it.each(validSkylinkVariations)( 183 | "should return correctly formed skylink URL using skylink %s", 184 | async (fullSkylink) => { 185 | const path = extractNonSkylinkPath(fullSkylink, skylink); 186 | let expectedPathUrl = expectedUrl; 187 | if (path !== "") { 188 | expectedPathUrl = `${expectedUrl}${path}`; 189 | } 190 | expect(await client.getSkylinkUrl(fullSkylink)).toEqual(expectedPathUrl); 191 | } 192 | ); 193 | 194 | it("should return correctly formed URLs when path is given", async () => { 195 | expect(await client.getSkylinkUrl(skylink, { path: "foo/bar" })).toEqual(`${expectedUrl}/foo/bar`); 196 | expect(await client.getSkylinkUrl(skylink, { path: "foo?bar" })).toEqual(`${expectedUrl}/foo%3Fbar`); 197 | }); 198 | 199 | it("should return correctly formed URL with forced download", async () => { 200 | const url = await client.getSkylinkUrl(skylink, { download: true, endpointDownload: "skynet/skylink" }); 201 | 202 | expect(url).toEqual(`${portalUrl}/skynet/skylink/${skylink}${attachment}`); 203 | }); 204 | 205 | it("should return correctly formed URLs with forced download and path", async () => { 206 | const url = await client.getSkylinkUrl(skylink, { download: true, path: "foo?bar" }); 207 | 208 | expect(url).toEqual(`${expectedUrl}/foo%3Fbar${attachment}`); 209 | }); 210 | 211 | const expectedBase32 = `https://${skylinkBase32}.siasky.net/`; 212 | 213 | it.each(validSkylinkVariations)("should convert base64 skylink to base32 using skylink %s", async (fullSkylink) => { 214 | let path = extractNonSkylinkPath(fullSkylink, skylink); 215 | path = trimForwardSlash(path); 216 | const url = await client.getSkylinkUrl(fullSkylink, { subdomain: true }); 217 | 218 | expect(url).toEqual(`${expectedBase32}${path}`); 219 | }); 220 | 221 | it("should throw if passing a non-string path", async () => { 222 | // @ts-expect-error we only check this use case in case someone ignores typescript typing 223 | await expect(client.getSkylinkUrl(skylink, { path: true })).rejects.toThrowError( 224 | "opts.path has to be a string, boolean provided" 225 | ); 226 | }); 227 | 228 | const invalidCases = ["123", `${skylink}xxx`, `${skylink}xxx/foo`, `${skylink}xxx?foo`]; 229 | 230 | it.each(invalidCases)("should throw on invalid skylink %s", async (invalidSkylink) => { 231 | await expect(client.getSkylinkUrl(invalidSkylink)).rejects.toThrow(); 232 | await expect(client.getSkylinkUrl(invalidSkylink, { subdomain: true })).rejects.toThrow(); 233 | }); 234 | }); 235 | 236 | describe("getHnsUrl", () => { 237 | it.each(validHnsLinkVariations)( 238 | "should return correctly formed non-subdomain hns URL using hns link %s", 239 | async (input) => { 240 | expect(await client.getHnsUrl(input)).toEqual(expectedHnsUrl); 241 | expect(await client.getHnsUrl(input, { subdomain: false })).toEqual(expectedHnsUrlNoSubdomain); 242 | } 243 | ); 244 | 245 | it("should return correctly formed hns URL with forced download", async () => { 246 | const url = await client.getHnsUrl(hnsLink, { download: true }); 247 | 248 | expect(url).toEqual(`${expectedHnsUrl}${attachment}`); 249 | }); 250 | }); 251 | 252 | describe("getHnsresUrl", () => { 253 | it.each(validHnsLinkVariations)("should return correctly formed hnsres URL using hnsres link %s", async (input) => { 254 | expect(await client.getHnsresUrl(input)).toEqual(expectedHnsresUrl); 255 | }); 256 | }); 257 | 258 | describe("getMetadata", () => { 259 | const headers = { 260 | "skynet-portal-api": portalUrl, 261 | "skynet-skylink": skylink, 262 | }; 263 | 264 | const skynetFileMetadata = { filename: "sia.pdf" }; 265 | 266 | it("should successfully fetch skynet file metadata from skylink", async () => { 267 | axios.mockResolvedValue({ status: 200, data: skynetFileMetadata, headers: headers }); 268 | const { metadata } = await client.getMetadata(skylink); 269 | 270 | expect(metadata).toEqual(skynetFileMetadata); 271 | }); 272 | 273 | it("should throw if a path is supplied", async () => { 274 | axios.mockResolvedValue({ status: 200, data: skynetFileMetadata }); 275 | 276 | await expect(client.getMetadata(`${skylink}/path/file`)).rejects.toThrowError( 277 | "Skylink string should not contain a path" 278 | ); 279 | }); 280 | 281 | it("should throw if no data was returned to getMetadata", async () => { 282 | axios.mockResolvedValue({ status: 200, headers: headers }); 283 | 284 | await expect(client.getMetadata(skylink)).rejects.toThrowError( 285 | "Metadata response invalid despite a successful request. Please try again and report this issue to the devs if it persists. Error: 'response.data' field missing" 286 | ); 287 | }); 288 | 289 | it("should throw if no headers were returned", async () => { 290 | axios.mockResolvedValue({ status: 200, data: {} }); 291 | 292 | await expect(client.getMetadata(skylink)).rejects.toThrowError( 293 | "Metadata response invalid despite a successful request. Please try again and report this issue to the devs if it persists. Error: 'response.headers' field missing" 294 | ); 295 | }); 296 | 297 | it("should throw if skynet-portal-api header is missing", async () => { 298 | const incompleteHeaders = { 299 | "skynet-portal-api": undefined, 300 | "skynet-skylink": skylink, 301 | }; 302 | axios.mockResolvedValue({ status: 200, data: {}, headers: incompleteHeaders }); 303 | 304 | await expect(client.getMetadata(skylink)).rejects.toThrowError( 305 | "Metadata response invalid despite a successful request. Please try again and report this issue to the devs if it persists. Error: 'skynet-portal-api' header missing" 306 | ); 307 | }); 308 | 309 | it("should throw if skynet-skylink header is missing", async () => { 310 | const incompleteHeaders = { 311 | "skynet-portal-api": portalUrl, 312 | "skynet-skylink": undefined, 313 | }; 314 | axios.mockResolvedValue({ status: 200, data: {}, headers: incompleteHeaders }); 315 | 316 | await expect(client.getMetadata(skylink)).rejects.toThrowError( 317 | "Metadata response invalid despite a successful request. Please try again and report this issue to the devs if it persists. Error: 'skynet-skylink' header missing" 318 | ); 319 | }); 320 | }); 321 | 322 | describe("getFileContent", () => { 323 | const skynetFileContents = { arbitrary: "json string" }; 324 | 325 | it.each(validSkylinkVariations)("should successfully fetch skynet file content for %s", async (input) => { 326 | const headers = { 327 | "skynet-portal-api": portalUrl, 328 | "skynet-skylink": skylink, 329 | "content-type": "application/json", 330 | }; 331 | axios.mockResolvedValue({ status: 200, data: skynetFileContents, headers: headers }); 332 | 333 | const { data, contentType, skylink: skylink2 } = await client.getFileContent(input); 334 | 335 | expect(data).toEqual(skynetFileContents); 336 | expect(contentType).toEqual("application/json"); 337 | expect(skylink2).toEqual(sialink); 338 | }); 339 | 340 | it("should throw if data is not returned", async () => { 341 | const headers = { 342 | "skynet-portal-api": portalUrl, 343 | "skynet-skylink": skylink, 344 | "content-type": "application/json", 345 | }; 346 | axios.mockResolvedValue({ status: 200, headers: headers }); 347 | 348 | await expect(client.getFileContent(skylink)).rejects.toThrowError( 349 | "File content response invalid despite a successful request. Please try again and report this issue to the devs if it persists. Error: 'response.data' field missing" 350 | ); 351 | }); 352 | 353 | it("should throw if no headers are returned", async () => { 354 | axios.mockResolvedValue({ status: 200, data: {} }); 355 | 356 | await expect(client.getFileContent(skylink)).rejects.toThrowError( 357 | "File content response invalid despite a successful request. Please try again and report this issue to the devs if it persists. Error: 'response.headers' field missing" 358 | ); 359 | }); 360 | 361 | it("should throw if content-type header is missing", async () => { 362 | const incompleteHeaders = { 363 | "skynet-portal-api": portalUrl, 364 | "skynet-skylink": skylink, 365 | "content-type": undefined, 366 | }; 367 | axios.mockResolvedValue({ status: 200, data: {}, headers: incompleteHeaders }); 368 | 369 | await expect(client.getFileContent(skylink)).rejects.toThrowError( 370 | "File content response invalid despite a successful request. Please try again and report this issue to the devs if it persists. Error: 'content-type' header missing" 371 | ); 372 | }); 373 | 374 | it("should throw if skynet-portal-api header is missing", async () => { 375 | const incompleteHeaders = { 376 | "skynet-portal-api": undefined, 377 | "skynet-skylink": skylink, 378 | "content-type": "application/json", 379 | }; 380 | axios.mockResolvedValue({ status: 200, data: {}, headers: incompleteHeaders }); 381 | 382 | await expect(client.getFileContent(skylink)).rejects.toThrowError( 383 | "File content response invalid despite a successful request. Please try again and report this issue to the devs if it persists. Error: 'skynet-portal-api' header missing" 384 | ); 385 | }); 386 | 387 | it("should throw if skynet-skylink header is missing", async () => { 388 | const incompleteHeaders = { 389 | "skynet-portal-api": portalUrl, 390 | "skynet-skylink": undefined, 391 | "content-type": "application/json", 392 | }; 393 | axios.mockResolvedValue({ status: 200, data: {}, headers: incompleteHeaders }); 394 | 395 | await expect(client.getFileContent(skylink)).rejects.toThrowError( 396 | "File content response invalid despite a successful request. Please try again and report this issue to the devs if it persists. Error: 'skynet-skylink' header missing" 397 | ); 398 | }); 399 | 400 | it("should set range header if range option is set", async () => { 401 | const headers = { 402 | "skynet-portal-api": portalUrl, 403 | "skynet-skylink": skylink, 404 | "content-type": "application/json", 405 | }; 406 | const skynetFileContents = { arbitrary: "json string", headers: { range: "4000-5000" } }; 407 | axios.mockResolvedValue({ status: 200, data: skynetFileContents, headers: headers }); 408 | const range = "4000-5000"; 409 | const request = await client.getFileContent(skylink, { range }); 410 | 411 | expect(request.data.headers["range"]).toEqual(range); 412 | }); 413 | 414 | describe("proof validation", () => { 415 | const headers = { 416 | "skynet-portal-api": portalUrl, 417 | "skynet-skylink": skylink, 418 | "content-type": "application/json", 419 | }; 420 | 421 | it("should throw if skynet-proof header is not valid JSON", async () => { 422 | const headersWithProof = { ...headers }; 423 | headersWithProof["skynet-proof"] = "foo"; 424 | axios.mockResolvedValue({ status: 200, data: {}, headers: headersWithProof }); 425 | 426 | await expect(client.getFileContent(skylink)).rejects.toThrowError( 427 | "File content response invalid despite a successful request. Please try again and report this issue to the devs if it persists. Error: Could not parse 'skynet-proof' header as JSON: SyntaxError: Unexpected token o in JSON at position 1" 428 | ); 429 | }); 430 | 431 | it("should throw if skynet-proof header is null", async () => { 432 | const headersWithProof = { ...headers }; 433 | headersWithProof["skynet-proof"] = "null"; 434 | axios.mockResolvedValue({ status: 200, data: {}, headers: headersWithProof }); 435 | 436 | await expect(client.getFileContent(skylink)).rejects.toThrowError( 437 | "File content response invalid despite a successful request. Please try again and report this issue to the devs if it persists. Error: Could not parse 'skynet-proof' header as JSON: Error: Could not parse 'skynet-proof' header as JSON" 438 | ); 439 | }); 440 | 441 | it("should throw if skynet-skylink does not match input data link", async () => { 442 | const headersWithProof = { ...headers }; 443 | headersWithProof["skynet-proof"] = "[]"; 444 | headersWithProof["skynet-skylink"] = entryLink; 445 | axios.mockResolvedValue({ status: 200, data: {}, headers: headersWithProof }); 446 | await expect(client.getFileContent(skylink)).rejects.toThrowError( 447 | "File content response invalid despite a successful request. Please try again and report this issue to the devs if it persists. Error: Expected returned skylink ('AQDwh1jnoZas9LaLHC_D4-2yP9XYDdZzNtz62H4Dww1jDA') to be the same as input data link ('XABvi7JtJbQSMAcDwnUnmp2FKDPjg8_tTTFP4BwMSxVdEg')" 448 | ); 449 | }); 450 | 451 | it("should throw if proof is present for data link", async () => { 452 | const headersWithProof = { ...headers }; 453 | headersWithProof["skynet-proof"] = "[1, 2]"; 454 | axios.mockResolvedValue({ status: 200, data: {}, headers: headersWithProof }); 455 | 456 | await expect(client.getFileContent(skylink)).rejects.toThrowError( 457 | "File content response invalid despite a successful request. Please try again and report this issue to the devs if it persists. Error: Expected 'skynet-proof' header to be empty for data link" 458 | ); 459 | }); 460 | 461 | it("should throw if skynet-skylink matches input entry link", async () => { 462 | const headersWithProof = { ...headers }; 463 | headersWithProof["skynet-proof"] = "[]"; 464 | headersWithProof["skynet-skylink"] = entryLink; 465 | axios.mockResolvedValue({ status: 200, data: {}, headers: headersWithProof }); 466 | await expect(client.getFileContent(entryLink)).rejects.toThrowError( 467 | "File content response invalid despite a successful request. Please try again and report this issue to the devs if it persists. Error: Expected returned skylink ('AQDwh1jnoZas9LaLHC_D4-2yP9XYDdZzNtz62H4Dww1jDA') to be different from input entry link" 468 | ); 469 | }); 470 | 471 | it("should throw if proof is empty for entry link", async () => { 472 | const headersWithProof = { ...headers }; 473 | headersWithProof["skynet-proof"] = "[]"; 474 | axios.mockResolvedValue({ status: 200, data: {}, headers: headersWithProof }); 475 | 476 | await expect(client.getFileContent(entryLink)).rejects.toThrowError( 477 | "File content response invalid despite a successful request. Please try again and report this issue to the devs if it persists. Error: Expected registry proof not to be empty" 478 | ); 479 | }); 480 | 481 | it("should throw if proof contains unsupported registry type", async () => { 482 | const headersWithProof = { ...headers }; 483 | // Corrupt the type. 484 | headersWithProof[ 485 | "skynet-proof" 486 | ] = `[{"data":"5c006f8bb26d25b412300703c275279a9d852833e383cfed4d314fe01c0c4b155d12","revision":0,"datakey":"43c8a9b01609544ab152dad397afc3b56c1518eb546750dbc6cad5944fec0292","publickey":{"algorithm":"ed25519","key":"y/l99FyfFm6JPhZL5xSkruhA06Qh9m5S9rnipQCc+rw="},"signature":"5a1437508eedb6f5352d7f744693908a91bb05c01370ce4743de9c25f761b4e87760b8172448c073a4ddd9d58d1a2bf978b3227e57e4fa8cbe830a2353be2207","type":0}]`; 487 | axios.mockResolvedValue({ status: 200, data: {}, headers: headersWithProof }); 488 | 489 | await expect(client.getFileContent(entryLink)).rejects.toThrowError( 490 | "File content response invalid despite a successful request. Please try again and report this issue to the devs if it persists. Error: Unsupported registry type in proof: '0'" 491 | ); 492 | }); 493 | 494 | it("should throw if proof chain is invalid", async () => { 495 | // Corrupt the input skylink. 496 | const newSkylink = entryLink.replace("-", "_"); 497 | 498 | const headersWithProof = { ...headers }; 499 | headersWithProof[ 500 | "skynet-proof" 501 | ] = `[{"data":"5c006f8bb26d25b412300703c275279a9d852833e383cfed4d314fe01c0c4b155d12","revision":0,"datakey":"43c8a9b01609544ab152dad397afc3b56c1518eb546750dbc6cad5944fec0292","publickey":{"algorithm":"ed25519","key":"y/l99FyfFm6JPhZL5xSkruhA06Qh9m5S9rnipQCc+rw="},"signature":"5a1437508eedb6f5352d7f744693908a91bb05c01370ce4743de9c25f761b4e87760b8172448c073a4ddd9d58d1a2bf978b3227e57e4fa8cbe830a2353be2207","type":1}]`; 502 | axios.mockResolvedValue({ status: 200, data: {}, headers: headersWithProof }); 503 | 504 | await expect(client.getFileContent(newSkylink)).rejects.toThrowError( 505 | "File content response invalid despite a successful request. Please try again and report this issue to the devs if it persists. Error: Could not verify registry proof chain" 506 | ); 507 | }); 508 | 509 | it("should throw if signature is invalid", async () => { 510 | const headersWithProof = { ...headers }; 511 | // Use a corrupted signature. 512 | headersWithProof[ 513 | "skynet-proof" 514 | ] = `[{"data":"5c006f8bb26d25b412300703c275279a9d852833e383cfed4d314fe01c0c4b155d12","revision":0,"datakey":"43c8a9b01609544ab152dad397afc3b56c1518eb546750dbc6cad5944fec0292","publickey":{"algorithm":"ed25519","key":"y/l99FyfFm6JPhZL5xSkruhA06Qh9m5S9rnipQCc+rw="},"signature":"4a1437508eedb6f5352d7f744693908a91bb05c01370ce4743de9c25f761b4e87760b8172448c073a4ddd9d58d1a2bf978b3227e57e4fa8cbe830a2353be2207","type":1}]`; 515 | axios.mockResolvedValue({ status: 200, data: {}, headers: headersWithProof }); 516 | 517 | await expect(client.getFileContent(entryLink)).rejects.toThrowError( 518 | "File content response invalid despite a successful request. Please try again and report this issue to the devs if it persists. Error: Could not verify signature from retrieved, signed registry entry in registry proof" 519 | ); 520 | }); 521 | 522 | it("should throw if proof chain results in different data link", async () => { 523 | const dataLink = "EAAFgq17B-MKsi0ARYKUMmf9vxbZlDpZkA6EaVBCG4YBAQ"; 524 | 525 | const headersWithProof = { ...headers }; 526 | headersWithProof[ 527 | "skynet-proof" 528 | ] = `[{"data":"5c006f8bb26d25b412300703c275279a9d852833e383cfed4d314fe01c0c4b155d12","revision":0,"datakey":"43c8a9b01609544ab152dad397afc3b56c1518eb546750dbc6cad5944fec0292","publickey":{"algorithm":"ed25519","key":"y/l99FyfFm6JPhZL5xSkruhA06Qh9m5S9rnipQCc+rw="},"signature":"5a1437508eedb6f5352d7f744693908a91bb05c01370ce4743de9c25f761b4e87760b8172448c073a4ddd9d58d1a2bf978b3227e57e4fa8cbe830a2353be2207","type":1}]`; 529 | headersWithProof["skynet-skylink"] = dataLink; 530 | axios.mockResolvedValue({ status: 200, data: {}, headers: headersWithProof }); 531 | 532 | await expect(client.getFileContent(entryLink)).rejects.toThrowError( 533 | "File content response invalid despite a successful request. Please try again and report this issue to the devs if it persists. Error: Could not verify registry proof chain" 534 | ); 535 | }); 536 | }); 537 | }); 538 | 539 | describe("getFileContentBinary", () => { 540 | const headers = { 541 | "skynet-portal-api": portalUrl, 542 | "skynet-skylink": skylink, 543 | "content-type": "application/json", 544 | }; 545 | 546 | it('should throw if responseType option is not "arraybuffer"', async () => { 547 | // This should throw. 548 | await expect(client.getFileContentBinary(skylink, { responseType: "json" })).rejects.toThrowError( 549 | "Unexpected 'responseType' option found for 'getFileContentBinary': 'json'" 550 | ); 551 | }); 552 | 553 | it('should not throw if responseType option is "arraybuffer"', async () => { 554 | const binaryData = [0, 1, 2, 3]; 555 | axios.mockResolvedValue({ status: 200, data: binaryData, headers: headers }); 556 | 557 | // Should not throw if "arraybuffer" is passed. 558 | const { 559 | data, 560 | contentType, 561 | skylink: skylink2, 562 | } = await client.getFileContentBinary(skylink, { responseType: "arraybuffer" }); 563 | 564 | expect(data).toEqual(new Uint8Array(binaryData)); 565 | expect(contentType).toEqual("application/json"); 566 | expect(skylink2).toEqual(sialink); 567 | }); 568 | }); 569 | 570 | describe("getFileContentHns", () => { 571 | const skynetFileContents = { arbitrary: "json string" }; 572 | const headers = { 573 | "skynet-portal-api": portalUrl, 574 | "skynet-skylink": skylink, 575 | "content-type": "application/json", 576 | }; 577 | 578 | it.each(validHnsLinkVariations)("should successfully fetch skynet file content for domain '%s'", async (domain) => { 579 | axios.mockResolvedValueOnce({ status: 200, data: skynetFileContents, headers: headers }); 580 | axios.mockResolvedValueOnce({ status: 200, data: { skylink }, headers: headers }); 581 | 582 | const { data } = await client.getFileContentHns(domain); 583 | 584 | expect(data).toEqual(skynetFileContents); 585 | }); 586 | }); 587 | 588 | describe("getFileContentBinaryHns", () => { 589 | const headers = { 590 | "skynet-portal-api": portalUrl, 591 | "skynet-skylink": skylink, 592 | "content-type": "application/json", 593 | }; 594 | 595 | it('should throw if responseType option is not "arraybuffer"', async () => { 596 | // This should throw. 597 | await expect(client.getFileContentBinaryHns(skylink, { responseType: "json" })).rejects.toThrowError( 598 | "Unexpected 'responseType' option found for 'getFileContentBinary': 'json'" 599 | ); 600 | }); 601 | 602 | it("should succeed with given domain", async () => { 603 | const binaryData = [0, 1, 2, 3]; 604 | axios.mockResolvedValueOnce({ status: 200, data: binaryData, headers: headers }); 605 | axios.mockResolvedValueOnce({ status: 200, data: { skylink }, headers: headers }); 606 | 607 | // Should not throw if "arraybuffer" is passed. 608 | const { data } = await client.getFileContentBinaryHns(hnsLink); 609 | 610 | expect(data).toEqual(new Uint8Array(binaryData)); 611 | }); 612 | }); 613 | 614 | describe("resolveHns", () => { 615 | it.each(validHnsLinkVariations)( 616 | "should call axios.get with the portal and hnsres link for %s and return the json body", 617 | async (hnsLink) => { 618 | axios.mockResolvedValueOnce({ status: 200, data: { skylink } }); 619 | const data = await client.resolveHns(hnsLink); 620 | 621 | expect(data.skylink).toEqual(skylink); 622 | } 623 | ); 624 | 625 | it("should throw if no data was returned to resolveHns", async () => { 626 | axios.mockResolvedValueOnce({ status: 200 }); 627 | 628 | await expect(client.resolveHns(hnsLink)).rejects.toThrowError( 629 | "Did not get a complete resolve HNS response despite a successful request. Please try again and report this issue to the devs if it persists. Error: 'response.data' field missing" 630 | ); 631 | }); 632 | 633 | it("should throw if unexpected data was returned to resolveHns", async () => { 634 | axios.mockResolvedValueOnce({ status: 200, data: { foo: "foo" } }); 635 | 636 | await expect(client.resolveHns(hnsLink)).rejects.toThrowError( 637 | "Did not get a complete resolve HNS response despite a successful request. Please try again and report this issue to the devs if it persists. Error: Expected response data object 'response.data' to be object containing skylink or registry field, was type 'object', value '[object Object]'" 638 | ); 639 | }); 640 | }); 641 | -------------------------------------------------------------------------------- /src/pin.test.js: -------------------------------------------------------------------------------- 1 | const axios = require("axios"); 2 | const MockAdapter = require("axios-mock-adapter"); 3 | 4 | const { SkynetClient } = require("../index"); 5 | const { DEFAULT_SKYNET_PORTAL_URL, URI_SKYNET_PREFIX } = require("skynet-js"); 6 | 7 | const portalUrl = DEFAULT_SKYNET_PORTAL_URL; 8 | const client = new SkynetClient(portalUrl); 9 | const skylink = "XABvi7JtJbQSMAcDwnUnmp2FKDPjg8_tTTFP4BwMSxVdEg"; 10 | const sialink = `${URI_SKYNET_PREFIX}${skylink}`; 11 | const expectedUrl = `${portalUrl}/skynet/pin/${skylink}`; 12 | 13 | describe("pinSkylink", () => { 14 | let mock = MockAdapter; 15 | 16 | beforeEach(() => { 17 | mock = new MockAdapter(axios); 18 | mock.onHead(portalUrl).replyOnce(200, {}, { "skynet-portal-api": portalUrl }); 19 | }); 20 | const headers = { 21 | "skynet-skylink": skylink, 22 | }; 23 | 24 | it("Should pin the skylink using the correct URL", async () => { 25 | mock.onPost(expectedUrl).replyOnce(200, "", headers); 26 | 27 | const { skylink: skylink2 } = await client.pinSkylink(skylink); 28 | expect(skylink2).toEqual(sialink); 29 | }); 30 | 31 | it("should throw if a path is supplied", async () => { 32 | mock.onPost(expectedUrl).replyOnce(200, ""); 33 | 34 | await expect(client.pinSkylink(`${skylink}/path/file`)).rejects.toThrowError( 35 | "Skylink string should not contain a path" 36 | ); 37 | }); 38 | 39 | it("should throw if a skylink was not returned", async () => { 40 | mock.onPost(expectedUrl).replyOnce(200, "", {}); 41 | 42 | await expect(client.pinSkylink(skylink)).rejects.toThrowError( 43 | "Did not get a complete pin response despite a successful request. Please try again and report this issue to the devs if it persists. Error: Expected pin response field 'response.headers[\"skynet-skylink\"]' to be type 'string', was type 'undefined'" 44 | ); 45 | }); 46 | 47 | it("should throw if no data was returned to pinSkylink", async () => { 48 | mock.onPost(expectedUrl).replyOnce(200, ""); 49 | 50 | await expect(client.pinSkylink(skylink)).rejects.toThrowError( 51 | "Did not get a complete pin response despite a successful request. Please try again and report this issue to the devs if it persists. Error: response.headers field missing" 52 | ); 53 | }); 54 | }); 55 | -------------------------------------------------------------------------------- /src/skydb.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const { URI_SKYNET_PREFIX, MAX_REVISION } = require("skynet-js"); 4 | 5 | const { 6 | DEFAULT_GET_ENTRY_OPTIONS, 7 | DEFAULT_SET_ENTRY_OPTIONS, 8 | DEFAULT_GET_JSON_OPTIONS, 9 | DEFAULT_SET_JSON_OPTIONS, 10 | DEFAULT_UPLOAD_OPTIONS, 11 | } = require("./defaults"); 12 | 13 | const { RAW_SKYLINK_SIZE } = require("./skylink_sia"); 14 | const { extractOptions, getPublicKeyFromPrivateKey, formatSkylink } = require("./utils"); 15 | const { trimPrefix } = require("./utils_string"); 16 | const { decodeSkylinkBase64 } = require("./utils_encoding"); 17 | const { buildSkynetJsonObject } = require("./skydb_v2"); 18 | 19 | /** 20 | * Sets a JSON object at the registry entry corresponding to the privateKey and dataKey using SkyDB V1. 21 | * 22 | * @param privateKey - The user private key. 23 | * @param dataKey - The key of the data to fetch for the given user. 24 | * @param json - The JSON data to set. 25 | * @param [customOptions] - Additional settings that can optionally be set. 26 | * @returns - The returned JSON and corresponding data link. 27 | * @throws - Will throw if the input keys are not valid strings. 28 | * @deprecated - Use of this method may result in data race bugs. Reworking your application to use `client.dbV2.setJSON` is recommended. 29 | */ 30 | const setJSONdbV1 = async function (privateKey, dataKey, json, customOptions = {}) { 31 | const opts = { ...DEFAULT_SET_JSON_OPTIONS, ...this.customOptions, ...customOptions }; 32 | 33 | const publicKey = getPublicKeyFromPrivateKey(privateKey); 34 | const { entry, dataLink } = await getOrCreateRegistryEntry(this, publicKey, dataKey, json, opts); 35 | 36 | // Update the registry. 37 | const setEntryOpts = extractOptions(opts, DEFAULT_SET_ENTRY_OPTIONS); 38 | await this.registry.setEntry(privateKey, entry, setEntryOpts); 39 | 40 | return { data: json, dataLink: formatSkylink(dataLink) }; 41 | }; 42 | 43 | /** 44 | * Gets the registry entry and data link or creates the entry if it doesn't exist. 45 | * 46 | * @param client - The Skynet client. 47 | * @param publicKey - The user public key. 48 | * @param dataKey - The dat akey. 49 | * @param json - The JSON to set. 50 | * @param [customOptions] - Additional settings that can optionally be set. 51 | * @returns - The registry entry and corresponding data link. 52 | * @throws - Will throw if the revision is already the maximum value. 53 | */ 54 | const getOrCreateRegistryEntry = async function (client, publicKey, dataKey, json, customOptions = {}) { 55 | const opts = { ...DEFAULT_GET_JSON_OPTIONS, ...client.customOptions, ...customOptions }; 56 | 57 | // Set the hidden _data and _v fields. 58 | const skynetJson = await buildSkynetJsonObject(json); 59 | const fullData = JSON.stringify(skynetJson); 60 | 61 | // uploads in-memory data to skynet 62 | const uploadOpts = extractOptions(opts, DEFAULT_UPLOAD_OPTIONS); 63 | const skylink = await client.uploadData(fullData, dataKey, uploadOpts); 64 | 65 | // Fetch the current value to find out the revision. 66 | const getEntryOpts = extractOptions(opts, DEFAULT_GET_ENTRY_OPTIONS); 67 | const signedEntry = await client.registry.getEntry(publicKey, dataKey, getEntryOpts); 68 | 69 | const revision = getNextRevisionFromEntry(signedEntry.entry); 70 | 71 | // Build the registry entry. 72 | const dataLink = trimPrefix(skylink, URI_SKYNET_PREFIX); 73 | const rawDataLink = decodeSkylinkBase64(dataLink); 74 | if (rawDataLink.length !== RAW_SKYLINK_SIZE) { 75 | throw new Error("rawDataLink is not 34 bytes long."); 76 | } 77 | 78 | const entry = { 79 | dataKey, 80 | data: rawDataLink, 81 | revision, 82 | }; 83 | 84 | return { entry: entry, dataLink: formatSkylink(skylink) }; 85 | }; 86 | 87 | /** 88 | * Gets the next revision from a returned entry (or 0 if the entry was not found). 89 | * 90 | * @param entry - The returned registry entry. 91 | * @returns - The revision. 92 | * @throws - Will throw if the next revision would be beyond the maximum allowed value. 93 | */ 94 | const getNextRevisionFromEntry = function (entry) { 95 | let revision; 96 | if (entry === null || entry === undefined) { 97 | revision = BigInt(0); 98 | } else { 99 | revision = entry.revision + BigInt(1); 100 | } 101 | // Throw if the revision is already the maximum value. 102 | if (revision > MAX_REVISION) { 103 | throw new Error("Current entry already has maximum allowed revision, could not update the entry"); 104 | } 105 | return revision; 106 | }; 107 | 108 | module.exports = { setJSONdbV1 }; 109 | -------------------------------------------------------------------------------- /src/skydb.test.js: -------------------------------------------------------------------------------- 1 | const axios = require("axios"); 2 | const MockAdapter = require("axios-mock-adapter"); 3 | 4 | const { SkynetClient } = require("../index"); 5 | 6 | const { 7 | getSkylinkUrlForPortal, 8 | MAX_REVISION, 9 | DEFAULT_SKYNET_PORTAL_URL, 10 | URI_SKYNET_PREFIX, 11 | getEntryUrlForPortal, 12 | DELETION_ENTRY_DATA, 13 | MAX_ENTRY_LENGTH, 14 | } = require("skynet-js"); 15 | 16 | const { REGEX_REVISION_NO_QUOTES } = require("./utils_registry"); 17 | const { checkCachedDataLink } = require("./skydb_v2"); 18 | 19 | // Generated with genKeyPairFromSeed("insecure test seed") 20 | const [publicKey, privateKey] = [ 21 | "658b900df55e983ce85f3f9fb2a088d568ab514e7bbda51cfbfb16ea945378d9", 22 | "7caffac49ac914a541b28723f11776d36ce81e7b9b0c96ccacd1302db429c79c658b900df55e983ce85f3f9fb2a088d568ab514e7bbda51cfbfb16ea945378d9", 23 | ]; 24 | 25 | const dataKey = "app"; 26 | const skylink = "CABAB_1Dt0FJsxqsu_J4TodNCbCGvtFf1Uys_3EgzOlTcg"; 27 | const sialink = `${URI_SKYNET_PREFIX}${skylink}`; 28 | const jsonData = { data: "thisistext" }; 29 | const fullJsonData = { _data: jsonData, _v: 2 }; 30 | const legacyJsonData = jsonData; 31 | const merkleroot = "QAf9Q7dBSbMarLvyeE6HTQmwhr7RX9VMrP9xIMzpU3I"; 32 | const bitfield = 2048; 33 | 34 | const portalUrl = DEFAULT_SKYNET_PORTAL_URL; 35 | const client = new SkynetClient(portalUrl); 36 | const registryUrl = `${portalUrl}/skynet/registry`; 37 | const registryLookupUrl = getEntryUrlForPortal(portalUrl, publicKey, dataKey); 38 | const uploadUrl = `${portalUrl}/skynet/skyfile`; 39 | const skylinkUrl = getSkylinkUrlForPortal(portalUrl, skylink); 40 | 41 | // Hex-encoded skylink. 42 | const data = "43414241425f31447430464a73787173755f4a34546f644e4362434776744666315579735f3345677a4f6c546367"; 43 | const revision = "11"; 44 | const entryData = { 45 | data: data, 46 | revision: revision, 47 | signature: 48 | "33d14d2889cb292142614da0e0ff13a205c4867961276001471d13b779fc9032568ddd292d9e0dff69d7b1f28be07972cc9d86da3cecf3adecb6f9b7311af809", 49 | }; 50 | 51 | describe("getJSON", () => { 52 | let mock = MockAdapter; 53 | 54 | beforeEach(() => { 55 | mock = new MockAdapter(axios); 56 | mock.onHead(portalUrl).replyOnce(200, {}, { "skynet-portal-api": portalUrl }); 57 | mock.resetHistory(); 58 | }); 59 | 60 | const headers = { 61 | "skynet-portal-api": portalUrl, 62 | "skynet-skylink": skylink, 63 | "content-type": "application/json", 64 | }; 65 | 66 | it("should perform a lookup and skylink GET", async () => { 67 | // mock a successful registry lookup 68 | mock.onGet(registryLookupUrl).replyOnce(200, JSON.stringify(entryData)); 69 | mock.onGet(skylinkUrl).replyOnce(200, fullJsonData, headers); 70 | 71 | const { data, dataLink } = await client.db.getJSON(publicKey, dataKey); 72 | expect(data).toEqual(jsonData); 73 | expect(dataLink).toEqual(sialink); 74 | expect(mock.history.get.length).toBe(2); 75 | }); 76 | 77 | it("should perform a lookup but not a skylink GET if the cachedDataLink is a hit", async () => { 78 | // mock a successful registry lookup 79 | mock.onGet(registryLookupUrl).replyOnce(200, JSON.stringify(entryData)); 80 | 81 | const { data, dataLink } = await client.db.getJSON(publicKey, dataKey, { cachedDataLink: skylink }); 82 | expect(data).toBeNull(); 83 | expect(dataLink).toEqual(sialink); 84 | expect(mock.history.get.length).toBe(1); 85 | }); 86 | 87 | it("should perform a lookup and a skylink GET if the cachedDataLink is not a hit", async () => { 88 | const skylinkNoHit = "XABvi7JtJbQSMAcDwnUnmp2FKDPjg8_tTTFP4BwMSxVdEg"; 89 | 90 | // mock a successful registry lookup 91 | mock.onGet(registryLookupUrl).replyOnce(200, JSON.stringify(entryData)); 92 | mock.onGet(skylinkUrl).replyOnce(200, fullJsonData, headers); 93 | 94 | const { data, dataLink } = await client.db.getJSON(publicKey, dataKey, { cachedDataLink: skylinkNoHit }); 95 | expect(data).toEqual(jsonData); 96 | expect(dataLink).toEqual(sialink); 97 | expect(mock.history.get.length).toBe(2); 98 | }); 99 | 100 | it("should throw if the cachedDataLink is not a valid skylink", async () => { 101 | // mock a successful registry lookup 102 | mock.onGet(registryLookupUrl).replyOnce(200, JSON.stringify(entryData)); 103 | mock.onGet(skylinkUrl).replyOnce(200, fullJsonData, {}); 104 | 105 | await expect(client.db.getJSON(publicKey, dataKey, { cachedDataLink: "asdf" })).rejects.toThrowError( 106 | "Expected optional parameter 'cachedDataLink' to be valid skylink of type 'string', was type 'string', value 'asdf'" 107 | ); 108 | }); 109 | 110 | it("should perform a lookup and skylink GET on legacy pre-v4 data", async () => { 111 | // mock a successful registry lookup 112 | mock.onGet(registryLookupUrl).replyOnce(200, JSON.stringify(entryData)); 113 | mock.onGet(skylinkUrl).replyOnce(200, legacyJsonData, headers); 114 | 115 | const jsonReturned = await client.db.getJSON(publicKey, dataKey); 116 | expect(jsonReturned.data).toEqual(jsonData); 117 | expect(mock.history.get.length).toBe(2); 118 | }); 119 | 120 | it("should return null if no entry is found", async () => { 121 | mock.onGet(registryLookupUrl).reply(404); 122 | 123 | const { data, dataLink } = await client.db.getJSON(publicKey, dataKey); 124 | expect(data).toBeNull(); 125 | expect(dataLink).toBeNull(); 126 | }); 127 | 128 | it("should throw if the returned file data is not JSON", async () => { 129 | // mock a successful registry lookup 130 | mock.onGet(registryLookupUrl).reply(200, JSON.stringify(entryData)); 131 | mock.onGet(skylinkUrl).reply(200, "thisistext", { ...headers, "content-type": "text/plain" }); 132 | 133 | await expect(client.db.getJSON(publicKey, dataKey)).rejects.toThrowError( 134 | `File data for the entry at data key '${dataKey}' is not JSON.` 135 | ); 136 | }); 137 | 138 | it("should throw if the returned _data field in the file data is not JSON", async () => { 139 | // mock a successful registry lookup 140 | mock.onGet(registryLookupUrl).reply(200, JSON.stringify(entryData)); 141 | mock.onGet(skylinkUrl).reply(200, { _data: "thisistext", _v: 1 }, headers); 142 | 143 | await expect(client.db.getJSON(publicKey, dataKey)).rejects.toThrowError( 144 | "File data '_data' for the entry at data key 'app' is not JSON." 145 | ); 146 | }); 147 | 148 | it("should throw if invalid entry data is returned", async () => { 149 | const client = new SkynetClient(portalUrl); 150 | const mockedFn = jest.fn(); 151 | mockedFn.mockReturnValueOnce({ entry: { data: new Uint8Array() } }); 152 | client.registry.getEntry = mockedFn; 153 | await expect(await client.db.getJSON(publicKey, dataKey)).toEqual({ data: null, dataLink: null }); 154 | }); 155 | }); 156 | 157 | describe("setJSON", () => { 158 | let mock = MockAdapter; 159 | 160 | beforeEach(() => { 161 | mock = new MockAdapter(axios); 162 | mock.onHead(portalUrl).replyOnce(200, {}, { "skynet-portal-api": portalUrl }); 163 | mock.resetHistory(); 164 | // mock a successful upload 165 | mock.onPost(uploadUrl).reply(200, { skylink, merkleroot, bitfield }); 166 | }); 167 | 168 | it("should perform an upload, lookup and registry update", async () => { 169 | // mock a successful registry lookup 170 | mock.onGet(registryLookupUrl).replyOnce(200, JSON.stringify(entryData)); 171 | 172 | // mock a successful registry update 173 | mock.onPost(registryUrl).replyOnce(204); 174 | 175 | // set data 176 | const { data: returnedData, dataLink: returnedSkylink } = await client.db.setJSON(privateKey, dataKey, jsonData); 177 | expect(returnedData).toEqual(jsonData); 178 | expect(returnedSkylink).toEqual(sialink); 179 | 180 | // assert our request history contains the expected amount of requests 181 | expect(mock.history.get.length).toBe(1); 182 | expect(mock.history.post.length).toBe(2); 183 | 184 | const data = JSON.parse(mock.history.post[1].data); 185 | expect(data).toBeDefined(); 186 | // change "revision + 1" to "12" 187 | expect(data.revision).toEqual(12); 188 | }); 189 | 190 | it("should use a revision number of 0 if the lookup failed", async () => { 191 | mock.onGet(registryLookupUrl).reply(404); 192 | 193 | // mock a successful registry update 194 | mock.onPost(registryUrl).reply(204); 195 | 196 | // call `setJSON` on the client 197 | await client.db.setJSON(privateKey, dataKey, jsonData); 198 | 199 | // assert our request history contains the expected amount of requests 200 | expect(mock.history.get.length).toBe(1); 201 | expect(mock.history.post.length).toBe(2); 202 | 203 | const data = JSON.parse(mock.history.post[1].data); 204 | expect(data).toBeDefined(); 205 | expect(data.revision).toEqual(0); 206 | }); 207 | 208 | it("should fail if the entry has the maximum allowed revision", async () => { 209 | // mock a successful registry lookup 210 | const entryData = { 211 | data, 212 | // String the bigint since JS doesn't support 64-bit numbers. 213 | revision: MAX_REVISION.toString(), 214 | signature: 215 | "18c76e88141c7cc76d8a77abcd91b5d64d8fc3833eae407ab8a5339e5fcf7940e3fa5830a8ad9439a0c0cc72236ed7b096ae05772f81eee120cbd173bfd6600e", 216 | }; 217 | // Replace the quotes around the stringed bigint. 218 | const json = JSON.stringify(entryData).replace(REGEX_REVISION_NO_QUOTES, '"revision":"$1"'); 219 | mock.onGet(registryLookupUrl).reply(200, json); 220 | 221 | // mock a successful registry update 222 | mock.onPost(registryUrl).reply(204); 223 | 224 | // Try to set data, should fail. 225 | await expect(client.db.setJSON(privateKey, dataKey, entryData)).rejects.toThrowError( 226 | "Current entry already has maximum allowed revision, could not update the entry" 227 | ); 228 | }); 229 | 230 | it("Should throw an error if the private key is not hex-encoded", async () => { 231 | await expect(client.db.setJSON("foo", dataKey, {})).rejects.toThrowError( 232 | // change the message 233 | "bad secret key size" 234 | ); 235 | }); 236 | 237 | it("Should throw an error if the data key is not provided", async () => { 238 | // @ts-expect-error We do not pass the data key on purpose. 239 | await expect(client.db.setJSON(privateKey)).rejects.toThrowError( 240 | "Expected parameter 'dataKey' to be type 'string', was type 'undefined'" 241 | ); 242 | }); 243 | 244 | it("Should throw an error if the json is not provided", async () => { 245 | // @ts-expect-error We do not pass the json on purpose. 246 | await expect(client.db.setJSON(privateKey, dataKey)).rejects.toThrowError( 247 | // change the message 248 | "Request failed with status code 404" 249 | ); 250 | }); 251 | }); 252 | 253 | describe("setEntryData", () => { 254 | it("should throw if trying to set entry data > 70 bytes", async () => { 255 | await expect( 256 | client.db.setEntryData(privateKey, dataKey, new Uint8Array(MAX_ENTRY_LENGTH + 1)) 257 | ).rejects.toThrowError( 258 | "Expected parameter 'data' to be 'Uint8Array' of length <= 70, was length 71, was type 'object', value '0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0'" 259 | ); 260 | }); 261 | 262 | it("should throw if trying to set the deletion entry data", async () => { 263 | await expect(client.db.setEntryData(privateKey, dataKey, DELETION_ENTRY_DATA)).rejects.toThrowError( 264 | "Tried to set 'Uint8Array' entry data that is the deletion sentinel ('Uint8Array(RAW_SKYLINK_SIZE)'), please use the 'deleteEntryData' method instead`" 265 | ); 266 | }); 267 | }); 268 | 269 | describe("checkCachedDataLink", () => { 270 | const differentSkylink = "XABvi7JtJbQSMAcDwnUnmp2FKDPjg8_tTTFP4BwMSxVdEg"; 271 | // change 272 | const inputs = [ 273 | [skylink, undefined, false], 274 | [skylink, skylink, true], 275 | [skylink, differentSkylink, false], 276 | [differentSkylink, skylink, false], 277 | ]; 278 | 279 | it.each(inputs)("checkCachedDataLink(%s, %s) should return %s", (rawDataLink, cachedDataLink, output) => { 280 | expect(checkCachedDataLink(rawDataLink, cachedDataLink)).toEqual(output); 281 | }); 282 | 283 | it("Should throw on invalid cachedDataLink", () => { 284 | expect(() => checkCachedDataLink(skylink, "asdf")).toThrowError( 285 | "Expected optional parameter 'cachedDataLink' to be valid skylink of type 'string', was type 'string', value 'asdf'" 286 | ); 287 | }); 288 | }); 289 | -------------------------------------------------------------------------------- /src/skydb_v2.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const { URI_SKYNET_PREFIX, MAX_REVISION } = require("skynet-js"); 4 | 5 | const { 6 | DEFAULT_SET_ENTRY_OPTIONS, 7 | DEFAULT_GET_JSON_OPTIONS, 8 | DEFAULT_SET_JSON_OPTIONS, 9 | DEFAULT_UPLOAD_OPTIONS, 10 | } = require("./defaults"); 11 | 12 | const { RAW_SKYLINK_SIZE } = require("./skylink_sia"); 13 | const { extractOptions, getPublicKeyFromPrivateKey, formatSkylink } = require("./utils"); 14 | const { trimPrefix } = require("./utils_string"); 15 | const { validateSkylinkString } = require("./utils_validation"); 16 | const { decodeSkylinkBase64 } = require("./utils_encoding"); 17 | 18 | const JSON_RESPONSE_VERSION = 2; 19 | 20 | /** 21 | * Sets a JSON object at the registry entry corresponding to the privateKey and dataKey using SkyDB V2. 22 | * 23 | * This will use the entry revision number from the cache, so getJSON must 24 | * always be called first for existing entries. 25 | * 26 | * @param privateKey - The user private key. 27 | * @param dataKey - The key of the data to fetch for the given user. 28 | * @param json - The JSON data to set. 29 | * @param [customOptions] - Additional settings that can optionally be set. 30 | * @returns - The returned JSON and corresponding data link. 31 | * @throws - Will throw if the input keys are not valid strings. 32 | */ 33 | const setJSONdbV2 = async function (privateKey, dataKey, json, customOptions = {}) { 34 | const opts = { ...DEFAULT_SET_JSON_OPTIONS, ...this.customOptions, ...customOptions }; 35 | 36 | const publicKey = getPublicKeyFromPrivateKey(privateKey); 37 | 38 | // Immediately fail if the mutex is not available. 39 | return await this.dbV2.revisionNumberCache.withCachedEntryLock(publicKey, dataKey, async (cachedRevisionEntry) => { 40 | // Get the cached revision number before doing anything else. Increment it. 41 | const newRevision = await incrementRevision(cachedRevisionEntry.revision); 42 | 43 | const { entry, dataLink } = await getOrCreateSkyDBRegistryEntry(this, dataKey, json, newRevision, opts); 44 | 45 | // Update the registry. 46 | const setEntryOpts = extractOptions(opts, DEFAULT_SET_ENTRY_OPTIONS); 47 | await this.registry.setEntry(privateKey, entry, setEntryOpts); 48 | 49 | // Update the cached revision number. 50 | cachedRevisionEntry.revision = newRevision; 51 | 52 | return { data: json, dataLink: formatSkylink(dataLink) }; 53 | }); 54 | }; 55 | 56 | /** 57 | * Gets the registry entry and data link or creates the entry if it doesn't 58 | * exist. Uses the cached revision number for the entry, or 0 if the entry has 59 | * not been cached. 60 | * 61 | * @param client - The Skynet client. 62 | * @param dataKey - The data key. 63 | * @param data - The JSON or raw byte data to set. 64 | * @param revision - The revision number to set. 65 | * @param [customOptions] - Additional settings that can optionally be set. 66 | * @returns - The registry entry and corresponding data link. 67 | * @throws - Will throw if the revision is already the maximum value. 68 | */ 69 | const getOrCreateSkyDBRegistryEntry = async function (client, dataKey, json, newRevision, customOptions = {}) { 70 | const opts = { ...DEFAULT_GET_JSON_OPTIONS, ...client.customOptions, ...customOptions }; 71 | 72 | // Set the hidden _data and _v fields. 73 | const skynetJson = await buildSkynetJsonObject(json); 74 | const fullData = JSON.stringify(skynetJson); 75 | 76 | // uploads in-memory data to skynet 77 | const uploadOpts = extractOptions(opts, DEFAULT_UPLOAD_OPTIONS); 78 | const skylink = await client.uploadData(fullData, dataKey, uploadOpts); 79 | 80 | // Build the registry entry. 81 | const revision = newRevision; 82 | const dataLink = trimPrefix(skylink, URI_SKYNET_PREFIX); 83 | const rawDataLink = decodeSkylinkBase64(dataLink); 84 | if (rawDataLink.length !== RAW_SKYLINK_SIZE) { 85 | throw new Error("rawDataLink is not 34 bytes long."); 86 | } 87 | 88 | const entry = { 89 | dataKey, 90 | data: rawDataLink, 91 | revision, 92 | }; 93 | 94 | return { entry: entry, dataLink: formatSkylink(skylink) }; 95 | }; 96 | 97 | /** 98 | * Increments the given revision number and checks to make sure it is not 99 | * greater than the maximum revision. 100 | * 101 | * @param revision - The given revision number. 102 | * @returns - The incremented revision number. 103 | * @throws - Will throw if the incremented revision number is greater than the maximum revision. 104 | */ 105 | const incrementRevision = function (revision) { 106 | revision = revision + BigInt(1); 107 | // Throw if the revision is already the maximum value. 108 | if (revision > MAX_REVISION) { 109 | throw new Error("Current entry already has maximum allowed revision, could not update the entry"); 110 | } 111 | return revision; 112 | }; 113 | 114 | /** 115 | * Sets the hidden _data and _v fields on the given raw JSON data. 116 | * 117 | * @param data - The given JSON data. 118 | * @returns - The Skynet JSON data. 119 | */ 120 | const buildSkynetJsonObject = function (data) { 121 | return { _data: data, _v: JSON_RESPONSE_VERSION }; 122 | }; 123 | 124 | /** 125 | * Checks whether the raw data link matches the cached data link, if provided. 126 | * 127 | * @param rawDataLink - The raw, unformatted data link. 128 | * @param cachedDataLink - The cached data link, if provided. 129 | * @returns - Whether the cached data link is a match. 130 | * @throws - Will throw if the given cached data link is not a valid skylink. 131 | */ 132 | const checkCachedDataLink = function (rawDataLink, cachedDataLink) { 133 | if (cachedDataLink) { 134 | cachedDataLink = validateSkylinkString("cachedDataLink", cachedDataLink, "optional parameter"); 135 | return rawDataLink === cachedDataLink; 136 | } 137 | return false; 138 | }; 139 | 140 | module.exports = { setJSONdbV2, buildSkynetJsonObject, checkCachedDataLink }; 141 | -------------------------------------------------------------------------------- /src/skydb_v2.test.js: -------------------------------------------------------------------------------- 1 | const axios = require("axios"); 2 | const MockAdapter = require("axios-mock-adapter"); 3 | 4 | const { SkynetClient } = require("../index"); 5 | 6 | const { 7 | stringToUint8ArrayUtf8, 8 | getSkylinkUrlForPortal, 9 | MAX_REVISION, 10 | DEFAULT_SKYNET_PORTAL_URL, 11 | URI_SKYNET_PREFIX, 12 | getEntryUrlForPortal, 13 | DELETION_ENTRY_DATA, 14 | MAX_ENTRY_LENGTH, 15 | JsonData, 16 | JSONResponse, 17 | } = require("skynet-js"); 18 | 19 | const { getSettledValues } = require("./utils_testing"); 20 | const { checkCachedDataLink } = require("./skydb_v2"); 21 | const { toHexString } = require("./utils_string"); 22 | const { decodeSkylink } = require("./utils_encoding"); 23 | 24 | // Generated with genKeyPairFromSeed("insecure test seed") 25 | const [publicKey, privateKey] = [ 26 | "658b900df55e983ce85f3f9fb2a088d568ab514e7bbda51cfbfb16ea945378d9", 27 | "7caffac49ac914a541b28723f11776d36ce81e7b9b0c96ccacd1302db429c79c658b900df55e983ce85f3f9fb2a088d568ab514e7bbda51cfbfb16ea945378d9", 28 | ]; 29 | 30 | const dataKey = "app"; 31 | const skylink = "CABAB_1Dt0FJsxqsu_J4TodNCbCGvtFf1Uys_3EgzOlTcg"; 32 | const sialink = `${URI_SKYNET_PREFIX}${skylink}`; 33 | const jsonData = { data: "thisistext" }; 34 | const fullJsonData = { _data: jsonData, _v: 2 }; 35 | const legacyJsonData = jsonData; 36 | const merkleroot = "QAf9Q7dBSbMarLvyeE6HTQmwhr7RX9VMrP9xIMzpU3I"; 37 | const bitfield = 2048; 38 | 39 | const portalUrl = DEFAULT_SKYNET_PORTAL_URL; 40 | const client = new SkynetClient(portalUrl); 41 | const uploadUrl = `${portalUrl}/skynet/skyfile`; 42 | const skylinkUrl = getSkylinkUrlForPortal(portalUrl, skylink); 43 | const registryPostUrl = `${portalUrl}/skynet/registry`; 44 | const registryGetUrl = getEntryUrlForPortal(portalUrl, publicKey, dataKey); 45 | 46 | // Hex-encoded skylink. 47 | const data = "43414241425f31447430464a73787173755f4a34546f644e4362434776744666315579735f3345677a4f6c546367"; 48 | const revision = "11"; 49 | const entryData = { 50 | data: data, 51 | revision: revision, 52 | signature: 53 | "33d14d2889cb292142614da0e0ff13a205c4867961276001471d13b779fc9032568ddd292d9e0dff69d7b1f28be07972cc9d86da3cecf3adecb6f9b7311af809", 54 | }; 55 | 56 | const headers = { 57 | "skynet-portal-api": portalUrl, 58 | "skynet-skylink": skylink, 59 | "content-type": "application/json", 60 | }; 61 | 62 | describe("getJSON", () => { 63 | let mock = MockAdapter; 64 | 65 | beforeEach(() => { 66 | mock = new MockAdapter(axios); 67 | mock.onHead(portalUrl).replyOnce(200, {}, { "skynet-portal-api": portalUrl }); 68 | mock.resetHistory(); 69 | }); 70 | 71 | it("should perform a lookup and skylink GET", async () => { 72 | // Mock a successful registry lookup. 73 | mock.onGet(registryGetUrl).replyOnce(200, JSON.stringify(entryData)); 74 | // Mock a successful data download. 75 | mock.onGet(skylinkUrl).replyOnce(200, fullJsonData, headers); 76 | 77 | const { data, dataLink } = await client.dbV2.getJSON(publicKey, dataKey); 78 | expect(data).toEqual(jsonData); 79 | expect(dataLink).toEqual(sialink); 80 | expect(mock.history.get.length).toBe(2); 81 | }); 82 | 83 | it("should fail properly with a too low error", async () => { 84 | // Use a custom data key for this test to get a fresh cache. 85 | const dataKey = "testTooLowError"; 86 | const registryGetUrl = getEntryUrlForPortal(portalUrl, publicKey, dataKey); 87 | const skylinkData = toHexString(decodeSkylink(skylink)); 88 | const entryData = { 89 | data: skylinkData, 90 | revision: 1, 91 | signature: 92 | "18d2b5f64042db39c4c591c21bd93015f7839eefab487ef8e27086cdb95b190732211b9a23d38c33f4f9a4e5219de55a80f75ff7e437713732ecdb4ccddb0804", 93 | }; 94 | const entryDataTooLow = { 95 | data: skylinkData, 96 | revision: 0, 97 | signature: 98 | "4d7b26923f4211794eaf5c13230e62618ea3bebcb3fa6511ec8772b1f1e1a675b5244e7c33f89daf31999aeabe46c3a1e324a04d2f35c6ba902c75d35ceba00d", 99 | }; 100 | 101 | // Cache the revision. 102 | 103 | // Mock a successful registry lookup. 104 | mock.onGet(registryGetUrl).replyOnce(200, JSON.stringify(entryData)); 105 | // Mock a successful data download. 106 | mock.onGet(skylinkUrl).replyOnce(200, fullJsonData, headers); 107 | 108 | const { data } = await client.dbV2.getJSON(publicKey, dataKey); 109 | expect(data).toEqual(jsonData); 110 | 111 | // The cache should contain revision 1. 112 | const cachedRevisionEntry = await client.dbV2.revisionNumberCache.getRevisionAndMutexForEntry(publicKey, dataKey); 113 | expect(cachedRevisionEntry.revision.toString()).toEqual("1"); 114 | 115 | // Return a revision that's too low. 116 | 117 | // Mock a successful registry lookup. 118 | mock.onGet(registryGetUrl).replyOnce(200, JSON.stringify(entryDataTooLow)); 119 | // Mock a successful data download. 120 | mock.onGet(skylinkUrl).replyOnce(200, fullJsonData, headers); 121 | 122 | await expect(client.dbV2.getJSON(publicKey, dataKey)).rejects.toThrowError( 123 | "Returned revision number too low. A higher revision number for this userID and path is already cached" 124 | ); 125 | 126 | // The cache should still contain revision 1. 127 | expect(cachedRevisionEntry.revision.toString()).toEqual("1"); 128 | }); 129 | 130 | it("should perform a lookup but not a skylink GET if the cachedDataLink is a hit", async () => { 131 | // mock a successful registry lookup 132 | mock.onGet(registryGetUrl).replyOnce(200, JSON.stringify(entryData)); 133 | 134 | const { data, dataLink } = await client.dbV2.getJSON(publicKey, dataKey, { cachedDataLink: skylink }); 135 | expect(data).toBeNull(); 136 | expect(dataLink).toEqual(sialink); 137 | expect(mock.history.get.length).toBe(1); 138 | }); 139 | 140 | it("should perform a lookup and a skylink GET if the cachedDataLink is not a hit", async () => { 141 | const skylinkNoHit = "XABvi7JtJbQSMAcDwnUnmp2FKDPjg8_tTTFP4BwMSxVdEg"; 142 | 143 | // mock a successful registry lookup 144 | mock.onGet(registryGetUrl).replyOnce(200, JSON.stringify(entryData)); 145 | mock.onGet(skylinkUrl).replyOnce(200, fullJsonData, headers); 146 | 147 | const { data, dataLink } = await client.dbV2.getJSON(publicKey, dataKey, { cachedDataLink: skylinkNoHit }); 148 | expect(data).toEqual(jsonData); 149 | expect(dataLink).toEqual(sialink); 150 | expect(mock.history.get.length).toBe(2); 151 | }); 152 | 153 | it("should throw if the cachedDataLink is not a valid skylink", async () => { 154 | // mock a successful registry lookup 155 | mock.onGet(registryGetUrl).replyOnce(200, JSON.stringify(entryData)); 156 | mock.onGet(skylinkUrl).replyOnce(200, fullJsonData, {}); 157 | 158 | await expect(client.dbV2.getJSON(publicKey, dataKey, { cachedDataLink: "asdf" })).rejects.toThrowError( 159 | "Expected optional parameter 'cachedDataLink' to be valid skylink of type 'string', was type 'string', value 'asdf'" 160 | ); 161 | }); 162 | 163 | it("should perform a lookup and skylink GET on legacy pre-v4 data", async () => { 164 | // mock a successful registry lookup 165 | mock.onGet(registryGetUrl).replyOnce(200, JSON.stringify(entryData)); 166 | mock.onGet(skylinkUrl).replyOnce(200, legacyJsonData, headers); 167 | 168 | const jsonReturned = await client.dbV2.getJSON(publicKey, dataKey); 169 | expect(jsonReturned.data).toEqual(jsonData); 170 | expect(mock.history.get.length).toBe(2); 171 | }); 172 | 173 | it("should return null if no entry is found", async () => { 174 | mock.onGet(registryGetUrl).replyOnce(404); 175 | 176 | const { data, dataLink } = await client.dbV2.getJSON(publicKey, dataKey); 177 | expect(data).toBeNull(); 178 | expect(dataLink).toBeNull(); 179 | }); 180 | 181 | it("should throw if the returned file data is not JSON", async () => { 182 | // mock a successful registry lookup 183 | mock.onGet(registryGetUrl).replyOnce(200, JSON.stringify(entryData)); 184 | mock.onGet(skylinkUrl).replyOnce(200, "thisistext", { ...headers, "content-type": "text/plain" }); 185 | 186 | await expect(client.dbV2.getJSON(publicKey, dataKey)).rejects.toThrowError( 187 | `File data for the entry at data key '${dataKey}' is not JSON.` 188 | ); 189 | }); 190 | 191 | it("should throw if the returned _data field in the file data is not JSON", async () => { 192 | // mock a successful registry lookup 193 | mock.onGet(registryGetUrl).replyOnce(200, JSON.stringify(entryData)); 194 | mock.onGet(skylinkUrl).replyOnce(200, { _data: "thisistext", _v: 1 }, headers); 195 | 196 | await expect(client.dbV2.getJSON(publicKey, dataKey)).rejects.toThrowError( 197 | "File data '_data' for the entry at data key 'app' is not JSON." 198 | ); 199 | }); 200 | 201 | it("should throw if invalid entry data is returned", async () => { 202 | const client = new SkynetClient(portalUrl); 203 | const mockedFn = jest.fn(); 204 | mockedFn.mockReturnValueOnce({ entry: { data: new Uint8Array() } }); 205 | client.registry.getEntry = mockedFn; 206 | await expect(await client.dbV2.getJSON(publicKey, dataKey)).toEqual({ data: null, dataLink: null }); 207 | }); 208 | }); 209 | 210 | describe("setJSON", () => { 211 | let mock = MockAdapter; 212 | 213 | beforeEach(() => { 214 | mock = new MockAdapter(axios); 215 | mock.onHead(portalUrl).replyOnce(200, {}, { "skynet-portal-api": portalUrl }); 216 | mock.resetHistory(); 217 | // mock a successful upload 218 | mock.onPost(uploadUrl).reply(200, { skylink, merkleroot, bitfield }); 219 | }); 220 | 221 | it("should perform an upload, lookup and registry update", async () => { 222 | // mock a successful registry update 223 | mock.onPost(registryPostUrl).replyOnce(204); 224 | 225 | // set data 226 | const { data: returnedData, dataLink: returnedSkylink } = await client.dbV2.setJSON(privateKey, dataKey, jsonData); 227 | expect(returnedData).toEqual(jsonData); 228 | expect(returnedSkylink).toEqual(sialink); 229 | 230 | // assert our request history contains the expected amount of requests 231 | expect(mock.history.get.length).toBe(0); 232 | expect(mock.history.post.length).toBe(2); 233 | 234 | const data = JSON.parse(mock.history.post[1].data); 235 | expect(data).toBeDefined(); 236 | expect(data.revision).toBeGreaterThanOrEqual(12); 237 | }); 238 | 239 | it("should use a revision number of 0 if the entry is not cached", async () => { 240 | // mock a successful registry update 241 | mock.onPost(registryPostUrl).replyOnce(204); 242 | 243 | // call `setJSON` on the client 244 | await client.dbV2.setJSON(privateKey, "inexistent entry", jsonData); 245 | 246 | // assert our request history contains the expected amount of requests 247 | expect(mock.history.get.length).toBe(0); 248 | expect(mock.history.post.length).toBe(2); 249 | 250 | const data = JSON.parse(mock.history.post[1].data); 251 | expect(data).toBeDefined(); 252 | expect(data.revision).toEqual(0); 253 | }); 254 | 255 | it("should fail if the entry has the maximum allowed revision", async () => { 256 | const dataKey = "maximum revision"; 257 | const cachedRevisionEntry = await client.dbV2.revisionNumberCache.getRevisionAndMutexForEntry(publicKey, dataKey); 258 | cachedRevisionEntry.revision = MAX_REVISION; 259 | 260 | // mock a successful registry update 261 | mock.onPost(registryPostUrl).replyOnce(204); 262 | 263 | // Try to set data, should fail. 264 | await expect(client.dbV2.setJSON(privateKey, dataKey, entryData)).rejects.toThrowError( 265 | "Current entry already has maximum allowed revision, could not update the entry" 266 | ); 267 | }); 268 | 269 | it("Should throw an error if the private key is not hex-encoded", async () => { 270 | await expect(client.dbV2.setJSON("foo", dataKey, {})).rejects.toThrowError("bad secret key size"); 271 | }); 272 | 273 | it("Should throw an error if the data key is not provided", async () => { 274 | // @ts-expect-error We do not pass the data key on purpose. 275 | await expect(client.dbV2.setJSON(privateKey)).rejects.toThrowError( 276 | "Expected parameter field 'entry.dataKey' to be type 'string', was type 'undefined'" 277 | ); 278 | }); 279 | 280 | it("Should throw an error if the json is not provided", async () => { 281 | // @ts-expect-error We do not pass the json on purpose. 282 | await expect(client.dbV2.setJSON(privateKey, dataKey)).rejects.toThrowError("Request failed with status code 404"); 283 | }); 284 | 285 | it("Should not update the cached revision if the registry update fails.", async () => { 286 | const dataKey = "registry failure"; 287 | const json = { foo: "bar" }; 288 | 289 | // mock a successful registry update 290 | mock.onPost(registryPostUrl).replyOnce(204); 291 | 292 | await client.dbV2.setJSON(privateKey, dataKey, json); 293 | 294 | const cachedRevisionEntry = await client.dbV2.revisionNumberCache.getRevisionAndMutexForEntry(publicKey, dataKey); 295 | const revision1 = cachedRevisionEntry.revision; 296 | 297 | // mock a failed registry update 298 | mock.onPost(registryPostUrl).replyOnce(400, JSON.stringify({ message: "foo" })); 299 | 300 | await expect(client.dbV2.setJSON(privateKey, dataKey, json)).rejects.toEqual( 301 | new Error("Request failed with status code 400: foo") 302 | ); 303 | 304 | const revision2 = cachedRevisionEntry.revision; 305 | 306 | expect(revision1.toString()).toEqual(revision2.toString()); 307 | }); 308 | }); 309 | 310 | describe("setEntryData", () => { 311 | it("should throw if trying to set entry data > 70 bytes", async () => { 312 | await expect( 313 | client.dbV2.setEntryData(privateKey, dataKey, new Uint8Array(MAX_ENTRY_LENGTH + 1)) 314 | ).rejects.toThrowError( 315 | "Expected parameter 'data' to be 'Uint8Array' of length <= 70, was length 71, was type 'object', value '0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0'" 316 | ); 317 | }); 318 | 319 | it("should throw if trying to set the deletion entry data", async () => { 320 | await expect(client.dbV2.setEntryData(privateKey, dataKey, DELETION_ENTRY_DATA)).rejects.toThrowError( 321 | "Tried to set 'Uint8Array' entry data that is the deletion sentinel ('Uint8Array(RAW_SKYLINK_SIZE)'), please use the 'deleteEntryData' method instead`" 322 | ); 323 | }); 324 | }); 325 | 326 | describe("checkCachedDataLink", () => { 327 | const differentSkylink = "XABvi7JtJbQSMAcDwnUnmp2FKDPjg8_tTTFP4BwMSxVdEg"; 328 | const inputs = [ 329 | [skylink, undefined, false], 330 | [skylink, skylink, true], 331 | [skylink, differentSkylink, false], 332 | [differentSkylink, skylink, false], 333 | ]; 334 | 335 | it.each(inputs)("checkCachedDataLink(%s, %s) should return %s", (rawDataLink, cachedDataLink, output) => { 336 | expect(checkCachedDataLink(rawDataLink, cachedDataLink)).toEqual(output); 337 | }); 338 | 339 | it("Should throw on invalid cachedDataLink", () => { 340 | expect(() => checkCachedDataLink(skylink, "asdf")).toThrowError( 341 | "Expected optional parameter 'cachedDataLink' to be valid skylink of type 'string', was type 'string', value 'asdf'" 342 | ); 343 | }); 344 | }); 345 | 346 | // REGRESSION TESTS: By creating a gap between setJSON and getJSON, a user 347 | // could call getJSON, get outdated data, then call setJSON, and overwrite 348 | // more up to date data with outdated data, but still use a high enough 349 | // revision number. 350 | // 351 | // The fix is that you cannot retrieve the revision number while calling 352 | // setJSON. You have to use the same revision number that you had when you 353 | // called getJSON. 354 | describe("getJSON/setJSON data race regression unit tests", () => { 355 | let mock = MockAdapter; 356 | 357 | beforeEach(() => { 358 | // Add a delay to responses to simulate actual calls that use the network. 359 | mock = new MockAdapter(axios, { delayResponse: 100 }); 360 | mock.reset(); 361 | mock.onHead(portalUrl).replyOnce(200, {}, { "skynet-portal-api": portalUrl }); 362 | }); 363 | 364 | const skylinkOld = "XABvi7JtJbQSMAcDwnUnmp2FKDPjg8_tTTFP4BwMSxVdEg"; 365 | const skylinkOldUrl = getSkylinkUrlForPortal(portalUrl, skylinkOld); 366 | const dataOld = toHexString(stringToUint8ArrayUtf8(skylinkOld)); // hex-encoded skylink 367 | const revisionOld = 0; 368 | const entryDataOld = { 369 | data: dataOld, 370 | revision: revisionOld, 371 | signature: 372 | "921d30e860d51f13d1065ea221b29fc8d11cfe7fa0e32b5d5b8e13bee6f91cfa86fe6b12ca4cef7a90ba52d2c50efb62b241f383e9d7bb264558280e564faa0f", 373 | }; 374 | const headersOld = { ...headers, "skynet-skylink": skylinkOld }; 375 | 376 | const skylinkNew = skylink; 377 | const skylinkNewUrl = skylinkUrl; 378 | const dataNew = data; // hex-encoded skylink 379 | const revisionNew = 1; 380 | const entryDataNew = { 381 | data: dataNew, 382 | revision: revisionNew, 383 | signature: 384 | "2a9889915f06d414e8cde51eb17db565410d20b2b50214e8297f7f4a0cb5c77e0edc62a319607dfaa042e0cc16ed0d7e549cca2abd11c2f86a335009936f150d", 385 | }; 386 | const headersNew = { ...headers, "skynet-skylink": skylinkNew }; 387 | 388 | const jsonOld = { message: 1 }; 389 | const jsonNew = { message: 2 }; 390 | const skynetJsonOld = { _data: jsonOld, _v: 2 }; 391 | const skynetJsonNew = { _data: jsonNew, _v: 2 }; 392 | 393 | const concurrentAccessError = "Concurrent access prevented in SkyDB"; 394 | const higherRevisionError = "A higher revision number for this userID and path is already cached"; 395 | 396 | it("should not get old data when getJSON and setJSON are called simultaneously on the same client and getJSON doesn't fail", async () => { 397 | // Create a new client with a fresh revision cache. 398 | const client = new SkynetClient(portalUrl); 399 | 400 | // Mock setJSON with the old skylink. 401 | mock.onPost(uploadUrl).replyOnce(200, { skylink: skylinkOld, merkleroot, bitfield }); 402 | mock.onPost(registryPostUrl).replyOnce(204); 403 | 404 | // Set the data. 405 | await client.dbV2.setJSON(privateKey, dataKey, jsonOld); 406 | 407 | // Mock getJSON with the new entry data and the new skylink. 408 | mock.onGet(registryGetUrl).replyOnce(200, JSON.stringify(entryDataNew)); 409 | mock.onGet(skylinkNewUrl).replyOnce(200, skynetJsonNew, headers); 410 | 411 | // Mock setJSON with the new skylink. 412 | mock.onPost(uploadUrl).replyOnce(200, { skylink: skylinkNew, merkleroot, bitfield }); 413 | mock.onPost(registryPostUrl).replyOnce(204); 414 | 415 | // Try to invoke the data race. 416 | // Get the data while also calling setJSON. 417 | // 418 | // Use Promise.allSettled to wait for all promises to finish, or some mocked 419 | // requests will hang around and interfere with the later tests. 420 | const settledResults = await Promise.allSettled([ 421 | client.dbV2.getJSON(publicKey, dataKey), 422 | client.dbV2.setJSON(privateKey, dataKey, jsonNew), 423 | ]); 424 | 425 | let data = JsonData | null; 426 | try { 427 | const values = getSettledValues(settledResults); 428 | data = values[0].data; 429 | } catch (e) { 430 | // If any promises were rejected, check the error message. 431 | if (e.message.includes(concurrentAccessError)) { 432 | // The data race condition was avoided and we received the expected 433 | // error. Return from test early. 434 | return; 435 | } 436 | 437 | throw e; 438 | } 439 | 440 | // Data race did not occur, getJSON should have latest JSON. 441 | expect(data).toEqual(jsonNew); 442 | 443 | // assert our request history contains the expected amount of requests 444 | expect(mock.history.get.length).toBe(2); 445 | expect(mock.history.post.length).toBe(4); 446 | }); 447 | 448 | it("should not get old data when getJSON and setJSON are called simultaneously on different clients and getJSON doesn't fail", async () => { 449 | // Create two new clients with a fresh revision cache. 450 | const client1 = new SkynetClient(portalUrl); 451 | const client2 = new SkynetClient(portalUrl); 452 | 453 | // Mock setJSON with the old skylink. 454 | mock.onPost(uploadUrl).replyOnce(200, { skylink: skylinkOld, merkleroot, bitfield }); 455 | mock.onPost(registryPostUrl).replyOnce(204); 456 | 457 | // Set the data. 458 | await client1.dbV2.setJSON(privateKey, dataKey, jsonOld); 459 | 460 | // Mock getJSON with the new entry data and the new skylink. 461 | mock.onGet(registryGetUrl).replyOnce(200, JSON.stringify(entryDataNew)); 462 | mock.onGet(skylinkNewUrl).replyOnce(200, skynetJsonNew, headersNew); 463 | 464 | // Mock setJSON with the new skylink. 465 | mock.onPost(uploadUrl).replyOnce(200, { skylink: skylinkNew, merkleroot, bitfield }); 466 | mock.onPost(registryPostUrl).replyOnce(204); 467 | 468 | // Try to invoke the data race. 469 | // Get the data while also calling setJSON. 470 | // 471 | // Use Promise.allSettled to wait for all promises to finish, or some mocked requests will hang around and interfere with the later tests. 472 | const settledResults = await Promise.allSettled([ 473 | client1.dbV2.getJSON(publicKey, dataKey), 474 | client2.dbV2.setJSON(privateKey, dataKey, jsonNew), 475 | ]); 476 | 477 | let data = JsonData | null; 478 | try { 479 | const values = getSettledValues(settledResults); 480 | data = values[0].data; 481 | } catch (e) { 482 | // If any promises were rejected, check the error message. 483 | if (e.message.includes(higherRevisionError)) { 484 | // The data race condition was avoided and we received the expected 485 | // error. Return from test early. 486 | return; 487 | } 488 | 489 | throw e; 490 | } 491 | 492 | // Data race did not occur, getJSON should have latest JSON. 493 | expect(data).toEqual(jsonNew); 494 | 495 | // assert our request history contains the expected amount of requests. 496 | expect(mock.history.get.length).toBe(2); 497 | expect(mock.history.post.length).toBe(4); 498 | }); 499 | 500 | it("should not mess up cache when two setJSON calls are made simultaneously and one fails", async () => { 501 | // Create a new client with a fresh revision cache. 502 | const client = new SkynetClient(portalUrl); 503 | 504 | // Mock a successful setJSON. 505 | mock.onPost(uploadUrl).replyOnce(200, { skylink: skylinkOld, merkleroot, bitfield }); 506 | mock.onPost(registryPostUrl).replyOnce(204); 507 | 508 | // Use Promise.allSettled to wait for all promises to finish, or some mocked 509 | // requests will hang around and interfere with the later tests. 510 | const values = await Promise.allSettled([ 511 | client.dbV2.setJSON(privateKey, dataKey, jsonOld), 512 | client.dbV2.setJSON(privateKey, dataKey, jsonOld), 513 | ]); 514 | 515 | try { 516 | getSettledValues < JSONResponse > values; 517 | } catch (e) { 518 | if (e.message.includes(concurrentAccessError)) { 519 | // The data race condition was avoided and we received the expected 520 | // error. Return from test early. 521 | return; 522 | } 523 | 524 | throw e; 525 | } 526 | 527 | const cachedRevisionEntry = await client.dbV2.revisionNumberCache.getRevisionAndMutexForEntry(publicKey, dataKey); 528 | expect(cachedRevisionEntry.revision.toString()).toEqual("0"); 529 | 530 | // Make a getJSON call. 531 | mock.onGet(registryGetUrl).replyOnce(200, JSON.stringify(entryDataOld)); 532 | mock.onGet(skylinkOldUrl).replyOnce(200, skynetJsonOld, headersOld); 533 | const { data: receivedJson1 } = await client.dbV2.getJSON(publicKey, dataKey); 534 | 535 | expect(receivedJson1).toEqual(jsonOld); 536 | 537 | // Make another setJSON call - it should still work. 538 | mock.onPost(uploadUrl).replyOnce(200, { skylink: skylinkNew, merkleroot, bitfield }); 539 | mock.onPost(registryPostUrl).replyOnce(204); 540 | await client.dbV2.setJSON(privateKey, dataKey, jsonNew); 541 | 542 | expect(cachedRevisionEntry.revision.toString()).toEqual("1"); 543 | 544 | // Make a getJSON call. 545 | mock.onGet(registryGetUrl).replyOnce(200, JSON.stringify(entryDataNew)); 546 | mock.onGet(skylinkNewUrl).replyOnce(200, skynetJsonNew, headersNew); 547 | const { data: receivedJson2 } = await client.dbV2.getJSON(publicKey, dataKey); 548 | 549 | expect(receivedJson2).toEqual(jsonNew); 550 | 551 | expect(mock.history.get.length).toBe(4); 552 | expect(mock.history.post.length).toBe(4); 553 | }); 554 | 555 | it("should not mess up cache when two setJSON calls are made simultaneously on different clients and one fails", async () => { 556 | // Create two new clients with a fresh revision cache. 557 | const client1 = new SkynetClient(portalUrl); 558 | const client2 = new SkynetClient(portalUrl); 559 | 560 | // Run two simultaneous setJSONs on two different clients - one should work, 561 | // one should fail due to bad revision number. 562 | 563 | // Mock a successful setJSON. 564 | mock.onPost(uploadUrl).replyOnce(200, { skylink: skylinkOld, merkleroot, bitfield }); 565 | mock.onPost(registryPostUrl).replyOnce(204); 566 | // Mock a failed setJSON (bad revision number). 567 | mock.onPost(uploadUrl).replyOnce(200, { skylink: skylinkOld, merkleroot, bitfield }); 568 | mock.onPost(registryPostUrl).replyOnce(400); 569 | 570 | // Use Promise.allSettled to wait for all promises to finish, or some mocked 571 | // requests will hang around and interfere with the later tests. 572 | const values = await Promise.allSettled([ 573 | client1.dbV2.setJSON(privateKey, dataKey, jsonOld), 574 | client2.dbV2.setJSON(privateKey, dataKey, jsonOld), 575 | ]); 576 | 577 | let successClient; 578 | let failClient; 579 | if (values[0].status === "rejected") { 580 | successClient = client2; 581 | failClient = client1; 582 | } else { 583 | successClient = client1; 584 | failClient = client2; 585 | } 586 | 587 | // Test that the client that succeeded has a consistent cache. 588 | 589 | const cachedRevisionEntrySuccess = await successClient.dbV2.revisionNumberCache.getRevisionAndMutexForEntry( 590 | publicKey, 591 | dataKey 592 | ); 593 | expect(cachedRevisionEntrySuccess.revision.toString()).toEqual("0"); 594 | 595 | // Make a getJSON call. 596 | mock.onGet(registryGetUrl).replyOnce(200, JSON.stringify(entryDataOld)); 597 | mock.onGet(skylinkOldUrl).replyOnce(200, skynetJsonOld, headersOld); 598 | const { data: receivedJson1 } = await successClient.dbV2.getJSON(publicKey, dataKey); 599 | 600 | expect(receivedJson1).toEqual(jsonOld); 601 | 602 | // Make another setJSON call - it should still work. 603 | mock.onPost(uploadUrl).replyOnce(200, { skylink: skylinkNew, merkleroot, bitfield }); 604 | mock.onPost(registryPostUrl).replyOnce(204); 605 | await successClient.dbV2.setJSON(privateKey, dataKey, jsonNew); 606 | 607 | expect(cachedRevisionEntrySuccess.revision.toString()).toEqual("1"); 608 | 609 | // Make a getJSON call. 610 | mock.onGet(registryGetUrl).replyOnce(200, JSON.stringify(entryDataNew)); 611 | mock.onGet(skylinkNewUrl).replyOnce(200, skynetJsonNew, headersNew); 612 | const { data: receivedJson2 } = await successClient.dbV2.getJSON(publicKey, dataKey); 613 | 614 | expect(receivedJson2).toEqual(jsonNew); 615 | 616 | // Test that the client that failed has a consistent cache. 617 | 618 | const cachedRevisionEntryFail = await failClient.dbV2.revisionNumberCache.getRevisionAndMutexForEntry( 619 | publicKey, 620 | dataKey 621 | ); 622 | expect(cachedRevisionEntryFail.revision.toString()).toEqual("-1"); 623 | 624 | // Make a getJSON call. 625 | mock.onGet(registryGetUrl).replyOnce(200, JSON.stringify(entryDataOld)); 626 | mock.onGet(skylinkOldUrl).replyOnce(200, skynetJsonOld, headersOld); 627 | const { data: receivedJsonFail1 } = await failClient.dbV2.getJSON(publicKey, dataKey); 628 | 629 | expect(receivedJsonFail1).toEqual(jsonOld); 630 | 631 | // Make another setJSON call - it should still work. 632 | mock.onPost(uploadUrl).replyOnce(200, { skylink: skylinkNew, merkleroot, bitfield }); 633 | mock.onPost(registryPostUrl).replyOnce(204); 634 | await failClient.dbV2.setJSON(privateKey, dataKey, jsonNew); 635 | 636 | expect(cachedRevisionEntrySuccess.revision.toString()).toEqual("1"); 637 | 638 | // Make a getJSON call. 639 | mock.onGet(registryGetUrl).replyOnce(200, JSON.stringify(entryDataNew)); 640 | mock.onGet(skylinkNewUrl).replyOnce(200, skynetJsonNew, headersNew); 641 | const { data: receivedJsonFail2 } = await failClient.dbV2.getJSON(publicKey, dataKey); 642 | 643 | expect(receivedJsonFail2).toEqual(jsonNew); 644 | 645 | // Check final request counts. 646 | 647 | expect(mock.history.get.length).toBe(8); 648 | expect(mock.history.post.length).toBe(8); 649 | }); 650 | }); 651 | -------------------------------------------------------------------------------- /src/skylink_sia.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | /** 4 | * The raw size in bytes of the data that gets put into a link. 5 | */ 6 | const RAW_SKYLINK_SIZE = 34; 7 | 8 | /** 9 | * The string length of the Skylink after it has been encoded using base32. 10 | */ 11 | const BASE32_ENCODED_SKYLINK_SIZE = 55; 12 | 13 | /** 14 | * The string length of the Skylink after it has been encoded using base64. 15 | */ 16 | const BASE64_ENCODED_SKYLINK_SIZE = 46; 17 | 18 | /** 19 | * Returned when a string could not be decoded into a Skylink due to it having 20 | * an incorrect size. 21 | */ 22 | const ERR_SKYLINK_INCORRECT_SIZE = "skylink has incorrect size"; 23 | 24 | module.exports = { 25 | RAW_SKYLINK_SIZE, 26 | BASE32_ENCODED_SKYLINK_SIZE, 27 | BASE64_ENCODED_SKYLINK_SIZE, 28 | ERR_SKYLINK_INCORRECT_SIZE, 29 | }; 30 | -------------------------------------------------------------------------------- /src/upload.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const FormData = require("form-data"); 4 | const fs = require("fs"); 5 | const p = require("path"); 6 | 7 | const { DetailedError, Upload } = require("@skynetlabs/tus-js-client"); 8 | 9 | const { buildRequestHeaders, SkynetClient } = require("./client"); 10 | const { DEFAULT_UPLOAD_OPTIONS, TUS_CHUNK_SIZE } = require("./defaults"); 11 | const { getFileMimeType, makeUrl, walkDirectory, uriSkynetPrefix, formatSkylink } = require("./utils"); 12 | const { throwValidationError, validateInteger } = require("./utils_validation"); 13 | const { splitSizeIntoChunkAlignedParts } = require("./utils_testing"); 14 | 15 | /** 16 | * Uploads in-memory data to Skynet. 17 | * 18 | * @param {string|Buffer} data - The data to upload, either a string or raw bytes. 19 | * @param {string} filename - The filename to use on Skynet. 20 | * @param {Object} [customOptions={}] - Configuration options. 21 | * @returns - The skylink. 22 | */ 23 | SkynetClient.prototype.uploadData = async function (data, filename, customOptions = {}) { 24 | const opts = { ...DEFAULT_UPLOAD_OPTIONS, ...this.customOptions, ...customOptions }; 25 | 26 | const sizeInBytes = data.length; 27 | 28 | if (sizeInBytes < opts.largeFileSize * opts.chunkSizeMultiplier) { 29 | return await uploadSmallFile(this, data, filename, opts); 30 | } 31 | return await uploadLargeFile(this, data, filename, sizeInBytes, opts); 32 | }; 33 | 34 | SkynetClient.prototype.uploadFile = async function (path, customOptions = {}) { 35 | const opts = { ...DEFAULT_UPLOAD_OPTIONS, ...this.customOptions, ...customOptions }; 36 | 37 | const stat = await fs.promises.stat(path); 38 | const sizeInBytes = stat.size; 39 | const filename = opts.customFilename ? opts.customFilename : p.basename(path); 40 | const stream = fs.createReadStream(path); 41 | 42 | if (sizeInBytes < opts.largeFileSize * opts.chunkSizeMultiplier) { 43 | return await uploadSmallFile(this, stream, filename, opts); 44 | } 45 | return await uploadLargeFile(this, stream, filename, sizeInBytes, opts); 46 | }; 47 | 48 | async function uploadSmallFile(client, stream, filename, opts) { 49 | const params = {}; 50 | if (opts.dryRun) params.dryrun = true; 51 | 52 | const formData = new FormData(); 53 | formData.append(opts.portalFileFieldname, stream, filename); 54 | const headers = formData.getHeaders(); 55 | 56 | const response = await client.executeRequest({ 57 | ...opts, 58 | method: "post", 59 | data: formData, 60 | headers, 61 | params, 62 | }); 63 | 64 | const responsedSkylink = response.data.skylink; 65 | // Format the skylink. 66 | const skylink = formatSkylink(responsedSkylink); 67 | 68 | return `${skylink}`; 69 | } 70 | 71 | async function uploadLargeFile(client, stream, filename, filesize, opts) { 72 | let upload = null; 73 | let uploadIsRunning = false; 74 | 75 | // Validation. 76 | if ( 77 | opts.staggerPercent !== undefined && 78 | opts.staggerPercent !== null && 79 | (opts.staggerPercent < 0 || opts.staggerPercent > 100) 80 | ) { 81 | throw new Error(`Expected 'staggerPercent' option to be between 0 and 100, was '${opts.staggerPercent}`); 82 | } 83 | if (opts.chunkSizeMultiplier < 1) { 84 | throwValidationError("opts.chunkSizeMultiplier", opts.chunkSizeMultiplier, "option", "greater than or equal to 1"); 85 | } 86 | // It's crucial that we only use strict multiples of the base chunk size. 87 | validateInteger("opts.chunkSizeMultiplier", opts.chunkSizeMultiplier, "option"); 88 | if (opts.numParallelUploads < 1) { 89 | throwValidationError("opts.numParallelUploads", opts.numParallelUploads, "option", "greater than or equal to 1"); 90 | } 91 | validateInteger("opts.numParallelUploads", opts.numParallelUploads, "option"); 92 | 93 | const url = makeUrl(opts.portalUrl, opts.endpointLargeUpload); 94 | // Build headers. 95 | const headers = buildRequestHeaders({}, opts.customUserAgent, opts.customCookie); 96 | 97 | // Set the number of parallel uploads as well as the part-split function. Note 98 | // that each part has to be chunk-aligned, so we may limit the number of 99 | // parallel uploads. 100 | let parallelUploads = opts.numParallelUploads; 101 | const chunkSize = TUS_CHUNK_SIZE * opts.chunkSizeMultiplier; 102 | // If we use `parallelUploads: 1` then these have to be set to null. 103 | let splitSizeIntoParts = null; 104 | let staggerPercent = null; 105 | 106 | // Limit the number of parallel uploads if some parts would end up empty, 107 | // e.g. 50mib would be split into 1 chunk-aligned part, one unaligned part, 108 | // and one empty part. 109 | const numChunks = Math.ceil(filesize / TUS_CHUNK_SIZE); 110 | if (parallelUploads > numChunks) { 111 | parallelUploads = numChunks; 112 | } 113 | 114 | if (parallelUploads > 1) { 115 | // Officially doing a parallel upload, set the parallel upload options. 116 | splitSizeIntoParts = (totalSize, partCount) => { 117 | return splitSizeIntoChunkAlignedParts(totalSize, partCount, chunkSize); 118 | }; 119 | staggerPercent = opts.staggerPercent; 120 | } 121 | 122 | const askToResumeUpload = function (previousUploads, currentUpload) { 123 | if (previousUploads.length === 0) return; 124 | 125 | let text = "You tried to upload this file previously at these times:\n\n"; 126 | previousUploads.forEach((previousUpload, index) => { 127 | text += `[${index}] ${previousUpload.creationTime}\n`; 128 | }); 129 | text += "\nEnter the corresponding number to resume an upload or press Cancel to start a new upload"; 130 | 131 | const answer = text + "yes"; 132 | const index = parseInt(answer, 10); 133 | 134 | if (!Number.isNaN(index) && previousUploads[index]) { 135 | currentUpload.resumeFromPreviousUpload(previousUploads[index]); 136 | } 137 | }; 138 | 139 | const reset = function () { 140 | upload = null; 141 | uploadIsRunning = false; 142 | }; 143 | 144 | const onProgress = 145 | opts.onUploadProgress && 146 | function (bytesSent, bytesTotal) { 147 | const progress = bytesSent / bytesTotal; 148 | 149 | // @ts-expect-error TS complains. 150 | opts.onUploadProgress(progress, { loaded: bytesSent, total: bytesTotal }); 151 | }; 152 | 153 | return new Promise((resolve, reject) => { 154 | const tusOpts = { 155 | endpoint: url, 156 | chunkSize: chunkSize, 157 | retryDelays: opts.retryDelays, 158 | metadata: { 159 | filename, 160 | filetype: getFileMimeType(filename), 161 | }, 162 | parallelUploads, 163 | staggerPercent, 164 | splitSizeIntoParts, 165 | headers, 166 | onProgress, 167 | onError: (error = Error | DetailedError) => { 168 | // Return error body rather than entire error. 169 | const res = error.originalResponse; 170 | const newError = res ? new Error(res.getBody().trim()) || error : error; 171 | reset(); 172 | reject(newError); 173 | }, 174 | onSuccess: async () => { 175 | if (!upload.url) { 176 | reject(new Error("'upload.url' was not set")); 177 | return; 178 | } 179 | 180 | // Call HEAD to get the metadata, including the skylink. 181 | const resp = await client.executeRequest({ 182 | ...opts, 183 | url: upload.url, 184 | endpointPath: opts.endpointLargeUpload, 185 | method: "head", 186 | headers: { ...headers, "Tus-Resumable": "1.0.0" }, 187 | }); 188 | const skylink = resp.headers["skynet-skylink"]; 189 | reset(); 190 | resolve(`${uriSkynetPrefix}${skylink}`); 191 | }, 192 | }; 193 | 194 | upload = new Upload(stream, tusOpts); 195 | upload.findPreviousUploads().then((previousUploads) => { 196 | askToResumeUpload(previousUploads, upload); 197 | 198 | upload.start(); 199 | uploadIsRunning = true; 200 | if (uploadIsRunning === false) { 201 | console.log("Upload Is Running: " + uploadIsRunning); 202 | } 203 | }); 204 | }); 205 | } 206 | 207 | /** 208 | * Uploads a directory from the local filesystem to Skynet. 209 | * 210 | * @param {string} path - The path of the directory to upload. 211 | * @param {Object} [customOptions] - Configuration options. 212 | * @param {Object} [customOptions.disableDefaultPath=false] - If the value of `disableDefaultPath` is `true` no content is served if the skyfile is accessed at its root path. 213 | * @returns - The skylink. 214 | */ 215 | SkynetClient.prototype.uploadDirectory = async function (path, customOptions = {}) { 216 | const opts = { ...DEFAULT_UPLOAD_OPTIONS, ...this.customOptions, ...customOptions }; 217 | 218 | // Check if there is a directory at given path. 219 | const stat = await fs.promises.stat(path); 220 | if (!stat.isDirectory()) { 221 | throw new Error(`Given path is not a directory: ${path}`); 222 | } 223 | 224 | const formData = new FormData(); 225 | path = p.resolve(path); 226 | let basepath = path; 227 | // Ensure the basepath ends in a slash. 228 | if (!basepath.endsWith("/")) { 229 | basepath += "/"; 230 | // Normalize the slash on non-Unix filesystems. 231 | basepath = p.normalize(basepath); 232 | } 233 | 234 | for (const file of walkDirectory(path)) { 235 | // Remove the dir path from the start of the filename if it exists. 236 | let filename = file; 237 | if (file.startsWith(basepath)) { 238 | filename = file.replace(basepath, ""); 239 | } 240 | formData.append(opts.portalDirectoryFileFieldname, fs.createReadStream(file), { filepath: filename }); 241 | } 242 | 243 | // Use either the custom dirname, or the last portion of the path. 244 | let filename = opts.customDirname || p.basename(path); 245 | if (filename.startsWith("/")) { 246 | filename = filename.slice(1); 247 | } 248 | const params = { filename }; 249 | if (opts.tryFiles) { 250 | params.tryfiles = JSON.stringify(opts.tryFiles); 251 | } 252 | if (opts.errorPages) { 253 | params.errorpages = JSON.stringify(opts.errorPages); 254 | } 255 | if (opts.disableDefaultPath) { 256 | params.disableDefaultPath = true; 257 | } 258 | 259 | if (opts.dryRun) params.dryrun = true; 260 | 261 | const response = await this.executeRequest({ 262 | ...opts, 263 | method: "post", 264 | data: formData, 265 | headers: formData.getHeaders(), 266 | params, 267 | }); 268 | 269 | return `${uriSkynetPrefix}${response.data.skylink}`; 270 | }; 271 | -------------------------------------------------------------------------------- /src/upload.test.js: -------------------------------------------------------------------------------- 1 | const axios = require("axios"); 2 | const fs = require("fs"); 3 | const tmp = require("tmp"); 4 | 5 | const { SkynetClient, defaultPortalUrl, uriSkynetPrefix } = require("../index"); 6 | const { TUS_CHUNK_SIZE } = require("./defaults"); 7 | const { splitSizeIntoChunkAlignedParts } = require("./utils_testing"); 8 | 9 | jest.mock("axios"); 10 | 11 | const portalUrl = defaultPortalUrl(); 12 | const skylink = "XABvi7JtJbQSMAcDwnUnmp2FKDPjg8_tTTFP4BwMSxVdEg"; 13 | const sialink = `${uriSkynetPrefix}${skylink}`; 14 | const client = new SkynetClient(); 15 | 16 | describe("uploadFile", () => { 17 | const filename = "testdata/file1.txt"; 18 | 19 | beforeEach(() => { 20 | axios.mockResolvedValue({ data: { skylink } }); 21 | }); 22 | 23 | it("should send post request to default portal", async () => { 24 | await client.uploadFile(filename); 25 | 26 | expect(axios).toHaveBeenCalledWith( 27 | expect.objectContaining({ 28 | url: `${portalUrl}/skynet/skyfile`, 29 | data: expect.objectContaining({ 30 | _streams: expect.arrayContaining([ 31 | expect.stringContaining( 32 | 'Content-Disposition: form-data; name="file"; filename="file1.txt"\r\nContent-Type: text/plain' 33 | ), 34 | ]), 35 | }), 36 | headers: expect.objectContaining({ "content-type": expect.stringContaining("multipart/form-data") }), 37 | params: expect.anything(), 38 | }) 39 | ); 40 | }); 41 | 42 | it("should send post request to client portal", async () => { 43 | const newPortalUrl = "https://siasky.net"; 44 | const client = new SkynetClient(newPortalUrl); 45 | 46 | await client.uploadFile(filename); 47 | 48 | expect(axios).toHaveBeenCalledWith( 49 | expect.objectContaining({ 50 | url: `${newPortalUrl}/skynet/skyfile`, 51 | data: expect.objectContaining({ 52 | _streams: expect.arrayContaining([ 53 | expect.stringContaining( 54 | 'Content-Disposition: form-data; name="file"; filename="file1.txt"\r\nContent-Type: text/plain' 55 | ), 56 | ]), 57 | }), 58 | headers: expect.objectContaining({ "content-type": expect.stringContaining("multipart/form-data") }), 59 | params: expect.anything(), 60 | }) 61 | ); 62 | }); 63 | 64 | it("should use custom upload options if defined", async () => { 65 | await client.uploadFile(filename, { 66 | portalUrl: "https://localhost", 67 | endpointPath: "/skynet/file", 68 | portalFileFieldname: "filetest", 69 | customFilename: "test.jpg", 70 | dryRun: true, 71 | }); 72 | 73 | expect(axios).toHaveBeenCalledWith( 74 | expect.objectContaining({ 75 | url: `https://localhost/skynet/file`, 76 | data: expect.objectContaining({ 77 | _streams: expect.arrayContaining([ 78 | expect.stringContaining('Content-Disposition: form-data; name="filetest"; filename="test.jpg"'), 79 | ]), 80 | }), 81 | headers: expect.anything(), 82 | params: { dryrun: true }, 83 | }) 84 | ); 85 | }); 86 | 87 | it("should use custom connection options if defined on the client", async () => { 88 | const client = new SkynetClient("", { 89 | APIKey: "foobar", 90 | skynetApiKey: "api-key", 91 | customUserAgent: "Sia-Agent", 92 | customCookie: "skynet-jwt=foo", 93 | }); 94 | 95 | await client.uploadFile(filename); 96 | 97 | expect(axios).toHaveBeenCalledWith( 98 | expect.objectContaining({ 99 | url: `${portalUrl}/skynet/skyfile`, 100 | data: expect.objectContaining({ 101 | _streams: expect.arrayContaining([ 102 | expect.stringContaining(`Content-Disposition: form-data; name="file"; filename="file1.txt"`), 103 | ]), 104 | }), 105 | auth: { username: "", password: "foobar" }, 106 | headers: expect.objectContaining({ 107 | "User-Agent": "Sia-Agent", 108 | Cookie: "skynet-jwt=foo", 109 | "Skynet-Api-Key": "api-key", 110 | }), 111 | params: expect.anything(), 112 | }) 113 | ); 114 | }); 115 | 116 | it("should use custom connection options if defined on the API call", async () => { 117 | const client = new SkynetClient("", { 118 | APIKey: "foobar", 119 | customUserAgent: "Sia-Agent", 120 | customCookie: "skynet-jwt=foo", 121 | }); 122 | 123 | await client.uploadFile(filename, { 124 | APIKey: "barfoo", 125 | customUserAgent: "Sia-Agent-2", 126 | customCookie: "skynet-jwt=bar", 127 | }); 128 | 129 | expect(axios).toHaveBeenCalledWith( 130 | expect.objectContaining({ 131 | url: `${portalUrl}/skynet/skyfile`, 132 | data: expect.objectContaining({ 133 | _streams: expect.arrayContaining([ 134 | expect.stringContaining(`Content-Disposition: form-data; name="file"; filename="file1.txt"`), 135 | ]), 136 | }), 137 | auth: { username: "", password: "barfoo" }, 138 | headers: expect.objectContaining({ "User-Agent": "Sia-Agent-2", Cookie: "skynet-jwt=bar" }), 139 | params: expect.anything(), 140 | }) 141 | ); 142 | }); 143 | 144 | it("should upload tmp files", async () => { 145 | const file = tmp.fileSync({ postfix: ".json" }); 146 | fs.writeFileSync(file.fd, JSON.stringify("testing")); 147 | 148 | const data = await client.uploadFile(file.name); 149 | 150 | expect(data).toEqual(sialink); 151 | }); 152 | 153 | it("should return skylink on success", async () => { 154 | const data = await client.uploadFile(filename); 155 | 156 | expect(data).toEqual(sialink); 157 | }); 158 | 159 | it("should return skylink on success with dryRun", async () => { 160 | const data = await client.uploadFile(filename, { dryRun: true }); 161 | 162 | expect(data).toEqual(sialink); 163 | }); 164 | }); 165 | 166 | describe("uploadLargeFile", () => { 167 | const file = tmp.fileSync({ postfix: ".txt" }); 168 | fs.writeFileSync(file.fd, Buffer.alloc(1024 * 1024 * 42 * 3).fill(0)); 169 | const filename = file.name; 170 | 171 | it("should throw if the chunk size multiplier is less than 1", async () => { 172 | // @ts-expect-error Using protected method. 173 | await expect(client.uploadFile(filename, { chunkSizeMultiplier: 0 })).rejects.toThrowError( 174 | "Expected option 'opts.chunkSizeMultiplier' to be greater than or equal to 1, was type 'number', value '0'" 175 | ); 176 | }); 177 | 178 | it("should throw if the chunk size multiplier is not an integer", async () => { 179 | // @ts-expect-error Using protected method. 180 | await expect(client.uploadFile(filename, { chunkSizeMultiplier: 1.5 })).rejects.toThrowError( 181 | "Expected option 'opts.chunkSizeMultiplier' to be an integer value, was type 'number', value '1.5'" 182 | ); 183 | }); 184 | 185 | it("should throw if the number of parallel uploads is less than 1", async () => { 186 | // @ts-expect-error Using protected method. 187 | await expect(client.uploadFile(filename, { numParallelUploads: 0.5 })).rejects.toThrowError( 188 | "Expected option 'opts.numParallelUploads' to be greater than or equal to 1, was type 'number', value '0.5'" 189 | ); 190 | }); 191 | 192 | it("should throw if the number of parallel uploads is not an integer", async () => { 193 | // @ts-expect-error Using protected method. 194 | await expect(client.uploadFile(filename, { numParallelUploads: 1.5 })).rejects.toThrowError( 195 | "Expected option 'opts.numParallelUploads' to be an integer value, was type 'number', value '1.5'" 196 | ); 197 | }); 198 | }); 199 | 200 | describe("uploadDirectory", () => { 201 | const dirname = "testdata"; 202 | const directory = ["file1.txt", "file2.txt", "dir1/file3.txt"]; 203 | 204 | beforeEach(() => { 205 | axios.mockResolvedValue({ data: { skylink } }); 206 | }); 207 | 208 | it("should send post request to default portal", async () => { 209 | await client.uploadDirectory(dirname); 210 | 211 | for (const file of directory) { 212 | expect(axios).toHaveBeenCalledWith( 213 | expect.objectContaining({ 214 | url: `${portalUrl}/skynet/skyfile`, 215 | data: expect.objectContaining({ 216 | _streams: expect.arrayContaining([ 217 | expect.stringContaining(`Content-Disposition: form-data; name="files[]"; filename="${file}"`), 218 | ]), 219 | }), 220 | headers: expect.anything(), 221 | params: { filename: dirname }, 222 | }) 223 | ); 224 | } 225 | }); 226 | 227 | // Test that . and .. get resolved as these are not allowed in Sia paths. 228 | it("should resolve paths containing . and ..", async () => { 229 | await client.uploadDirectory(`${dirname}/./../${dirname}/.`); 230 | 231 | expect(axios).toHaveBeenCalledWith( 232 | expect.objectContaining({ 233 | params: { filename: dirname }, 234 | }) 235 | ); 236 | }); 237 | 238 | it("should send post request to client portal", async () => { 239 | const newPortalUrl = "https://siasky.xyz"; 240 | const client = new SkynetClient(newPortalUrl); 241 | 242 | await client.uploadDirectory(dirname); 243 | 244 | for (const file of directory) { 245 | expect(axios).toHaveBeenCalledWith( 246 | expect.objectContaining({ 247 | url: `${newPortalUrl}/skynet/skyfile`, 248 | data: expect.objectContaining({ 249 | _streams: expect.arrayContaining([ 250 | expect.stringContaining(`Content-Disposition: form-data; name="files[]"; filename="${file}"`), 251 | ]), 252 | }), 253 | headers: expect.anything(), 254 | params: { filename: dirname }, 255 | }) 256 | ); 257 | } 258 | }); 259 | 260 | it("should use custom options if defined", async () => { 261 | await client.uploadDirectory(dirname, { 262 | portalUrl: "http://localhost", 263 | endpointPath: "/skynet/file", 264 | portalDirectoryFileFieldname: "filetest", 265 | customDirname: "/testpath", 266 | disableDefaultPath: true, 267 | dryRun: true, 268 | }); 269 | 270 | for (const file of directory) { 271 | expect(axios).toHaveBeenCalledWith( 272 | expect.objectContaining({ 273 | url: `http://localhost/skynet/file`, 274 | data: expect.objectContaining({ 275 | _streams: expect.arrayContaining([ 276 | expect.stringContaining(`Content-Disposition: form-data; name="filetest"; filename="${file}"`), 277 | ]), 278 | }), 279 | headers: expect.anything(), 280 | params: { 281 | filename: "testpath", 282 | disableDefaultPath: true, 283 | dryrun: true, 284 | }, 285 | }) 286 | ); 287 | } 288 | }); 289 | 290 | it("should set errorpages if defined", async () => { 291 | const errorPages = { 404: "404.html", 500: "500.html" }; 292 | 293 | await client.uploadDirectory(dirname, { errorPages }); 294 | 295 | expect(axios).toHaveBeenCalledWith( 296 | expect.objectContaining({ 297 | params: expect.objectContaining({ 298 | errorpages: JSON.stringify(errorPages), 299 | }), 300 | }) 301 | ); 302 | }); 303 | 304 | it("should set tryfiles if defined", async () => { 305 | const tryFiles = ["foo", "bar"]; 306 | 307 | await client.uploadDirectory(dirname, { tryFiles }); 308 | 309 | expect(axios).toHaveBeenCalledWith( 310 | expect.objectContaining({ 311 | params: expect.objectContaining({ 312 | tryfiles: JSON.stringify(tryFiles), 313 | }), 314 | }) 315 | ); 316 | }); 317 | 318 | it("should return single skylink on success", async () => { 319 | const data = await client.uploadDirectory(dirname); 320 | 321 | expect(data).toEqual(sialink); 322 | }); 323 | 324 | it("should return single skylink on success with dryRun", async () => { 325 | const data = await client.uploadDirectory(dirname, { dryRun: true }); 326 | 327 | expect(data).toEqual(sialink); 328 | }); 329 | }); 330 | 331 | describe("uploadData", () => { 332 | const filename = "testdata/file1.txt"; 333 | const data = "asdf"; 334 | 335 | beforeEach(() => { 336 | axios.mockResolvedValue({ data: { skylink } }); 337 | }); 338 | 339 | it("should send post request to default portal", async () => { 340 | const receivedSkylink = await client.uploadData(data, filename); 341 | 342 | expect(axios).toHaveBeenCalledWith( 343 | expect.objectContaining({ 344 | url: `${portalUrl}/skynet/skyfile`, 345 | data: expect.objectContaining({ 346 | _streams: expect.arrayContaining([ 347 | expect.stringContaining( 348 | 'Content-Disposition: form-data; name="file"; filename="file1.txt"\r\nContent-Type: text/plain' 349 | ), 350 | ]), 351 | }), 352 | headers: expect.objectContaining({ "content-type": expect.stringContaining("multipart/form-data") }), 353 | params: expect.anything(), 354 | }) 355 | ); 356 | 357 | expect(receivedSkylink).toEqual(`sia://${skylink}`); 358 | }); 359 | }); 360 | 361 | describe("splitSizeIntoChunkAlignedParts", () => { 362 | const mib = 1 << 20; 363 | const chunk = TUS_CHUNK_SIZE; 364 | const cases = [ 365 | [ 366 | 41 * mib, 367 | 2, 368 | chunk, 369 | [ 370 | { start: 0, end: 40 * mib }, 371 | { start: 40 * mib, end: 41 * mib }, 372 | ], 373 | ], 374 | [ 375 | 80 * mib, 376 | 2, 377 | chunk, 378 | [ 379 | { start: 0, end: 40 * mib }, 380 | { start: 40 * mib, end: 80 * mib }, 381 | ], 382 | ], 383 | [ 384 | 50 * mib, 385 | 2, 386 | chunk, 387 | [ 388 | { start: 0, end: 40 * mib }, 389 | { start: 40 * mib, end: 50 * mib }, 390 | ], 391 | ], 392 | [ 393 | 100 * mib, 394 | 2, 395 | chunk, 396 | [ 397 | { start: 0, end: 40 * mib }, 398 | { start: 40 * mib, end: 100 * mib }, 399 | ], 400 | ], 401 | [ 402 | 50 * mib, 403 | 3, 404 | chunk, 405 | [ 406 | { start: 0, end: 40 * mib }, 407 | { start: 40 * mib, end: 50 * mib }, 408 | { start: 50 * mib, end: 50 * mib }, 409 | ], 410 | ], 411 | [ 412 | 100 * mib, 413 | 3, 414 | chunk, 415 | [ 416 | { start: 0, end: 40 * mib }, 417 | { start: 40 * mib, end: 80 * mib }, 418 | { start: 80 * mib, end: 100 * mib }, 419 | ], 420 | ], 421 | [ 422 | 500 * mib, 423 | 6, 424 | chunk, 425 | [ 426 | { start: 0 * mib, end: 80 * mib }, 427 | { start: 80 * mib, end: 160 * mib }, 428 | { start: 160 * mib, end: 240 * mib }, 429 | { start: 240 * mib, end: 320 * mib }, 430 | { start: 320 * mib, end: 400 * mib }, 431 | { start: 400 * mib, end: 500 * mib }, 432 | ], 433 | ], 434 | 435 | // Use larger chunk size. 436 | [ 437 | 81 * mib, 438 | 2, 439 | chunk * 2, 440 | [ 441 | { start: 0, end: 80 * mib }, 442 | { start: 80 * mib, end: 81 * mib }, 443 | ], 444 | ], 445 | [ 446 | 121 * mib, 447 | 2, 448 | chunk * 3, 449 | [ 450 | { start: 0, end: 120 * mib }, 451 | { start: 120 * mib, end: 121 * mib }, 452 | ], 453 | ], 454 | [ 455 | 121 * mib, 456 | 3, 457 | chunk * 3, 458 | [ 459 | { start: 0, end: 120 * mib }, 460 | { start: 120 * mib, end: 121 * mib }, 461 | { start: 121 * mib, end: 121 * mib }, 462 | ], 463 | ], 464 | ]; 465 | 466 | it.each(cases)( 467 | "(totalSize: '%s', partCount: '%s', chunkSize: '%s') should result in '%s'", 468 | (totalSize, partCount, chunkSize, expectedParts) => { 469 | const parts = splitSizeIntoChunkAlignedParts(totalSize, partCount, chunkSize); 470 | expect(parts).toEqual(expectedParts); 471 | } 472 | ); 473 | 474 | const sizeTooSmallCases = [ 475 | [40 * mib, 2, chunk * 2], 476 | [41 * mib, 2, chunk * 2], 477 | [40 * mib, 3, chunk * 3], 478 | [80 * mib, 2, chunk * 2], 479 | [40 * mib, 2, chunk], 480 | [40 * mib, 3, chunk], 481 | [0, 2, chunk], 482 | ]; 483 | 484 | it.each(sizeTooSmallCases)( 485 | "(totalSize: '%s', partCount: '%s', chunkSize: '%s') should throw", 486 | (totalSize, partCount, chunkSize) => { 487 | expect(() => splitSizeIntoChunkAlignedParts(totalSize, partCount, chunkSize)).toThrowError( 488 | `Expected parameter 'totalSize' to be greater than the size of a chunk ('${chunkSize}'), was type 'number', value '${totalSize}'` 489 | ); 490 | } 491 | ); 492 | 493 | it("should throw if the partCount is 0", () => { 494 | expect(() => splitSizeIntoChunkAlignedParts(1, 0, 1)).toThrowError( 495 | "Expected parameter 'partCount' to be greater than or equal to 1, was type 'number', value '0'" 496 | ); 497 | }); 498 | 499 | it("should throw if the chunkSize is 0", () => { 500 | expect(() => splitSizeIntoChunkAlignedParts(1, 1, 0)).toThrowError( 501 | "Expected parameter 'chunkSize' to be greater than or equal to 1, was type 'number', value '0'" 502 | ); 503 | }); 504 | }); 505 | -------------------------------------------------------------------------------- /src/utils.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const path = require("path"); 4 | const fs = require("fs"); 5 | const mime = require("mime/lite"); 6 | const urljoin = require("url-join"); 7 | const { sign } = require("tweetnacl"); 8 | 9 | const { URI_SKYNET_PREFIX, DEFAULT_SKYNET_PORTAL_URL } = require("skynet-js"); 10 | const { trimPrefix } = require("./utils_string"); 11 | const { validateString } = require("./utils_validation"); 12 | 13 | BigInt.prototype.toJSON = function () { 14 | return this.toString(); 15 | }; 16 | 17 | /** 18 | * The default URL of the Skynet portal to use in the absence of configuration. 19 | */ 20 | const defaultSkynetPortalUrl = DEFAULT_SKYNET_PORTAL_URL; 21 | 22 | /** 23 | * The URI prefix for Skynet. 24 | */ 25 | const uriSkynetPrefix = URI_SKYNET_PREFIX; 26 | 27 | function defaultOptions(endpointPath) { 28 | return { 29 | portalUrl: defaultPortalUrl(), 30 | endpointPath: endpointPath, 31 | 32 | APIKey: "", 33 | skynetApiKey: "", 34 | customUserAgent: "", 35 | onUploadProgress: undefined, 36 | }; 37 | } 38 | 39 | /** 40 | * Selects the default portal URL to use when initializing a client. May involve network queries to several candidate portals. 41 | */ 42 | function defaultPortalUrl() { 43 | return defaultSkynetPortalUrl; 44 | } 45 | 46 | /** 47 | * Extract only the model's custom options from the given options. 48 | * 49 | * @param opts - The given options. 50 | * @param model - The model options. 51 | * @returns - The extracted custom options. 52 | * @throws - If the given opts don't contain all properties of the model. 53 | */ 54 | function extractOptions(opts, model) { 55 | const result = {}; 56 | for (const property in model) { 57 | if (!Object.prototype.hasOwnProperty.call(model, property)) { 58 | continue; 59 | } 60 | // Throw if the given options don't contain the model's property. 61 | if (!Object.prototype.hasOwnProperty.call(opts, property)) { 62 | throw new Error(`Property '${property}' not found`); 63 | } 64 | result[property] = opts[property]; 65 | } 66 | 67 | return result; 68 | } 69 | 70 | /** 71 | * Get the file mime type. Try to guess the file type based on the extension. 72 | * 73 | * @param filename - The filename. 74 | * @returns - The mime type. 75 | */ 76 | function getFileMimeType(filename) { 77 | let ext = path.extname(filename); 78 | ext = trimPrefix(ext, "."); 79 | if (ext !== "") { 80 | const mimeType = mime.getType(ext); 81 | if (mimeType) { 82 | return mimeType; 83 | } 84 | } 85 | return ""; 86 | } 87 | 88 | /** 89 | * Properly joins paths together to create a URL. Takes a variable number of 90 | * arguments. 91 | */ 92 | function makeUrl() { 93 | let args = Array.from(arguments); 94 | return args.reduce(function (acc, cur) { 95 | return urljoin(acc, cur); 96 | }); 97 | } 98 | 99 | function walkDirectory(filepath, out) { 100 | let files = []; 101 | if (!fs.existsSync(filepath)) { 102 | return files; 103 | } 104 | 105 | for (const subpath of fs.readdirSync(filepath)) { 106 | const fullpath = path.join(filepath, subpath); 107 | if (fs.statSync(fullpath).isDirectory()) { 108 | files = files.concat(walkDirectory(fullpath, out)); 109 | continue; 110 | } 111 | files.push(fullpath); 112 | } 113 | return files; 114 | } 115 | 116 | /** 117 | * Get the publicKey from privateKey. 118 | * 119 | * @param privateKey - The privateKey. 120 | * @returns - The publicKey. 121 | */ 122 | const getPublicKeyFromPrivateKey = function (privateKey) { 123 | const publicKey = Buffer.from( 124 | sign.keyPair.fromSecretKey(Uint8Array.from(Buffer.from(privateKey, "hex"))).publicKey 125 | ).toString("hex"); 126 | return publicKey; 127 | }; 128 | 129 | /** 130 | * Formats the skylink by adding the sia: prefix. 131 | * 132 | * @param skylink - The skylink. 133 | * @returns - The formatted skylink. 134 | */ 135 | const formatSkylink = function (skylink) { 136 | validateString("skylink", skylink, "parameter"); 137 | 138 | if (skylink === "") { 139 | return skylink; 140 | } 141 | if (!skylink.startsWith(URI_SKYNET_PREFIX)) { 142 | skylink = `${URI_SKYNET_PREFIX}${skylink}`; 143 | } 144 | return skylink; 145 | }; 146 | 147 | module.exports = { 148 | defaultSkynetPortalUrl, 149 | uriSkynetPrefix, 150 | defaultOptions, 151 | defaultPortalUrl, 152 | extractOptions, 153 | getFileMimeType, 154 | makeUrl, 155 | walkDirectory, 156 | getPublicKeyFromPrivateKey, 157 | formatSkylink, 158 | }; 159 | -------------------------------------------------------------------------------- /src/utils.test.js: -------------------------------------------------------------------------------- 1 | const { defaultPortalUrl, makeUrl, uriSkynetPrefix } = require("./utils.js"); 2 | const { trimSiaPrefix } = require("./utils_string"); 3 | 4 | const portalUrl = defaultPortalUrl(); 5 | const skylink = "XABvi7JtJbQSMAcDwnUnmp2FKDPjg8_tTTFP4BwMSxVdEg"; 6 | 7 | describe("makeUrl", () => { 8 | it("should return correctly formed URLs", () => { 9 | expect(makeUrl(portalUrl, "/")).toEqual(`${portalUrl}/`); 10 | expect(makeUrl(portalUrl, "/skynet")).toEqual(`${portalUrl}/skynet`); 11 | expect(makeUrl(portalUrl, "/skynet/")).toEqual(`${portalUrl}/skynet/`); 12 | 13 | expect(makeUrl(portalUrl, "/", skylink)).toEqual(`${portalUrl}/${skylink}`); 14 | expect(makeUrl(portalUrl, "/skynet", skylink)).toEqual(`${portalUrl}/skynet/${skylink}`); 15 | expect(makeUrl(portalUrl, "//skynet/", skylink)).toEqual(`${portalUrl}/skynet/${skylink}`); 16 | }); 17 | }); 18 | 19 | describe("trimSiaPrefix", () => { 20 | it("should correctly trim the sia prefix", () => { 21 | expect(trimSiaPrefix(skylink)).toEqual(skylink); 22 | expect(trimSiaPrefix(`${uriSkynetPrefix}${skylink}`)).toEqual(skylink); 23 | }); 24 | }); 25 | -------------------------------------------------------------------------------- /src/utils_encoding.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const { fromByteArray, toByteArray } = require("base64-js"); 4 | const base32Decode = require("base32-decode"); 5 | const base32Encode = require("base32-encode"); 6 | 7 | const { URI_SKYNET_PREFIX } = require("skynet-js"); 8 | const { 9 | RAW_SKYLINK_SIZE, 10 | BASE32_ENCODED_SKYLINK_SIZE, 11 | BASE64_ENCODED_SKYLINK_SIZE, 12 | ERR_SKYLINK_INCORRECT_SIZE, 13 | } = require("./skylink_sia"); 14 | const { trimUriPrefix } = require("./utils_string"); 15 | const { validateStringLen } = require("./utils_validation"); 16 | 17 | const BASE32_ENCODING_VARIANT = "RFC4648-HEX"; 18 | 19 | /** 20 | * Decodes the skylink encoded using base32 encoding to bytes. 21 | * 22 | * @param skylink - The encoded skylink. 23 | * @returns - The decoded bytes. 24 | */ 25 | const decodeSkylinkBase32 = function (skylink) { 26 | validateStringLen("skylink", skylink, "parameter", BASE32_ENCODED_SKYLINK_SIZE); 27 | skylink = skylink.toUpperCase(); 28 | const bytes = base32Decode(skylink, BASE32_ENCODING_VARIANT); 29 | return new Uint8Array(bytes); 30 | }; 31 | 32 | /** 33 | * Encodes the bytes to a skylink encoded using base32 encoding. 34 | * 35 | * @param bytes - The bytes to encode. 36 | * @returns - The encoded skylink. 37 | */ 38 | const encodeSkylinkBase32 = function (bytes) { 39 | return base32Encode(bytes, BASE32_ENCODING_VARIANT, { padding: false }).toLowerCase(); 40 | }; 41 | 42 | /** 43 | * Decodes the skylink encoded using base64 raw URL encoding to bytes. 44 | * 45 | * @param skylink - The encoded skylink. 46 | * @returns - The decoded bytes. 47 | */ 48 | function decodeSkylinkBase64(skylink) { 49 | // Check if Skylink is 46 bytes long. 50 | if (skylink.length !== BASE64_ENCODED_SKYLINK_SIZE) { 51 | throw new Error("Skylink is not 46 bytes long."); 52 | } 53 | // Add padding. 54 | skylink = `${skylink}==`; 55 | // Convert from URL encoding. 56 | skylink = skylink.replace(/-/g, "+").replace(/_/g, "/"); 57 | return toByteArray(skylink); 58 | } 59 | 60 | /** 61 | * Encodes the bytes to a skylink encoded using base64 raw URL encoding. 62 | * 63 | * @param bytes - The bytes to encode. 64 | * @returns - The encoded skylink. 65 | */ 66 | const encodeSkylinkBase64 = function (bytes) { 67 | let base64 = fromByteArray(bytes); 68 | // Convert to URL encoding. 69 | base64 = base64.replace(/\+/g, "-").replace(/\//g, "_"); 70 | // Remove trailing "==". This will always be present as the skylink encoding 71 | // gets padded so that the string is a multiple of 4 characters in length. 72 | return base64.slice(0, -2); 73 | }; 74 | 75 | /** 76 | * A helper function that decodes the given string representation of a skylink 77 | * into raw bytes. It either performs a base32 decoding, or base64 decoding, 78 | * depending on the length. 79 | * 80 | * @param encoded - The encoded string. 81 | * @returns - The decoded raw bytes. 82 | * @throws - Will throw if the skylink is not a V1 or V2 skylink string. 83 | */ 84 | const decodeSkylink = function (encoded) { 85 | encoded = trimUriPrefix(encoded, URI_SKYNET_PREFIX); 86 | 87 | let bytes; 88 | if (encoded.length === BASE32_ENCODED_SKYLINK_SIZE) { 89 | bytes = decodeSkylinkBase32(encoded); 90 | } else if (encoded.length === BASE64_ENCODED_SKYLINK_SIZE) { 91 | bytes = decodeSkylinkBase64(encoded); 92 | } else { 93 | throw new Error(ERR_SKYLINK_INCORRECT_SIZE); 94 | } 95 | 96 | // Sanity check the size of the given data. 97 | if (bytes.length != RAW_SKYLINK_SIZE) { 98 | throw new Error("failed to load skylink data"); 99 | } 100 | 101 | return bytes; 102 | }; 103 | 104 | module.exports = { 105 | decodeSkylinkBase32, 106 | encodeSkylinkBase32, 107 | decodeSkylinkBase64, 108 | encodeSkylinkBase64, 109 | decodeSkylink, 110 | }; 111 | -------------------------------------------------------------------------------- /src/utils_registry.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | /** 4 | * Regex for JSON revision value without quotes. 5 | */ 6 | const REGEX_REVISION_NO_QUOTES = /"revision":\s*([0-9]+)/; 7 | 8 | module.exports = { 9 | REGEX_REVISION_NO_QUOTES, 10 | }; 11 | -------------------------------------------------------------------------------- /src/utils_string.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const { uriSkynetPrefix } = require("skynet-js"); 4 | const { validateString } = require("./utils_validation"); 5 | 6 | /** 7 | * Removes a prefix from the beginning of the string. 8 | * 9 | * @param str - The string to process. 10 | * @param prefix - The prefix to remove. 11 | * @param [limit] - Maximum amount of times to trim. No limit by default. 12 | * @returns - The processed string. 13 | */ 14 | function trimPrefix(str, prefix, limit) { 15 | if (typeof limit !== "number" && typeof limit !== "undefined") { 16 | throw new Error(`Invalid input: 'limit' must be type 'number | undefined', was '${typeof limit}'`); 17 | } 18 | 19 | while (str.startsWith(prefix)) { 20 | if (limit !== undefined && limit <= 0) { 21 | break; 22 | } 23 | str = str.slice(prefix.length); 24 | if (limit) { 25 | limit -= 1; 26 | } 27 | } 28 | return str; 29 | } 30 | 31 | /** 32 | * Removes a suffix from the end of the string. 33 | * 34 | * @param str - The string to process. 35 | * @param suffix - The suffix to remove. 36 | * @param [limit] - Maximum amount of times to trim. No limit by default. 37 | * @returns - The processed string. 38 | */ 39 | const trimSuffix = function (str, suffix, limit) { 40 | while (str.endsWith(suffix)) { 41 | if (limit !== undefined && limit <= 0) { 42 | break; 43 | } 44 | str = str.substring(0, str.length - suffix.length); 45 | if (limit) { 46 | limit -= 1; 47 | } 48 | } 49 | return str; 50 | }; 51 | 52 | /** 53 | * Removes slashes from the beginning and end of the string. 54 | * 55 | * @param str - The string to process. 56 | * @returns - The processed string. 57 | */ 58 | const trimForwardSlash = function (str) { 59 | return trimPrefix(trimSuffix(str, "/"), "/"); 60 | }; 61 | 62 | /** 63 | * Removes a URI prefix from the beginning of the string. 64 | * 65 | * @param str - The string to process. 66 | * @param prefix - The prefix to remove. Should contain double slashes, e.g. sia://. 67 | * @returns - The processed string. 68 | */ 69 | const trimUriPrefix = function (str, prefix) { 70 | const longPrefix = prefix.toLowerCase(); 71 | const shortPrefix = trimSuffix(longPrefix, "/"); 72 | // Make sure the trimming is case-insensitive. 73 | const strLower = str.toLowerCase(); 74 | if (strLower.startsWith(longPrefix)) { 75 | // longPrefix is exactly at the beginning 76 | return str.slice(longPrefix.length); 77 | } 78 | if (strLower.startsWith(shortPrefix)) { 79 | // else shortPrefix is exactly at the beginning 80 | return str.slice(shortPrefix.length); 81 | } 82 | return str; 83 | }; 84 | 85 | function trimSiaPrefix(str) { 86 | return trimPrefix(str, uriSkynetPrefix); 87 | } 88 | 89 | /** 90 | * Returns true if the input is a valid hex-encoded string. 91 | * 92 | * @param str - The input string. 93 | * @returns - True if the input is hex-encoded. 94 | * @throws - Will throw if the input is not a string. 95 | */ 96 | const isHexString = function (str) { 97 | validateString("str", str, "parameter"); 98 | 99 | return /^[0-9A-Fa-f]*$/g.test(str); 100 | }; 101 | 102 | /** 103 | * Convert a byte array to a hex string. 104 | * 105 | * @param byteArray - The byte array to convert. 106 | * @returns - The hex string. 107 | * @see {@link https://stackoverflow.com/a/44608819|Stack Overflow} 108 | */ 109 | const toHexString = function (byteArray) { 110 | let s = ""; 111 | byteArray.forEach(function (byte) { 112 | s += ("0" + (byte & 0xff).toString(16)).slice(-2); 113 | }); 114 | return s; 115 | }; 116 | 117 | module.exports = { 118 | trimPrefix, 119 | trimSuffix, 120 | trimForwardSlash, 121 | trimUriPrefix, 122 | trimSiaPrefix, 123 | isHexString, 124 | toHexString, 125 | }; 126 | -------------------------------------------------------------------------------- /src/utils_testing.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const parse = require("url-parse"); 4 | const { trimForwardSlash } = require("./utils_string"); 5 | const { throwValidationError } = require("./utils_validation"); 6 | const cliProgress = require("cli-progress"); 7 | 8 | let fileSizeCounter = 0; 9 | let endProgressCounter = 0; 10 | 11 | /** 12 | * Extracts the non-skylink part of the path from the url. 13 | * 14 | * @param url - The input URL. 15 | * @param skylink - The skylink to remove, if it is present. 16 | * @returns - The non-skylink part of the path. 17 | */ 18 | const extractNonSkylinkPath = function (url, skylink) { 19 | const parsed = parse(url, {}); 20 | let path = parsed.pathname.replace(skylink, ""); // Remove skylink to get the path. 21 | // Ensure there are no leading or trailing slashes. 22 | path = trimForwardSlash(path); 23 | // Add back the slash, unless there is no path. 24 | if (path !== "") { 25 | path = `/${path}`; 26 | } 27 | return path; 28 | }; 29 | 30 | /** 31 | * Gets the settled values from `Promise.allSettled`. Throws if an error is 32 | * found. Returns all settled values if no errors were found. 33 | * 34 | * @param values - The settled values. 35 | * @returns - The settled value if no errors were found. 36 | * @throws - Will throw if an unexpected error occurred. 37 | */ 38 | const getSettledValues = function (values) { 39 | const receivedValues = []; 40 | 41 | for (const value of values) { 42 | if (value.status === "rejected") { 43 | throw value.reason; 44 | } else if (value.value) { 45 | receivedValues.push(value.value); 46 | } 47 | } 48 | 49 | return receivedValues; 50 | }; 51 | 52 | const splitSizeIntoChunkAlignedParts = function (totalSize, partCount, chunkSize) { 53 | if (partCount < 1) { 54 | throwValidationError("partCount", partCount, "parameter", "greater than or equal to 1"); 55 | } 56 | if (chunkSize < 1) { 57 | throwValidationError("chunkSize", chunkSize, "parameter", "greater than or equal to 1"); 58 | } 59 | // NOTE: Unexpected code flow. `uploadLargeFileRequest` should not enable 60 | // parallel uploads for this case. 61 | if (totalSize <= chunkSize) { 62 | throwValidationError("totalSize", totalSize, "parameter", `greater than the size of a chunk ('${chunkSize}')`); 63 | } 64 | 65 | const partSizes = new Array(partCount).fill(0); 66 | 67 | // Assign chunks to parts in order, looping back to the beginning if we get to 68 | // the end of the parts array. 69 | const numFullChunks = Math.floor(totalSize / chunkSize); 70 | for (let i = 0; i < numFullChunks; i++) { 71 | partSizes[i % partCount] += chunkSize; 72 | } 73 | 74 | // The leftover size that must go into the last part. 75 | const leftover = totalSize % chunkSize; 76 | // If there is non-chunk-aligned leftover, add it. 77 | if (leftover > 0) { 78 | // Assign the leftover to the part after the last part that was visited, or 79 | // the last part in the array if all parts were used. 80 | // 81 | // NOTE: We don't need to worry about empty parts, tus ignores those. 82 | const lastIndex = Math.min(numFullChunks, partCount - 1); 83 | partSizes[lastIndex] += leftover; 84 | } 85 | 86 | // Convert sizes into parts. 87 | const parts = []; 88 | let lastBoundary = 0; 89 | for (let i = 0; i < partCount; i++) { 90 | parts.push({ 91 | start: lastBoundary, 92 | end: lastBoundary + partSizes[i], 93 | }); 94 | lastBoundary = parts[i].end; 95 | } 96 | 97 | return parts; 98 | }; 99 | 100 | // Set the custom upload progress tracker. 101 | const onUploadProgress = (progress, { loaded, total }) => { 102 | let progressOutput = Math.floor((loaded * 100) / total); 103 | 104 | if (progressOutput === 0 && fileSizeCounter === 0) { 105 | process.stdout.write(" The uploading File size is " + total + " bytes.\n\n"); 106 | process.stdout.moveCursor(0, -1); 107 | fileSizeCounter++; 108 | } 109 | 110 | // create new progress bar with custom token "speed" 111 | const bar = new cliProgress.Bar({ 112 | format: "-> uploading [{bar}] " + progressOutput + "% {eta_formatted} ", 113 | fps: 1, 114 | }); 115 | 116 | // initialize the bar - set payload token "speed" with the default value "N/A" 117 | bar.start(100, 0, { 118 | speed: "N/A", 119 | }); 120 | bar.updateETA(Buffer); 121 | // update bar value. set custom token "speed" to 125 122 | bar.update(progressOutput, { 123 | speed: "122", 124 | }); 125 | 126 | // stop the bar 127 | bar.stop(); 128 | 129 | if (progressOutput === 100) { 130 | process.stdout.write(`\n`); 131 | process.stdout.moveCursor(0, -1); 132 | process.stdout.write(`\n`); 133 | endProgressCounter++; 134 | if (endProgressCounter === 2) { 135 | process.stdout.write(`\n\n`); 136 | } 137 | } 138 | process.stdout.moveCursor(0, -1); 139 | }; 140 | 141 | module.exports = { 142 | extractNonSkylinkPath, 143 | getSettledValues, 144 | splitSizeIntoChunkAlignedParts, 145 | onUploadProgress, 146 | }; 147 | -------------------------------------------------------------------------------- /src/utils_validation.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const { parseSkylink } = require("skynet-js"); 4 | 5 | /** 6 | * Returns an error for the given value 7 | * 8 | * @param name - The name of the value. 9 | * @param value - The actual value. 10 | * @param valueKind - The kind of value that is being checked (e.g. "parameter", "response field", etc.) 11 | * @param expected - The expected aspect of the value that could not be validated (e.g. "type 'string'" or "non-null"). 12 | * @returns - The validation error. 13 | */ 14 | const validationError = function (name, value, valueKind, expected) { 15 | let actualValue; 16 | if (value === undefined) { 17 | actualValue = "type 'undefined'"; 18 | } else if (value === null) { 19 | actualValue = "type 'null'"; 20 | } else { 21 | actualValue = `type '${typeof value}', value '${value}'`; 22 | } 23 | return new Error(`Expected ${valueKind} '${name}' to be ${expected}, was ${actualValue}`); 24 | }; 25 | 26 | /** 27 | * Throws an error for the given value 28 | * 29 | * @param name - The name of the value. 30 | * @param value - The actual value. 31 | * @param valueKind - The kind of value that is being checked (e.g. "parameter", "response field", etc.) 32 | * @param expected - The expected aspect of the value that could not be validated (e.g. "type 'string'" or "non-null"). 33 | * @throws - Will always throw. 34 | */ 35 | const throwValidationError = function (name, value, valueKind, expected) { 36 | throw validationError(name, value, valueKind, expected); 37 | }; 38 | 39 | /** 40 | * Validates the given value as a string. 41 | * 42 | * @param name - The name of the value. 43 | * @param value - The actual value. 44 | * @param valueKind - The kind of value that is being checked (e.g. "parameter", "response field", etc.) 45 | * @throws - Will throw if not a valid string. 46 | */ 47 | const validateString = function (name, value, valueKind) { 48 | if (typeof value !== "string") { 49 | throwValidationError(name, value, valueKind, "type 'string'"); 50 | } 51 | }; 52 | 53 | /** 54 | * Validates the given value as a string of the given length. 55 | * 56 | * @param name - The name of the value. 57 | * @param value - The actual value. 58 | * @param valueKind - The kind of value that is being checked (e.g. "parameter", "response field", etc.) 59 | * @param len - The length to check. 60 | * @throws - Will throw if not a valid string of the given length. 61 | */ 62 | const validateStringLen = function (name, value, valueKind, len) { 63 | validateString(name, value, valueKind); 64 | const actualLen = value.length; 65 | if (actualLen !== len) { 66 | throwValidationError(name, value, valueKind, `type 'string' of length ${len}, was length ${actualLen}`); 67 | } 68 | }; 69 | 70 | /** 71 | * Validates the given value as a skylink string. 72 | * 73 | * @param name - The name of the value. 74 | * @param value - The actual value. 75 | * @param valueKind - The kind of value that is being checked (e.g. "parameter", "response field", etc.) 76 | * @returns - The validated and parsed skylink. 77 | * @throws - Will throw if not a valid skylink string. 78 | */ 79 | const validateSkylinkString = function (name, value, valueKind) { 80 | validateString(name, value, valueKind); 81 | 82 | const parsedSkylink = parseSkylink(value); 83 | if (parsedSkylink === null) { 84 | throw validationError(name, value, valueKind, `valid skylink of type 'string'`); 85 | } 86 | 87 | return parsedSkylink; 88 | }; 89 | 90 | /** 91 | * Validates the given value as a number. 92 | * 93 | * @param name - The name of the value. 94 | * @param value - The actual value. 95 | * @param valueKind - The kind of value that is being checked (e.g. "parameter", "response field", etc.) 96 | * @throws - Will throw if not a valid number. 97 | */ 98 | const validateNumber = function (name, value, valueKind) { 99 | if (typeof value !== "number") { 100 | throwValidationError(name, value, valueKind, "type 'number'"); 101 | } 102 | }; 103 | 104 | /** 105 | * Validates the given value as a integer. 106 | * 107 | * @param name - The name of the value. 108 | * @param value - The actual value. 109 | * @param valueKind - The kind of value that is being checked (e.g. "parameter", "response field", etc.) 110 | * @throws - Will throw if not a valid integer. 111 | */ 112 | const validateInteger = function (name, value, valueKind) { 113 | validateNumber(name, value, valueKind); 114 | if (!Number.isInteger(value)) { 115 | throwValidationError(name, value, valueKind, "an integer value"); 116 | } 117 | }; 118 | 119 | /** 120 | * Validates the given value as an object. 121 | * 122 | * @param name - The name of the value. 123 | * @param value - The actual value. 124 | * @param valueKind - The kind of value that is being checked (e.g. "parameter", "response field", etc.) 125 | * @throws - Will throw if not a valid object. 126 | */ 127 | const validateObject = function (name, value, valueKind) { 128 | if (typeof value !== "object") { 129 | throwValidationError(name, value, valueKind, "type 'object'"); 130 | } 131 | if (value === null) { 132 | throwValidationError(name, value, valueKind, "non-null"); 133 | } 134 | }; 135 | 136 | /** 137 | * Validates the given value as an optional object. 138 | * 139 | * @param name - The name of the value. 140 | * @param value - The actual value. 141 | * @param valueKind - The kind of value that is being checked (e.g. "parameter", "response field", etc.) 142 | * @param model - A model object that contains all possible fields. 'value' does not need to have all fields, but it may not have any fields not contained in 'model'. 143 | * @throws - Will throw if not a valid optional object. 144 | */ 145 | const validateOptionalObject = function validateOptionalObject(name, value, valueKind, model) { 146 | if (!value) { 147 | // This is okay, the object is optional. 148 | return; 149 | } 150 | validateObject(name, value, valueKind); 151 | 152 | // Check if all given properties of value also exist in the model. 153 | for (const property in value) { 154 | if (!(property in model)) { 155 | throw new Error(`Object ${valueKind} '${name}' contains unexpected property '${property}'`); 156 | } 157 | } 158 | }; 159 | 160 | module.exports = { 161 | validationError, 162 | throwValidationError, 163 | validateString, 164 | validateStringLen, 165 | validateSkylinkString, 166 | validateNumber, 167 | validateInteger, 168 | validateObject, 169 | validateOptionalObject, 170 | }; 171 | -------------------------------------------------------------------------------- /testdata/data.json: -------------------------------------------------------------------------------- 1 | { "example": "This is some example testdata/data.json" } 2 | -------------------------------------------------------------------------------- /testdata/dir1/file3.txt: -------------------------------------------------------------------------------- 1 | test 2 | -------------------------------------------------------------------------------- /testdata/file1.txt: -------------------------------------------------------------------------------- 1 | test 2 | -------------------------------------------------------------------------------- /testdata/file2.txt: -------------------------------------------------------------------------------- 1 | testing 2 | --------------------------------------------------------------------------------