├── .editorconfig ├── .env.example ├── .github ├── dependabot.yml └── workflows │ ├── create-sentry-release.yml │ ├── lint.yml │ └── test.yml ├── .gitignore ├── LICENSE ├── README.md ├── jest.config.ts ├── package.json ├── src ├── aws.ts ├── gateways.json ├── index.ts ├── metrics.ts ├── middlewares │ └── useProxyCache.ts ├── providers │ ├── 4everland.ts │ ├── fleek.ts │ ├── index.ts │ ├── infura.ts │ ├── pinata.ts │ ├── utils.ts │ └── web3storage.ts ├── proxy.ts ├── rpc.ts ├── upload.ts └── utils.ts ├── test ├── .env.test ├── e2e │ ├── fixtures │ │ ├── file.json │ │ ├── json-file.png │ │ ├── too-heavy.jpg │ │ └── valid.png │ ├── proxy.test.ts │ ├── rpc.test.ts │ └── upload.test.ts └── unit │ └── providers │ ├── fixtures │ └── sample.webp │ └── index.test.ts ├── tsconfig.json └── yarn.lock /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | indent_style = space 5 | indent_size = 2 6 | end_of_line = LF 7 | charset = utf-8 8 | trim_trailing_whitespace = true 9 | insert_final_newline = true 10 | 11 | [*.md] 12 | trim_trailing_whitespace = false 13 | -------------------------------------------------------------------------------- /.env.example: -------------------------------------------------------------------------------- 1 | # Amazon AWS keys for cache 2 | AWS_ACCESS_KEY_ID= 3 | AWS_REGION= 4 | AWS_SECRET_ACCESS_KEY= 5 | AWS_BUCKET_NAME= 6 | 7 | # IPFS providers keys 8 | PINATA_API_KEY= 9 | PINATA_API_SECRET= 10 | FLEEK_API_KEY= 11 | FLEEK_API_SECRET= 12 | WEB3STORAGE_API_TOKEN= 13 | INFURA_PROJECT_ID= 14 | INFURA_PROJECT_SECRET= 15 | EVER_API_KEY= 16 | EVER_API_SECRET= 17 | EVER_BUCKET_NAME= 18 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "npm" 4 | directory: "/" 5 | schedule: 6 | interval: "daily" 7 | allow: 8 | # Allow updates for snapshot.js only 9 | - dependency-name: "@snapshot-labs/*" 10 | -------------------------------------------------------------------------------- /.github/workflows/create-sentry-release.yml: -------------------------------------------------------------------------------- 1 | name: Create a Sentry release 2 | 3 | on: 4 | push: 5 | branches: 6 | - 'master' 7 | 8 | jobs: 9 | create-sentry-release: 10 | uses: snapshot-labs/actions/.github/workflows/create-sentry-release.yml@main 11 | secrets: inherit 12 | with: 13 | project: pineapple 14 | -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: Lint 2 | 3 | on: [push] 4 | 5 | jobs: 6 | lint: 7 | uses: snapshot-labs/actions/.github/workflows/lint.yml@main 8 | secrets: inherit 9 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | on: [push] 3 | 4 | jobs: 5 | test: 6 | runs-on: ubuntu-20.04 7 | env: 8 | FLEEK_API_KEY: ${{ secrets.FLEEK_API_KEY }} 9 | FLEEK_API_SECRET: ${{ secrets.FLEEK_API_SECRET }} 10 | INFURA_PROJECT_ID: ${{ secrets.INFURA_PROJECT_ID }} 11 | INFURA_PROJECT_SECRET: ${{ secrets.INFURA_PROJECT_SECRET }} 12 | PINATA_API_KEY: ${{ secrets.PINATA_API_KEY }} 13 | PINATA_API_SECRET: ${{ secrets.PINATA_API_SECRET }} 14 | WEB3STORAGE_API_TOKEN: ${{ secrets.WEB3STORAGE_API_TOKEN }} 15 | EVER_API_KEY: ${{ secrets.EVER_API_KEY }} 16 | EVER_API_SECRET: ${{ secrets.EVER_API_SECRET }} 17 | EVER_BUCKET_NAME: ${{ secrets.EVER_BUCKET_NAME }} 18 | AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} 19 | AWS_REGION: ${{ secrets.AWS_REGION }} 20 | AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 21 | AWS_BUCKET_NAME: ${{ secrets.AWS_BUCKET_NAME }} 22 | steps: 23 | - uses: actions/checkout@v3 24 | - uses: actions/setup-node@v3 25 | with: 26 | node-version: '16' 27 | cache: 'yarn' 28 | - run: yarn 29 | - run: yarn test 30 | - name: Upload coverage reports to Codecov 31 | uses: codecov/codecov-action@v3 32 | with: 33 | token: ${{ secrets.CODECOV_TOKEN }} 34 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | node_modules 3 | dist 4 | build 5 | .env 6 | coverage 7 | uploads 8 | 9 | # Remove some common IDE working directories 10 | .idea 11 | .vscode 12 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) Snapshot Labs 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Pineapple 2 | 3 | ## Error codes 4 | 5 | All endpoints will respond with a [JSON-RPC 2.0](https://www.jsonrpc.org/specification) error response on error: 6 | 7 | ```bash 8 | { 9 | "jsonrpc":"2.0", 10 | "error":{ 11 | "code": CODE, 12 | "message": MESSAGE 13 | }, 14 | "id": ID 15 | } 16 | ``` 17 | 18 | | Description | `CODE` | `MESSAGE` | 19 | | --------------------------------------------------------------- | ------ | ------------------------------------------- | 20 | | Uploaded file exceed 1MB | 400 | File too large | 21 | | Uploaded image file is not an image | 415 | Unsupported file type | 22 | | Uploaded payload does not contain a fileSize | 400 | No file submitted | 23 | | Server error | 500 | (Will depend on the error) | | 24 | -------------------------------------------------------------------------------- /jest.config.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * For a detailed explanation regarding each configuration property and type check, visit: 3 | * https://jestjs.io/docs/configuration 4 | */ 5 | 6 | export default { 7 | clearMocks: true, 8 | collectCoverage: true, 9 | collectCoverageFrom: ['./src/**'], 10 | coverageDirectory: 'coverage', 11 | coverageProvider: 'v8', 12 | coveragePathIgnorePatterns: ['/node_modules/', '/dist/'], 13 | 14 | preset: 'ts-jest', 15 | testEnvironment: 'node', 16 | setupFiles: ['dotenv/config'], 17 | testPathIgnorePatterns: ['/node_modules/', '/dist/'], 18 | moduleFileExtensions: ['js', 'ts'], 19 | verbose: true 20 | }; 21 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pineapple", 3 | "version": "0.1.0", 4 | "license": "MIT", 5 | "scripts": { 6 | "lint:fix": "yarn lint --fix", 7 | "lint": "eslint src/ test/ --ext .ts", 8 | "typecheck": "tsc --noEmit", 9 | "build": "tsc", 10 | "dev": "nodemon src/index.ts", 11 | "start": "node dist/src/index.js", 12 | "start:test": "dotenv -e test/.env.test yarn dev", 13 | "test": "PORT=3003 start-server-and-test 'yarn start:test' 3003 'dotenv -e test/.env.test jest --runInBand'", 14 | "test:unit": "dotenv -e test/.env.test jest test/unit/", 15 | "test:e2e": "PORT=3003 start-server-and-test 'yarn start:test' 3003 'dotenv -e test/.env.test jest --runInBand --collectCoverage=false test/e2e/'" 16 | }, 17 | "eslintConfig": { 18 | "extends": "@snapshot-labs" 19 | }, 20 | "prettier": "@snapshot-labs/prettier-config", 21 | "dependencies": { 22 | "@aws-sdk/client-s3": "^3.75.0", 23 | "@fleekhq/fleek-storage-js": "^1.0.21", 24 | "@pinata/sdk": "^1.1.25", 25 | "@snapshot-labs/snapshot-metrics": "^1.4.1", 26 | "@snapshot-labs/snapshot-sentry": "^1.5.5", 27 | "compression": "^1.7.4", 28 | "cors": "^2.8.5", 29 | "dotenv": "^16.0.1", 30 | "express": "^4.18.2", 31 | "ipfs-http-client": "^56.0.0", 32 | "multer": "^1.4.5-lts.1", 33 | "node-fetch": "<3", 34 | "nodemon": "^3.0.1", 35 | "sharp": "^0.32.4", 36 | "ts-node": "^10.9.1", 37 | "typescript": "^5.1.6", 38 | "web3.storage": "^4.2.0" 39 | }, 40 | "devDependencies": { 41 | "@snapshot-labs/eslint-config": "^0.1.0-beta.18", 42 | "@snapshot-labs/prettier-config": "^0.1.0-beta.7", 43 | "@types/express": "^4.17.17", 44 | "@types/jest": "^29.5.3", 45 | "@types/multer": "^1.4.7", 46 | "@types/node": "^20.4.9", 47 | "dotenv-cli": "^7.2.1", 48 | "eslint": "^8.45.0", 49 | "eslint-plugin-prettier": "^5.0.0", 50 | "jest": "^29.6.2", 51 | "prettier": "^3.0.1", 52 | "start-server-and-test": "^2.0.0", 53 | "supertest": "^6.3.3", 54 | "ts-jest": "^29.1.1" 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/aws.ts: -------------------------------------------------------------------------------- 1 | import { Readable } from 'stream'; 2 | import * as AWS from '@aws-sdk/client-s3'; 3 | 4 | let client; 5 | const region = process.env.AWS_REGION; 6 | const endpoint = process.env.AWS_ENDPOINT || undefined; 7 | if (region) client = new AWS.S3({ region, endpoint }); 8 | const dir = 'pineapple'; 9 | 10 | async function streamToString(stream: Readable): Promise { 11 | return await new Promise((resolve, reject) => { 12 | const chunks: Uint8Array[] = []; 13 | stream.on('data', chunk => chunks.push(chunk)); 14 | stream.on('error', reject); 15 | stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf-8'))); 16 | }); 17 | } 18 | 19 | export async function set(key, value) { 20 | if (!client) return; 21 | try { 22 | return await client.putObject({ 23 | Bucket: process.env.AWS_BUCKET_NAME, 24 | Key: `public/${dir}/${key}`, 25 | Body: JSON.stringify(value), 26 | ContentType: 'application/json; charset=utf-8' 27 | }); 28 | } catch (e) { 29 | console.log('Store cache failed', e); 30 | throw e; 31 | } 32 | } 33 | 34 | export async function get(key) { 35 | if (!client) return false; 36 | try { 37 | const { Body } = await client.getObject({ 38 | Bucket: process.env.AWS_BUCKET_NAME, 39 | Key: `public/${dir}/${key}` 40 | }); 41 | const str = await streamToString(Body); 42 | return JSON.parse(str); 43 | } catch (e) { 44 | return false; 45 | } 46 | } 47 | 48 | export async function remove(key) { 49 | if (!client) return false; 50 | try { 51 | return await client.deleteObject({ 52 | Bucket: process.env.AWS_BUCKET_NAME, 53 | Key: `public/${dir}/${key}` 54 | }); 55 | } catch (e: any) { 56 | return false; 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/gateways.json: -------------------------------------------------------------------------------- 1 | [ 2 | "cloudflare-ipfs.com", 3 | "cf-ipfs.com", 4 | "ipfs.io", 5 | "ipfs.fleek.co", 6 | "gateway.pinata.cloud", 7 | "dweb.link", 8 | "ipfs.infura.io", 9 | "snapshot.4everland.link" 10 | ] 11 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import 'dotenv/config'; 2 | import { fallbackLogger, initLogger } from '@snapshot-labs/snapshot-sentry'; 3 | import compression from 'compression'; 4 | import cors from 'cors'; 5 | import express from 'express'; 6 | import initMetrics from './metrics'; 7 | import proxy from './proxy'; 8 | import rpc from './rpc'; 9 | import upload from './upload'; 10 | import { version } from '../package.json'; 11 | 12 | const app = express(); 13 | const PORT = process.env.PORT || 3000; 14 | 15 | const commit = process.env.COMMIT_HASH || ''; 16 | const v = commit ? `${version}#${commit.substr(0, 7)}` : version; 17 | 18 | initLogger(app); 19 | initMetrics(app); 20 | 21 | app.disable('x-powered-by'); 22 | app.use(express.json({ limit: '10mb' })); 23 | app.use(express.urlencoded({ limit: '4mb', extended: false })); 24 | app.use(cors({ maxAge: 86400 })); 25 | app.use(compression()); 26 | app.use('/', rpc); 27 | app.use('/', upload); 28 | app.use('/', proxy); 29 | app.get('/', (req, res) => res.json({ version: v, port: PORT })); 30 | 31 | fallbackLogger(app); 32 | 33 | app.listen(PORT, () => console.log(`Listening at http://localhost:${PORT}`)); 34 | -------------------------------------------------------------------------------- /src/metrics.ts: -------------------------------------------------------------------------------- 1 | import init, { client } from '@snapshot-labs/snapshot-metrics'; 2 | import { capture } from '@snapshot-labs/snapshot-sentry'; 3 | import { Express } from 'express'; 4 | import gateways from './gateways.json'; 5 | import { IMAGE_PROVIDERS, JSON_PROVIDERS, providersMap } from './providers/utils'; 6 | 7 | export default function initMetrics(app: Express) { 8 | init(app, { 9 | normalizedPath: [['^/ipfs/.*', '/ipfs/#url']], 10 | whitelistedPath: [/^\/$/, /^\/upload$/, /^\/ipfs\/.*$/], 11 | errorHandler: capture 12 | }); 13 | 14 | app.use(providersInstrumentation); 15 | } 16 | 17 | const gatewaysCount = new client.Gauge({ 18 | name: 'ipfs_gateways_count', 19 | help: 'Number of IPFS gateways.' 20 | }); 21 | gatewaysCount.set(gateways.length); 22 | 23 | const providersJsonCount = new client.Gauge({ 24 | name: 'providers_json_count', 25 | help: 'Number of providers used for JSON pinning.' 26 | }); 27 | providersJsonCount.set(JSON_PROVIDERS.filter(p => providersMap[p].isConfigured()).length); 28 | 29 | const providersImageCount = new client.Gauge({ 30 | name: 'providers_image_count', 31 | help: 'Number of providers used for image pinning.' 32 | }); 33 | providersImageCount.set(IMAGE_PROVIDERS.filter(p => providersMap[p].isConfigured()).length); 34 | 35 | export const timeProvidersUpload = new client.Histogram({ 36 | name: 'providers_upload_duration_seconds', 37 | help: "Duration in seconds of provider's upload requests.", 38 | labelNames: ['name', 'type', 'status'], 39 | buckets: [0.5, 1, 2, 5, 10, 15] 40 | }); 41 | 42 | export const providersUploadSize = new client.Counter({ 43 | name: 'providers_upload_size', 44 | help: "Total size of each provider's upload file.", 45 | labelNames: ['name', 'type'] 46 | }); 47 | 48 | const providersReturnCount = new client.Counter({ 49 | name: 'providers_return_count', 50 | help: 'Number of times each provider have been used.', 51 | labelNames: ['name', 'type'] 52 | }); 53 | 54 | export const timeIpfsGatewaysResponse = new client.Histogram({ 55 | name: 'ipfs_gateways_response_duration_seconds', 56 | help: "Duration in seconds of each IPFS gateway's reponse.", 57 | labelNames: ['name', 'status'], 58 | buckets: [0.5, 1, 2, 5, 10, 15] 59 | }); 60 | 61 | export const ipfsGatewaysReturnCount = new client.Counter({ 62 | name: 'ipfs_gateways_return_count', 63 | help: 'Number of times each gateway have been used.', 64 | labelNames: ['name'] 65 | }); 66 | 67 | export const ipfsGatewaysCacheHitCount = new client.Counter({ 68 | name: 'ipfs_gateways_cache_hit_count', 69 | help: 'Number of hit/miss of the IPFS gateways cache layer', 70 | labelNames: ['status'] 71 | }); 72 | 73 | export const ipfsGatewaysCacheSize = new client.Counter({ 74 | name: 'ipfs_gateways_cache_size', 75 | help: 'Total size going through the IPFS gateways cache layer', 76 | labelNames: ['status'] 77 | }); 78 | 79 | export const countOpenProvidersRequest = new client.Gauge({ 80 | name: 'providers_open_connections_count', 81 | help: 'Number of open connections to providers.', 82 | labelNames: ['name', 'type'] 83 | }); 84 | 85 | export const countOpenGatewaysRequest = new client.Gauge({ 86 | name: 'ipfs_gateways_open_connections_count', 87 | help: 'Number of open connections to gateways.', 88 | labelNames: ['name'] 89 | }); 90 | 91 | const providersInstrumentation = (req, res, next) => { 92 | let type; 93 | if (req.method === 'POST' && req.originalUrl === '/') { 94 | type = 'json'; 95 | } else if (req.method === 'POST' && req.originalUrl === '/upload') { 96 | type = 'image'; 97 | } 98 | 99 | if (!type) { 100 | return next(); 101 | } 102 | 103 | const oldJson = res.json; 104 | res.json = body => { 105 | if (res.statusCode === 200 && body) { 106 | providersReturnCount.inc({ name: body.result?.provider || body.provider, type }); 107 | } 108 | 109 | res.locals.body = body; 110 | return oldJson.call(res, body); 111 | }; 112 | next(); 113 | }; 114 | -------------------------------------------------------------------------------- /src/middlewares/useProxyCache.ts: -------------------------------------------------------------------------------- 1 | import { capture } from '@snapshot-labs/snapshot-sentry'; 2 | import { get, set } from '../aws'; 3 | import { ipfsGatewaysCacheHitCount, ipfsGatewaysCacheSize } from '../metrics'; 4 | import { MAX } from '../utils'; 5 | 6 | /** 7 | * This middleware serves a cache if it exists, else it will process the controller 8 | * and caches its results if it's less than 1MB 9 | */ 10 | export default async function useProxyCache(req, res, next) { 11 | const { cid } = req.params; 12 | 13 | const cache = await get(cid); 14 | if (cache) { 15 | const cachedSize = Buffer.from(JSON.stringify(cache)).length; 16 | ipfsGatewaysCacheHitCount.inc({ status: 'HIT' }); 17 | ipfsGatewaysCacheSize.inc({ status: 'HIT' }, cachedSize); 18 | return res.json(cache); 19 | } 20 | 21 | const oldJson = res.json; 22 | res.json = async body => { 23 | res.locals.body = body; 24 | 25 | if (res.statusCode === 200 && body) { 26 | try { 27 | const size = Buffer.from(JSON.stringify(body)).length; 28 | if (size <= MAX) { 29 | ipfsGatewaysCacheHitCount.inc({ status: 'MISS' }); 30 | ipfsGatewaysCacheSize.inc({ status: 'MISS' }, size); 31 | await set(cid, body); 32 | } 33 | } catch (e) { 34 | capture(e); 35 | } 36 | } 37 | 38 | return oldJson.call(res, body); 39 | }; 40 | 41 | next(); 42 | } 43 | -------------------------------------------------------------------------------- /src/providers/4everland.ts: -------------------------------------------------------------------------------- 1 | import { S3 } from '@aws-sdk/client-s3'; 2 | import { sha256 } from '../utils'; 3 | 4 | const { EVER_API_KEY, EVER_API_SECRET } = process.env; 5 | 6 | export const provider = '4everland'; 7 | const client = new S3({ 8 | endpoint: 'https://endpoint.4everland.co', 9 | region: 'eu-west-2', 10 | credentials: { 11 | accessKeyId: EVER_API_KEY || '', 12 | secretAccessKey: EVER_API_SECRET || '' 13 | } 14 | }); 15 | 16 | export async function set(data: Buffer | object) { 17 | const payload = data instanceof Buffer ? data : JSON.stringify(data); 18 | const params = { 19 | Bucket: process.env.EVER_BUCKET_NAME, 20 | Key: sha256(payload) 21 | }; 22 | await client.putObject({ 23 | ...params, 24 | Body: payload, 25 | ContentType: data instanceof Buffer ? undefined : 'application/json; charset=utf-8' 26 | }); 27 | const result = await client.headObject(params); 28 | const cid = JSON.parse(result.ETag || 'null'); 29 | 30 | return { cid, provider }; 31 | } 32 | 33 | export function isConfigured() { 34 | return EVER_API_KEY && EVER_API_SECRET; 35 | } 36 | -------------------------------------------------------------------------------- /src/providers/fleek.ts: -------------------------------------------------------------------------------- 1 | import fleek from '@fleekhq/fleek-storage-js'; 2 | import { sha256 } from '../utils'; 3 | 4 | const { FLEEK_API_KEY, FLEEK_API_SECRET } = process.env; 5 | 6 | export const provider = 'fleek'; 7 | const config: any = { 8 | apiKey: FLEEK_API_KEY, 9 | apiSecret: FLEEK_API_SECRET 10 | }; 11 | 12 | export async function set(data: Buffer | object) { 13 | const input = config; 14 | input.data = data instanceof Buffer ? data : JSON.stringify(data); 15 | input.key = sha256(input.data); 16 | const result = await fleek.upload(input); 17 | const cid = result.hashV0; 18 | 19 | return { cid, provider }; 20 | } 21 | 22 | export function isConfigured() { 23 | return FLEEK_API_KEY && FLEEK_API_SECRET; 24 | } 25 | -------------------------------------------------------------------------------- /src/providers/index.ts: -------------------------------------------------------------------------------- 1 | import { capture } from '@snapshot-labs/snapshot-sentry'; 2 | import { providersMap } from './utils'; 3 | import { countOpenProvidersRequest, providersUploadSize, timeProvidersUpload } from '../metrics'; 4 | type ProviderType = 'image' | 'json'; 5 | 6 | export default function uploadToProviders(providers: string[], type: ProviderType, params: any) { 7 | const configuredProviders = providers.filter(p => providersMap[p].isConfigured()); 8 | 9 | return Promise.any( 10 | configuredProviders.map(async name => { 11 | const type: ProviderType = params instanceof Buffer ? 'image' : 'json'; 12 | const end = timeProvidersUpload.startTimer({ name, type }); 13 | let status = 0; 14 | 15 | try { 16 | countOpenProvidersRequest.inc({ name, type }); 17 | 18 | const result = await providersMap[name].set(params); 19 | const size = (params instanceof Buffer ? params : Buffer.from(JSON.stringify(params))) 20 | .length; 21 | providersUploadSize.inc({ name, type }, size); 22 | status = 1; 23 | console.log(`JSON pinned: ${result.provider} - ${result.cid}`); 24 | 25 | return result; 26 | } catch (e: any) { 27 | if (e instanceof Error) { 28 | if (e.message !== 'Request timed out') { 29 | capture(e, { name }); 30 | } 31 | } else { 32 | capture(new Error(`Error from ${name} provider`), { 33 | contexts: { provider_response: e } 34 | }); 35 | } 36 | return Promise.reject(e); 37 | } finally { 38 | end({ status }); 39 | countOpenProvidersRequest.dec({ name, type }); 40 | } 41 | }) 42 | ); 43 | } 44 | -------------------------------------------------------------------------------- /src/providers/infura.ts: -------------------------------------------------------------------------------- 1 | import { create } from 'ipfs-http-client'; 2 | 3 | const { INFURA_PROJECT_ID, INFURA_PROJECT_SECRET } = process.env; 4 | 5 | export const provider = 'infura'; 6 | const client = create({ 7 | host: 'ipfs.infura.io', 8 | port: 5001, 9 | protocol: 'https', 10 | timeout: 10e3, 11 | headers: { 12 | authorization: `Basic ${Buffer.from(`${INFURA_PROJECT_ID}:${INFURA_PROJECT_SECRET}`).toString( 13 | 'base64' 14 | )}` 15 | } 16 | }); 17 | 18 | export async function set(data: Buffer | object) { 19 | const input = data instanceof Buffer ? data : JSON.stringify(data); 20 | const result = await client.add(input, { 21 | pin: true 22 | }); 23 | const cid = result.cid.toV0().toString(); 24 | 25 | return { cid, provider }; 26 | } 27 | 28 | export function isConfigured() { 29 | return INFURA_PROJECT_ID && INFURA_PROJECT_SECRET; 30 | } 31 | -------------------------------------------------------------------------------- /src/providers/pinata.ts: -------------------------------------------------------------------------------- 1 | import { randomUUID } from 'crypto'; 2 | import { Readable } from 'stream'; 3 | import pinataSDK from '@pinata/sdk'; 4 | 5 | const { PINATA_API_KEY, PINATA_API_SECRET } = process.env; 6 | 7 | export const provider = 'pinata'; 8 | const client = pinataSDK(PINATA_API_KEY || '', PINATA_API_SECRET || ''); 9 | 10 | export async function set(data: Buffer | object) { 11 | let result; 12 | if (data instanceof Buffer) { 13 | const stream = Readable.from(data); 14 | // @ts-ignore 15 | stream.path = randomUUID(); 16 | result = await client.pinFileToIPFS(stream); 17 | } else { 18 | result = await client.pinJSONToIPFS(data); 19 | } 20 | 21 | const cid = result.IpfsHash; 22 | 23 | return { cid, provider }; 24 | } 25 | 26 | export function isConfigured() { 27 | return PINATA_API_KEY && PINATA_API_SECRET; 28 | } 29 | -------------------------------------------------------------------------------- /src/providers/utils.ts: -------------------------------------------------------------------------------- 1 | import * as fourEverland from './4everland'; 2 | import * as fleek from './fleek'; 3 | import * as infura from './infura'; 4 | import * as pinata from './pinata'; 5 | import * as web3storage from './web3storage'; 6 | 7 | // List of providers used for pinning images 8 | export const IMAGE_PROVIDERS = ['fleek', 'infura', 'pinata', '4everland']; 9 | 10 | // List of providers used for pinning json 11 | export const JSON_PROVIDERS = ['fleek', 'infura', 'web3storage', '4everland']; 12 | 13 | export const providersMap = { 14 | fleek, 15 | infura, 16 | pinata, 17 | web3storage, 18 | '4everland': fourEverland 19 | }; 20 | -------------------------------------------------------------------------------- /src/providers/web3storage.ts: -------------------------------------------------------------------------------- 1 | import { Blob, File, Web3Storage } from 'web3.storage'; 2 | import { sha256 } from '../utils'; 3 | 4 | const { WEB3STORAGE_API_TOKEN } = process.env; 5 | 6 | export const provider = 'web3storage'; 7 | const client = new Web3Storage({ token: WEB3STORAGE_API_TOKEN || '' }); 8 | 9 | export async function set(data: Buffer | object) { 10 | let file; 11 | if (data instanceof Buffer) { 12 | const blob = new Blob([data]); 13 | file = new File([blob], sha256(data)); 14 | } else { 15 | const content = JSON.stringify(data); 16 | const blob = new Blob([content]); 17 | file = new File([blob], sha256(content)); 18 | } 19 | 20 | const cid = await client.put([file], { wrapWithDirectory: false }); 21 | 22 | return { cid, provider }; 23 | } 24 | 25 | export function isConfigured() { 26 | return !!WEB3STORAGE_API_TOKEN; 27 | } 28 | -------------------------------------------------------------------------------- /src/proxy.ts: -------------------------------------------------------------------------------- 1 | import { capture } from '@snapshot-labs/snapshot-sentry'; 2 | import express from 'express'; 3 | import { create } from 'ipfs-http-client'; 4 | import fetch from 'node-fetch'; 5 | import gateways from './gateways.json'; 6 | import { 7 | countOpenGatewaysRequest, 8 | ipfsGatewaysReturnCount, 9 | timeIpfsGatewaysResponse 10 | } from './metrics'; 11 | import useProxyCache from './middlewares/useProxyCache'; 12 | 13 | const UNSUPPORTED_FILE_TYPE = 'unsupported file type'; 14 | 15 | const router = express.Router(); 16 | const graphIpfsClient = create({ url: 'https://api.thegraph.com/ipfs/api/v0' }); 17 | 18 | async function getFileFromGraphIpfs(cid: string) { 19 | const chunks: Uint8Array[] = []; 20 | for await (const chunk of graphIpfsClient.cat(cid)) { 21 | chunks.push(chunk); 22 | } 23 | 24 | try { 25 | const json = JSON.parse(Buffer.concat(chunks).toString('utf-8')); 26 | 27 | return { 28 | gateway: 'graph', 29 | json 30 | }; 31 | } catch (e) { 32 | return Promise.reject(UNSUPPORTED_FILE_TYPE); 33 | } 34 | } 35 | 36 | router.get('^/ipfs/:cid([0-9a-zA-Z]+)$', useProxyCache, async (req, res) => { 37 | try { 38 | const result = await Promise.any([ 39 | getFileFromGraphIpfs(req.params.cid), 40 | ...gateways.map(async gateway => { 41 | const end = timeIpfsGatewaysResponse.startTimer({ name: gateway }); 42 | let status = 0; 43 | 44 | try { 45 | countOpenGatewaysRequest.inc({ name: gateway }); 46 | 47 | const url = `https://${gateway}${req.originalUrl}`; 48 | const response = await fetch(url, { timeout: 15e3 }); 49 | 50 | if (!response.ok) { 51 | return Promise.reject(response.status); 52 | } 53 | 54 | if (!['text/plain', 'application/json'].includes(response.headers.get('content-type'))) { 55 | return Promise.reject(UNSUPPORTED_FILE_TYPE); 56 | } 57 | 58 | let json; 59 | try { 60 | json = await response.json(); 61 | } catch (e: any) { 62 | return Promise.reject(e); 63 | } 64 | 65 | status = 1; 66 | return { gateway, json }; 67 | } finally { 68 | end({ status }); 69 | countOpenGatewaysRequest.dec({ name: gateway }); 70 | } 71 | }) 72 | ]); 73 | ipfsGatewaysReturnCount.inc({ name: result.gateway }); 74 | 75 | return res.json(result.json); 76 | } catch (e) { 77 | if (e instanceof AggregateError) { 78 | return res.status(e.errors.includes(UNSUPPORTED_FILE_TYPE) ? 415 : 400).json(); 79 | } 80 | 81 | capture(e); 82 | return res.status(500).json(); 83 | } 84 | }); 85 | 86 | export default router; 87 | -------------------------------------------------------------------------------- /src/rpc.ts: -------------------------------------------------------------------------------- 1 | import { capture } from '@snapshot-labs/snapshot-sentry'; 2 | import express from 'express'; 3 | import { set as setAws } from './aws'; 4 | import uploadToProviders from './providers/'; 5 | import { JSON_PROVIDERS } from './providers/utils'; 6 | import { MAX, rpcError, rpcSuccess } from './utils'; 7 | 8 | const router = express.Router(); 9 | 10 | router.post('/', async (req, res) => { 11 | const { id, params } = req.body; 12 | 13 | if (!params) { 14 | return rpcError(res, 400, 'Malformed body', id); 15 | } 16 | 17 | try { 18 | const size = Buffer.from(JSON.stringify(params)).length; 19 | if (size > MAX) return rpcError(res, 400, 'File too large', id); 20 | 21 | const result = await uploadToProviders(JSON_PROVIDERS, 'json', params); 22 | try { 23 | await setAws(result.cid, params); 24 | } catch (e: any) { 25 | capture(e); 26 | } 27 | 28 | return rpcSuccess(res, { ...result, size }, id); 29 | } catch (e: any) { 30 | capture(e); 31 | return rpcError(res, 500, e, id); 32 | } 33 | }); 34 | 35 | export default router; 36 | -------------------------------------------------------------------------------- /src/upload.ts: -------------------------------------------------------------------------------- 1 | import fs from 'fs'; 2 | import { capture } from '@snapshot-labs/snapshot-sentry'; 3 | import express from 'express'; 4 | import multer from 'multer'; 5 | import sharp from 'sharp'; 6 | import uploadToProviders from './providers/'; 7 | import { IMAGE_PROVIDERS } from './providers/utils'; 8 | import { rpcError, rpcSuccess } from './utils'; 9 | 10 | const MAX_INPUT_SIZE = 1024 * 1024; 11 | const MAX_IMAGE_DIMENSION = 1500; 12 | 13 | const router = express.Router(); 14 | const upload = multer({ 15 | dest: 'uploads/', 16 | limits: { fileSize: MAX_INPUT_SIZE } 17 | }).single('file'); 18 | 19 | router.post('/upload', async (req, res) => { 20 | upload(req, res, async err => { 21 | try { 22 | if (err) return rpcError(res, 400, err.message); 23 | if (!req.file) return rpcError(res, 400, 'No file submitted'); 24 | 25 | const transformer = sharp() 26 | .resize({ 27 | width: MAX_IMAGE_DIMENSION, 28 | height: MAX_IMAGE_DIMENSION, 29 | fit: 'inside' 30 | }) 31 | .webp({ lossless: true }); 32 | 33 | const buffer = await fs 34 | .createReadStream(req.file?.path as string) 35 | .pipe(transformer) 36 | .toBuffer(); 37 | 38 | const result = await uploadToProviders(IMAGE_PROVIDERS, 'image', buffer); 39 | const file = { 40 | cid: result.cid, 41 | provider: result.provider 42 | }; 43 | 44 | return rpcSuccess(res, file); 45 | } catch (e: any) { 46 | if (e.message === 'Input buffer contains unsupported image format') { 47 | return rpcError(res, 415, 'Unsupported file type'); 48 | } 49 | 50 | capture(e); 51 | return rpcError(res, 500, e); 52 | } finally { 53 | if (req.file) { 54 | await fs.promises.unlink(req.file.path as string); 55 | } 56 | } 57 | }); 58 | }); 59 | 60 | export default router; 61 | -------------------------------------------------------------------------------- /src/utils.ts: -------------------------------------------------------------------------------- 1 | import { createHash } from 'crypto'; 2 | import { Response } from 'express'; 3 | 4 | export const MAX = 20e10; // 20 MB 5 | 6 | export function rpcSuccess(res: Response, result: any, id = '') { 7 | res.json({ 8 | jsonrpc: '2.0', 9 | result, 10 | id 11 | }); 12 | } 13 | 14 | export function rpcError(res: Response, code: number, e: Error | string, id = null) { 15 | res.status(code).json({ 16 | jsonrpc: '2.0', 17 | error: { 18 | code, 19 | message: e, 20 | data: {} 21 | }, 22 | id 23 | }); 24 | } 25 | 26 | export function sha256(input: string | Buffer) { 27 | return createHash('sha256').update(input).digest('hex'); 28 | } 29 | -------------------------------------------------------------------------------- /test/.env.test: -------------------------------------------------------------------------------- 1 | NODE_ENV=test 2 | -------------------------------------------------------------------------------- /test/e2e/fixtures/file.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Snapshot" 3 | } 4 | -------------------------------------------------------------------------------- /test/e2e/fixtures/json-file.png: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Snapshot" 3 | } 4 | -------------------------------------------------------------------------------- /test/e2e/fixtures/too-heavy.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snapshot-labs/pineapple/4023c623585e4cde2296922572d35c85d45cf940/test/e2e/fixtures/too-heavy.jpg -------------------------------------------------------------------------------- /test/e2e/fixtures/valid.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snapshot-labs/pineapple/4023c623585e4cde2296922572d35c85d45cf940/test/e2e/fixtures/valid.png -------------------------------------------------------------------------------- /test/e2e/proxy.test.ts: -------------------------------------------------------------------------------- 1 | import request from 'supertest'; 2 | import { get, remove, set } from '../../src/aws'; 3 | 4 | const HOST = `http://localhost:${process.env.PORT || 3003}`; 5 | 6 | describe('GET /ipfs/:cid', () => { 7 | describe('when the IPFS cid exists', () => { 8 | const cid = 'bafkreib5epjzumf3omr7rth5mtcsz4ugcoh3ut4d46hx5xhwm4b3pqr2vi'; 9 | const path = `/ipfs/${cid}`; 10 | const content = { status: 'OK' }; 11 | 12 | afterEach(async () => { 13 | await remove(cid); 14 | }); 15 | 16 | describe('when the file is cached', () => { 17 | if (process.env.AWS_REGION) { 18 | const cachedContent = { status: 'CACHED' }; 19 | 20 | it('returns the cache file', async () => { 21 | await set(cid, cachedContent); 22 | const response = await request(HOST).get(path); 23 | 24 | expect(response.body).toEqual(cachedContent); 25 | expect(response.statusCode).toBe(200); 26 | expect(response.headers['content-type']).toBe('application/json; charset=utf-8'); 27 | expect(await get(cid)).toEqual(cachedContent); 28 | }); 29 | } else { 30 | it.todo('needs to set AWS credentials to test the cache'); 31 | } 32 | }); 33 | 34 | describe('when the file is not cached', () => { 35 | if (process.env.AWS_REGION) { 36 | it('returns the file and caches it', async () => { 37 | const response = await request(HOST).get(path); 38 | 39 | expect(response.body).toEqual(content); 40 | expect(response.statusCode).toBe(200); 41 | expect(response.headers['content-type']).toBe('application/json; charset=utf-8'); 42 | expect(await get(cid)).toEqual(response.body); 43 | }); 44 | } else { 45 | it.todo('needs to set AWS credentials to test the cache'); 46 | } 47 | }); 48 | 49 | it('returns a 415 error when not a JSON file', async () => { 50 | const response = await request(HOST).get( 51 | '/ipfs/bafybeie2x4ptheqskiauhfz4w4pbq7o6742oupitganczhjanvffp2spti' 52 | ); 53 | 54 | expect(response.statusCode).toBe(415); 55 | }, 30e3); 56 | }); 57 | 58 | describe('when the IPFS cid does not exist', () => { 59 | it('returns a 400 error', async () => { 60 | const response = await request(HOST).get('/ipfs/test'); 61 | 62 | expect(response.statusCode).toBe(400); 63 | }, 30e3); 64 | }); 65 | }); 66 | -------------------------------------------------------------------------------- /test/e2e/rpc.test.ts: -------------------------------------------------------------------------------- 1 | import request from 'supertest'; 2 | 3 | const HOST = `http://localhost:${process.env.PORT || 3003}`; 4 | 5 | describe('POST /', () => { 6 | describe('when the payload is valid', () => { 7 | it('returns a 200 error', async () => { 8 | const response = await request(HOST) 9 | .post('/') 10 | .send({ params: { test: 'value' } }); 11 | 12 | expect(response.statusCode).toBe(200); 13 | expect(response.body.result.cid.length).toBeGreaterThan(10); 14 | expect(['4everland', 'fleek', 'pinata']).toContain(response.body.result.provider); 15 | }); 16 | }); 17 | 18 | describe('when the payload is no valid', () => { 19 | it('returns a 400 error on malformed json', async () => { 20 | const response = await request(HOST).post('/').send({ test: 'value' }); 21 | 22 | expect(response.statusCode).toBe(400); 23 | expect(response.body.error.message).toBe('Malformed body'); 24 | }); 25 | 26 | it('returns a 400 error on empty body', async () => { 27 | const response = await request(HOST).post('/'); 28 | 29 | expect(response.statusCode).toBe(400); 30 | expect(response.body.error.message).toBe('Malformed body'); 31 | }); 32 | }); 33 | }); 34 | -------------------------------------------------------------------------------- /test/e2e/upload.test.ts: -------------------------------------------------------------------------------- 1 | import path from 'path'; 2 | import request from 'supertest'; 3 | 4 | const HOST = `http://localhost:${process.env.PORT || 3003}`; 5 | 6 | describe('POST /upload', () => { 7 | describe('when the image exceed the maximum file size', () => { 8 | it('returns a 400 error', async () => { 9 | const response = await request(HOST) 10 | .post('/upload') 11 | .attach('file', path.join(__dirname, './fixtures/too-heavy.jpg')); 12 | 13 | expect(response.statusCode).toBe(400); 14 | expect(response.body.error.message).toBe('File too large'); 15 | }); 16 | }); 17 | 18 | describe('when the file is not an image', () => { 19 | it.each([ 20 | ['non-image file extension', 'file.json'], 21 | ['non-image impersonating image extension', 'json-file.png'] 22 | ])('returns a 415 error on %s', async (title, filename) => { 23 | const response = await request(HOST) 24 | .post('/upload') 25 | .attach('file', path.join(__dirname, `./fixtures/${filename}`)); 26 | 27 | expect(response.statusCode).toBe(415); 28 | expect(response.body.error.message).toBe('Unsupported file type'); 29 | }); 30 | }); 31 | 32 | describe('when the file is missing', () => { 33 | it('returns a 400 error', async () => { 34 | const response = await request(HOST).post('/upload'); 35 | 36 | expect(response.statusCode).toBe(400); 37 | expect(response.body.error.message).toBe('No file submitted'); 38 | }); 39 | }); 40 | 41 | describe('when the file is correct', () => { 42 | it('uploads the file and returns a JSON-RPC response with the CID and its provider', async () => { 43 | const response = await request(HOST) 44 | .post('/upload') 45 | .attach('file', path.join(__dirname, './fixtures/valid.png')); 46 | 47 | expect(response.statusCode).toBe(200); 48 | expect(response.body.jsonrpc).toBe('2.0'); 49 | expect(response.body.result.cid.length).toBeGreaterThan(10); 50 | expect(['4everland', 'fleek', 'pinata']).toContain(response.body.result.provider); 51 | }); 52 | }); 53 | }); 54 | -------------------------------------------------------------------------------- /test/unit/providers/fixtures/sample.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snapshot-labs/pineapple/4023c623585e4cde2296922572d35c85d45cf940/test/unit/providers/fixtures/sample.webp -------------------------------------------------------------------------------- /test/unit/providers/index.test.ts: -------------------------------------------------------------------------------- 1 | import fs from 'fs'; 2 | import path from 'path'; 3 | import * as FourEverland from '../../../src/providers/4everland'; 4 | import * as Fleek from '../../../src/providers/fleek'; 5 | import * as Pinata from '../../../src/providers/pinata'; 6 | 7 | describe('providers', () => { 8 | jest.retryTimes(2); 9 | 10 | const json = { 11 | input: { 12 | status: 'OK' 13 | }, 14 | output: { 15 | v0: 'QmTJzHxw4pPgcv6ZtA7hm9L32Evr2Z6A1LoEuTp9UGZC2n', 16 | v1: 'bafkreib5epjzumf3omr7rth5mtcsz4ugcoh3ut4d46hx5xhwm4b3pqr2vi' 17 | } 18 | }; 19 | 20 | const image = { 21 | input: fs.promises.readFile(path.join(__dirname, './fixtures/sample.webp')), 22 | output: { 23 | v0: 'QmY4XQ2qvrRwEZWr918BCjbW35Q7WJ7rsYZsDB1f8fhk7K', 24 | v1: 'bafkreickll7k76ciher5vfsly6ew6awqfa7ix74zww4pqkrrvyzbjwvr2a' 25 | } 26 | }; 27 | 28 | function buildProviderPayload(providers, version) { 29 | return providers.map(p => ({ 30 | name: p.provider, 31 | provider: p, 32 | idVersion: version 33 | })); 34 | } 35 | 36 | const providerPayload: { name: string; provider: any; idVersion: string }[] = 37 | buildProviderPayload([Fleek, Pinata], 'v0').concat(buildProviderPayload([FourEverland], 'v1')); 38 | 39 | describe.each(providerPayload)('$name', ({ name, provider, idVersion }) => { 40 | if (!provider.isConfigured()) { 41 | it.todo(`needs to set credentials for ${name}`); 42 | } else { 43 | it('should upload a JSON file', async () => { 44 | const result = await provider.set(json.input); 45 | 46 | expect(result.provider).toBe(name); 47 | expect(result.cid).toBe(json.output[idVersion]); 48 | }, 20e3); 49 | 50 | it('should upload an image file', async () => { 51 | const buffer = await image.input; 52 | const result = await provider.set(buffer); 53 | 54 | expect(result.provider).toBe(name); 55 | expect(result.cid).toBe(image.output[idVersion]); 56 | }, 20e3); 57 | } 58 | }); 59 | }); 60 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "esnext", 4 | "module": "commonjs", 5 | "rootDir": "./", 6 | "outDir": "./dist", 7 | "esModuleInterop": true, 8 | "strict": true, 9 | "noImplicitAny": false, 10 | "resolveJsonModule": true, 11 | "moduleResolution": "Node", 12 | "skipLibCheck": true, 13 | "sourceMap": true, 14 | "inlineSources": true, 15 | "sourceRoot": "/" 16 | } 17 | } 18 | --------------------------------------------------------------------------------