├── .npmrc ├── .gitattributes ├── .eslintignore ├── .env.example ├── lib ├── workerhelpers.ts └── fileCore.ts ├── .gitignore ├── .editorconfig ├── .npmignore ├── .github └── workflows │ ├── push.yml │ └── release.yml ├── .eslintrc ├── worker.webpack.config.js ├── LICENSE.md ├── worker ├── constants.ts ├── workerCode.ts └── tsconfig.json ├── cli ├── clihelpers.ts └── cli.ts ├── package.json ├── README.md └── tsconfig.json /.npmrc: -------------------------------------------------------------------------------- 1 | registry = https://registry.npmjs.org 2 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | * text=auto 2 | *.js text eol=lf 3 | -------------------------------------------------------------------------------- /.eslintignore: -------------------------------------------------------------------------------- 1 | .git 2 | .serverless 3 | .vscode 4 | .webpack 5 | coverage 6 | node_modules 7 | tmp 8 | -------------------------------------------------------------------------------- /.env.example: -------------------------------------------------------------------------------- 1 | CLOUDFLARE_API_TOKEN= 2 | CLOUDFLARE_ACCOUNT_ID= 3 | CLOUDFLARE_KV_NAMESPACE_ID= 4 | CLOUDFLARE_ZONE_ID= 5 | -------------------------------------------------------------------------------- /lib/workerhelpers.ts: -------------------------------------------------------------------------------- 1 | export function ArrayBufferToString(buffer, encoding = 'utf8') { 2 | return Buffer.from(buffer).toString(encoding) 3 | } 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .git 2 | .vscode 3 | .serverless 4 | .webpack 5 | .idea 6 | coverage 7 | node_modules 8 | *.tgz 9 | .env 10 | test/ 11 | lib/*.js 12 | dist/ 13 | worker/*.js 14 | .vs/ 15 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | [*] 3 | indent_style = space 4 | indent_size = 4 5 | end_of_line = lf 6 | charset = utf-8 7 | trim_trailing_whitespace = true 8 | insert_final_newline = true 9 | 10 | [*.json] 11 | indent_size = 4 12 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | .eslintrc 2 | .eslintignore 3 | .editorconfig 4 | .gitattributes 5 | .gitignore 6 | .prettierrc 7 | .prettierignore 8 | .serverless/ 9 | .travis.yml 10 | .vscode/ 11 | .idea/ 12 | *.tgz 13 | CODE_OF_CONDUCT.md 14 | jest-config.json 15 | __tests__/ 16 | example/ 17 | coverage/ 18 | .env 19 | test/ 20 | node_modules/ 21 | -------------------------------------------------------------------------------- /.github/workflows/push.yml: -------------------------------------------------------------------------------- 1 | on: push 2 | name: push 3 | jobs: 4 | build: 5 | runs-on: ubuntu-latest 6 | steps: 7 | - uses: actions/checkout@master 8 | - uses: actions/setup-node@v1 9 | with: 10 | node-version: '12.x' 11 | - run: npm ci 12 | - run: npm run dist 13 | - uses: actions/upload-artifact@master 14 | with: 15 | name: dist 16 | path: dist 17 | -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "root": true, 3 | "env": { 4 | "node": true, 5 | "es6": true 6 | }, 7 | "extends": ["airbnb-base", "plugin:prettier/recommended"], 8 | "plugins": ["prettier"], 9 | "settings": { 10 | "import/core-modules": ["aws-sdk"] 11 | }, 12 | "rules": { 13 | "no-console": "off", 14 | "max-len": ["error", { "code": 100, "ignoreUrls": true }], 15 | "comma-dangle": ["error", "always-multiline"] 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /worker.webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | 3 | module.exports = { 4 | devtool: 'hidden-source-map', // eval() doesn't work in CF's workers 5 | entry: './worker/workerCode.ts', 6 | target: 'webworker', 7 | mode: process.env.NODE_ENV || 'development', 8 | optimization: { 9 | minimize: false, 10 | }, 11 | module: { 12 | rules: [ 13 | { 14 | loader: 'ts-loader', 15 | exclude: [/node_modules/, /cli/], 16 | options: { configFile: 'worker/tsconfig.json' }, 17 | } 18 | ] 19 | }, 20 | resolve: { 21 | extensions: ['.ts', '.js' ] 22 | }, 23 | output: { 24 | filename: 'worker.js', 25 | path: path.resolve(__dirname, 'dist'), 26 | } 27 | }; 28 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | on: release 2 | name: release 3 | jobs: 4 | release: 5 | runs-on: ubuntu-latest 6 | if: github.event.action == 'published' 7 | steps: 8 | - uses: actions/checkout@master 9 | - uses: actions/setup-node@v1 10 | name: Setup node (npmjs.org) 11 | with: 12 | node-version: '12.x' 13 | registry-url: 'https://registry.npmjs.org' 14 | - run: npm ci 15 | - run: npm run dist 16 | - run: npm publish --access public 17 | env: 18 | NODE_AUTH_TOKEN: ${{ secrets.NPM_AUTH_TOKEN }} 19 | - uses: actions/setup-node@v1 20 | with: 21 | registry-url: 'https://npm.pkg.github.com' 22 | - run: npm publish 23 | env: 24 | NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} 25 | - uses: actions/upload-artifact@master 26 | with: 27 | name: dist 28 | path: dist 29 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright 2019 Hunter Ray 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /worker/constants.ts: -------------------------------------------------------------------------------- 1 | export const error_404 = "404 error: file not found"; 2 | // leading slash will be removed during execution 3 | export const default_index = "/index.html"; 4 | export const default_ct = "text/plain"; 5 | 6 | // non-exhaustive. Add more to your own deployment if you use other extensions. 7 | export const extensionToContentType = { 8 | 'css': 'text/css', 9 | 'html': 'text/html', 10 | 'js': 'application/javascript', 11 | 'json': 'application/json', 12 | 'acc': 'audio/acc', 13 | 'avi': 'video/x-msvideo', 14 | 'bin': 'application/octet-stream', 15 | 'bmp': 'image/bmp', 16 | 'bz2': 'application/x-bzip2', 17 | 'bz': 'application/x-bzip', 18 | 'csv': 'text/csv', 19 | 'epub': 'application/epub+zip', 20 | 'gif': 'image/gif', 21 | 'htm': 'text/html', 22 | 'ico': 'image/vnd.microsoft.icon', 23 | 'jar': 'application/java-archive', 24 | 'jpeg': 'image/jpeg', 25 | 'jpg': 'image/jpeg', 26 | 'mp3': 'audio/mpeg', 27 | 'mp4': 'video/mp4', 28 | 'mpeg': 'video/mpeg', 29 | 'pdf': 'application/pdf', 30 | 'png': 'image/png', 31 | 'rar': 'application/x-rar-compressed', 32 | 'rtf': 'application/rtf', 33 | 'sh': 'application/x-sh', 34 | 'swf': 'application/x-shockwave-flash', 35 | 'tar': 'application/x-tar', 36 | 'tif': 'image/tiff', 37 | 'tiff': 'image/tiff', 38 | 'ttf': 'font/ttf', 39 | 'wav': 'audio/wav', 40 | 'webm': 'video/webm', 41 | 'webp': 'image/webp', 42 | 'woff': 'font/woff', 43 | 'woff2': 'font/woff2', 44 | 'xml': 'text/xml', 45 | 'zip': 'application/zip', 46 | '7z': 'application/x-7z-compressed', 47 | }; 48 | -------------------------------------------------------------------------------- /worker/workerCode.ts: -------------------------------------------------------------------------------- 1 | import CloudflareWorkerGlobalScope, {CloudflareWorkerKV} from 'types-cloudflare-worker'; 2 | import fileCore from "../lib/fileCore"; 3 | 4 | declare var self: CloudflareWorkerGlobalScope; 5 | declare var STATIC_KV: CloudflareWorkerKV; 6 | 7 | export class Worker { 8 | public async handle(event: FetchEvent) { 9 | 10 | // First: use index.html for the root 11 | let path = new URL(event.request.url).pathname; 12 | if (path == '/') { 13 | path = '/index.html' 14 | } 15 | // 16 | // call the cache api-like API for fileCore 17 | // Param 1: file name/path 18 | // Param 2: the KV namespace CF is injecting 19 | // Param 3: whether or not to use cache 20 | // Param 4: request for cache matches and puts 21 | // Param 5: any custom response headers (accept-ranges is not supported) 22 | // 23 | // the custom headers (param 5) is not included in the sample 24 | // 25 | let _filecore = await fileCore.getFile(path, STATIC_KV, true, event.request); 26 | // Due to the streaming nature if the file is over 10mb, 27 | // be wary before trying to create your own response from this returned response 28 | if (_filecore) { 29 | return _filecore; 30 | } 31 | // no file was found, so do other code/handling 32 | // since this is the stand-alone worker, we 404 33 | return new Response("404 not found", {status: 404}); 34 | } 35 | } 36 | self.addEventListener('fetch', (event: FetchEvent) => { 37 | const worker = new Worker(); 38 | event.respondWith(worker.handle(event)); 39 | }); 40 | -------------------------------------------------------------------------------- /cli/clihelpers.ts: -------------------------------------------------------------------------------- 1 | import * as path from "path"; 2 | import slash from "slash"; 3 | import Axios from "axios"; 4 | import * as fssync from "fs"; 5 | 6 | const {CLOUDFLARE_API_TOKEN} = process.env; 7 | 8 | export function isDirectory(dirPath) { 9 | return fssync.statSync(dirPath).isDirectory(); 10 | } 11 | 12 | export function relativePath(fullPath) { 13 | return path.relative(process.cwd(), fullPath) 14 | } 15 | 16 | export function fileToUri(filePath: string, path: string): string { 17 | let uriPath = slash(filePath).replace(path, ''); 18 | if (!uriPath.startsWith('/')) { 19 | uriPath = `/${uriPath}`; 20 | } 21 | return uriPath 22 | } 23 | 24 | export function splitNChars(txt, num) { 25 | var result = []; 26 | for (var i = 0; i < txt.length; i += num) { 27 | // @ts-ignore 28 | result.push(txt.substr(i, num)); 29 | } 30 | return result; 31 | } 32 | 33 | export function splitBuffer(buf: Buffer, num) { 34 | let result: Buffer[] = []; 35 | for (let i = 0; i < buf.length; i += num) { 36 | if (i + num > buf.length) { 37 | result.push(buf.slice(i, buf.length)); 38 | } else { 39 | result.push(buf.slice(i, i + num)); 40 | } 41 | } 42 | return result; 43 | } 44 | 45 | // re-implemented this to use Axios because why not? 46 | export async function cfApiCall({url, method, contentType = '', body = null}) { 47 | return await Axios({ 48 | method: method, 49 | headers: { 50 | 'Authorization': `Bearer ${CLOUDFLARE_API_TOKEN}`, 51 | 'content-type': contentType 52 | }, 53 | data: body, 54 | url: url, 55 | responseType: "json" 56 | }); 57 | } 58 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cloudflare-file-hosting", 3 | "version": "0.2.0", 4 | "description": "Use Cloudflare as a file host with Workers and Workers KV", 5 | "main": "./dist/lib/fileCore.js", 6 | "bin": { 7 | "cfupload": "./dist/cli/cli.js" 8 | }, 9 | "types": "./lib/fileCore.ts", 10 | "scripts": { 11 | "testcli": "npm run buildcli && node ./dist/cli/cli.js --path test", 12 | "buildcli": "tsc", 13 | "buildlib": "tsc", 14 | "dist": "cross-env NODE_ENV=production npm run buildworker && cross-env NODE_ENV=production npm run buildcli && cross-env NODE_ENV=production npm run buildlib", 15 | "buildworker": "webpack --config worker.webpack.config.js", 16 | "watchworker": "webpack --config worker.webpack.config.js --watch" 17 | }, 18 | "repository": { 19 | "type": "git", 20 | "url": "https://github.com/judge2020/cloudflare-file-hosting" 21 | }, 22 | "keywords": [ 23 | "cloudflare", 24 | "serverless", 25 | "workers" 26 | ], 27 | "author": "judge2020", 28 | "license": "MIT", 29 | "devDependencies": { 30 | "@types/node": "^12.6.2", 31 | "@udacity/types-service-worker-mock": "^1.0.0", 32 | "cloudflare-worker-mock": "^1.0.0", 33 | "cross-dotenv": "^1.0.4", 34 | "cross-env": "^5.2.0", 35 | "ts-loader": "^6.0.4", 36 | "types-cloudflare-worker": "^1.0.0", 37 | "typescript": "^3.5.3", 38 | "webpack": "^4.35.3", 39 | "webpack-cli": "^3.3.6" 40 | }, 41 | "dependencies": { 42 | "arraybuffer-to-string": "^1.0.2", 43 | "axios": "^0.19.0", 44 | "cloudflare-workers-toolkit": "^0.1.0", 45 | "commander": "^2.20.0", 46 | "slash": "^3.0.0", 47 | "walkdir": "^0.4.0" 48 | } 49 | } 50 | 51 | -------------------------------------------------------------------------------- /cli/cli.ts: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env node 2 | import commander from "commander"; 3 | import {promises as fs} from "fs"; 4 | import * as helpers from "./clihelpers"; 5 | import walk from "walkdir"; 6 | 7 | const {CLOUDFLARE_API_TOKEN, CLOUDFLARE_ACCOUNT_ID, CLOUDFLARE_KV_NAMESPACE_ID} = process.env; 8 | 9 | // begin CLI 10 | const program = new commander.Command(); 11 | program.version('0.2.0') 12 | .description('Upload large files to Cloudflare using Workers and Workers KV') 13 | .option('-p, --path ', 'Path to root directory of hostname (eg. \"--path dist\" will mean dist/test.css is available at (hostname)/test.css)'); 14 | 15 | program.parse(process.argv); 16 | 17 | // ensure Cloudflare environment variables are passed 18 | 19 | if (!CLOUDFLARE_API_TOKEN) { 20 | console.log('The Environment variable CLOUDFLARE_API_TOKEN need to be set.'); 21 | process.exit(1) 22 | } 23 | 24 | let path = program.path; 25 | 26 | if (!path) { 27 | console.log("the \"--path\" argument is required."); 28 | process.exit(1) 29 | } 30 | 31 | // test if the path is a directory 32 | if (!helpers.isDirectory(path)) { 33 | console.log(`Path passed \"${path}\" is not a directory.`); 34 | process.exit(1) 35 | } 36 | 37 | 38 | export async function uploadFile(key, value) { 39 | return await helpers.cfApiCall({ 40 | url: `https://api.cloudflare.com/client/v4/accounts/${CLOUDFLARE_ACCOUNT_ID}/storage/kv/namespaces/${CLOUDFLARE_KV_NAMESPACE_ID}/values/${key}`, 41 | method: 'PUT', 42 | body: value, 43 | contentType: "application/json" 44 | }); 45 | } 46 | 47 | walk(path, {}, async (_filePath, stat) => { 48 | if (stat.isDirectory()) { 49 | return; 50 | } 51 | let filePath = helpers.relativePath(_filePath); 52 | let uriPath = helpers.fileToUri(filePath, path); 53 | 54 | let b64Contents = await fs.readFile(filePath, {encoding: null}); 55 | // handle <10mb files 56 | if (b64Contents.length < 10485760) { 57 | console.log(`Uploading ${uriPath}...`); 58 | await uploadFile(uriPath, b64Contents); 59 | return; 60 | } 61 | 62 | // file splitting logic for >10mb files 63 | let b64parts = helpers.splitBuffer(b64Contents, 10485760); 64 | 65 | await uploadFile(uriPath, `SPLIT_${b64parts.length}`); 66 | 67 | b64parts.forEach(async (value: Buffer, index) => { 68 | let _logName = `${uriPath} part ${index}`; 69 | console.log(`Uploading ${_logName}...`); 70 | let result = await uploadFile(`${uriPath}_${index}`, value); 71 | console.log(_logName, result.data.success ? 'success' : result.data) 72 | }); 73 | }); 74 | -------------------------------------------------------------------------------- /lib/fileCore.ts: -------------------------------------------------------------------------------- 1 | import {CloudflareWorkerKV} from "types-cloudflare-worker"; 2 | import {ArrayBufferToString} from "./workerhelpers"; 3 | 4 | import * as constants from "../worker/constants"; 5 | 6 | 7 | export default class FileCore { 8 | 9 | public static getContentType(fileName, defaultType = "text/plain", ct = constants.extensionToContentType) { 10 | let filenamesplit = fileName.split('.'); 11 | let fileExtension = filenamesplit[filenamesplit.length - 1]; 12 | return ct[fileExtension] || defaultType; 13 | } 14 | 15 | /** 16 | * Check if a file exists in the specified KV namespace. If so, return a response for that file 17 | * @param filePath Path to the file, eg `new URL(event.request.url).pathname` 18 | * @param KV_NAMESPACE a KV namespace binding for your files 19 | * @param useCache Whether or not to use the cache API. Will only cache files less than 10mb in size. 20 | * @param requestForCache If you set useCache to true, this is the event.request object to use for cache matching. 21 | * @param customHeaders any custom headers 22 | * @return {Response} Response when a file is found; immediately return this if it's found or streamed responses will not work 23 | * @return {null} null when a file is not found in KV 24 | */ 25 | public static async getFile(filePath: string, KV_NAMESPACE: CloudflareWorkerKV, useCache: boolean = false, requestForCache: Request | null = null, customHeaders = {}): Promise { 26 | // remove a leading slash 27 | if (filePath.startsWith("/")) { 28 | filePath = filePath.substr(1); 29 | } 30 | // replace %20 with space since the CLI uploads with the space 31 | filePath = filePath.replace('%20', ' '); 32 | 33 | // test cache before pulling from KV 34 | if (useCache && requestForCache) { 35 | let _cache = await caches.default.match(requestForCache); 36 | if (_cache) { 37 | return _cache; 38 | } 39 | } 40 | 41 | let arrayBufferValue = await KV_NAMESPACE.get(filePath, "arrayBuffer"); 42 | 43 | if (arrayBufferValue === null) { 44 | return null 45 | } 46 | 47 | // prep custom headers 48 | customHeaders["content-type"] = this.getContentType(filePath); 49 | customHeaders["accept-ranges"] = "none"; 50 | 51 | // we use an arrayBuffer so that, if the file is <10mb, we can instantly return it 52 | let _asStr = ArrayBufferToString(arrayBufferValue); 53 | if (!_asStr.startsWith('SPLIT_')) { 54 | let resp = new Response(arrayBufferValue, { 55 | headers: customHeaders 56 | }); 57 | if (useCache && requestForCache) { 58 | await caches.default.put(requestForCache, resp.clone()); 59 | } 60 | return resp; 61 | } 62 | 63 | // file stitching logic 64 | 65 | let {readable, writable} = new TransformStream(); 66 | 67 | // numberOfKeys is formatted as `SPLIT_`, 68 | // eg. a 5-part file will be represented as `SPLIT_5` and have KV values at `_0` to `_4` 69 | let numberOfKeys = _asStr.split('_')[1]; 70 | 71 | // no await so that we can stream the response 72 | // due to the streamed response, we can't cache the response. 73 | this.streamKv(numberOfKeys, writable, KV_NAMESPACE, filePath); 74 | 75 | return new Response(readable, { 76 | headers: customHeaders 77 | }) 78 | } 79 | 80 | public static async streamKv(numberOfKeys, writable: WritableStream, KV_NAMESPACE: CloudflareWorkerKV, reqpath) { 81 | let writer = writable.getWriter(); 82 | for (let i = 0; i < numberOfKeys; ++i) { 83 | writer.releaseLock(); 84 | let a = await KV_NAMESPACE.get(`${reqpath}_${i}`, "stream"); 85 | await a.pipeTo(writable, {preventClose: true}); 86 | writer = writable.getWriter() 87 | } 88 | await writer.close() 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Cloudflare file hosting 2 | 3 | Uses Workers KV to enable Cloudflare as a file host. MIT Licensed. 4 | 5 | ### This vs. [Workers Sites](https://developers.cloudflare.com/workers/sites/) 6 | 7 | This tool is intended for when you need to host files that are above Cloudflare Workers' size limitations (currently 10mb); however, this comes at a cost penalty. See the below pricing notice. 8 | 9 | ### Pricing notice 10 | 11 | You should know that **Cloudflare Workers always runs in front the CF Cache**. This means none of the files will be cached, and every request made to these files will count against your Workers quota and pricing ($0.50/million after 10 million). 12 | 13 | If you use files >10mb then this script will split the file into multiple 10mb parts. Due to this, the script will incur multiple `KV.get` calls (one for file split detection, others per each 10mb part), meaning you will go through your included KV read quota quicker. 14 | 15 | The $5/mo workers charge gets you **10 million free KV reads**. You will exhaust your quota depending on the size of files you have. See [this spreadsheet for the pricing quota usage](https://docs.google.com/spreadsheets/d/1seiWWouWcN1vc3RCoCDy0I5-WbZmAwIZRd7K3Ju8WCY/edit?usp=sharing). 16 | 17 | ### Usage 18 | 19 | #### Standalone 20 | 21 | See [the wiki](https://github.com/judge2020/cloudflare-file-hosting/wiki/Standalone-usage). 22 | 23 | #### Existing worker 24 | 25 | the "get file response" method is provided as a library in order to make integrating this into existing workers projects simple. 26 | 27 | The API is similar to that of the `caches.default` API where you call the function, then check if the return value is `null`, in which case you continue with other code, or a `Response`, in which case you would immediately return that response. 28 | 29 | Use it: 30 | 31 | `npm install cloudflare-file-hosting` 32 | 33 | ```js 34 | // ES2015 Modules / typescript 35 | import CF_FILES from "cloudflare-file-hosting"; 36 | 37 | // require / javascript 38 | let CF_FILES = require("cloudflare-file-hosting"); 39 | 40 | async function handleRequest(request) { 41 | // FILES_KV is a KV namespace dedicated to the static files 42 | let url = new URL(request.url) 43 | let _filesResponse = await CF_FILES.getFile(url.pathname, FILES_KV); 44 | if (_filesResponse) { 45 | return _filesResponse; 46 | } 47 | /* other code here for when the file wasn't found */ 48 | } 49 | ``` 50 | 51 | The standalone worker is a direct example of using this API, see [workerCode.ts](worker/workerCode.ts). 52 | 53 | ##### Uploading files 54 | 55 | To upload files, the following environment variables must be set: 56 | 57 | * CLOUDFLARE_API_TOKEN - the API token to use for uploading. Require permission `Workers KV Storage:edit` on the account that owns the namespace. 58 | * CLOUDFLARE_ACCOUNT_ID 59 | * CLOUDFLARE_ZONE_ID 60 | * CLOUDFLARE_KV_NAMESPACE_ID - the KV namespace ID for where to upload files 61 | 62 | With these set, run `cfupload --path path/to/root`. You might need to use `node_modules/.bin/cfupload --path path/to/root` instead depending on your PATH setup. 63 | 64 | ### Limitations 65 | 66 | #### Download speeds 67 | 68 | Not all Cloudflare regions are optimal for large file downloads. A download from the ATL datacenter with a fiber connection within the same state (Georgia) obtained ~6 MB/s download speeds. 69 | 70 | Argo smart routing likely won't have any effect due to the nature of Workers (Argo only routes CF <--> origin better). 71 | 72 | #### Upload size and frequency limits 73 | 74 | Your uploading may get rate limited by Cloudflare if you have a lot of data to upload and/or are frequently uploading files. 75 | 76 | The Cloudflare API request limit is 5000/hour. Each file upload is 1 request and big files that are being split will incur a request per every ~10mb, so you can upload at most 12.5gb before exhausting the limit. 77 | 78 | We cannot use bulk upload since it requires uploaded values be UTF-8 encoded strings, which isn't possible for binary data (base64 is not used due to it taking up memory in the worker, where there is a 128mb limit). 79 | 80 | #### Cache 81 | 82 | 83 | At the moment, responses under 10mb in size will be pulled from the Cloudflare cache if possible, but this will still incur a workers request charge (but not a KV get charge). The cache is on a per-datacenter basis. 84 | As far as I know, due to the fact that this uses streamed responses for >10mb files, we can't use the Cache API to prevent incurring charges for KV reads. This is a tradeoff to support files larger than 128mb. 85 | 86 | -------------------------------------------------------------------------------- /worker/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | /* Basic Options */ 4 | // "incremental": true, /* Enable incremental compilation */ 5 | "target": "ESNext", 6 | /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019' or 'ESNEXT'. */ 7 | "module": "commonjs", 8 | /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ 9 | "lib": [ 10 | "esnext", 11 | "webworker" 12 | ], 13 | /* Specify library files to be included in the compilation. */ 14 | // "allowJs": true, /* Allow javascript files to be compiled. */ 15 | // "checkJs": true, /* Report errors in .js files. */ 16 | // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */ 17 | // "declaration": true, /* Generates corresponding '.d.ts' file. */ 18 | // "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */ 19 | // "sourceMap": true, /* Generates corresponding '.map' file. */ 20 | // "outFile": "./", /* Concatenate and emit output to single file. */ 21 | // "outDir": "./", /* Redirect output structure to the directory. */ 22 | // "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ 23 | // "composite": true, /* Enable project compilation */ 24 | // "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */ 25 | // "removeComments": true, /* Do not emit comments to output. */ 26 | // "noEmit": true, /* Do not emit outputs. */ 27 | // "importHelpers": true, /* Import emit helpers from 'tslib'. */ 28 | // "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */ 29 | // "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */ 30 | 31 | /* Strict Type-Checking Options */ 32 | "strict": true, 33 | /* Enable all strict type-checking options. */ 34 | "noImplicitAny": false, 35 | /* Raise error on expressions and declarations with an implied 'any' type. */ 36 | // "strictNullChecks": true, /* Enable strict null checks. */ 37 | // "strictFunctionTypes": true, /* Enable strict checking of function types. */ 38 | // "strictBindCallApply": true, /* Enable strict 'bind', 'call', and 'apply' methods on functions. */ 39 | // "strictPropertyInitialization": true, /* Enable strict checking of property initialization in classes. */ 40 | // "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */ 41 | // "alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */ 42 | 43 | /* Additional Checks */ 44 | // "noUnusedLocals": true, /* Report errors on unused locals. */ 45 | // "noUnusedParameters": true, /* Report errors on unused parameters. */ 46 | // "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */ 47 | // "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */ 48 | 49 | /* Module Resolution Options */ 50 | // "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */ 51 | // "baseUrl": "./", /* Base directory to resolve non-absolute module names. */ 52 | // "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */ 53 | // "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */ 54 | // "typeRoots": [], /* List of folders to include type definitions from. */ 55 | // "types": [], /* Type declaration files to be included in compilation. */ 56 | // "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */ 57 | "esModuleInterop": true 58 | /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */ 59 | // "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */ 60 | // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ 61 | 62 | /* Source Map Options */ 63 | // "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */ 64 | // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ 65 | // "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */ 66 | // "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */ 67 | 68 | /* Experimental Options */ 69 | // "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */ 70 | // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | /* Basic Options */ 4 | // "incremental": true, /* Enable incremental compilation */ 5 | "target": "ESNext", 6 | /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019' or 'ESNEXT'. */ 7 | "module": "commonjs", 8 | /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ 9 | "lib": [ 10 | "esnext", 11 | "webworker" 12 | ], 13 | /* Specify library files to be included in the compilation. */ 14 | // "allowJs": true, /* Allow javascript files to be compiled. */ 15 | // "checkJs": true, /* Report errors in .js files. */ 16 | // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */ 17 | // "declaration": true, /* Generates corresponding '.d.ts' file. */ 18 | // "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */ 19 | // "sourceMap": true, /* Generates corresponding '.map' file. */ 20 | // "outFile": "./", /* Concatenate and emit output to single file. */ 21 | "outDir": "./dist", 22 | /* Redirect output structure to the directory. */ 23 | // "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ 24 | // "composite": true, /* Enable project compilation */ 25 | // "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */ 26 | // "removeComments": true, /* Do not emit comments to output. */ 27 | // "noEmit": true, /* Do not emit outputs. */ 28 | // "importHelpers": true, /* Import emit helpers from 'tslib'. */ 29 | // "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */ 30 | // "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */ 31 | 32 | /* Strict Type-Checking Options */ 33 | "strict": true, 34 | /* Enable all strict type-checking options. */ 35 | "noImplicitAny": false, 36 | /* Raise error on expressions and declarations with an implied 'any' type. */ 37 | // "strictNullChecks": true, /* Enable strict null checks. */ 38 | // "strictFunctionTypes": true, /* Enable strict checking of function types. */ 39 | // "strictBindCallApply": true, /* Enable strict 'bind', 'call', and 'apply' methods on functions. */ 40 | // "strictPropertyInitialization": true, /* Enable strict checking of property initialization in classes. */ 41 | // "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */ 42 | // "alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */ 43 | 44 | /* Additional Checks */ 45 | // "noUnusedLocals": true, /* Report errors on unused locals. */ 46 | // "noUnusedParameters": true, /* Report errors on unused parameters. */ 47 | // "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */ 48 | // "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */ 49 | 50 | /* Module Resolution Options */ 51 | "moduleResolution": "node", 52 | /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */ 53 | // "baseUrl": "./", /* Base directory to resolve non-absolute module names. */ 54 | // "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */ 55 | // "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */ 56 | // "typeRoots": [], /* List of folders to include type definitions from. */ 57 | // "types": [], /* Type declaration files to be included in compilation. */ 58 | // "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */ 59 | "esModuleInterop": true 60 | /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */ 61 | // "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */ 62 | // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ 63 | 64 | /* Source Map Options */ 65 | // "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */ 66 | // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ 67 | // "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */ 68 | // "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */ 69 | 70 | /* Experimental Options */ 71 | // "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */ 72 | // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ 73 | }, 74 | "exclude": [ 75 | "worker/*" 76 | ] 77 | } 78 | --------------------------------------------------------------------------------