├── .editorconfig ├── .github └── workflows │ ├── ci.yml │ └── release.yml ├── .gitignore ├── .prettierrc.json ├── README.md ├── eleventy-fetch.js ├── package.json ├── sample.js ├── src ├── AssetCache.js ├── DirectoryManager.js ├── ExistsCache.js ├── FileCache.js ├── RemoteAssetCache.js └── Sources.js └── test ├── AssetCacheTest.js ├── QueueTest.js ├── RemoteAssetCacheTest.js └── v5flattedcachefile /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | indent_style = tab 5 | end_of_line = lf 6 | insert_final_newline = true 7 | trim_trailing_whitespace = true 8 | charset = utf-8 9 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Node Unit Tests 2 | on: [push, pull_request] 3 | permissions: read-all 4 | jobs: 5 | build: 6 | runs-on: ${{ matrix.os }} 7 | strategy: 8 | matrix: 9 | os: ["ubuntu-latest", "macos-latest", "windows-latest"] 10 | node: ["18", "20", "22", "24"] 11 | name: Node.js ${{ matrix.node }} on ${{ matrix.os }} 12 | steps: 13 | - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # 4.1.7 14 | - name: Setup node 15 | uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # 4.0.3 16 | with: 17 | node-version: ${{ matrix.node }} 18 | # cache: npm 19 | - run: npm install 20 | - run: npm test 21 | env: 22 | YARN_GPG: no 23 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Publish Release to npm 2 | on: 3 | release: 4 | types: [published] 5 | permissions: read-all 6 | jobs: 7 | build: 8 | runs-on: ubuntu-latest 9 | permissions: 10 | contents: read 11 | id-token: write 12 | steps: 13 | - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # 4.1.7 14 | - uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # 4.0.3 15 | with: 16 | node-version: "20" 17 | registry-url: "https://registry.npmjs.org" 18 | - run: npm install 19 | - run: npm test 20 | - if: ${{ github.event.release.tag_name != '' && env.NPM_PUBLISH_TAG != '' }} 21 | run: npm publish --provenance --access=public --tag=${{ env.NPM_PUBLISH_TAG }} 22 | env: 23 | NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} 24 | NPM_PUBLISH_TAG: ${{ contains(github.event.release.tag_name, '-beta.') && 'beta' || 'latest' }} 25 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | package-lock.json 3 | .cache 4 | .customcache -------------------------------------------------------------------------------- /.prettierrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "useTabs": true, 3 | "singleQuote": false, 4 | "semi": true, 5 | "endOfLine": "lf", 6 | "arrowParens": "always", 7 | "printWidth": 100 8 | } 9 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

eleventy Logo

2 | 3 | # eleventy-fetch 4 | 5 | _Requires Node 18+_ 6 | 7 | Formerly known as [`@11ty/eleventy-cache-assets`](https://www.npmjs.com/package/@11ty/eleventy-cache-assets). 8 | 9 | Fetch network resources and cache them so you don’t bombard your API (or other resources). Do this at configurable intervals—not with every build! Once per minute, or once per hour, once per day, or however often you like! 10 | 11 | With the added benefit that if one successful request completes, you can now work offline! 12 | 13 | This plugin can save any kind of asset—JSON, HTML, images, videos, etc. 14 | 15 | ## [The full `eleventy-fetch` documentation is on 11ty.dev](https://www.11ty.dev/docs/plugins/cache/). 16 | 17 | - _This is a plugin for the [Eleventy static site generator](https://www.11ty.dev/)._ 18 | - Find more [Eleventy plugins](https://www.11ty.dev/docs/plugins/). 19 | - Please star [Eleventy on GitHub](https://github.com/11ty/eleventy/), follow [@eleven_ty](https://twitter.com/eleven_ty) on Twitter, and support [11ty on Open Collective](https://opencollective.com/11ty) 20 | 21 | [![npm Version](https://img.shields.io/npm/v/@11ty/eleventy-fetch.svg?style=for-the-badge)](https://www.npmjs.com/package/@11ty/eleventy-fetch) [![GitHub issues](https://img.shields.io/github/issues/11ty/eleventy-fetch.svg?style=for-the-badge)](https://github.com/11ty/eleventy/issues) 22 | 23 | ## Installation 24 | 25 | ``` 26 | npm install @11ty/eleventy-fetch 27 | ``` 28 | 29 | _[The full `eleventy-fetch` documentation is on 11ty.dev](https://www.11ty.dev/docs/plugins/cache/)._ 30 | 31 | ## Tests 32 | 33 | ``` 34 | npm run test 35 | ``` 36 | 37 | - We use the [ava JavaScript test runner](https://github.com/avajs/ava) ([Assertions documentation](https://github.com/avajs/ava/blob/master/docs/03-assertions.md)) 38 | - ℹ️ To keep tests fast, thou shalt try to avoid writing files in tests. 39 | 40 | 50 | 51 | ## Community Roadmap 52 | 53 | - [Top Feature Requests](https://github.com/11ty/eleventy-fetch/issues?q=label%3Aneeds-votes+sort%3Areactions-%2B1-desc+label%3Aenhancement) (Add your own votes using the 👍 reaction) 54 | - [Top Bugs 😱](https://github.com/11ty/eleventy-fetch/issues?q=is%3Aissue+is%3Aopen+label%3Abug+sort%3Areactions-%2B1-desc) (Add your own votes using the 👍 reaction) 55 | - [Newest Bugs 🙀](https://github.com/11ty/eleventy-fetch/issues?q=is%3Aopen+is%3Aissue+label%3Abug) 56 | -------------------------------------------------------------------------------- /eleventy-fetch.js: -------------------------------------------------------------------------------- 1 | const { default: PQueue } = require("p-queue"); 2 | const debug = require("debug")("Eleventy:Fetch"); 3 | 4 | const Sources = require("./src/Sources.js"); 5 | const RemoteAssetCache = require("./src/RemoteAssetCache.js"); 6 | const AssetCache = require("./src/AssetCache.js"); 7 | const DirectoryManager = require("./src/DirectoryManager.js"); 8 | 9 | const globalOptions = { 10 | type: "buffer", 11 | directory: ".cache", 12 | concurrency: 10, 13 | fetchOptions: {}, 14 | dryRun: false, // don’t write anything to the file system 15 | 16 | // *does* affect cache key hash 17 | removeUrlQueryParams: false, 18 | 19 | // runs after removeUrlQueryParams, does not affect cache key hash 20 | // formatUrlForDisplay: function(url) { 21 | // return url; 22 | // }, 23 | 24 | verbose: false, // Changed in 3.0+ 25 | 26 | hashLength: 30, 27 | }; 28 | 29 | /* Queue */ 30 | let queue = new PQueue({ 31 | concurrency: globalOptions.concurrency, 32 | }); 33 | 34 | queue.on("active", () => { 35 | debug(`Concurrency: ${queue.concurrency}, Size: ${queue.size}, Pending: ${queue.pending}`); 36 | }); 37 | 38 | let instCache = {}; 39 | 40 | let directoryManager = new DirectoryManager(); 41 | 42 | function createRemoteAssetCache(source, rawOptions = {}) { 43 | if (!Sources.isFullUrl(source) && !Sources.isValidSource(source)) { 44 | return Promise.reject(new Error("Invalid source. Received: " + source)); 45 | } 46 | 47 | let options = Object.assign({}, globalOptions, rawOptions); 48 | let sourceKey = RemoteAssetCache.getRequestId(source, options); 49 | if(!sourceKey) { 50 | return Promise.reject(Sources.getInvalidSourceError(source)); 51 | } 52 | 53 | if(instCache[sourceKey]) { 54 | return instCache[sourceKey]; 55 | } 56 | 57 | let inst = new RemoteAssetCache(source, options.directory, options); 58 | inst.setQueue(queue); 59 | inst.setDirectoryManager(directoryManager); 60 | 61 | instCache[sourceKey] = inst; 62 | 63 | return inst; 64 | } 65 | 66 | module.exports = function (source, options) { 67 | let instance = createRemoteAssetCache(source, options); 68 | return instance.queue(); 69 | }; 70 | 71 | Object.defineProperty(module.exports, "concurrency", { 72 | get: function () { 73 | return queue.concurrency; 74 | }, 75 | set: function (concurrency) { 76 | queue.concurrency = concurrency; 77 | }, 78 | }); 79 | 80 | module.exports.Fetch = createRemoteAssetCache; 81 | 82 | // Deprecated API kept for backwards compat, instead: use default export directly. 83 | // Intentional: queueCallback is ignored here 84 | module.exports.queue = function(source, queueCallback, options) { 85 | let instance = createRemoteAssetCache(source, options); 86 | return instance.queue(); 87 | }; 88 | 89 | module.exports.Util = { 90 | isFullUrl: Sources.isFullUrl, 91 | }; 92 | module.exports.RemoteAssetCache = RemoteAssetCache; 93 | module.exports.AssetCache = AssetCache; 94 | module.exports.Sources = Sources; 95 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@11ty/eleventy-fetch", 3 | "version": "5.1.0", 4 | "description": "Fetch and locally cache remote API calls and assets.", 5 | "publishConfig": { 6 | "access": "public" 7 | }, 8 | "repository": { 9 | "type": "git", 10 | "url": "git+https://github.com/11ty/eleventy-fetch.git" 11 | }, 12 | "main": "eleventy-fetch.js", 13 | "scripts": { 14 | "test": "ava", 15 | "sample": "node sample", 16 | "format": "prettier . --write" 17 | }, 18 | "files": [ 19 | "src/", 20 | "eleventy-fetch.js" 21 | ], 22 | "engines": { 23 | "node": ">=18" 24 | }, 25 | "funding": { 26 | "type": "opencollective", 27 | "url": "https://opencollective.com/11ty" 28 | }, 29 | "keywords": [ 30 | "eleventy", 31 | "eleventy-utility" 32 | ], 33 | "author": { 34 | "name": "Zach Leatherman", 35 | "email": "zachleatherman@gmail.com", 36 | "url": "https://zachleat.com/" 37 | }, 38 | "license": "MIT", 39 | "bugs": { 40 | "url": "https://github.com/11ty/eleventy-fetch/issues" 41 | }, 42 | "homepage": "https://github.com/11ty/eleventy-fetch#readme", 43 | "devDependencies": { 44 | "ava": "^6.3.0", 45 | "prettier": "^3.5.3" 46 | }, 47 | "dependencies": { 48 | "@11ty/eleventy-utils": "^2.0.7", 49 | "@rgrove/parse-xml": "^4.2.0", 50 | "debug": "^4.4.0", 51 | "flatted": "^3.3.3", 52 | "p-queue": "6.6.2" 53 | }, 54 | "ava": { 55 | "failFast": false, 56 | "files": [ 57 | "./test/*.js" 58 | ], 59 | "watchMode": { 60 | "ignoreChanges": [ 61 | "**/.cache/**", 62 | "**/.customcache/**" 63 | ] 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /sample.js: -------------------------------------------------------------------------------- 1 | const startCpu = process.cpuUsage(); 2 | const os = require("os"); 3 | const saveLocal = require("."); 4 | const AssetCache = saveLocal.AssetCache; 5 | 6 | (async () => { 7 | saveLocal.concurrency = 2; 8 | let options = { 9 | duration: "4h", 10 | }; 11 | 12 | let promises = []; 13 | 14 | // don’t await here to test concurrency 15 | let first = saveLocal("https://www.zachleat.com/img/avatar-2017-big.png", options); 16 | promises.push(first); 17 | 18 | promises.push(saveLocal("https://github.com/11ty/eleventy/releases.atom", { 19 | type: "text" 20 | })); 21 | 22 | // let second = saveLocal("https://www.zachleat.com/img/avatar-2017-big.png", options); 23 | // promises.push(second); 24 | 25 | // promises.push(saveLocal("https://www.zachleat.com/web/css/fonts/lato/2.0/LatoLatin-Regular.ttf", options)); 26 | 27 | // let json = saveLocal("https://opencollective.com/11ty/members/all.json", { 28 | // duration: options.duration, 29 | // type: "json" 30 | // }); 31 | // promises.push(json); 32 | 33 | let asset = new AssetCache("twitter-followers-eleven_ty"); 34 | if (asset.isCacheValid("4d")) { 35 | console.log(await asset.getCachedValue()); 36 | } else { 37 | asset.save({ followers: 42 }, "json"); 38 | } 39 | 40 | await Promise.all(promises); 41 | 42 | // console.log( JSON.stringify(await json).substr(0, 100), "… (truncated)" ); 43 | 44 | console.log(process.cpuUsage(startCpu)); 45 | console.log(os.freemem() / (1024 * 1024), os.totalmem() / (1024 * 1024)); 46 | // console.log( process.memoryUsage() ); 47 | // console.log( process.resourceUsage() ); 48 | })(); 49 | -------------------------------------------------------------------------------- /src/AssetCache.js: -------------------------------------------------------------------------------- 1 | const fs = require("node:fs"); 2 | const path = require("node:path"); 3 | const { DateCompare, createHashHexSync } = require("@11ty/eleventy-utils"); 4 | 5 | const FileCache = require("./FileCache.js"); 6 | const Sources = require("./Sources.js"); 7 | 8 | const debugUtil = require("debug"); 9 | const debug = debugUtil("Eleventy:Fetch"); 10 | 11 | class AssetCache { 12 | #source; 13 | #hash; 14 | #customFilename; 15 | #cache; 16 | #cacheDirectory; 17 | #cacheLocationDirty = false; 18 | #directoryManager; 19 | 20 | constructor(source, cacheDirectory, options = {}) { 21 | if(!Sources.isValidSource(source)) { 22 | throw Sources.getInvalidSourceError(source); 23 | } 24 | 25 | let uniqueKey = AssetCache.getCacheKey(source, options); 26 | this.uniqueKey = uniqueKey; 27 | this.hash = AssetCache.getHash(uniqueKey, options.hashLength); 28 | 29 | this.cacheDirectory = cacheDirectory || ".cache"; 30 | this.options = options; 31 | 32 | this.defaultDuration = "1d"; 33 | this.duration = options.duration || this.defaultDuration; 34 | 35 | // Compute the filename only once 36 | if (typeof this.options.filenameFormat === "function") { 37 | this.#customFilename = AssetCache.cleanFilename(this.options.filenameFormat(uniqueKey, this.hash)); 38 | 39 | if (typeof this.#customFilename !== "string" || this.#customFilename.length === 0) { 40 | throw new Error(`The provided filenameFormat callback function needs to return valid filename characters.`); 41 | } 42 | } 43 | } 44 | 45 | log(message) { 46 | if (this.options.verbose) { 47 | console.log(`[11ty/eleventy-fetch] ${message}`); 48 | } else { 49 | debug(message); 50 | } 51 | } 52 | 53 | static cleanFilename(filename) { 54 | // Ensure no illegal characters are present (Windows or Linux: forward/backslash, chevrons, colon, double-quote, pipe, question mark, asterisk) 55 | if (filename.match(/([\/\\<>:"|?*]+?)/)) { 56 | let sanitizedFilename = filename.replace(/[\/\\<>:"|?*]+/g, ""); 57 | debug( 58 | `[@11ty/eleventy-fetch] Some illegal characters were removed from the cache filename: ${filename} will be cached as ${sanitizedFilename}.`, 59 | ); 60 | return sanitizedFilename; 61 | } 62 | 63 | return filename; 64 | } 65 | 66 | static getCacheKey(source, options) { 67 | // RemoteAssetCache passes in a string here, which skips this check (requestId is already used upstream) 68 | if (Sources.isValidComplexSource(source)) { 69 | if(options.requestId) { 70 | return options.requestId; 71 | } 72 | 73 | if(typeof source.toString === "function") { 74 | // return source.toString(); 75 | let toStr = source.toString(); 76 | if(toStr !== "function() {}" && toStr !== "[object Object]") { 77 | return toStr; 78 | } 79 | } 80 | 81 | throw Sources.getInvalidSourceError(source); 82 | } 83 | 84 | return source; 85 | } 86 | 87 | // Defult hashLength also set in global options, duplicated here for tests 88 | // v5.0+ key can be Array or literal 89 | static getHash(key, hashLength = 30) { 90 | if (!Array.isArray(key)) { 91 | key = [key]; 92 | } 93 | 94 | let result = createHashHexSync(...key); 95 | return result.slice(0, hashLength); 96 | } 97 | 98 | get source() { 99 | return this.#source; 100 | } 101 | 102 | set source(source) { 103 | this.#source = source; 104 | } 105 | 106 | get hash() { 107 | return this.#hash; 108 | } 109 | 110 | set hash(value) { 111 | if (value !== this.#hash) { 112 | this.#cacheLocationDirty = true; 113 | } 114 | 115 | this.#hash = value; 116 | } 117 | 118 | get cacheDirectory() { 119 | return this.#cacheDirectory; 120 | } 121 | 122 | set cacheDirectory(dir) { 123 | if (dir !== this.#cacheDirectory) { 124 | this.#cacheLocationDirty = true; 125 | } 126 | 127 | this.#cacheDirectory = dir; 128 | } 129 | 130 | get cacheFilename() { 131 | if (typeof this.#customFilename === "string" && this.#customFilename.length > 0) { 132 | return this.#customFilename; 133 | } 134 | 135 | return `eleventy-fetch-${this.hash}`; 136 | } 137 | 138 | get rootDir() { 139 | // Work in an AWS Lambda (serverless) 140 | // https://docs.aws.amazon.com/lambda/latest/dg/configuration-envvars.html 141 | 142 | // Bad: LAMBDA_TASK_ROOT is /var/task/ on AWS so we must use ELEVENTY_ROOT 143 | // When using ELEVENTY_ROOT, cacheDirectory must be relative 144 | // (we are bundling the cache files into the serverless function) 145 | if ( 146 | process.env.LAMBDA_TASK_ROOT && 147 | process.env.ELEVENTY_ROOT && 148 | !this.cacheDirectory.startsWith("/") 149 | ) { 150 | return path.resolve(process.env.ELEVENTY_ROOT, this.cacheDirectory); 151 | } 152 | 153 | // otherwise, it is recommended to use somewhere in /tmp/ for serverless (otherwise it won’t write) 154 | return path.resolve(this.cacheDirectory); 155 | } 156 | 157 | get cachePath() { 158 | return path.join(this.rootDir, this.cacheFilename); 159 | } 160 | 161 | get cache() { 162 | if (!this.#cache || this.#cacheLocationDirty) { 163 | let cache = new FileCache(this.cacheFilename, { 164 | dir: this.rootDir, 165 | source: this.source, 166 | }); 167 | cache.setDefaultType(this.options.type); 168 | cache.setDryRun(this.options.dryRun); 169 | cache.setDirectoryManager(this.#directoryManager); 170 | 171 | this.#cache = cache; 172 | this.#cacheLocationDirty = false; 173 | } 174 | return this.#cache; 175 | } 176 | 177 | getDurationMs(duration = "0s") { 178 | return DateCompare.getDurationMs(duration); 179 | } 180 | 181 | setDirectoryManager(manager) { 182 | this.#directoryManager = manager; 183 | } 184 | 185 | async save(contents, type = "buffer", metadata = {}) { 186 | if(!contents) { 187 | throw new Error("save(contents) expects contents (was falsy)"); 188 | } 189 | 190 | this.cache.set(type, contents, metadata); 191 | 192 | // Dry-run handled downstream 193 | this.cache.save(); 194 | } 195 | 196 | getCachedContents() { 197 | return this.cache.getContents(); 198 | } 199 | 200 | getCachedValue() { 201 | if(this.options.returnType === "response") { 202 | return { 203 | ...this.cachedObject.metadata?.response, 204 | body: this.getCachedContents(), 205 | cache: "hit", 206 | } 207 | } 208 | 209 | return this.getCachedContents(); 210 | } 211 | 212 | getCachedTimestamp() { 213 | return this.cachedObject?.cachedAt; 214 | } 215 | 216 | isCacheValid(duration = this.duration) { 217 | if(!this.cachedObject || !this.cachedObject?.cachedAt) { 218 | return false; 219 | } 220 | 221 | if(this.cachedObject?.type && DateCompare.isTimestampWithinDuration(this.cachedObject?.cachedAt, duration)) { 222 | return this.cache.hasContents(this.cachedObject?.type); // check file system to make files haven’t been purged. 223 | } 224 | 225 | return false; 226 | } 227 | 228 | get cachedObject() { 229 | return this.cache.get(); 230 | } 231 | 232 | // Deprecated 233 | needsToFetch(duration) { 234 | return !this.isCacheValid(duration); 235 | } 236 | 237 | // This is only included for completenes—not on the docs. 238 | async fetch(optionsOverride = {}) { 239 | if (this.isCacheValid(optionsOverride.duration)) { 240 | // promise 241 | debug(`Using cached version of: ${this.uniqueKey}`); 242 | return this.getCachedValue(); 243 | } 244 | 245 | debug(`Saving ${this.uniqueKey} to ${this.cacheFilename}`); 246 | await this.save(this.source, optionsOverride.type); 247 | 248 | return this.source; 249 | } 250 | 251 | // for testing 252 | hasAnyCacheFiles() { 253 | for(let p of this.cache.getAllPossibleFilePaths()) { 254 | if(fs.existsSync(p)) { 255 | return true; 256 | } 257 | } 258 | return false; 259 | } 260 | 261 | // for testing 262 | async destroy() { 263 | await Promise.all(this.cache.getAllPossibleFilePaths().map(path => { 264 | if (fs.existsSync(path)) { 265 | return fs.unlinkSync(path); 266 | } 267 | })) 268 | } 269 | } 270 | module.exports = AssetCache; 271 | -------------------------------------------------------------------------------- /src/DirectoryManager.js: -------------------------------------------------------------------------------- 1 | const fs = require("node:fs"); 2 | const debugAssets = require("debug")("Eleventy:Assets"); 3 | 4 | class DirectoryManager { 5 | #dirs = new Set(); 6 | 7 | isCreated(dir) { 8 | return this.#dirs.has(dir); 9 | } 10 | 11 | create(dir) { 12 | if(this.isCreated(dir)) { 13 | return; 14 | } 15 | 16 | this.#dirs.add(dir); 17 | debugAssets("Creating directory %o", dir); 18 | fs.mkdirSync(dir, { recursive: true }); 19 | } 20 | } 21 | 22 | module.exports = DirectoryManager; 23 | -------------------------------------------------------------------------------- /src/ExistsCache.js: -------------------------------------------------------------------------------- 1 | const fs = require("node:fs"); 2 | // const debug = require("debug")("Eleventy:Assets"); 3 | 4 | class ExistsCache { 5 | #checks = new Map(); 6 | #count = 0; 7 | 8 | set(target, value) { 9 | this.#checks.set(target, Boolean(value)); 10 | } 11 | 12 | exists(target) { 13 | if(this.#checks.has(target)) { 14 | return this.#checks.get(target); 15 | } 16 | 17 | let exists = fs.existsSync(target); 18 | this.#count++; 19 | this.#checks.set(target, exists); 20 | return exists; 21 | } 22 | } 23 | 24 | module.exports = ExistsCache; 25 | -------------------------------------------------------------------------------- /src/FileCache.js: -------------------------------------------------------------------------------- 1 | const fs = require("node:fs"); 2 | const path = require("node:path"); 3 | const debugUtil = require("debug"); 4 | const { parse } = require("flatted"); 5 | 6 | const debug = debugUtil("Eleventy:Fetch"); 7 | const debugAssets = debugUtil("Eleventy:Assets"); 8 | 9 | const DirectoryManager = require("./DirectoryManager.js"); 10 | const ExistsCache = require("./ExistsCache.js"); 11 | 12 | let existsCache = new ExistsCache(); 13 | 14 | class FileCache { 15 | #source; 16 | #directoryManager; 17 | #metadata; 18 | #defaultType; 19 | #contents; 20 | #dryRun = false; 21 | #cacheDirectory = ".cache"; 22 | #savePending = false; 23 | #counts = { 24 | read: 0, 25 | write: 0, 26 | }; 27 | 28 | constructor(cacheFilename, options = {}) { 29 | this.cacheFilename = cacheFilename; 30 | if(options.dir) { 31 | this.#cacheDirectory = options.dir; 32 | } 33 | if(options.source) { 34 | this.#source = options.source; 35 | } 36 | } 37 | 38 | setDefaultType(type) { 39 | if(type) { 40 | this.#defaultType = type; 41 | } 42 | } 43 | 44 | setDryRun(val) { 45 | this.#dryRun = Boolean(val); 46 | } 47 | 48 | setDirectoryManager(manager) { 49 | this.#directoryManager = manager; 50 | } 51 | 52 | ensureDir() { 53 | if (this.#dryRun || existsCache.exists(this.#cacheDirectory)) { 54 | return; 55 | } 56 | 57 | if(!this.#directoryManager) { 58 | // standalone fallback (for tests) 59 | this.#directoryManager = new DirectoryManager(); 60 | } 61 | 62 | this.#directoryManager.create(this.#cacheDirectory); 63 | } 64 | 65 | set(type, contents, extraMetadata = {}) { 66 | this.#savePending = true; 67 | 68 | this.#metadata = { 69 | cachedAt: Date.now(), 70 | type, 71 | // source: this.#source, 72 | metadata: extraMetadata, 73 | }; 74 | 75 | this.#contents = contents; 76 | } 77 | 78 | get fsPath() { 79 | return path.join(this.#cacheDirectory, this.cacheFilename); 80 | } 81 | 82 | getContentsPath(type) { 83 | if(!type) { 84 | throw new Error("Missing cache type for " + this.fsPath); 85 | } 86 | 87 | // normalize to storage type 88 | if(type === "xml") { 89 | type = "text"; 90 | } else if(type === "parsed-xml") { 91 | type = "json"; 92 | } 93 | 94 | return `${this.fsPath}.${type}`; 95 | } 96 | 97 | // only when side loaded (buffer content) 98 | get contentsPath() { 99 | return this.getContentsPath(this.#metadata?.type); 100 | } 101 | 102 | get() { 103 | if(this.#metadata) { 104 | return this.#metadata; 105 | } 106 | 107 | if(!existsCache.exists(this.fsPath)) { 108 | return; 109 | } 110 | 111 | debug(`Fetching from cache ${this.fsPath}`); 112 | if(this.#source) { 113 | debugAssets("[11ty/eleventy-fetch] Reading via %o", this.#source); 114 | } else { 115 | debugAssets("[11ty/eleventy-fetch] Reading %o", this.fsPath); 116 | } 117 | 118 | this.#counts.read++; 119 | let data = fs.readFileSync(this.fsPath, "utf8"); 120 | 121 | let json; 122 | // Backwards compatibility with previous caches usingn flat-cache and `flatted` 123 | if(data.startsWith(`[["1"],`)) { 124 | let flattedParsed = parse(data); 125 | if(flattedParsed?.[0]?.value) { 126 | json = flattedParsed?.[0]?.value 127 | } 128 | } else { 129 | json = JSON.parse(data); 130 | } 131 | 132 | this.#metadata = json; 133 | 134 | return json; 135 | } 136 | 137 | _backwardsCompatGetContents(rawData, type) { 138 | if (type === "json") { 139 | return rawData.contents; 140 | } else if (type === "text") { 141 | return rawData.contents.toString(); 142 | } 143 | 144 | // buffer 145 | return Buffer.from(rawData.contents); 146 | } 147 | 148 | hasContents(type) { 149 | if(this.#contents) { 150 | return true; 151 | } 152 | if(this.get()?.contents) { // backwards compat with very old caches 153 | return true; 154 | } 155 | return existsCache.exists(this.getContentsPath(type)); 156 | } 157 | 158 | getType() { 159 | return this.#metadata?.type || this.#defaultType; 160 | } 161 | 162 | getContents() { 163 | if(this.#contents) { 164 | return this.#contents; 165 | } 166 | 167 | let metadata = this.get(); 168 | // backwards compat with old caches 169 | if(metadata?.contents) { 170 | // already parsed, part of the top level file 171 | let normalizedContent = this._backwardsCompatGetContents(this.get(), this.getType()); 172 | this.#contents = normalizedContent; 173 | return normalizedContent; 174 | } 175 | 176 | if(!existsCache.exists(this.contentsPath)) { 177 | return; 178 | } 179 | 180 | debug(`Fetching from cache ${this.contentsPath}`); 181 | if(this.#source) { 182 | debugAssets("[11ty/eleventy-fetch] Reading (side loaded) via %o", this.#source); 183 | } else { 184 | debugAssets("[11ty/eleventy-fetch] Reading (side loaded) %o", this.contentsPath); 185 | } 186 | 187 | // It is intentional to store contents in a separate file from the metadata: we don’t want to 188 | // have to read the entire contents via JSON.parse (or otherwise) to check the cache validity. 189 | this.#counts.read++; 190 | let type = metadata?.type || this.getType(); 191 | let data = fs.readFileSync(this.contentsPath); 192 | if (type === "json" || type === "parsed-xml") { 193 | data = JSON.parse(data); 194 | } 195 | this.#contents = data; 196 | return data; 197 | } 198 | 199 | save() { 200 | if(this.#dryRun || !this.#savePending || this.#metadata && Object.keys(this.#metadata) === 0) { 201 | return; 202 | } 203 | 204 | this.ensureDir(); // doesn’t add to counts (yet?) 205 | 206 | // contents before metadata 207 | debugAssets("[11ty/eleventy-fetch] Writing %o (side loaded) from %o", this.contentsPath, this.#source); 208 | 209 | this.#counts.write++; 210 | // the contents must exist before the cache metadata are saved below 211 | let contents = this.#contents; 212 | let type = this.getType(); 213 | if (type === "json" || type === "parsed-xml") { 214 | contents = JSON.stringify(contents); 215 | } 216 | fs.writeFileSync(this.contentsPath, contents); 217 | debug(`Writing ${this.contentsPath}`); 218 | 219 | this.#counts.write++; 220 | debugAssets("[11ty/eleventy-fetch] Writing %o from %o", this.fsPath, this.#source); 221 | fs.writeFileSync(this.fsPath, JSON.stringify(this.#metadata), "utf8"); 222 | debug(`Writing ${this.fsPath}`); 223 | } 224 | 225 | // for testing 226 | getAllPossibleFilePaths() { 227 | let types = ["text", "buffer", "json"]; 228 | let paths = new Set(); 229 | paths.add(this.fsPath); 230 | for(let type of types) { 231 | paths.add(this.getContentsPath(type)); 232 | } 233 | return Array.from(paths); 234 | } 235 | } 236 | 237 | module.exports = FileCache; 238 | -------------------------------------------------------------------------------- /src/RemoteAssetCache.js: -------------------------------------------------------------------------------- 1 | const debugUtil = require("debug"); 2 | const { parseXml } = require('@rgrove/parse-xml'); 3 | 4 | const Sources = require("./Sources.js"); 5 | const AssetCache = require("./AssetCache.js"); 6 | 7 | const debug = debugUtil("Eleventy:Fetch"); 8 | const debugAssets = debugUtil("Eleventy:Assets"); 9 | 10 | class RemoteAssetCache extends AssetCache { 11 | #queue; 12 | #queuePromise; 13 | #fetchPromise; 14 | #lastFetchType; 15 | 16 | constructor(source, cacheDirectory, options = {}) { 17 | let requestId = RemoteAssetCache.getRequestId(source, options); 18 | super(requestId, cacheDirectory, options); 19 | 20 | this.source = source; 21 | this.options = options; 22 | this.displayUrl = RemoteAssetCache.convertUrlToString(source, options); 23 | this.fetchCount = 0; 24 | } 25 | 26 | static getRequestId(source, options = {}) { 27 | if (Sources.isValidComplexSource(source)) { 28 | return this.getCacheKey(source, options); 29 | } 30 | 31 | if (options.removeUrlQueryParams) { 32 | let cleaned = this.cleanUrl(source); 33 | return this.getCacheKey(cleaned, options); 34 | } 35 | 36 | return this.getCacheKey(source, options); 37 | } 38 | 39 | static getCacheKey(source, options) { 40 | let cacheKey = { 41 | source: AssetCache.getCacheKey(source, options), 42 | }; 43 | 44 | if(options.type === "xml" || options.type === "parsed-xml") { 45 | cacheKey.type = options.type; 46 | } 47 | 48 | if (options.fetchOptions) { 49 | if (options.fetchOptions.method && options.fetchOptions.method !== "GET") { 50 | cacheKey.method = options.fetchOptions.method; 51 | } 52 | if (options.fetchOptions.body) { 53 | cacheKey.body = options.fetchOptions.body; 54 | } 55 | } 56 | 57 | if(Object.keys(cacheKey).length > 1) { 58 | return JSON.stringify(cacheKey); 59 | } 60 | 61 | return cacheKey.source; 62 | } 63 | 64 | static cleanUrl(url) { 65 | if(!Sources.isFullUrl(url)) { 66 | return url; 67 | } 68 | 69 | let cleanUrl; 70 | if(typeof url === "string" || typeof url.toString === "function") { 71 | cleanUrl = new URL(url); 72 | } else if(url instanceof URL) { 73 | cleanUrl = url; 74 | } else { 75 | throw new Error("Invalid source for cleanUrl: " + url) 76 | } 77 | 78 | cleanUrl.search = new URLSearchParams([]); 79 | 80 | return cleanUrl.toString(); 81 | } 82 | 83 | static convertUrlToString(source, options = {}) { 84 | // removes query params 85 | source = RemoteAssetCache.cleanUrl(source); 86 | 87 | let { formatUrlForDisplay } = options; 88 | if (formatUrlForDisplay && typeof formatUrlForDisplay === "function") { 89 | return "" + formatUrlForDisplay(source); 90 | } 91 | 92 | return "" + source; 93 | } 94 | 95 | async getResponseValue(response, type) { 96 | if (type === "json") { 97 | return response.json(); 98 | } else if (type === "text" || type === "xml") { 99 | return response.text(); 100 | } else if(type === "parsed-xml") { 101 | return parseXml(await response.text()); 102 | } 103 | return Buffer.from(await response.arrayBuffer()); 104 | } 105 | 106 | setQueue(queue) { 107 | this.#queue = queue; 108 | } 109 | 110 | // Returns raw Promise 111 | queue() { 112 | if(!this.#queue) { 113 | throw new Error("Missing `#queue` instance."); 114 | } 115 | 116 | if(!this.#queuePromise) { 117 | // optionsOverride not supported on fetch here for re-use 118 | this.#queuePromise = this.#queue.add(() => this.fetch()).catch((e) => { 119 | this.#queuePromise = undefined; 120 | throw e; 121 | }); 122 | } 123 | 124 | return this.#queuePromise; 125 | } 126 | 127 | isCacheValid(duration = undefined) { 128 | // uses this.options.duration if not explicitly defined here 129 | return super.isCacheValid(duration); 130 | } 131 | 132 | // if last fetch was a cache hit (no fetch occurred) or a cache miss (fetch did occur) 133 | // used by Eleventy Image in disk cache checks. 134 | wasLastFetchCacheHit() { 135 | return this.#lastFetchType === "hit"; 136 | } 137 | 138 | async #fetch(optionsOverride = {}) { 139 | // Important: no disk writes when dryRun 140 | // As of Fetch v4, reads are now allowed! 141 | if (this.isCacheValid(optionsOverride.duration)) { 142 | debug(`Cache hit for ${this.displayUrl}`); 143 | this.#lastFetchType = "hit"; 144 | return super.getCachedValue(); 145 | } 146 | 147 | this.#lastFetchType = "miss"; 148 | 149 | try { 150 | let isDryRun = optionsOverride.dryRun || this.options.dryRun; 151 | this.log(`Fetching ${this.displayUrl}`); 152 | 153 | let body; 154 | let metadata = {}; 155 | let type = optionsOverride.type || this.options.type; 156 | if (typeof this.source === "object" && typeof this.source.then === "function") { 157 | body = await this.source; 158 | } else if (typeof this.source === "function") { 159 | // sync or async function 160 | body = await this.source(); 161 | } else { 162 | let fetchOptions = optionsOverride.fetchOptions || this.options.fetchOptions || {}; 163 | if(!Sources.isFullUrl(this.source)) { 164 | throw Sources.getInvalidSourceError(this.source); 165 | } 166 | 167 | this.fetchCount++; 168 | 169 | debugAssets("[11ty/eleventy-fetch] Fetching %o", this.source); 170 | 171 | // v5: now using global (Node-native or otherwise) fetch instead of node-fetch 172 | let response = await fetch(this.source, fetchOptions); 173 | if (!response.ok) { 174 | throw new Error( 175 | `Bad response for ${this.displayUrl} (${response.status}): ${response.statusText}`, 176 | { cause: response }, 177 | ); 178 | } 179 | 180 | metadata.response = { 181 | url: response.url, 182 | status: response.status, 183 | headers: Object.fromEntries(response.headers.entries()), 184 | }; 185 | 186 | body = await this.getResponseValue(response, type); 187 | } 188 | 189 | if (!isDryRun) { 190 | await super.save(body, type, metadata); 191 | } 192 | 193 | if(this.options.returnType === "response") { 194 | return { 195 | ...metadata.response, 196 | body, 197 | cache: "miss", 198 | } 199 | } 200 | 201 | return body; 202 | } catch (e) { 203 | if (this.cachedObject && this.getDurationMs(this.duration) > 0) { 204 | debug(`Error fetching ${this.displayUrl}. Message: ${e.message}`); 205 | debug(`Failing gracefully with an expired cache entry.`); 206 | return super.getCachedValue(); 207 | } else { 208 | return Promise.reject(e); 209 | } 210 | } 211 | } 212 | 213 | // async but not explicitly declared for promise equality checks 214 | // returns a Promise 215 | async fetch(optionsOverride = {}) { 216 | if(!this.#fetchPromise) { 217 | // one at a time. clear when finished 218 | this.#fetchPromise = this.#fetch(optionsOverride).finally(() => { 219 | this.#fetchPromise = undefined; 220 | }); 221 | } 222 | 223 | return this.#fetchPromise; 224 | } 225 | } 226 | module.exports = RemoteAssetCache; 227 | -------------------------------------------------------------------------------- /src/Sources.js: -------------------------------------------------------------------------------- 1 | class Sources { 2 | static isFullUrl(url) { 3 | try { 4 | if(url instanceof URL) { 5 | return true; 6 | } 7 | 8 | new URL(url); 9 | return true; 10 | } catch (e) { 11 | // invalid url OR already a local path 12 | return false; 13 | } 14 | } 15 | 16 | static isValidSource(source) { 17 | // String (url?) 18 | if(typeof source === "string") { 19 | return true; 20 | } 21 | if(this.isValidComplexSource(source)) { 22 | return true; 23 | } 24 | return false; 25 | } 26 | 27 | static isValidComplexSource(source) { 28 | // Async/sync Function 29 | if(typeof source === "function") { 30 | return true; 31 | } 32 | if(typeof source === "object") { 33 | // Raw promise 34 | if(typeof source.then === "function") { 35 | return true; 36 | } 37 | // anything string-able 38 | if(typeof source.toString === "function") { 39 | return true; 40 | } 41 | } 42 | return false; 43 | } 44 | 45 | static getInvalidSourceError(source, errorCause) { 46 | return new Error("Invalid source: must be a string, function, or Promise. If a function or Promise, you must provide a `toString()` method or an `options.requestId` unique key. Received: " + source, { cause: errorCause }); 47 | } 48 | } 49 | 50 | module.exports = Sources; 51 | -------------------------------------------------------------------------------- /test/AssetCacheTest.js: -------------------------------------------------------------------------------- 1 | const test = require("ava"); 2 | const path = require("node:path"); 3 | const fs = require("node:fs"); 4 | const AssetCache = require("../src/AssetCache"); 5 | 6 | function normalizePath(pathStr) { 7 | if (typeof pathStr !== "string") { 8 | return pathStr; 9 | } 10 | 11 | if (pathStr.match(/^[A-Z]\:/)) { 12 | pathStr = pathStr.substr(2); 13 | } 14 | return pathStr.split(path.sep).join("/"); 15 | } 16 | 17 | test("Absolute path cache directory", (t) => { 18 | let cache = new AssetCache("lksdjflkjsdf", "/tmp/.cache"); 19 | let cachePath = normalizePath(cache.cachePath); 20 | 21 | t.is(cachePath, "/tmp/.cache/eleventy-fetch-73015bafd152bccf9929e0f4dcbe36"); 22 | }); 23 | 24 | test("Relative path cache directory", (t) => { 25 | let cache = new AssetCache("lksdjflkjsdf", ".cache"); 26 | let cachePath = normalizePath(cache.cachePath); 27 | 28 | t.not(cachePath, ".cache/eleventy-fetch-73015bafd152bccf9929e0f4dcbe36"); 29 | t.true(cachePath.endsWith(".cache/eleventy-fetch-73015bafd152bccf9929e0f4dcbe36")); 30 | }); 31 | 32 | test("AWS Lambda root directory resolves correctly", (t) => { 33 | let cwd = normalizePath(process.cwd()); 34 | process.env.ELEVENTY_ROOT = cwd; 35 | process.env.LAMBDA_TASK_ROOT = "/var/task/z/"; 36 | let cache = new AssetCache("lksdjflkjsdf", ".cache"); 37 | let cachePath = normalizePath(cache.cachePath); 38 | 39 | t.is(cachePath, `${cwd}/.cache/eleventy-fetch-73015bafd152bccf9929e0f4dcbe36`); 40 | (delete "ELEVENTY_ROOT") in process.env; 41 | (delete "LAMBDA_TASK_ROOT") in process.env; 42 | }); 43 | 44 | test("Test a save", async (t) => { 45 | let asset = new AssetCache("zachleat_twitter_followers", ".customcache"); 46 | let cachePath = normalizePath(asset.cachePath); 47 | 48 | await asset.save({ followers: 10 }, "json"); 49 | 50 | t.truthy(fs.existsSync(cachePath)); 51 | 52 | fs.unlinkSync(cachePath); 53 | }); 54 | 55 | test("Cache path should handle slashes without creating directories, issue #14", (t) => { 56 | let cache = new AssetCache("lksdjflk/jsdf", "/tmp/.cache"); 57 | let cachePath = normalizePath(cache.cachePath); 58 | 59 | t.is(cachePath, "/tmp/.cache/eleventy-fetch-135797dbf5ab1187e5003c49162602"); 60 | }); 61 | 62 | test("Uses `requestId` property when caching a promise", async (t) => { 63 | let asset = new AssetCache(Promise.resolve(), ".customcache", { 64 | requestId: "mock-display-url-2", 65 | }); 66 | 67 | await asset.save({ name: "Sophia Smith" }, "json"); 68 | 69 | await asset.destroy(); 70 | 71 | t.falsy(asset.hasAnyCacheFiles()); 72 | }); 73 | 74 | test("Uses `requestId` property when caching a function", async (t) => { 75 | let asset = new AssetCache(function() {}, ".cache", { 76 | requestId: "mock-function", 77 | }); 78 | await asset.save({ name: "Sophia Smith" }, "json"); 79 | 80 | await asset.destroy(); 81 | 82 | t.falsy(asset.hasAnyCacheFiles()); 83 | }); 84 | 85 | test("Uses `requestId` property when caching an async function", async (t) => { 86 | let asset = new AssetCache(async function() {}, ".cache", { 87 | requestId: "mock-async-function", 88 | }); 89 | 90 | await asset.save({ name: "Sophia Smith" }, "json"); 91 | 92 | await asset.destroy(); 93 | 94 | t.falsy(asset.hasAnyCacheFiles()); 95 | }); 96 | 97 | test("Uses filenameFormat", async (t) => { 98 | let asset = new AssetCache("some-thing", undefined, { 99 | filenameFormat() { 100 | // don’t include the file extension 101 | return "testing"; 102 | }, 103 | }); 104 | 105 | let cachePath = normalizePath(asset.cachePath); 106 | 107 | t.truthy(cachePath.endsWith("/.cache/testing")); 108 | 109 | await asset.save({ name: "Sophia Smith" }, "json"); 110 | 111 | t.truthy(fs.existsSync(cachePath)); 112 | 113 | await asset.destroy(); 114 | 115 | t.falsy(asset.hasAnyCacheFiles()); 116 | }); 117 | 118 | test("v5 flatted cache file", async (t) => { 119 | let asset = new AssetCache("some-thing", "test", { 120 | filenameFormat() { 121 | // don’t include the file extension 122 | return "v5flattedcachefile"; 123 | }, 124 | }); 125 | 126 | let cachePath = normalizePath(asset.cachePath); 127 | t.truthy(cachePath.endsWith("test/v5flattedcachefile")); 128 | t.truthy(asset.cachedObject.cachedAt); 129 | t.truthy(asset.cachedObject.metadata); 130 | t.is(asset.cachedObject.type, "json"); 131 | t.deepEqual(asset.getCachedValue(), undefined); 132 | t.deepEqual(asset.getCachedContents(), undefined); 133 | }); 134 | -------------------------------------------------------------------------------- /test/QueueTest.js: -------------------------------------------------------------------------------- 1 | const test = require("ava"); 2 | const Cache = require("../eleventy-fetch.js"); 3 | const { queue, Fetch } = Cache; 4 | 5 | test("Queue without options", async (t) => { 6 | let example = "https://example.com/"; 7 | let req = await queue(example, () => { 8 | let asset = new RemoteAssetCache(example); 9 | return asset.fetch(); 10 | }); 11 | 12 | t.truthy(Buffer.isBuffer(req)) 13 | 14 | try { 15 | await req.destroy(); 16 | } catch (e) {} 17 | }); 18 | 19 | test("Double Fetch", async (t) => { 20 | let pngUrl = "https://www.zachleat.com/img/avatar-2017-big.png"; 21 | let ac1 = Cache(pngUrl); 22 | let ac2 = Cache(pngUrl); 23 | 24 | // Make sure we only fetch once! 25 | t.is(ac1, ac2); 26 | 27 | await ac1; 28 | await ac2; 29 | 30 | let forDestroyOnly = Fetch(pngUrl); 31 | // file is now accessible 32 | try { 33 | await forDestroyOnly.destroy(); 34 | } catch (e) {} 35 | }); 36 | 37 | test("Double Fetch (dry run)", async (t) => { 38 | let pngUrl = "https://www.zachleat.com/img/avatar-2017-88.png"; 39 | let ac1 = Cache(pngUrl, { dryRun: true }); 40 | let ac2 = Cache(pngUrl, { dryRun: true }); 41 | 42 | // Make sure we only fetch once! 43 | t.is(ac1, ac2); 44 | 45 | await ac1; 46 | await ac2; 47 | 48 | let forTestOnly = Fetch(pngUrl, { 49 | cacheDirectory: ".cache", 50 | dryRun: true, 51 | }); 52 | // file is now accessible 53 | t.false(forTestOnly.hasAnyCacheFiles()); 54 | }); 55 | 56 | test("Double Fetch async function (dry run)", async (t) => { 57 | let expected = { mockKey: "mockValue" }; 58 | 59 | async function fetch() { 60 | return Promise.resolve(expected); 61 | }; 62 | 63 | let ac1 = Cache(fetch, { 64 | dryRun: true, 65 | requestId: "fetch-1", 66 | }); 67 | let ac2 = Cache(fetch, { 68 | dryRun: true, 69 | requestId: "fetch-2", 70 | }); 71 | 72 | // two distinct fetches 73 | t.not(ac1, ac2); 74 | 75 | let result1 = await ac1; 76 | let result2 = await ac2; 77 | 78 | t.deepEqual(result1, result2); 79 | t.deepEqual(result1, expected); 80 | t.deepEqual(result2, expected); 81 | }); 82 | 83 | test("Double Fetch 404 errors should only fetch once", async (t) => { 84 | let ac1 = Cache("https://httpstat.us/404", { 85 | dryRun: true, 86 | }); 87 | let ac2 = Cache("https://httpstat.us/404", { 88 | dryRun: true, 89 | }); 90 | 91 | // Make sure we only fetch once! 92 | t.is(ac1, ac2); 93 | 94 | await t.throwsAsync(async () => await ac1); 95 | await t.throwsAsync(async () => await ac2); 96 | }); 97 | 98 | test("Docs example https://www.11ty.dev/docs/plugins/fetch/#manually-store-your-own-data-in-the-cache", async (t) => { 99 | t.plan(2); 100 | 101 | async function fn() { 102 | t.true(true); 103 | return new Promise(resolve => { 104 | setTimeout(() => { 105 | resolve({ followerCount: 1000 }) 106 | }); 107 | }); 108 | } 109 | 110 | let fakeFollowers = Cache(fn, { 111 | type: "json", 112 | dryRun: true, 113 | requestId: "zachleat_twitter_followers" 114 | }); 115 | 116 | t.deepEqual(await fakeFollowers, { 117 | followerCount: 1000 118 | }); 119 | }); 120 | 121 | test("Raw Fetch using queue method", async (t) => { 122 | let pngUrl = "https://www.zachleat.com/img/avatar-2017.png?q=1"; 123 | let ac1 = Fetch(pngUrl); 124 | let ac2 = Fetch(pngUrl); 125 | 126 | // Destroy to clear any existing cache 127 | try { 128 | await ac1.destroy(); 129 | } catch (e) {} 130 | try { 131 | await ac2.destroy(); 132 | } catch (e) {} 133 | 134 | // Make sure the instance is the same 135 | t.is(ac1, ac2); 136 | 137 | let result1 = await ac1.queue(); 138 | t.false(ac1.wasLastFetchCacheHit()) 139 | 140 | let result2 = await ac1.queue(); 141 | // reuses the same fetch 142 | t.false(ac1.wasLastFetchCacheHit()) 143 | 144 | t.is(result1, result2); 145 | 146 | // file is now accessible 147 | try { 148 | await ac1.destroy(); 149 | } catch (e) {} 150 | try { 151 | await ac2.destroy(); 152 | } catch (e) {} 153 | }); 154 | 155 | 156 | test("Raw Fetch using fetch method", async (t) => { 157 | let pngUrl = "https://www.zachleat.com/img/avatar-2017.png?q=2"; 158 | let ac1 = Fetch(pngUrl); 159 | let ac2 = Fetch(pngUrl); 160 | 161 | // Destroy to clear any existing cache 162 | try { 163 | await ac1.destroy(); 164 | } catch (e) {} 165 | try { 166 | await ac2.destroy(); 167 | } catch (e) {} 168 | 169 | // Make sure the instance is the same 170 | t.is(ac1, ac2); 171 | 172 | let result1 = await ac1.fetch(); 173 | t.false(ac1.wasLastFetchCacheHit()) 174 | 175 | let result2 = await ac1.fetch(); 176 | t.true(ac1.wasLastFetchCacheHit()) 177 | 178 | t.is(result1, result2); 179 | 180 | // file is now accessible 181 | try { 182 | await ac1.destroy(); 183 | } catch (e) {} 184 | try { 185 | await ac2.destroy(); 186 | } catch (e) {} 187 | }); 188 | 189 | test("Raw Fetch using fetch method (check parallel fetch promise reuse)", async (t) => { 190 | let pngUrl = "https://www.zachleat.com/img/avatar-2017.png?q=3"; 191 | let ac1 = Fetch(pngUrl); 192 | let ac2 = Fetch(pngUrl); 193 | 194 | // Destroy to clear any existing cache 195 | try { 196 | await ac1.destroy(); 197 | } catch (e) {} 198 | try { 199 | await ac2.destroy(); 200 | } catch (e) {} 201 | 202 | // Make sure the instance is the same 203 | t.is(ac1, ac2); 204 | 205 | let [result1, result2] = await Promise.all([ac1.fetch(), ac1.fetch()]); 206 | 207 | t.is(result1, result2); 208 | 209 | t.false(ac1.wasLastFetchCacheHit()) 210 | 211 | // file is now accessible 212 | try { 213 | await ac1.destroy(); 214 | } catch (e) {} 215 | try { 216 | await ac2.destroy(); 217 | } catch (e) {} 218 | }); 219 | 220 | test("Refetches data on transient failures", async (t) => { 221 | let firstFetch = true; 222 | let successData = "Good data"; 223 | let failData = "Transient error"; 224 | let ac1 = Fetch(() => { 225 | if (firstFetch) { 226 | firstFetch = false; 227 | throw new Error(failData) 228 | } 229 | return successData; 230 | }, {duration: "0s"}) 231 | 232 | await t.throwsAsync(async () => await ac1.queue()) 233 | t.is(await ac1.queue(), successData); 234 | }) 235 | -------------------------------------------------------------------------------- /test/RemoteAssetCacheTest.js: -------------------------------------------------------------------------------- 1 | const test = require("ava"); 2 | const path = require("node:path"); 3 | const { Util } = require("../"); 4 | const AssetCache = require("../src/AssetCache"); 5 | const RemoteAssetCache = require("../src/RemoteAssetCache"); 6 | 7 | test("getDurationMs", (t) => { 8 | let cache = new RemoteAssetCache("https://example.com/"); 9 | // t.is(cache.getDurationMs("0"), 0); 10 | t.is(cache.getDurationMs("0s"), 0); 11 | t.is(cache.getDurationMs("1s"), 1000); 12 | t.is(cache.getDurationMs("1m"), 60 * 1000); 13 | t.is(cache.getDurationMs("1h"), 60 * 60 * 1000); 14 | t.is(cache.getDurationMs("1d"), 24 * 60 * 60 * 1000); 15 | t.is(cache.getDurationMs("1w"), 7 * 24 * 60 * 60 * 1000); 16 | t.is(cache.getDurationMs("1y"), 365 * 24 * 60 * 60 * 1000); 17 | 18 | t.is(cache.getDurationMs("5s"), 5000); 19 | t.is(cache.getDurationMs("0m"), 0); 20 | t.is(cache.getDurationMs("7m"), 60 * 7000); 21 | t.is(cache.getDurationMs("9h"), 60 * 60 * 9000); 22 | t.is(cache.getDurationMs("0h"), 0); 23 | t.is(cache.getDurationMs("11d"), 24 * 60 * 60 * 11000); 24 | t.is(cache.getDurationMs("0d"), 0); 25 | t.is(cache.getDurationMs("13w"), 7 * 24 * 60 * 60 * 13000); 26 | t.is(cache.getDurationMs("0w"), 0); 27 | t.is(cache.getDurationMs("15y"), 365 * 24 * 60 * 60 * 15000); 28 | t.is(cache.getDurationMs("0y"), 0); 29 | }); 30 | 31 | test("Local hash file names", async (t) => { 32 | let pngUrl = "https://www.zachleat.com/img/avatar-2017-big.png"; 33 | t.is( 34 | new RemoteAssetCache(pngUrl).cachePath, 35 | path.resolve(".", `.cache/eleventy-fetch-${AssetCache.getHash(pngUrl)}`), 36 | ); 37 | 38 | let fontUrl = "https://www.zachleat.com/font.woff"; 39 | t.is( 40 | new RemoteAssetCache(fontUrl).cachePath, 41 | path.resolve(".", `.cache/eleventy-fetch-${AssetCache.getHash(fontUrl)}`), 42 | ); 43 | 44 | let fontUrl2 = "https://www.zachleat.com/font.woff2"; 45 | t.is( 46 | new RemoteAssetCache(fontUrl2).cachePath, 47 | path.resolve(".", `.cache/eleventy-fetch-${AssetCache.getHash(fontUrl2)}`), 48 | ); 49 | }); 50 | 51 | test("Clean url", async (t) => { 52 | let shortUrl = "https://example.com/207115/photos/243-0-1.jpg"; 53 | let longUrl = 54 | "https://example.com/207115/photos/243-0-1.jpg?Policy=FAKE_THING~2123ksjhd&Signature=FAKE_THING~2123ksjhd&Key-Pair-Id=FAKE_THING~2123ksjhd"; 55 | t.is( 56 | new RemoteAssetCache(longUrl, ".cache", { 57 | removeUrlQueryParams: true, 58 | }).displayUrl, 59 | shortUrl, 60 | ); 61 | }); 62 | 63 | test("Local hash without file extension in URL", async (t) => { 64 | let noExt = "https://twitter.com/zachleat/profile_image?size=bigger"; 65 | t.is( 66 | new RemoteAssetCache(noExt).cachePath, 67 | path.resolve(".", `.cache/eleventy-fetch-${AssetCache.getHash(noExt)}`), 68 | ); 69 | }); 70 | 71 | test("Unique hashes for URLs", async (t) => { 72 | let apiURL1 = "https://api.zooniverse.org/projects/illustratedlife/talk/subjects/ASC0000qu3"; 73 | let apiURL2 = "https://api.zooniverse.org/projects/illustratedlife/talk/subjects/ASC0000q71"; 74 | let cachePath1 = new RemoteAssetCache(apiURL1).cachePath; 75 | let cachePath2 = new RemoteAssetCache(apiURL2).cachePath; 76 | t.not(cachePath1, cachePath2); 77 | }); 78 | 79 | test("Same hashes for implicit and explicit HTTP GET", async (t) => { 80 | let sameURL = "https://example.com/"; 81 | let cachePath1 = new RemoteAssetCache(sameURL, ".cache", { 82 | fetchOptions: { method: "GET" }, 83 | }).cachePath; 84 | let cachePath2 = new RemoteAssetCache(sameURL, ".cache", { 85 | fetchOptions: {}, 86 | }).cachePath; 87 | t.is(cachePath1, cachePath2); 88 | }); 89 | 90 | test("Unique hashes for different HTTP methods", async (t) => { 91 | let sameURL = "https://example.com/"; 92 | let cachePath1 = new RemoteAssetCache(sameURL, ".cache", { 93 | fetchOptions: { method: "POST" }, 94 | }).cachePath; 95 | let cachePath2 = new RemoteAssetCache(sameURL, ".cache", { 96 | fetchOptions: { method: "DELETE" }, 97 | }).cachePath; 98 | t.not(cachePath1, cachePath2); 99 | }); 100 | 101 | test("Unique hashes for different HTTP bodies", async (t) => { 102 | let sameURL = "https://example.com/"; 103 | let cachePath1 = new RemoteAssetCache(sameURL, ".cache", { 104 | fetchOptions: { body: "123" }, 105 | }).cachePath; 106 | let cachePath2 = new RemoteAssetCache(sameURL, ".cache", { 107 | fetchOptions: { body: "456" }, 108 | }).cachePath; 109 | t.not(cachePath1, cachePath2); 110 | }); 111 | 112 | test("Fetching!", async (t) => { 113 | let pngUrl = "https://www.zachleat.com/img/avatar-2017-big.png"; 114 | let ac = new RemoteAssetCache(pngUrl); 115 | let buffer = await ac.fetch(); 116 | try { 117 | await ac.destroy(); 118 | } catch (e) {} 119 | 120 | t.is(ac.fetchCount, 1); 121 | t.is(Buffer.isBuffer(buffer), true); 122 | }); 123 | 124 | test("Fetching (dry run)!", async (t) => { 125 | let svgUrl = "https://www.zachleat.com/img/avatar-2017-88.png"; 126 | let ac = new RemoteAssetCache(svgUrl, ".cache", { 127 | dryRun: true, 128 | }); 129 | let buffer = await ac.fetch(); 130 | t.is(Buffer.isBuffer(buffer), true); 131 | t.is(ac.fetchCount, 1); 132 | t.false(ac.hasAnyCacheFiles()); 133 | }); 134 | 135 | test("Fetching pass in URL", async (t) => { 136 | let pngUrl = new URL("https://www.zachleat.com/img/avatar-2017-big.png"); 137 | let ac = new RemoteAssetCache(pngUrl); 138 | let buffer = await ac.fetch(); 139 | try { 140 | await ac.destroy(); 141 | } catch (e) {} 142 | 143 | t.is(Buffer.isBuffer(buffer), true); 144 | }); 145 | 146 | test("Fetching pass non-stringable", async (t) => { 147 | let e = await t.throwsAsync(async () => { 148 | class B {} 149 | let source = new B(); 150 | 151 | let ac = new RemoteAssetCache(source, undefined, { 152 | dryRun: true, 153 | }); 154 | await ac.fetch(); 155 | }); 156 | 157 | t.is(e.message, "Invalid source: must be a string, function, or Promise. If a function or Promise, you must provide a `toString()` method or an `options.requestId` unique key. Received: [object Object]"); 158 | }); 159 | 160 | test("Fetching pass class with toString()", async (t) => { 161 | class B { 162 | toString() { 163 | return "https://www.zachleat.com/img/avatar-2017-big.png"; 164 | } 165 | } 166 | 167 | let ac = new RemoteAssetCache(new B()); 168 | let buffer = await ac.fetch(); 169 | try { 170 | await ac.destroy(); 171 | } catch (e) {} 172 | 173 | t.is(Buffer.isBuffer(buffer), true); 174 | }); 175 | 176 | test("formatUrlForDisplay (manual query param removal)", async (t) => { 177 | let finalUrl = "https://example.com/207115/photos/243-0-1.jpg"; 178 | let longUrl = 179 | "https://example.com/207115/photos/243-0-1.jpg?Policy=FAKE_THING~2123ksjhd&Signature=FAKE_THING~2123ksjhd&Key-Pair-Id=FAKE_THING~2123ksjhd"; 180 | t.is( 181 | new RemoteAssetCache(longUrl, ".cache", { 182 | removeUrlQueryParams: false, 183 | formatUrlForDisplay(url) { 184 | let [rest, queryParams] = url.split("?"); 185 | return rest; 186 | }, 187 | }).displayUrl, 188 | finalUrl, 189 | ); 190 | }); 191 | 192 | test("formatUrlForDisplay (using removeUrlQueryParams)", async (t) => { 193 | let finalUrl = "https://example.com/207115/photos/243-0-1.jpg"; 194 | let longUrl = 195 | "https://example.com/207115/photos/243-0-1.jpg?Policy=FAKE_THING~2123ksjhd&Signature=FAKE_THING~2123ksjhd&Key-Pair-Id=FAKE_THING~2123ksjhd"; 196 | t.is( 197 | new RemoteAssetCache(longUrl, ".cache", { 198 | removeUrlQueryParams: true, 199 | formatUrlForDisplay(url) { 200 | return url; 201 | }, 202 | }).displayUrl, 203 | finalUrl, 204 | ); 205 | }); 206 | 207 | test("formatUrlForDisplay (using removeUrlQueryParams and requestId)", async (t) => { 208 | let longUrl = 209 | "https://example.com/207115/photos/243-0-1.jpg?Policy=FAKE_THING~2123ksjhd&Signature=FAKE_THING~2123ksjhd&Key-Pair-Id=FAKE_THING~2123ksjhd"; 210 | t.is( 211 | new RemoteAssetCache(function() {}, ".cache", { 212 | requestId: longUrl, 213 | removeUrlQueryParams: true, 214 | formatUrlForDisplay(url) { 215 | return url; 216 | }, 217 | }).displayUrl, 218 | "function() {}", 219 | ); 220 | }); 221 | 222 | test("Issue #6, URLs with HTTP Auth", async (t) => { 223 | let url = "https://${USERNAME}:${PASSWORD}@api.pinboard.in/v1/posts/all?format=json&tag=read"; 224 | t.true(Util.isFullUrl(url)); 225 | }); 226 | 227 | test("Error with `cause`", async (t) => { 228 | let finalUrl = "https://example.com/207115/photos/243-0-1.jpg"; 229 | let asset = new RemoteAssetCache(finalUrl); 230 | 231 | try { 232 | await asset.fetch(); 233 | } catch (e) { 234 | t.is( 235 | e.message, 236 | `Bad response for https://example.com/207115/photos/243-0-1.jpg (404): Not Found`, 237 | ); 238 | t.truthy(e.cause); 239 | } finally { 240 | await asset.destroy(); 241 | } 242 | }); 243 | 244 | test("supports promises that resolve", async (t) => { 245 | let expected = { mockKey: "mockValue" }; 246 | let promise = Promise.resolve(expected); 247 | let asset = new RemoteAssetCache(promise, undefined, { 248 | type: "json", 249 | requestId: "resolve-promise", 250 | }); 251 | 252 | let actual = await asset.fetch(); 253 | try { 254 | await asset.destroy(); 255 | } catch (e) {} 256 | 257 | t.deepEqual(actual, expected); 258 | 259 | }); 260 | 261 | test("supports promises that reject", async (t) => { 262 | let expected = "mock error message"; 263 | let cause = new Error("mock cause"); 264 | let promise = Promise.reject(new Error(expected, { cause })); 265 | 266 | let asset = new RemoteAssetCache(promise, undefined, { 267 | requestId: "reject-promise", 268 | }); 269 | 270 | try { 271 | await asset.fetch(); 272 | } catch (e) { 273 | t.is(e.message, expected); 274 | t.is(e.cause, cause); 275 | } finally { 276 | try { 277 | await asset.destroy(); 278 | } catch (e) {} 279 | } 280 | }); 281 | 282 | test("supports async functions that return data", async (t) => { 283 | let expected = { mockKey: "mockValue" }; 284 | let asyncFunction = async () => { 285 | return Promise.resolve(expected); 286 | }; 287 | let asset = new RemoteAssetCache(asyncFunction, undefined, { 288 | type: "json", 289 | requestId: "async-return", 290 | }); 291 | 292 | let actual = await asset.fetch(); 293 | try { 294 | await asset.destroy(); 295 | } catch (e) {} 296 | 297 | t.deepEqual(actual, expected); 298 | 299 | }); 300 | 301 | test("supports async functions that throw", async (t) => { 302 | let expected = "mock error message"; 303 | let cause = new Error("mock cause"); 304 | let asyncFunction = async () => { 305 | throw new Error(expected, { cause }); 306 | }; 307 | 308 | try { 309 | let ac = new RemoteAssetCache(asyncFunction, undefined, { 310 | requestId: "async-throws", 311 | }); 312 | await ac.fetch(); 313 | } catch (e) { 314 | t.is(e.message, expected); 315 | t.is(e.cause, cause); 316 | } finally { 317 | try { 318 | await asset.destroy(); 319 | } catch (e) {} 320 | } 321 | 322 | 323 | }); 324 | 325 | test("type: xml", async (t) => { 326 | let feedUrl = "https://www.11ty.dev/blog/feed.xml"; 327 | let ac = new RemoteAssetCache(feedUrl, ".cache", { 328 | type: "xml" 329 | }); 330 | let xml = await ac.fetch(); 331 | try { 332 | await ac.destroy(); 333 | } catch (e) {} 334 | 335 | t.true(xml.length > 50) 336 | }); 337 | 338 | test("type: parsed-xml", async (t) => { 339 | let feedUrl = "https://www.11ty.dev/blog/feed.xml"; 340 | let ac = new RemoteAssetCache(feedUrl, ".cache", { 341 | type: "parsed-xml" 342 | }); 343 | let xml = await ac.fetch(); 344 | try { 345 | await ac.destroy(); 346 | } catch (e) {} 347 | 348 | t.is(xml.children.length, 1) 349 | }); 350 | 351 | test("returnType: response, cache miss", async (t) => { 352 | let feedUrl = "https://www.11ty.dev/blog/feed.xml"; 353 | let ac = new RemoteAssetCache(feedUrl, ".cache", { 354 | type: "xml", 355 | returnType: "response", 356 | }); 357 | let response = await ac.fetch(); 358 | 359 | try { 360 | await ac.destroy(); 361 | } catch (e) {} 362 | 363 | t.is(ac.fetchCount, 1); 364 | t.is(response.url, feedUrl); 365 | t.is(response.status, 200); 366 | t.truthy(response.headers.server); 367 | t.truthy(response.headers['last-modified']); 368 | t.truthy(response.body); 369 | t.is(response.cache, "miss"); 370 | }); 371 | 372 | 373 | test("returnType: response, cache hit", async (t) => { 374 | let feedUrl = "https://www.11ty.dev/blog/feed.xml"; 375 | let ac = new RemoteAssetCache(feedUrl, ".cache", { 376 | type: "xml", 377 | returnType: "response", 378 | }); 379 | 380 | // miss 381 | await ac.fetch(); 382 | 383 | // hit 384 | let response = await ac.fetch(); 385 | 386 | try { 387 | await ac.destroy(); 388 | } catch (e) {} 389 | 390 | t.is(response.cache, "hit"); 391 | }); 392 | 393 | -------------------------------------------------------------------------------- /test/v5flattedcachefile: -------------------------------------------------------------------------------- 1 | [["1"],{"key":"2","value":"3"},"0ca70533982928b147b5fdc377559c",{"cachedAt":1741622350514,"type":"4","metadata":"5"},"json",{"response":"6"},{"url":"7","status":200,"headers":"8"},"https://11tybundle.dev/api/global-data.json"] 2 | --------------------------------------------------------------------------------