├── .eslintrc.cjs ├── .github └── workflows │ └── test.yaml ├── .gitignore ├── LICENSE ├── README.md ├── package-lock.json ├── package.json ├── src └── ffetch.js └── test └── ffetch.js /.eslintrc.cjs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Adobe. All rights reserved. 3 | * This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. You may obtain a copy 5 | * of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | * 7 | * Unless required by applicable law or agreed to in writing, software distributed under 8 | * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | * OF ANY KIND, either express or implied. See the License for the specific language 10 | * governing permissions and limitations under the License. 11 | */ 12 | 13 | module.exports = { 14 | root: true, 15 | extends: 'airbnb-base', 16 | env: { 17 | browser: true, 18 | }, 19 | parser: '@babel/eslint-parser', 20 | parserOptions: { 21 | allowImportExportEverywhere: true, 22 | sourceType: 'module', 23 | requireConfigFile: false, 24 | }, 25 | rules: { 26 | // allow reassigning param 27 | 'no-param-reassign': [2, { props: false }], 28 | 'linebreak-style': ['error', 'unix'], 29 | 'import/extensions': ['error', { 30 | js: 'always', 31 | }], 32 | 'no-use-before-define': 'off', 33 | 'no-restricted-syntax': 'off', 34 | }, 35 | } 36 | -------------------------------------------------------------------------------- /.github/workflows/test.yaml: -------------------------------------------------------------------------------- 1 | name: Linting & Tests 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: main 7 | 8 | jobs: 9 | build: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v3 13 | - name: Use Node.js 14 | uses: actions/setup-node@v3 15 | with: 16 | node-version: '18' #required for npm 8 or later. 17 | - run: npm ci 18 | - run: npm run lint 19 | env: 20 | CI: true 21 | - run: npm run test -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | coverage/ 2 | node_modules/ 3 | .vscode/ -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # `ffetch` – `fetch` for Edge Delivery Services (Franklin) 2 | 3 | `ffetch` is a small wrapper around the JavaScript `fetch` function that helps you deal with the AEM (.live) Content API when 4 | building a composable application. It makes it easy to `fetch` content from 5 | [an Index](https://www.aem.live/developer/indexing), apply lazy pagination, follow links to pages, and even pull 6 | [page metadata](https://www.aem.live/developer/block-collection/metadata). With `ffetch` you get all the ease of creating 7 | a headless application without the peformance baggage of headless SDKs and the complexity of headless APIs. 8 | 9 | ## Why `ffetch`? 10 | 11 | - minimal: less than [200 lines of code](https://github.com/Buuhuu/ffetch/blob/main/src/ffetch.js) 12 | - dependency free, just copy it into your project 13 | - high performance: uses your browser cache 14 | - works in browsers and node.js 15 | - fun to use 16 | 17 | ## Usage 18 | 19 | Check the [tests for detailed examples](https://github.com/Buuhuu/ffetch/blob/main/test/ffetch.js): 20 | 21 | ### Get Entries from an Index 22 | 23 | ```javascript 24 | const entries = ffetch('/query-index.json'); 25 | let i = 0; 26 | for await (const entry of entries) { 27 | console.log(entry.title); 28 | } 29 | ``` 30 | 31 | `ffetch` will return a generator, so you can just iterate over the return value. If pagination is necessary, `ffetch` will 32 | fetch additional pages from the server as you exhaust the available records. 33 | 34 | ### Get the first entry 35 | 36 | ```javascript 37 | console.log(await ffetch('/query-index.json').first()); 38 | ``` 39 | 40 | ### Get all entries as an array (so you can `.map` and `.filter`) 41 | 42 | Using `.all()` you can change the return value from a generator to a plain array. 43 | 44 | ```javascript 45 | const allentries = await ffetch('/query-index.json').all(); 46 | allentries.forEach((e) => { 47 | console.log(e); 48 | }); 49 | ``` 50 | 51 | But if you prefer to use `.map` and `.filter`, you can do this right on the generator: 52 | 53 | ```javascript 54 | const someentries = ffetch('/query-index.json') 55 | .map(({title}) => title) 56 | .filter(title => title.indexOf('Helix')); 57 | for await (title of someentries) { 58 | console.log(title); 59 | } 60 | ``` 61 | 62 | ### Tune performance with `.chunks` and `.limit` 63 | 64 | If you want to control the size of the chunks that are loaded using pagination, use `ffetch(...).chunks(100)`. 65 | 66 | To limit the result set based on the number of entries you need to show on the page, use `ffetch(...).limit(5)`. The `limit()` 67 | applies after all `.filter()`s, so it is an effective way to only process what needs to be shown. 68 | 69 | If you need to skip a couple of entries, then `.slice(start, end)` is your friend. It works exactly like 70 | [`Array.prototype.slice()`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/slice) 71 | 72 | ### Work with multi-sheets 73 | 74 | AEM JSON resources can contain multiple sheets. With `.sheet(name)` you can specify, which sheet you want to access. 75 | 76 | ```javascript 77 | const entries = ffetch('/query-index.json') 78 | .sheet('products'); 79 | let i = 0; 80 | for await (const entry of entries) { 81 | console.log(entry.sku); 82 | } 83 | ``` 84 | 85 | ### Work with HTML pages 86 | 87 | In AEM, the Hypertext is the API, so you can get a [Document](https://developer.mozilla.org/en-US/docs/Web/API/Document) for 88 | each HTML document referenced from an index sheet. 89 | 90 | ```javascript 91 | const docs = ffetch('/query-index.json') 92 | // assumes that the path property holds the reference to our document 93 | // stores the returned document in a new field (optional) 94 | .follow('path', 'document') 95 | .map(({document}) => document.querySelector('img')) // get the first image 96 | .filter(i => !!i) // drop entries that don't have an image 97 | .limit(10); // that's enough 98 | 99 | for await (const img of docs) { 100 | document.append(img); // take the image from the linked document and place it here 101 | } 102 | ``` 103 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ffetch", 3 | "version": "0.0.1", 4 | "description": "Franklin wrapper for fetch", 5 | "main": "src/index.js", 6 | "type": "module", 7 | "scripts": { 8 | "test": "c8 mocha", 9 | "lint": "eslint src/**/*.js test/**/*.js" 10 | }, 11 | "keywords": [ 12 | "fetch", 13 | "franklin" 14 | ], 15 | "author": "Dirk Rudolph", 16 | "license": "Apache-2.0", 17 | "devDependencies": { 18 | "@adobe/eslint-config-helix": "^2.0.2", 19 | "@adobe/fetch": "^4.0.4", 20 | "@babel/eslint-parser": "^7.22.15", 21 | "c8": "^7.13.0", 22 | "eslint": "^8.35.0", 23 | "eslint-plugin-import": "^2.27.5", 24 | "htmlparser2": "^8.0.1", 25 | "mocha": "^10.2.0", 26 | "nock": "^13.3.0" 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/ffetch.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Adobe. All rights reserved. 3 | * This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. You may obtain a copy 5 | * of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | * 7 | * Unless required by applicable law or agreed to in writing, software distributed under 8 | * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | * OF ANY KIND, either express or implied. See the License for the specific language 10 | * governing permissions and limitations under the License. 11 | */ 12 | 13 | /* eslint-disable no-restricted-syntax, no-await-in-loop */ 14 | 15 | async function* request(url, context) { 16 | const { 17 | chunkSize, cacheReload, sheetName, fetch, 18 | } = context; 19 | for (let offset = 0, total = Infinity; offset < total; offset += chunkSize) { 20 | const params = new URLSearchParams(`offset=${offset}&limit=${chunkSize}`); 21 | if (sheetName) params.append('sheet', sheetName); 22 | const resp = await fetch(`${url}?${params.toString()}`, { cache: cacheReload ? 'reload' : 'default' }); 23 | if (resp.ok) { 24 | const json = await resp.json(); 25 | total = json.total; 26 | context.total = total; 27 | for (const entry of json.data) yield entry; 28 | } else { 29 | return; 30 | } 31 | } 32 | } 33 | 34 | // Operations: 35 | 36 | function withFetch(upstream, context, fetch) { 37 | context.fetch = fetch; 38 | return upstream; 39 | } 40 | 41 | function withHtmlParser(upstream, context, parseHtml) { 42 | context.parseHtml = parseHtml; 43 | return upstream; 44 | } 45 | 46 | function chunks(upstream, context, chunkSize) { 47 | context.chunkSize = chunkSize; 48 | return upstream; 49 | } 50 | 51 | function sheet(upstream, context, sheetName) { 52 | context.sheetName = sheetName; 53 | return upstream; 54 | } 55 | 56 | async function* skip(upstream, context, from) { 57 | let skipped = 0; 58 | for await (const entry of upstream) { 59 | if (skipped < from) { 60 | skipped += 1; 61 | } else { 62 | yield entry; 63 | } 64 | } 65 | } 66 | 67 | async function* limit(upstream, context, aLimit) { 68 | let yielded = 0; 69 | for await (const entry of upstream) { 70 | yield entry; 71 | yielded += 1; 72 | if (yielded === aLimit) { 73 | return; 74 | } 75 | } 76 | } 77 | 78 | async function* map(upstream, context, fn, maxInFlight = 5) { 79 | const promises = []; 80 | for await (let entry of upstream) { 81 | promises.push(fn(entry)); 82 | if (promises.length === maxInFlight) { 83 | for (entry of promises) { 84 | entry = await entry; 85 | if (entry) yield entry; 86 | } 87 | promises.splice(0, promises.length); 88 | } 89 | } 90 | for (let entry of promises) { 91 | entry = await entry; 92 | if (entry) yield entry; 93 | } 94 | } 95 | 96 | async function* filter(upstream, context, fn) { 97 | for await (const entry of upstream) { 98 | if (fn(entry)) { 99 | yield entry; 100 | } 101 | } 102 | } 103 | 104 | function slice(upstream, context, from, to) { 105 | return limit(skip(upstream, context, from), context, to - from); 106 | } 107 | 108 | function follow(upstream, context, name, newName, maxInFlight = 5) { 109 | const { fetch, parseHtml } = context; 110 | return map(upstream, context, async (entry) => { 111 | const value = entry[name]; 112 | if (value) { 113 | const resp = await fetch(value); 114 | return { ...entry, [newName || name]: resp.ok ? parseHtml(await resp.text()) : null }; 115 | } 116 | return entry; 117 | }, maxInFlight); 118 | } 119 | 120 | async function all(upstream) { 121 | const result = []; 122 | for await (const entry of upstream) { 123 | result.push(entry); 124 | } 125 | return result; 126 | } 127 | 128 | async function first(upstream) { 129 | /* eslint-disable-next-line no-unreachable-loop */ 130 | for await (const entry of upstream) { 131 | return entry; 132 | } 133 | return null; 134 | } 135 | 136 | // Helper 137 | 138 | function assignOperations(generator, context) { 139 | // operations that return a new generator 140 | function createOperation(fn) { 141 | return (...rest) => assignOperations(fn.apply(null, [generator, context, ...rest]), context); 142 | } 143 | const operations = { 144 | skip: createOperation(skip), 145 | limit: createOperation(limit), 146 | slice: createOperation(slice), 147 | map: createOperation(map), 148 | filter: createOperation(filter), 149 | follow: createOperation(follow), 150 | }; 151 | 152 | // functions that either return the upstream generator or no generator at all 153 | const functions = { 154 | chunks: chunks.bind(null, generator, context), 155 | all: all.bind(null, generator, context), 156 | first: first.bind(null, generator, context), 157 | withFetch: withFetch.bind(null, generator, context), 158 | withHtmlParser: withHtmlParser.bind(null, generator, context), 159 | sheet: sheet.bind(null, generator, context), 160 | }; 161 | 162 | Object.assign(generator, operations, functions); 163 | Object.defineProperty(generator, 'total', { get: () => context.total }); 164 | return generator; 165 | } 166 | 167 | export default function ffetch(url) { 168 | let chunkSize = 255; 169 | let cacheReload = false; 170 | const fetch = (...rest) => window.fetch.apply(null, rest); 171 | const parseHtml = (html) => new window.DOMParser().parseFromString(html, 'text/html'); 172 | 173 | try { 174 | if ('connection' in window.navigator && window.navigator.connection.saveData === true) { 175 | // request smaller chunks in save data mode 176 | chunkSize = 64; 177 | } 178 | // detect page reloads and set cacheReload to true 179 | const entries = performance.getEntriesByType('navigation'); 180 | const reloads = entries.filter((entry) => entry.type === 'reload'); 181 | if (reloads.length > 0) cacheReload = true; 182 | } catch (e) { /* ignore */ } 183 | 184 | const context = { 185 | chunkSize, cacheReload, fetch, parseHtml, 186 | }; 187 | const generator = request(url, context); 188 | 189 | return assignOperations(generator, context); 190 | } 191 | -------------------------------------------------------------------------------- /test/ffetch.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Adobe. All rights reserved. 3 | * This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. You may obtain a copy 5 | * of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | * 7 | * Unless required by applicable law or agreed to in writing, software distributed under 8 | * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | * OF ANY KIND, either express or implied. See the License for the specific language 10 | * governing permissions and limitations under the License. 11 | */ 12 | /* eslint-disable no-undef */ 13 | import { fetch as adobeFetch } from '@adobe/fetch'; 14 | import { promises as fs } from 'node:fs'; 15 | import { URL } from 'node:url'; 16 | import assert from 'node:assert/strict'; 17 | import { parseDocument } from 'htmlparser2'; 18 | import nock from 'nock'; 19 | import ffetch from '../src/ffetch.js'; 20 | 21 | // enforce http/1.1 because nock does not support h2 yet 22 | const testDomain = 'http://example.com'; 23 | 24 | function mockDocumentRequest(path, body = 'Document

Hello World

') { 25 | nock(testDomain).get(path).reply(200, body); 26 | } 27 | 28 | function mockIndexRequests(path, total, chunks = 255, sheet = undefined, generatorFn = (i) => ({ title: `Entry ${i}` })) { 29 | for (let offset = 0; offset < total; offset += chunks) { 30 | const data = Array.from( 31 | { length: offset + chunks < total ? chunks : 555 - offset }, 32 | (_, i) => generatorFn(offset + i), 33 | ); 34 | const response = { 35 | total, offset, limit: chunks, data, 36 | }; 37 | const params = { offset, limit: chunks }; 38 | if (sheet) { 39 | params.sheet = sheet; 40 | } 41 | 42 | nock(testDomain).get(path).query(params).reply(200, response); 43 | } 44 | } 45 | 46 | function mockNotFound(path) { 47 | nock(testDomain).get(path).query(() => true).reply(404); 48 | } 49 | 50 | describe('ffetch', () => { 51 | // wrap fetch to make all calls absolute 52 | const fetch = (url) => (url.charAt(0) === '/' 53 | ? adobeFetch(`${testDomain}${url}`) 54 | : adobeFetch(url)); 55 | 56 | it('has no dependencies', async () => { 57 | const packageJson = await fs.readFile(new URL('../package.json', import.meta.url), { encoding: 'utf-8' }); 58 | const { dependencies } = JSON.parse(packageJson); 59 | 60 | assert(!dependencies || dependencies.length === 0); 61 | }); 62 | 63 | it('returns a generator for all entries', async () => { 64 | mockIndexRequests('/query-index.json', 555); 65 | 66 | const entries = ffetch('/query-index.json').withFetch(fetch); 67 | let i = 0; 68 | 69 | assert.equal(undefined, entries.total); 70 | for await (const entry of entries) { 71 | assert.deepStrictEqual(entry, { title: `Entry ${i}` }); 72 | i += 1; 73 | } 74 | 75 | assert.equal(555, entries.total); 76 | assert.equal(555, i); 77 | }); 78 | 79 | describe('failure handling', () => { 80 | it('returns an empty generator for a 404', async () => { 81 | mockNotFound('/not-found.json'); 82 | 83 | const entries = ffetch('/not-found.json').withFetch(fetch); 84 | 85 | /* eslint-disable-next-line no-unused-vars */ 86 | for await (const entry of entries) assert(false); 87 | }); 88 | 89 | it('returns an empty array for a 404', async () => { 90 | mockNotFound('/not-found.json'); 91 | 92 | const entries = await ffetch('/not-found.json').withFetch(fetch).all(); 93 | 94 | assert.equal(entries.length, 0); 95 | }); 96 | 97 | it('returns null for the first enrty of a 404', async () => { 98 | mockNotFound('/not-found.json'); 99 | 100 | const entry = await ffetch('/not-found.json').withFetch(fetch).first(); 101 | 102 | assert.equal(null, entry); 103 | }); 104 | }); 105 | 106 | describe('operations', () => { 107 | describe('chunks', () => { 108 | it('returns a generator for all entries with custom chunk size', async () => { 109 | mockIndexRequests('/query-index.json', 555, 1000); 110 | 111 | const entries = ffetch('/query-index.json').withFetch(fetch).chunks(1000); 112 | let i = 0; 113 | for await (const entry of entries) { 114 | assert.deepStrictEqual(entry, { title: `Entry ${i}` }); 115 | i += 1; 116 | } 117 | 118 | assert.equal(555, i); 119 | }); 120 | }); 121 | 122 | describe('map', () => { 123 | it('returns a generator that maps each entry', async () => { 124 | mockIndexRequests('/query-index.json', 555); 125 | 126 | const entries = ffetch('/query-index.json').withFetch(fetch) 127 | .map(({ title }) => title); 128 | let i = 0; 129 | for await (const entry of entries) { 130 | assert.equal(entry, `Entry ${i}`); 131 | i += 1; 132 | } 133 | 134 | assert.equal(555, i); 135 | }); 136 | 137 | it('returns the first enrty after applying multiple mappings', async () => { 138 | mockIndexRequests('/query-index.json', 555); 139 | 140 | const entry = await ffetch('/query-index.json').withFetch(fetch) 141 | .map(({ title }) => title) 142 | .map((title) => title.toUpperCase()) 143 | .first(); 144 | 145 | assert.equal(entry, 'ENTRY 0'); 146 | }); 147 | }); 148 | 149 | describe('filter', () => { 150 | it('returns a generator that filters entries', async () => { 151 | mockIndexRequests('/query-index.json', 555); 152 | 153 | const expectedEntries = ['Entry 99', 'Entry 199', 'Entry 299', 'Entry 399', 'Entry 499']; 154 | const entries = ffetch('/query-index.json').withFetch(fetch) 155 | .filter(({ title }) => expectedEntries.indexOf(title) >= 0); 156 | let i = 0; 157 | for await (const entry of entries) { 158 | assert.equal(entry.title, expectedEntries[i]); 159 | i += 1; 160 | } 161 | 162 | assert.equal(5, i); 163 | }); 164 | 165 | it('returns the first enrty after multiple filters', async () => { 166 | mockIndexRequests('/query-index.json', 555); 167 | 168 | const entry = await ffetch('/query-index.json').withFetch(fetch) 169 | .filter(({ title }) => title.indexOf('9') > 0) 170 | .filter(({ title }) => title.indexOf('8') > 0) 171 | .filter(({ title }) => title.indexOf('4') > 0) 172 | .first(); 173 | 174 | assert.deepStrictEqual(entry, { title: 'Entry 489' }); 175 | }); 176 | }); 177 | 178 | describe('limit', () => { 179 | it('returns a generator for a limited set entries', async () => { 180 | mockIndexRequests('/query-index.json', 555); 181 | 182 | const entries = ffetch('/query-index.json').withFetch(fetch) 183 | .limit(10); 184 | let i = 0; 185 | for await (const entry of entries) { 186 | assert.deepStrictEqual(entry, { title: `Entry ${i}` }); 187 | i += 1; 188 | } 189 | 190 | assert.equal(10, i); 191 | }); 192 | 193 | it('returns an array of all entries', async () => { 194 | mockIndexRequests('/query-index.json', 555); 195 | 196 | const entries = await ffetch('/query-index.json').withFetch(fetch) 197 | .limit(5) 198 | .all(); 199 | 200 | assert.deepStrictEqual(entries, [ 201 | { title: 'Entry 0' }, 202 | { title: 'Entry 1' }, 203 | { title: 'Entry 2' }, 204 | { title: 'Entry 3' }, 205 | { title: 'Entry 4' }, 206 | ]); 207 | }); 208 | }); 209 | 210 | describe('slice', () => { 211 | it('returns a generator that filters a sliced set of entries', async () => { 212 | mockIndexRequests('/query-index.json', 555); 213 | 214 | const expectedEntries = ['Entry 99', 'Entry 199', 'Entry 299', 'Entry 399', 'Entry 499']; 215 | const entries = ffetch('/query-index.json').withFetch(fetch) 216 | .filter(({ title }) => expectedEntries.indexOf(title) >= 0) 217 | .slice(2, 4); 218 | let i = 0; 219 | for await (const entry of entries) { 220 | assert.equal(entry.title, expectedEntries[i + 2]); 221 | i += 1; 222 | } 223 | 224 | assert.equal(2, i); 225 | }); 226 | 227 | it('returns an array of a slice of entries', async () => { 228 | mockIndexRequests('/query-index.json', 555); 229 | 230 | const entries = await ffetch('/query-index.json').withFetch(fetch) 231 | .slice(300, 305) 232 | .all(); 233 | 234 | assert.deepStrictEqual(entries, [ 235 | { title: 'Entry 300' }, 236 | { title: 'Entry 301' }, 237 | { title: 'Entry 302' }, 238 | { title: 'Entry 303' }, 239 | { title: 'Entry 304' }, 240 | ]); 241 | }); 242 | }); 243 | 244 | describe('sheet', () => { 245 | it('returns a generator for all entries of a given sheet', async () => { 246 | mockIndexRequests('/query-index.json', 555, 255, 'test'); 247 | 248 | const entries = ffetch('/query-index.json').withFetch(fetch).sheet('test'); 249 | let i = 0; 250 | for await (const entry of entries) { 251 | assert.deepStrictEqual(entry, { title: `Entry ${i}` }); 252 | i += 1; 253 | } 254 | 255 | assert.equal(555, i); 256 | }); 257 | }); 258 | 259 | describe('follow', () => { 260 | it('returns the html parsed as document when following a reference', async () => { 261 | mockDocumentRequest('/document'); 262 | mockIndexRequests('/query-index.json', 1, 255, null, () => ({ path: '/document' })); 263 | 264 | const entry = await ffetch('/query-index.json').withFetch(fetch).withHtmlParser(parseDocument) 265 | .follow('path') 266 | .first(); 267 | 268 | assert(entry); 269 | assert(entry.path); 270 | }); 271 | 272 | it('stores the document result in a a new field', async () => { 273 | mockDocumentRequest('/document'); 274 | mockIndexRequests('/query-index.json', 1, 255, null, () => ({ path: '/document' })); 275 | 276 | const entry = await ffetch('/query-index.json').withFetch(fetch).withHtmlParser(parseDocument) 277 | .follow('path', 'content') 278 | .first(); 279 | 280 | assert(entry); 281 | assert(entry.path === '/document'); 282 | assert(entry.content); 283 | }); 284 | 285 | it('returns null if the reference does not exist', async () => { 286 | mockIndexRequests('/query-index.json', 1, 255, null, () => ({ ref: '/document' })); 287 | 288 | const entry = await ffetch('/query-index.json').withFetch(fetch).withHtmlParser(parseDocument) 289 | .follow('path') 290 | .first(); 291 | 292 | assert(entry); 293 | assert(!entry.path); 294 | }); 295 | 296 | it('returns null if the referenced document is not found', async () => { 297 | mockNotFound('/document'); 298 | mockIndexRequests('/query-index.json', 1, 255, null, () => ({ path: '/document' })); 299 | 300 | const entry = await ffetch('/query-index.json').withFetch(fetch).withHtmlParser(parseDocument) 301 | .follow('path') 302 | .first(); 303 | 304 | assert(entry); 305 | assert(!entry.path); 306 | }); 307 | }); 308 | }); 309 | 310 | it('implements array-like semantics for chaining operations', async () => { 311 | mockIndexRequests('/query-index.json', 555); 312 | 313 | const entries = await ffetch('/query-index.json').withFetch(fetch) 314 | .slice(100, 500) // entry 199 to 499 315 | .map(({ title }) => title) // map to title 316 | .filter((title) => title.indexOf('99') > 0) // filter now applied on title 317 | .map((title) => title.toUpperCase()) // map applied on title as well 318 | .slice(1, 2) // slice of ENTRY 199, ENTRY 299, ENTRY 399 319 | .all(); 320 | 321 | assert.equal(1, entries.length); 322 | assert.equal('ENTRY 299', entries[0]); 323 | }); 324 | }); 325 | --------------------------------------------------------------------------------