├── .github ├── ISSUE_TEMPLATE │ ├── ---bug-report.md │ └── --anything-else.md └── workflows │ ├── npm-publish.yml │ └── test.yml ├── .gitignore ├── LICENSE ├── README.md ├── __tests__ ├── content-types.mocks.js ├── default-usage.test.js ├── media-content-type.test.js └── option-switches.test.js ├── gatsby-node.js ├── index.js ├── jest.config.js ├── package-lock.json ├── package.json └── src ├── data-loader.js ├── utils.js └── workers.js /.github/ISSUE_TEMPLATE/---bug-report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "Bug report" 3 | about: Report reproducible software issues, so we can improve 4 | 5 | --- 6 | 7 | Welcome to the Flotiq Source Plugin for GatsbyJS GitHub repo! 8 | 9 | For questions related to the usage of Gatsby or GraphQL, please check out their docs at https://www.gatsbyjs.org/ and https://graphql.org/ 10 | 11 | For support, help, questions, requests and ideas use https://discord.gg/FwXcHnX 12 | 13 | If your issue is with Gatsby.js itself, please report it at the Gatsby repo https://github.com/gatsbyjs/gatsby/issues/new. 14 | 15 | 16 | ### Issue Summary 17 | 18 | A summary of the issue and the browser/OS environment in which it occurs. 19 | 20 | ### To Reproduce 21 | 22 | 1. This is the first step 23 | 2. This is the second step, etc. 24 | 25 | Any other info e.g. Why do you consider this to be a bug? What did you expect to happen instead? 26 | 27 | ### Technical details: 28 | 29 | * Gatsby Version: 30 | * Node Version: 31 | * OS: 32 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/--anything-else.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "Anything else" 3 | about: "For help, support, features & ideas - please use https://discord.gg/FwXcHnX" 4 | 5 | --- 6 | 7 | Click "Preview" for a nicer view! 8 | 9 | For support, help, questions, requests and ideas use https://discord.gg/FwXcHnX 10 | 11 | Alternatively, check out these resources below. Thanks! 12 | 13 | - [Discord](https://discord.gg/FwXcHnX) 14 | - [Content API Docs](https://flotiq.com/docs/) 15 | - [Flotiq blog](https://blog.flotiq.com/) 16 | - [Flotiq website](https://flotiq.com) 17 | -------------------------------------------------------------------------------- /.github/workflows/npm-publish.yml: -------------------------------------------------------------------------------- 1 | name: npm-publish 2 | on: 3 | push: 4 | branches: 5 | - master 6 | jobs: 7 | npm-publish: 8 | runs-on: ubuntu-latest 9 | permissions: 10 | contents: write 11 | steps: 12 | - uses: actions/checkout@v4 13 | - uses: actions/setup-node@v3 14 | with: 15 | node-version: "20" 16 | - run: npm ci 17 | - uses: JS-DevTools/npm-publish@v3 18 | with: 19 | token: ${{ secrets.NPM_AUTH_TOKEN }} 20 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v2 10 | - name: Install modules 11 | run: npm install 12 | - name: Run tests 13 | run: npm test -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 flotiq 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | Flotiq logo 3 | 4 | 5 | gatsby-source-flotiq 6 | ==================== 7 | 8 | ![](https://img.shields.io/npm/v/gatsby-source-flotiq) 9 | 10 | Source plugin for pulling data from [Flotiq](https://flotiq.com) into [Gatsby](https://www.gatsbyjs.org/) websites. 11 | 12 | Get up and running in minutes with a starter project: 13 | * [Simple blog with Gatsby](https://github.com/flotiq/gatsby-starter-blog) 14 | * [Projects portfolio](https://github.com/flotiq/gatsby-starter-projects) 15 | * [Events calendar](https://github.com/flotiq/gatsby-starter-event-calendar) 16 | * [Products showcase](https://github.com/flotiq/gatsby-starter-products) 17 | * [Products with categories showcase](https://github.com/flotiq/gatsby-starter-products-with-categories) 18 | * [Blog with Gatsby](https://github.com/flotiq/flotiq-blog) 19 | 20 | ## Table of contents 21 | 22 | - [Install](#install) 23 | - [Parameters](#parameters) 24 | - [Collaboration](#collaboration) 25 | 26 | 27 | ## Install 28 | 29 | Add Gatsby Source Flotiq plugin to your project: 30 | ```bash 31 | npm install --save gatsby-source-flotiq gatsby-plugin-image gatsby-plugin-sharp gatsby-transformer-sharp 32 | ``` 33 | 34 | Enable and configure plugin: 35 | ```js 36 | // in your gatsby-config.js in root of the project 37 | require('dotenv').config(); 38 | 39 | module.exports = { 40 | // ... 41 | plugins: [ 42 | { 43 | resolve: 'gatsby-source-flotiq', 44 | options: { 45 | baseUrl: process.env.GATSBY_FLOTIQ_BASE_URL, 46 | authToken: process.env.GATSBY_FLOTIQ_API_KEY, 47 | forceReload: false, //(optional) 48 | includeTypes: ['contettype1', 'contettype2', ... ], //(optional) List of used contenttypes identified by API Name. If ommitted, all content types will be synchronized. Make sure to include all referenced content types as well 49 | objectLimit: 100000, //optional, limit total number of objects downloaded for every type 50 | singleFetchLimit: 1000, //optional, limit the number of objects downloaded in single api call. Min: 1, Max 1000, Default 1000 51 | maxConcurrentDataDownloads: 10, //optional, limit the number of concurrent api calls. Default: 10, Min: 1, Max: 50 52 | timeout: 5000, //optional 53 | resolveMissingRelations: true, //optional, if the limit of objects is small some of the objects in relations could not be obtained from server, it this option is true they will be obtained as the graphQL queries in project would be resolved, if false, the missing object would resolve to null 54 | downloadMediaFile: false //optional, should media files be cached and be available for gatsby-image and gatsby-sharp 55 | }, 56 | }, 57 | 'gatsby-plugin-image', 58 | 'gatsby-plugin-sharp', 59 | 'gatsby-transformer-sharp' 60 | ], 61 | // ... 62 | } 63 | ``` 64 | 65 | ### Parameters 66 | 67 | * `baseUrl` - url to the Flotiq API (in most cases `https://api.flotiq.com`) 68 | * `authToken` - API token, if you wish to only pull data from Flotiq it can be Red-only key, if you need to put data it has to be Read-write key, more about Flotiq API keys [here](https://flotiq.com/docs/API/) 69 | * `forceRelaod` - indicates if the data should be pulled in full or plugin should use cache (`true` for full pull, `false` for cache usage) 70 | * `includeTypes` - array of Content Type Definitions used in the project (if you use images or files pulled from Flotiq, you must include `_media` CTD) 71 | * `objectsLimit` - if you wish to not pull all objects from Flotiq (e.g. in development to speed up reload), you can limit it using this parameter. This will limit the total number of downloaded objects. In production it should be higher than number of object in any Content Type pulled to project 72 | * `singleFetchLimit` - if you experience timeuts, or any other problems with download, you can change the default number of objects downloaded in a single API call. It has to be in integer from `1` to `1000`. The default value is `1000`. 73 | * `maxConcurrentDataDownloads` - If you have a large number of content types, or many objects in a single content type, you can change the default number of concurrent connections. It has to be in integer from `1` to `50`. The default value is `10`. 74 | * `timeout` - time (in milliseconds) after which connection to Flotiq should timed out 75 | * `resolveMissingRelations` - when the `objectsLimit` is smaller than number of objects in CTDs to avoid nulls on objects connected to other objects plugin make additional calls to pull missing data, if you want to suppress this behavior set this parameter to `false` 76 | * `downloadMediaFile` - should media files be downloaded and cached and be available fully for gatsby-image and gatsby-image-sharp 77 | 78 | please make sure to put your API credentials in your `.env` file: 79 | 80 | ``` 81 | GATSBY_FLOTIQ_BASE_URL="https://api.flotiq.com" 82 | GATSBY_FLOTIQ_API_KEY=XXXX-YOUR-API-KEY-XXXX 83 | ``` 84 | 85 | At this point you should have added Content Type Definitions required by your project/starter, more about adding Content Types ond Objects in [the Flotiq documentation](https://flotiq.com/docs/API/content-types/). 86 | 87 | ## Media 88 | 89 | If you are using default `downloadMediaFile` parameter (`false`), the fixed and fluid images are limited (no base46, automatic webp translation and tracedSVG). You can use them like that (assuming you have blogpost Content Type with headerImage media property): 90 | 91 | ``` 92 | query MyQuery { 93 | allBlogpost { 94 | nodes { 95 | headerImage { 96 | gatsbyImageData(height: 1000, width: 1000) 97 | extension 98 | url 99 | } 100 | } 101 | } 102 | } 103 | ``` 104 | 105 | ``` 106 | import { GatsbyImage, getImage } from "gatsby-plugin-image" 107 | //... 108 | const post = this.props.data.blogpost; 109 | const image = getImage(post.headerImage[0]) 110 | //... 111 | {post.headerImage[0].extension !== 'svg' ? 112 | () : 113 | (post image) 114 | } 115 | ``` 116 | You need a fallback for svg images because gatsby-plugin-image do not display them correctly. 117 | 118 | If you are using `downloadMediaFile` as `true`, you can use full potential of gatsby-plugin-image and gatsby-image-sharp. You can use them like that (assuming you have blogpost Content Type with headerImage media property): 119 | ``` 120 | query MyQuery { 121 | allBlogpost { 122 | nodes { 123 | headerImage { 124 | extension 125 | url 126 | localFile { 127 | childImageSharp { 128 | gatsbyImageData(width: 1000, height: 1000) 129 | } 130 | } 131 | } 132 | } 133 | } 134 | } 135 | ``` 136 | 137 | ``` 138 | import { GatsbyImage, getImage } from "gatsby-plugin-image" 139 | //... 140 | const post = this.props.data.blogpost; 141 | //... 142 | {post.headerImage[0].localFile.extension !== 'svg' ? 143 | () : 144 | (post image) 145 | } 146 | ``` 147 | 148 | You need a fallback for svg images because gatsby-plugin-image do not display them correctly. 149 | 150 | You can learn more about [Gatsby Image plugin here](https://www.gatsbyjs.com/docs/reference/built-in-components/gatsby-plugin-image/). 151 | 152 | ## Collaboration 153 | 154 | If you wish to talk with us about this project, feel free to hop on [![Discord Chat](https://img.shields.io/discord/682699728454025410.svg)](https://discord.gg/FwXcHnX). 155 | 156 | If you found a bug, please report it in [issues](https://github.com/flotiq/gatsby-source-flotiq/issues). 157 | 158 | ## NPM publish 159 | To publish a new package in NPM, you need to update the version in the packages.json and packages-lock.json files and then commit the changes with the message "Release x.y.z". Where x.y.z is the new version of the package. Commit about this on the master branch will start building a tag about this version and publishing a new version to npm. 160 | -------------------------------------------------------------------------------- /__tests__/content-types.mocks.js: -------------------------------------------------------------------------------- 1 | module.exports.CTD1 = { 2 | "id": "Type-1-id", 3 | "name": "Type-1-name", 4 | "label": "Type-1-label", 5 | "internal": false, 6 | "schemaDefinition": { 7 | "type": "object", 8 | "allOf": [ 9 | {"$ref": "#/components/schemas/AbstractContentTypeSchemaDefinition"}, 10 | { 11 | "type": "object", 12 | "properties": { 13 | "data": { 14 | "type": "string", 15 | "minLength": 1 16 | }, 17 | "name": { 18 | "type": "string", 19 | "minLength": 1 20 | } 21 | } 22 | } 23 | ], 24 | "required": [ 25 | "name", 26 | "data" 27 | ], 28 | "additionalProperties": false 29 | }, 30 | "metaDefinition": { 31 | "order": [ 32 | "name", 33 | "data" 34 | ], 35 | "propertiesConfig": { 36 | "data": { 37 | "label": "Data", 38 | "unique": true, 39 | "helpText": "", 40 | "inputType": "text" 41 | }, 42 | "name": { 43 | "label": "Name", 44 | "unique": false, 45 | "helpText": "", 46 | "inputType": "text" 47 | } 48 | } 49 | }, 50 | "deletedAt": null, 51 | "createdAt": "2020-02-20T09:25:54.000000+0000", 52 | "updatedAt": null 53 | } 54 | 55 | module.exports.CTD1_STR = JSON.stringify(module.exports.CTD1); 56 | 57 | module.exports.CTD1_OBJECT1_DATA = { 58 | name: "Object 1 name", 59 | data: "Object 1 data", 60 | } 61 | 62 | module.exports.CTD1_OBJECT1 = { 63 | id: "CTD1-Object-1", 64 | ...module.exports.CTD1_OBJECT1_DATA, 65 | internal: { 66 | "deletedAt": null, 67 | "createdAt": "2020-02-20T09:25:54.000000+0000", 68 | "updatedAt": null 69 | } 70 | } 71 | 72 | module.exports.CTD1_OBJECT1_STR = JSON.stringify(module.exports.CTD1_OBJECT1) 73 | 74 | 75 | module.exports.CTD1_OBJECT2_DATA = { 76 | name: "Object 2 name", 77 | data: "Object 2 data", 78 | } 79 | 80 | module.exports.CTD1_OBJECT2 = { 81 | id: "CTD1-Object-2", 82 | ...module.exports.CTD1_OBJECT2_DATA, 83 | internal: { 84 | "deletedAt": null, 85 | "createdAt": "2020-02-20T09:25:54.000000+0000", 86 | "updatedAt": null 87 | } 88 | } 89 | 90 | module.exports.CTD1_OBJECT2_STR = JSON.stringify(module.exports.CTD1_OBJECT2) -------------------------------------------------------------------------------- /__tests__/default-usage.test.js: -------------------------------------------------------------------------------- 1 | const { 2 | CTD1, 3 | CTD1_STR, 4 | CTD1_OBJECT1, 5 | CTD1_OBJECT1_DATA, 6 | CTD1_OBJECT1_STR, 7 | CTD1_OBJECT2_DATA, 8 | CTD1_OBJECT2 } = require('./content-types.mocks') 9 | const { when , verifyAllWhenMocksCalled, resetAllWhenMocks} = require('jest-when'); 10 | 11 | jest.mock('node-fetch'); 12 | const fetch = require('node-fetch'); 13 | const {Response} = jest.requireActual('node-fetch'); 14 | 15 | const {sourceNodes, onPluginInit} = require('../gatsby-node'); 16 | 17 | function createObjectWithMethods(functionNames) { 18 | return functionNames.reduce((acc, name) => { 19 | acc[name] = jest.fn().mockName(name) 20 | return acc; 21 | }, {}) 22 | } 23 | 24 | beforeEach(() => { 25 | resetAllWhenMocks() 26 | }) 27 | describe('onPluginInit', () => { 28 | test('Success Init plugin', async () => { 29 | const gatsbyFunctions = { 30 | actions: createObjectWithMethods(['createNode']), 31 | reporter: createObjectWithMethods(['panic', 'info', 'success']), 32 | schema: createObjectWithMethods(['buildObjectType']) 33 | }; 34 | const options = { 35 | baseUrl: "https://api.flotiq.com", 36 | authToken: 'qweasdzxcrtyfghvbnqweasdzxcrtyfg', 37 | contentTypeDefinitions: [], 38 | }; 39 | 40 | const expectedHeaders = expect.objectContaining({ 41 | headers: expect.objectContaining({ 42 | 'X-AUTH-TOKEN': options.authToken 43 | }) 44 | }) 45 | when(fetch) 46 | .expectCalledWith( 47 | expect.stringContaining(`${options.baseUrl}/api/v1/internal/contenttype`), 48 | expectedHeaders 49 | ) 50 | .mockReturnValueOnce(Promise.resolve(new Response(`{"data": [${CTD1_STR}]}`))) 51 | 52 | await onPluginInit(gatsbyFunctions, options); 53 | 54 | verifyAllWhenMocksCalled() 55 | }); 56 | 57 | test('Failed init plugin when no api key', async () => { 58 | const reporter = createObjectWithMethods(['panic', 'info']); 59 | const gatsbyFunctions = { 60 | actions: createObjectWithMethods(['createNode']), 61 | reporter: reporter, 62 | schema: createObjectWithMethods(['buildObjectType']) 63 | }; 64 | const options = { 65 | authToken: '', 66 | contentTypeDefinitions: [], 67 | }; 68 | 69 | await onPluginInit(gatsbyFunctions, options); 70 | 71 | expect(reporter.panic).toHaveBeenCalledTimes(1); 72 | }); 73 | }); 74 | 75 | describe('sourceNodes', () => { 76 | test('Downloads the data from scratch', async () => { 77 | const actions = createObjectWithMethods(['createNode','setPluginStatus','touchNode','deleteNode']); 78 | const gatsbyFunctions = { 79 | actions, 80 | store: {getState: jest.fn(_ => {return { status: {plugins: {}} }})}, 81 | getNodes: jest.fn().mockName('getNodes').mockReturnValue([]), 82 | reporter: createObjectWithMethods(['info','panic','warn']), 83 | schema: createObjectWithMethods(['buildObjectType']) 84 | }; 85 | const baseUrl = 'https://api.flotiq.com'; 86 | const options = { 87 | authToken: 'qweasdzxcrtyfghvbnqweasdzxcrtyfg', 88 | contentTypeDefinitions: [CTD1] 89 | }; 90 | 91 | const expectedHeaders = expect.objectContaining({ 92 | headers: expect.objectContaining({ 93 | 'X-AUTH-TOKEN': options.authToken 94 | }) 95 | }) 96 | 97 | when(fetch) 98 | .expectCalledWith( 99 | expect.stringContaining(`${baseUrl}/api/v1/content/Type-1-name?limit=1000&page=1`), 100 | expectedHeaders 101 | ) 102 | .mockReturnValueOnce(Promise.resolve(new Response(`{"data": [${CTD1_OBJECT1_STR}]}`))); 103 | 104 | await sourceNodes(gatsbyFunctions, options) 105 | 106 | verifyAllWhenMocksCalled() 107 | expect(actions.createNode).toHaveBeenCalledTimes(1); 108 | expect(actions.setPluginStatus).toHaveBeenCalledTimes(1); 109 | }); 110 | 111 | describe('When launched second time', () => { 112 | test('Removes outdated data', async () => { 113 | const actions = createObjectWithMethods(['createNode','setPluginStatus','touchNode','deleteNode']); 114 | const LAST_UPDATE = '2020-01-01T00:00:00Z'; 115 | const gatsbyFunctions = { 116 | actions, 117 | store: {getState: jest.fn(_ => {return { status: {plugins: { 118 | 'gatsby-source-flotiq': { 119 | updated_at: LAST_UPDATE 120 | } 121 | }} }})}, 122 | getNodes: jest.fn().mockName('getNodes').mockReturnValue([ 123 | {id: `${CTD1.name}_${CTD1_OBJECT1.id}`, ...CTD1_OBJECT1_DATA, internal: {owner: 'gatsby-source-flotiq'}} 124 | ]), 125 | getNodesByType: jest.fn().mockName('getNodesByType').mockReturnValue([ 126 | {id: `${CTD1.name}_${CTD1_OBJECT1.id}`, ...CTD1_OBJECT1_DATA, internal: {owner: 'gatsby-source-flotiq'}} 127 | ]), 128 | reporter: createObjectWithMethods(['info','panic','warn']), 129 | schema: createObjectWithMethods(['buildObjectType']) 130 | }; 131 | 132 | const baseUrl = 'https://api.flotiq.com'; 133 | const options = { 134 | authToken: 'qweasdzxcrtyfghvbnqweasdzxcrtyfg', 135 | contentTypeDefinitions: [CTD1], 136 | }; 137 | 138 | const expectedHeaders = expect.objectContaining({ 139 | headers: expect.objectContaining({ 140 | 'X-AUTH-TOKEN': options.authToken 141 | }) 142 | }) 143 | 144 | when(fetch) 145 | .calledWith(expect.stringMatching(`${baseUrl}/api/v1/content/${CTD1.name}.*updatedAt.*${encodeURIComponent(LAST_UPDATE)}`), expectedHeaders) 146 | .mockReturnValueOnce(Promise.resolve(new Response(`{"data": []}`))) 147 | 148 | when(fetch) 149 | .calledWith(expect.stringMatching(`${baseUrl}/api/v1/content/${CTD1.name}/removed\\?deletedAfter=${encodeURIComponent(LAST_UPDATE)}`), expectedHeaders) 150 | .mockReturnValueOnce(Promise.resolve(new Response(`["${CTD1_OBJECT1.id}"]`))) 151 | 152 | await sourceNodes(gatsbyFunctions, options) 153 | 154 | verifyAllWhenMocksCalled() 155 | expect(actions.deleteNode).toBeCalledWith(expect.objectContaining({id: expect.stringContaining(CTD1_OBJECT1.id)})) 156 | }); 157 | 158 | test('Updates only new data', async () => { 159 | const actions = createObjectWithMethods(['createNode','setPluginStatus','touchNode','deleteNode']); 160 | const LAST_UPDATE = '2020-01-01T00:00:00Z'; 161 | 162 | const gatsbyFunctions = { 163 | actions, 164 | store: {getState: jest.fn(_ => {return { status: {plugins: { 165 | 'gatsby-source-flotiq': { 166 | updated_at: LAST_UPDATE 167 | } 168 | }} }})}, 169 | getNodes: jest.fn().mockName('getNodes').mockReturnValue([ 170 | {id: `${CTD1.name}_${CTD1_OBJECT1.id}`, ...CTD1_OBJECT1_DATA, internal: {owner: 'gatsby-source-flotiq'}}, 171 | {id: `${CTD1.name}_${CTD1_OBJECT2.id}`, ...CTD1_OBJECT2_DATA, internal: {owner: 'gatsby-source-flotiq'}} 172 | ]), 173 | getNodesByType: jest.fn().mockName('getNodesByType').mockReturnValue([ 174 | {id: `${CTD1.name}_${CTD1_OBJECT1.id}`, ...CTD1_OBJECT1_DATA, internal: {owner: 'gatsby-source-flotiq'}}, 175 | {id: `${CTD1.name}_${CTD1_OBJECT2.id}`, ...CTD1_OBJECT2_DATA, internal: {owner: 'gatsby-source-flotiq'}} 176 | ]), 177 | reporter: createObjectWithMethods(['info','panic','warn']), 178 | schema: createObjectWithMethods(['buildObjectType']) 179 | }; 180 | const baseUrl = 'https://api.flotiq.com'; 181 | const options = { 182 | authToken: 'qweasdzxcrtyfghvbnqweasdzxcrtyfg', 183 | contentTypeDefinitions: [CTD1], 184 | }; 185 | 186 | const expectedHeaders = expect.objectContaining({ 187 | headers: expect.objectContaining({ 188 | 'X-AUTH-TOKEN': options.authToken 189 | }) 190 | }) 191 | 192 | when(fetch) 193 | .calledWith( 194 | expect.stringMatching(`${baseUrl}/api/v1/content/${CTD1.name}.*updatedAt.*${encodeURIComponent(LAST_UPDATE)}`), 195 | expectedHeaders 196 | ).mockReturnValueOnce(Promise.resolve(new Response(`{"data": [${CTD1_OBJECT1_STR}]}`))) 197 | 198 | when(fetch) 199 | .calledWith( 200 | expect.stringMatching(`${baseUrl}/api/v1/content/${CTD1.name}/removed\\?deletedAfter=${encodeURIComponent(LAST_UPDATE)}`), 201 | expectedHeaders 202 | ).mockReturnValueOnce(Promise.resolve(new Response(`[]`))) 203 | 204 | await sourceNodes(gatsbyFunctions, options) 205 | 206 | verifyAllWhenMocksCalled() 207 | expect(actions.touchNode).toBeCalledTimes(2) 208 | expect(actions.createNode).toHaveBeenCalledWith(expect.objectContaining(CTD1_OBJECT1_DATA)) 209 | }); 210 | }) 211 | }) 212 | 213 | -------------------------------------------------------------------------------- /__tests__/media-content-type.test.js: -------------------------------------------------------------------------------- 1 | 2 | describe('Media content type:', () => { 3 | test.todo('Creates media type') 4 | 5 | test.todo('Downloads media as remote file') 6 | test.todo('Generates media srcSet when using remote medias') 7 | 8 | test.todo('Does not download more than objectLimit') 9 | }) -------------------------------------------------------------------------------- /__tests__/option-switches.test.js: -------------------------------------------------------------------------------- 1 | 2 | describe('forceReload:', () => { 3 | test.todo('Reloads data when true') 4 | }) 5 | 6 | describe('includeTypes:', () => { 7 | test.todo('Downloads only requested content types') 8 | }) 9 | -------------------------------------------------------------------------------- /gatsby-node.js: -------------------------------------------------------------------------------- 1 | const fetch = require('node-fetch'); 2 | const {createContentDigest} = require(`gatsby-core-utils`); 3 | const {getGatsbyImageResolver} = require("gatsby-plugin-image/graphql-utils"); 4 | const {generateImageData, getLowResolutionImageURL} = require("gatsby-plugin-image"); 5 | const {getContentTypes, getDeletedObjects, getContentObjects} = require('./src/data-loader'); 6 | const {capitalize, createHeaders} = require('./src/utils') 7 | const CTD_MEDIA = '_media'; 8 | const digest = str => createContentDigest(str); 9 | 10 | let apiUrl; 11 | 12 | let typeDefinitionsDeferred; 13 | let typeDefinitionsPromise = new Promise((resolve, reject) => { 14 | typeDefinitionsDeferred = {resolve: resolve, reject: reject}; 15 | }); 16 | 17 | let createNodeGlobal; 18 | let resolveMissingRelationsGlobal; 19 | let downloadMediaFileGlobal = false; 20 | let headers = {}; 21 | let globalSchema = {}; 22 | let contentTypeDefsData = []; 23 | 24 | exports.onPluginInit = async ({actions, schema, reporter}, options) => { 25 | const {createNode} = actions; 26 | const { 27 | baseUrl = "https://api.flotiq.com", 28 | authToken, 29 | includeTypes = null, 30 | resolveMissingRelations = true, 31 | downloadMediaFile = false 32 | } = options; 33 | headers = createHeaders(options); 34 | 35 | createNodeGlobal = createNode; 36 | resolveMissingRelationsGlobal = resolveMissingRelations; 37 | downloadMediaFileGlobal = downloadMediaFile; 38 | apiUrl = baseUrl; 39 | globalSchema = schema; 40 | if (authToken) { 41 | contentTypeDefsData = await getContentTypes(reporter, options, apiUrl); 42 | } 43 | 44 | if (!apiUrl) { 45 | reporter.panic('FLOTIQ: You must specify API url ' + 46 | '(in most cases it is "https://api.flotiq.com")'); 47 | } 48 | if (!authToken) { 49 | reporter.panic("FLOTIQ: You must specify API token " + 50 | "(if you don't know what it is check: https://flotiq.com/docs/API/)"); 51 | } 52 | 53 | if (includeTypes && (!Array.isArray(includeTypes) || typeof includeTypes[0] !== "string")) { 54 | reporter.panic("FLOTIQ: `includeTypes` should be an array of content type api names. It cannot be empty."); 55 | } 56 | } 57 | 58 | exports.sourceNodes = async (gatsbyFunctions, options) => { 59 | 60 | const {actions, store, getNodes, reporter, schema} = gatsbyFunctions; 61 | const {createNode, setPluginStatus, touchNode, deleteNode} = actions; 62 | const {forceReload} = options; 63 | 64 | try { 65 | if (forceReload) { 66 | setPluginStatus({'updated_at': null}); 67 | } 68 | let lastUpdate = store.getState().status.plugins['gatsby-source-flotiq']; 69 | 70 | const existingNodes = getNodes().filter( 71 | n => n.internal.owner === `gatsby-source-flotiq` 72 | ); 73 | existingNodes.forEach(n => touchNode(n)); 74 | if (!existingNodes.length) { 75 | lastUpdate = undefined; 76 | } 77 | 78 | let changed = 0; 79 | let removed = 0; 80 | 81 | if (lastUpdate && lastUpdate.updated_at) { 82 | removed = await getDeletedObjects(gatsbyFunctions, options, lastUpdate.updated_at, contentTypeDefsData, apiUrl, async (ctd, id) => { 83 | let node = existingNodes.find(n => n.id === `${ctd.name}_${id}`); 84 | return await deleteNode(node); 85 | }); 86 | } 87 | 88 | changed = await getContentObjects(gatsbyFunctions, options, lastUpdate && lastUpdate.updated_at, contentTypeDefsData, apiUrl, async (ctd, datum) => { 89 | return createNode({ 90 | ...datum, 91 | // custom 92 | flotiqInternal: datum.internal, 93 | // required 94 | id: ctd.name === CTD_MEDIA ? datum.id : `${ctd.name}_${datum.id}`, 95 | parent: null, 96 | children: [], 97 | internal: { 98 | type: capitalize(ctd.name), 99 | contentDigest: digest(JSON.stringify(datum)), 100 | }, 101 | }); 102 | }) 103 | 104 | if (changed) { 105 | reporter.info(`Updated entries ${changed}`); 106 | } 107 | if (removed) { 108 | reporter.info(`Removed entries ${removed}`); 109 | } 110 | setPluginStatus({ 111 | 'updated_at': 112 | (new Date()).toISOString().replace(/T/, ' ').replace(/\..+/, '') 113 | }); 114 | } catch (e) { 115 | reporter.panic(`FLOTIQ: ${e.message}`) 116 | } 117 | 118 | return {}; 119 | }; 120 | 121 | exports.createSchemaCustomization = ({actions}, options) => { 122 | const {createTypes} = actions; 123 | createTypeDefs(contentTypeDefsData, globalSchema, options.includeTypes); 124 | 125 | typeDefinitionsPromise.then(typeDefs => { 126 | typeDefs.push(`type FlotiqInternal { 127 | createdAt: String! 128 | deletedAt: String! 129 | updatedAt: String! 130 | contentType: String! 131 | } 132 | type FlotiqGeo { 133 | lat: Float 134 | lon: Float 135 | } 136 | type FlotiqBlock { 137 | time: String 138 | version: String 139 | blocks: [FlotiqBlock2] 140 | } 141 | type FlotiqBlock2 { 142 | id: String 143 | type: String 144 | data: FlotiqBlockData 145 | tunes: FlotiqBlockTunes 146 | } 147 | type FlotiqBlockData { 148 | text: String 149 | level: Float 150 | anchor: String 151 | items: [FlotiqBlockItems] 152 | style: String 153 | url: String 154 | width: String 155 | height: String 156 | fileName: String 157 | extension: String 158 | caption: String 159 | stretched: String 160 | withBorder: String 161 | withBackground: String 162 | message: String 163 | title: String 164 | alignment: String 165 | code: String 166 | withHeadings: Boolean 167 | content: [[String]] 168 | } 169 | type FlotiqBlockTunes { 170 | alignmentTuneTool: FlotiqBlockAlignementTune 171 | } 172 | type FlotiqBlockAlignementTune { 173 | alignment: String 174 | } 175 | type FlotiqBlockItems { 176 | items: [FlotiqBlockItems] 177 | content: String 178 | } 179 | type FlotiqImageFixed implements Node { 180 | aspectRatio: Float 181 | width: Float 182 | height: Float 183 | src: String 184 | srcSet: String 185 | originalName: String 186 | } 187 | type FlotiqImageFluid implements Node { 188 | aspectRatio: Float 189 | src: String 190 | srcSet: String 191 | originalName: String 192 | sizes: String 193 | } 194 | type DataSource { 195 | dataUrl: String! 196 | type: String! 197 | }` 198 | ); 199 | createTypes(typeDefs); 200 | }) 201 | 202 | }; 203 | 204 | exports.createResolvers = ({ 205 | actions, 206 | cache, 207 | createNodeId, 208 | createResolvers, 209 | store, 210 | reporter, 211 | }) => { 212 | if (downloadMediaFileGlobal) { 213 | const {createRemoteFileNode} = require(`gatsby-source-filesystem`) 214 | const {createNode} = actions 215 | createResolvers({ 216 | _media: { 217 | localFile: { 218 | type: `File`, 219 | resolve(source, args, context, info) { 220 | return createRemoteFileNode({ 221 | url: apiUrl + source.url, 222 | store, 223 | cache, 224 | createNode, 225 | createNodeId, 226 | reporter, 227 | ext: `.${source.extension}` 228 | }); 229 | } 230 | } 231 | } 232 | }) 233 | } else { 234 | createResolvers({ 235 | _media: { 236 | gatsbyImageData: getGatsbyImageResolver(resolveGatsbyImageData), 237 | }, 238 | }) 239 | } 240 | } 241 | 242 | 243 | const createAditionalDef = ( 244 | name, 245 | property, 246 | properties, 247 | includeTypes, 248 | schema, 249 | processedDefs=[], 250 | ) => { 251 | const defName = capitalize(property) + capitalize(name); 252 | let typeDefs=[]; 253 | 254 | let additionalDef = { 255 | name: defName, 256 | fields: {}, 257 | interfaces: ["Node"], 258 | }; 259 | Object.keys(properties[property].items.propertiesConfig).forEach(prop => { 260 | let propConfig = properties[property].items.propertiesConfig[prop]; 261 | 262 | if (propConfig.inputType === 'object' && !processedDefs.includes(defName)) { 263 | typeDefs.push(...createAditionalDef( 264 | name, 265 | prop, 266 | properties[property].items.propertiesConfig, 267 | includeTypes, 268 | schema, 269 | processedDefs 270 | )); 271 | } 272 | additionalDef.fields[prop] = getType( 273 | propConfig, 274 | false, 275 | prop, 276 | capitalize(name), 277 | includeTypes 278 | ); 279 | }); 280 | additionalDef.fields.flotiqInternal = `FlotiqInternal!`; 281 | 282 | if(!processedDefs.includes(defName)){ 283 | typeDefs.push(schema.buildObjectType(additionalDef)); 284 | processedDefs.push(defName) 285 | } 286 | 287 | return typeDefs; 288 | } 289 | 290 | const createTypeDefs = (contentTypesDefinitions, schema, includeTypes) => { 291 | let typeDefs = []; 292 | const names = contentTypesDefinitions.map(ctd => capitalize(ctd.name)); 293 | typeDefs.push(`union AllTypes = ${names.join(' | ')}`); 294 | contentTypesDefinitions.forEach(ctd => { 295 | let tmpDef = { 296 | name: capitalize(ctd.name), 297 | fields: {}, 298 | interfaces: ["Node"], 299 | }; 300 | Object.keys(ctd.schemaDefinition.allOf[1].properties).forEach(property => { 301 | if(['if', 'else', 'then'].includes(property) || !ctd.metaDefinition.propertiesConfig[property]) { 302 | return; 303 | } 304 | tmpDef.fields[property] = getType( 305 | ctd.metaDefinition.propertiesConfig[property], 306 | ctd.schemaDefinition.required.indexOf(property) > -1, 307 | property, 308 | capitalize(ctd.name), 309 | includeTypes 310 | ); 311 | if (ctd.metaDefinition.propertiesConfig[property].inputType === 'object') { 312 | let additionalDef = createAditionalDef( 313 | ctd.name, 314 | property, 315 | ctd.metaDefinition.propertiesConfig, 316 | includeTypes, 317 | schema 318 | ); 319 | 320 | typeDefs.push(...additionalDef); 321 | } 322 | }); 323 | 324 | tmpDef.fields.flotiqInternal = `FlotiqInternal!`; 325 | typeDefs.push(schema.buildObjectType(tmpDef)); 326 | }); 327 | typeDefinitionsDeferred.resolve(typeDefs); 328 | }; 329 | 330 | 331 | const getType = (propertyConfig, required, property, ctdName, includeTypes) => { 332 | 333 | switch (propertyConfig.inputType) { 334 | case 'text': 335 | case 'textarea': 336 | case 'richtext': 337 | case 'textMarkdown': 338 | case 'email': 339 | case 'radio': 340 | default: 341 | return 'String' + (required ? '!' : ''); 342 | case 'select': 343 | return propertyConfig.multiple ? ['String' + (required ? '!' : '')] : 'String' + (required ? '!' : ''); 344 | case 'number': 345 | return 'Float' + (required ? '!' : ''); 346 | case 'checkbox': 347 | return 'Boolean' + (required ? '!' : ''); 348 | case 'geo': 349 | return 'FlotiqGeo' + (required ? '!' : ''); 350 | case 'simpleList': 351 | return ['String' + (required ? '!' : '')]; 352 | case 'datasource': 353 | if ( 354 | includeTypes 355 | && propertyConfig.validation.relationContenttype !== '' 356 | && includeTypes.indexOf(propertyConfig.validation.relationContenttype) === -1 357 | && propertyConfig.validation.relationContenttype !== CTD_MEDIA 358 | ) { 359 | return 'DataSource'; 360 | } 361 | let type = 362 | propertyConfig.validation.relationContenttype 363 | ? (propertyConfig.validation.relationContenttype !== CTD_MEDIA 364 | ? capitalize(propertyConfig.validation.relationContenttype) 365 | : CTD_MEDIA) 366 | : 'AllTypes'; 367 | return { 368 | type: `[${type}]`, 369 | resolve: async (source, args, context, info) => { 370 | if (source[property]) { 371 | let nodes = await Promise.all(source[property].map(async (prop) => { 372 | if (typeof (prop.dataUrl) === 'undefined') { 373 | return; 374 | } 375 | let node = { 376 | id: prop.dataUrl.split('/')[4] === CTD_MEDIA 377 | ? prop.dataUrl.split('/')[5] 378 | : prop.dataUrl.split('/')[4] + '_' + prop.dataUrl.split('/')[5], 379 | type: type, 380 | }; 381 | let nodeModel = context.nodeModel.getNodeById(node); 382 | if (nodeModel === null && resolveMissingRelationsGlobal) { 383 | let url = apiUrl + prop.dataUrl; 384 | let response = await fetch(url, {headers: headers}); 385 | if (response.ok) { 386 | const json = await response.json(); 387 | await createNodeGlobal({ 388 | ...json, 389 | // custom 390 | flotiqInternal: json.internal, 391 | // required 392 | id: prop.dataUrl.split('/')[4] === CTD_MEDIA 393 | ? json.id 394 | : `${prop.dataUrl.split('/')[4]}_${json.id}`, 395 | parent: null, 396 | children: [], 397 | internal: { 398 | type: capitalize(prop.dataUrl.split('/')[4]), 399 | contentDigest: digest(JSON.stringify(json)), 400 | }, 401 | }); 402 | nodeModel = context.nodeModel.getNodeById(node); 403 | return nodeModel; 404 | } else { 405 | return nodeModel; 406 | } 407 | } else { 408 | return nodeModel 409 | } 410 | })); 411 | if (!nodes[0]) { 412 | return []; 413 | } 414 | return nodes; 415 | } 416 | return null; 417 | } 418 | }; 419 | case 'object': 420 | return `[${capitalize(property)}${ctdName}]`; 421 | case 'block': 422 | return 'FlotiqBlock' 423 | } 424 | }; 425 | 426 | const generateImageSource = (baseURL, width = 0, height = 0, format, fit, options) => { 427 | const src = `https://api.flotiq.com/image/${width || options.width}x${height || options.height}/${baseURL}.${format}` 428 | return {src, width, height, format} 429 | } 430 | 431 | 432 | const resolveGatsbyImageData = async (image, options) => { 433 | const filename = image.id 434 | const sourceMetadata = { 435 | width: image.width, 436 | height: image.height, 437 | format: image.extension 438 | } 439 | const imageDataArgs = { 440 | ...options, 441 | pluginName: `gatsby-source-flotiq`, 442 | sourceMetadata, 443 | filename, 444 | placeholderURL: '', 445 | generateImageSource, 446 | formats: [image.extension], 447 | options: {...options, extension: image.extension}, 448 | } 449 | // if(options.placeholder === "blurred") { 450 | // const lowResImage = getLowResolutionImageURL(imageDataArgs) 451 | // imageDataArgs.placeholderURL = await getBase64Image(lowResImage) 452 | // } 453 | return generateImageData(imageDataArgs) 454 | } 455 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flotiq/gatsby-source-flotiq/22ca18435c0a027fa05d29183476aaf59eced16d/index.js -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = async () => { 2 | return { 3 | verbose: true, 4 | testRegex: "__tests__/.*\\.(spec|test)\\.[jt]s?x?", 5 | rootDir: __dirname, 6 | clearMocks: true 7 | }; 8 | }; -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "gatsby-source-flotiq", 3 | "version": "5.4.0", 4 | "license": "MIT", 5 | "author": "Flotiq team ", 6 | "repository": { 7 | "type": "git", 8 | "url": "git+https://github.com/flotiq/gatsby-source-flotiq.git" 9 | }, 10 | "keywords": [ 11 | "flotiq", 12 | "gatsby", 13 | "gatsby-plugin" 14 | ], 15 | "dependencies": { 16 | "gatsby-core-utils": "^4.14.0", 17 | "gatsby-source-filesystem": "^5.14.0", 18 | "node-fetch": "^2.7.0" 19 | }, 20 | "devDependencies": { 21 | "gatsby": "^5.14.1", 22 | "gatsby-plugin-image": "^3.14.0", 23 | "gatsby-plugin-sharp": "^5.14.0", 24 | "jest": "^27.4.5", 25 | "jest-when": "^3.7.0", 26 | "react": "^18.3.1", 27 | "react-dom": "^18.3.1" 28 | }, 29 | "peerDependencies": { 30 | "gatsby": "^5.14.1", 31 | "gatsby-plugin-image": "^3.14.0", 32 | "gatsby-plugin-sharp": "^5.14.0", 33 | "react": "^18.3.1", 34 | "react-dom": "^18.3.1" 35 | }, 36 | "scripts": { 37 | "test": "jest" 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/data-loader.js: -------------------------------------------------------------------------------- 1 | const fetch = require('node-fetch'); 2 | const workers = require('./workers') 3 | const {capitalize, createHeaders} = require('./utils') 4 | 5 | module.exports.getContentTypes = async function (reporter, options, apiUrl) { 6 | const { 7 | timeout = 5000, 8 | includeTypes = null, 9 | } = options; 10 | 11 | reporter.info('Connecting to Flotiq backend to fetch Conent Type Definitions...'); 12 | 13 | let contentTypeDefinitionsResponse = await fetch( 14 | `${apiUrl}/api/v1/internal/contenttype?limit=10000&order_by=label`, 15 | { 16 | headers: createHeaders(options), 17 | timeout: timeout 18 | }); 19 | 20 | if (contentTypeDefinitionsResponse.ok) { 21 | reporter.success('Conent Type Definitions fetched'); 22 | 23 | const disallowedTypes = ['_page', '_layout', '_navigation', '_site']; 24 | let contentTypeDefinitions = await contentTypeDefinitionsResponse.json(); 25 | return contentTypeDefinitions.data.filter( 26 | contentTypeDef => disallowedTypes.indexOf(contentTypeDef.name) === -1 && (!includeTypes || includeTypes.indexOf(contentTypeDef.name) > -1)) 27 | } else { 28 | if (contentTypeDefinitionsResponse.status === 404) { 29 | throw new Error(`We couldn't connect to API. Check if you specified correct API url (in most cases it is "https://api.flotiq.com")`) 30 | } else if (contentTypeDefinitionsResponse.status === 403) { 31 | throw new Error(`We couldn't authorize you in API. Check if you specified correct API token (if you don't know what it is check: https://flotiq.com/docs/API/)`) 32 | } else throw new Error(await contentTypeDefinitionsResponse.text()) 33 | 34 | } 35 | } 36 | 37 | module.exports.getDeletedObjects = async function (gatsbyFunctions, options, since, contentTypes, apiUrl, handleDeletedId) { 38 | let removed = 0; 39 | const { reporter } = gatsbyFunctions; 40 | await Promise.all(contentTypes.map(async ctd => { 41 | 42 | let url = `${apiUrl}/api/v1/content/${ctd.name}/removed?deletedAfter=${encodeURIComponent(since)}`; 43 | let response = await fetch(url, {headers: createHeaders(options)}); 44 | reporter.info(`Fetching removed content type ${ctd.name}: ${url}`); 45 | if (response.ok) { 46 | const jsonRemoved = await response.json(); 47 | await Promise.all(jsonRemoved.map(async id => { 48 | removed++; 49 | return await handleDeletedId(ctd, id) 50 | })); 51 | } 52 | 53 | })); 54 | 55 | return removed; 56 | } 57 | 58 | module.exports.getContentObjects = async function (gatsbyFunctions, options, since, contentTypes, apiUrl, handleObject) { 59 | const { reporter, getNodesByType } = gatsbyFunctions; 60 | 61 | const { 62 | objectLimit = 100000, 63 | timeout = 5000 64 | } = options; 65 | 66 | let { 67 | singleFetchLimit = 1000, 68 | maxConcurrentDataDownloads = 10 69 | } = options; 70 | 71 | maxConcurrentDataDownloads = Math.max(Math.min(maxConcurrentDataDownloads, 50), 1) // 1 <= maxConcurrentDataDownloads <= 50 72 | singleFetchLimit = Math.max(Math.min(singleFetchLimit, 1000), 1); // 1 <= singleFetchLimit <= 1000 73 | 74 | let changed = 0; 75 | let downloadJobs = contentTypes.map(ctd => { 76 | let currentNodeCount = 0; 77 | let limitPerPage = Math.min(singleFetchLimit, objectLimit); 78 | let url = `${apiUrl}/api/v1/content/${ctd.name }?limit=${limitPerPage}`; 79 | 80 | if (since) { 81 | currentNodeCount = getNodesByType(capitalize(ctd.name)).count; 82 | url += '&filters=' + encodeURIComponent(JSON.stringify({ 83 | "internal.updatedAt": { 84 | "type": "greaterThan", 85 | "filter": since 86 | } 87 | })) 88 | } 89 | 90 | return { 91 | apiUrl: url, 92 | objectLimit: objectLimit - currentNodeCount, 93 | limitPerPage, 94 | page: 1, 95 | ctd 96 | } 97 | }); 98 | 99 | const dataLoadCount = {} 100 | 101 | await workers(downloadJobs, maxConcurrentDataDownloads, async ({apiUrl, objectLimit, page, ctd, totalPages, limitPerPage}) => { 102 | const url = `${apiUrl}&page=${page}`; 103 | const humanizedPageNumber = page === 1 ? 'first page' : `${page}/${totalPages}` 104 | 105 | reporter.info(`Fetching${since ? ' updates' : ''}: ${ctd.name} ${humanizedPageNumber}`) 106 | let response = await fetch(url, {headers: createHeaders(options), timeout: timeout}); 107 | if (!response.ok) 108 | { 109 | reporter.warn('Error fetching data', response); 110 | return; 111 | } 112 | 113 | const json = await response.json(); 114 | totalPages = json.total_pages; 115 | dataLoadCount[ctd.name] = dataLoadCount[ctd.name] || 0; 116 | await Promise.all(json.data.map(async datum => { 117 | changed++; 118 | dataLoadCount[ctd.name]++; 119 | return await handleObject(ctd, datum) 120 | })); 121 | 122 | // Now that we know the dataset size, we can try to queue the rest of the download 123 | if(page === 1 && json.total_pages > page) { 124 | let maxAllowedPages = Math.ceil(objectLimit/limitPerPage); 125 | let pageLimit = Math.min(json.total_pages, maxAllowedPages); 126 | for(let i = page + 1; i <= pageLimit; i++) { 127 | downloadJobs.push({ 128 | apiUrl, objectLimit, page: i, ctd, totalPages: pageLimit 129 | }) 130 | } 131 | } 132 | }) 133 | 134 | return changed; 135 | } 136 | -------------------------------------------------------------------------------- /src/utils.js: -------------------------------------------------------------------------------- 1 | module.exports.capitalize = (s) => { 2 | if (typeof s !== 'string') return ''; 3 | return s.charAt(0).toUpperCase() + s.slice(1); 4 | }; 5 | 6 | module.exports.createHeaders = (options) => { 7 | const { 8 | authToken 9 | } = options; 10 | 11 | return { 12 | 'accept': 'application/json', 13 | 'X-AUTH-TOKEN': authToken 14 | }; 15 | }; 16 | -------------------------------------------------------------------------------- /src/workers.js: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @param {Array} input 4 | * @param {int} workerLimit 5 | * @param {function} taskFN 6 | */ 7 | module.exports = async function workers(input, workerLimit = 10, taskFN) { 8 | let workerIndices = [... (new Array(workerLimit)).keys()]; // an array of 0,1,..,workerLimit - 1 9 | let results = []; 10 | let currentDataIdx = 0; 11 | await Promise.all(workerIndices.map(async function(n){ 12 | while (input.length) { 13 | let localDataIdx = currentDataIdx++; 14 | results[localDataIdx] = await taskFN(input.shift()); 15 | } 16 | })); 17 | return results; 18 | } 19 | --------------------------------------------------------------------------------