├── .gitignore ├── Readme.md ├── index.js ├── manifest.yml ├── package-lock.json └── package.json /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .netlify 3 | *.tgz 4 | -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | # Netlify Plugin - Fetch Feeds 2 | 3 | This [plugin](https://www.netlify.com/build/plugins-beta?utm_source=github&utm_medium=plugin-fetchfeeds-pnh&utm_campaign=devex) adds the ability to source content from remote feeds including RSS and JSON, and cache them between builds. 4 | 5 | ## Overview 6 | 7 | This plugin requests data from the RSS and JSON resources that you specify. It will save this data as JSON in the Netlify build cache and only re-request each feed after a specified time-to-live value has elapsed. Requests are skipped harmlessly if data for a feed was previously cached, adding greater resilience to builds which depend on remote data. 8 | 9 | Configure this plugin to present the gathered data in the appropriate location, so your chosen [static site generator](https://www.netlify.com/blog/2020/04/14/what-is-a-static-site-generator-and-3-ways-to-find-the-best-one/?utm_source=github&utm_medium=whatisanssg-pnh&utm_campaign=devex) can leverage it during the build. 10 | 11 | 12 | ## Demonstration 13 | 14 | See this plugin being used in this simplified demo site: https://demo-plugin-fetch-feeds.netlify.app/ 15 | 16 | 17 | ## Installation 18 | 19 | To include this plugin in your site deployment: 20 | 21 | 22 | ### 1. Add the plugin as a dependency 23 | 24 | ```bash 25 | 26 | # Add the plugin as a dependency of your build 27 | npm i --s netlify-plugin-fetch-feeds 28 | 29 | ``` 30 | 31 | 32 | ### 2. Add the plugin and its options to your netlify.toml 33 | 34 | This plugin will fetch the specified feeds and stash their data prior to the execution of the `build` command you have specified in your Netlify configuration. The desired feeds can be specified in the `netlify.toml` config file. 35 | 36 | 37 | ```toml 38 | # Config for the Netlify Build Plugin: netlify-plugin-fetch-feeds 39 | [[plugins]] 40 | package = "netlify-plugin-fetch-feeds" 41 | 42 | [plugins.inputs] 43 | # Where should data files reside 44 | dataDir = "site/_data" 45 | 46 | # All the feeds we wish to gather for use in the build 47 | 48 | [[plugins.inputs.feeds]] 49 | name = "hawksworx" 50 | url = "https://hawksworx.com/feed.json" 51 | ttl = 3600 52 | [[plugins.inputs.feeds]] 53 | name = "netlify" 54 | url = "https://www.netlify.com/blog/index.xml" 55 | ttl = 86400 56 | ``` 57 | 58 | 59 | 60 | ## Quick try-out 61 | 62 | You can try out this plugin by deploying [a simple site](https://demo-plugin-fetch-feeds.netlify.app/) which uses it. 63 | 64 | Clicking the button below will clone [a test site repo](https://github.com/philhawksworth/demo-netlify-plugin-fetch-feeds), setup a new site [on Netlify](https://netlify.com?utm_source=github&utm_medium=plugin-fetchfeeds-pnh&utm_campaign=devex) and deploy the site complete with the plugin configured and operational. 65 | 66 | [![Deploy to Netlify](https://www.netlify.com/img/deploy/button.svg)](https://app.netlify.com/start/deploy?repository=https://github.com/philhawksworth/demo-netlify-plugin-fetch-feeds) 67 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const fetch = require('node-fetch'); 3 | const parser = require('xml2json'); 4 | const chalk = require('chalk'); 5 | 6 | 7 | module.exports = { 8 | 9 | async onPreBuild({ inputs, utils }) { 10 | 11 | // Gather the data from all the specified feeds 12 | for (const feed of inputs.feeds) { 13 | 14 | // Where fetched data should reside in the buid 15 | let dataFilePath = `${inputs.dataDir}/${feed.name}.json`; 16 | 17 | // reinstate from cache if it is present 18 | if ( await utils.cache.has(dataFilePath) ) { 19 | await utils.cache.restore(dataFilePath); 20 | console.log('Restored from cache:', chalk.green(feed.url)); 21 | } 22 | // Or if it's not cached, let's fetch it and cache it. 23 | else { 24 | var data = await fetch(feed.url) 25 | .then(async function(res) { 26 | 27 | // Stash all data as JSON. 28 | let contentType = res.headers.get('content-type').toLowerCase(); 29 | if(contentType == 'application/json') { 30 | return res.json(); 31 | } else { 32 | let text = await res.text(); 33 | let json = parser.toJson(text); 34 | return JSON.parse(json); 35 | } 36 | }); 37 | 38 | // put the fetched data in the daa file, and then cahce it. 39 | // await saveFeed(JSON.stringify(data), dataFilePath); 40 | await fs.writeFileSync(dataFilePath, JSON.stringify(data)); 41 | await utils.cache.save(dataFilePath, { ttl: feed.ttl }); 42 | console.log('Fetched and cached: ', chalk.yellow(feed.url), chalk.gray(`(TTL:${feed.ttl} seconds)`)); 43 | 44 | } 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /manifest.yml: -------------------------------------------------------------------------------- 1 | name: netlify-plugin-fetch-feeds 2 | inputs: 3 | - name: dataDir 4 | required: true 5 | - name: feeds 6 | required: true 7 | -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "netlify-plugin-fetch-feeds", 3 | "version": "0.2.1", 4 | "lockfileVersion": 1, 5 | "requires": true, 6 | "dependencies": { 7 | "@types/color-name": { 8 | "version": "1.1.1", 9 | "resolved": "https://registry.npmjs.org/@types/color-name/-/color-name-1.1.1.tgz", 10 | "integrity": "sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ==" 11 | }, 12 | "ansi-styles": { 13 | "version": "4.2.1", 14 | "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz", 15 | "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==", 16 | "requires": { 17 | "@types/color-name": "^1.1.1", 18 | "color-convert": "^2.0.1" 19 | } 20 | }, 21 | "bindings": { 22 | "version": "1.5.0", 23 | "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", 24 | "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", 25 | "requires": { 26 | "file-uri-to-path": "1.0.0" 27 | } 28 | }, 29 | "chalk": { 30 | "version": "4.0.0", 31 | "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.0.0.tgz", 32 | "integrity": "sha512-N9oWFcegS0sFr9oh1oz2d7Npos6vNoWW9HvtCg5N1KRFpUhaAhvTv5Y58g880fZaEYSNm3qDz8SU1UrGvp+n7A==", 33 | "requires": { 34 | "ansi-styles": "^4.1.0", 35 | "supports-color": "^7.1.0" 36 | } 37 | }, 38 | "color-convert": { 39 | "version": "2.0.1", 40 | "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", 41 | "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", 42 | "requires": { 43 | "color-name": "~1.1.4" 44 | } 45 | }, 46 | "color-name": { 47 | "version": "1.1.4", 48 | "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", 49 | "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" 50 | }, 51 | "file-uri-to-path": { 52 | "version": "1.0.0", 53 | "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", 54 | "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==" 55 | }, 56 | "has-flag": { 57 | "version": "4.0.0", 58 | "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", 59 | "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==" 60 | }, 61 | "hoek": { 62 | "version": "4.2.1", 63 | "resolved": "https://registry.npmjs.org/hoek/-/hoek-4.2.1.tgz", 64 | "integrity": "sha512-QLg82fGkfnJ/4iy1xZ81/9SIJiq1NGFUMGs6ParyjBZr6jW2Ufj/snDqTHixNlHdPNwN2RLVD0Pi3igeK9+JfA==" 65 | }, 66 | "isemail": { 67 | "version": "3.2.0", 68 | "resolved": "https://registry.npmjs.org/isemail/-/isemail-3.2.0.tgz", 69 | "integrity": "sha512-zKqkK+O+dGqevc93KNsbZ/TqTUFd46MwWjYOoMrjIMZ51eU7DtQG3Wmd9SQQT7i7RVnuTPEiYEWHU3MSbxC1Tg==", 70 | "requires": { 71 | "punycode": "2.x.x" 72 | } 73 | }, 74 | "joi": { 75 | "version": "13.7.0", 76 | "resolved": "https://registry.npmjs.org/joi/-/joi-13.7.0.tgz", 77 | "integrity": "sha512-xuY5VkHfeOYK3Hdi91ulocfuFopwgbSORmIwzcwHKESQhC7w1kD5jaVSPnqDxS2I8t3RZ9omCKAxNwXN5zG1/Q==", 78 | "requires": { 79 | "hoek": "5.x.x", 80 | "isemail": "3.x.x", 81 | "topo": "3.x.x" 82 | }, 83 | "dependencies": { 84 | "hoek": { 85 | "version": "5.0.4", 86 | "resolved": "https://registry.npmjs.org/hoek/-/hoek-5.0.4.tgz", 87 | "integrity": "sha512-Alr4ZQgoMlnere5FZJsIyfIjORBqZll5POhDsF4q64dPuJR6rNxXdDxtHSQq8OXRurhmx+PWYEE8bXRROY8h0w==" 88 | } 89 | } 90 | }, 91 | "nan": { 92 | "version": "2.14.1", 93 | "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.1.tgz", 94 | "integrity": "sha512-isWHgVjnFjh2x2yuJ/tj3JbwoHu3UC2dX5G/88Cm24yB6YopVgxvBObDY7n5xW6ExmFhJpSEQqFPvq9zaXc8Jw==" 95 | }, 96 | "node-expat": { 97 | "version": "2.3.18", 98 | "resolved": "https://registry.npmjs.org/node-expat/-/node-expat-2.3.18.tgz", 99 | "integrity": "sha512-9dIrDxXePa9HSn+hhlAg1wXkvqOjxefEbMclGxk2cEnq/Y3U7Qo5HNNqeo3fQ4bVmLhcdt3YN1TZy7WMZy4MHw==", 100 | "requires": { 101 | "bindings": "^1.5.0", 102 | "nan": "^2.13.2" 103 | } 104 | }, 105 | "node-fetch": { 106 | "version": "2.6.0", 107 | "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz", 108 | "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==" 109 | }, 110 | "punycode": { 111 | "version": "2.1.1", 112 | "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", 113 | "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" 114 | }, 115 | "supports-color": { 116 | "version": "7.1.0", 117 | "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.1.0.tgz", 118 | "integrity": "sha512-oRSIpR8pxT1Wr2FquTNnGet79b3BWljqOuoW/h4oBhxJ/HUbX5nX6JSruTkvXDCFMwDPvsaTTbvMLKZWSy0R5g==", 119 | "requires": { 120 | "has-flag": "^4.0.0" 121 | } 122 | }, 123 | "topo": { 124 | "version": "3.0.3", 125 | "resolved": "https://registry.npmjs.org/topo/-/topo-3.0.3.tgz", 126 | "integrity": "sha512-IgpPtvD4kjrJ7CRA3ov2FhWQADwv+Tdqbsf1ZnPUSAtCJ9e1Z44MmoSGDXGk4IppoZA7jd/QRkNddlLJWlUZsQ==", 127 | "requires": { 128 | "hoek": "6.x.x" 129 | }, 130 | "dependencies": { 131 | "hoek": { 132 | "version": "6.1.3", 133 | "resolved": "https://registry.npmjs.org/hoek/-/hoek-6.1.3.tgz", 134 | "integrity": "sha512-YXXAAhmF9zpQbC7LEcREFtXfGq5K1fmd+4PHkBq8NUqmzW3G+Dq10bI/i0KucLRwss3YYFQ0fSfoxBZYiGUqtQ==" 135 | } 136 | } 137 | }, 138 | "xml2json": { 139 | "version": "0.12.0", 140 | "resolved": "https://registry.npmjs.org/xml2json/-/xml2json-0.12.0.tgz", 141 | "integrity": "sha512-EPJHRWJnJUYbJlzR4pBhZODwWdi2IaYGtDdteJi0JpZ4OD31IplWALuit8r73dJuM4iHZdDVKY1tLqY2UICejg==", 142 | "requires": { 143 | "hoek": "^4.2.1", 144 | "joi": "^13.1.2", 145 | "node-expat": "^2.3.18" 146 | } 147 | } 148 | } 149 | } 150 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "netlify-plugin-fetch-feeds", 3 | "version": "0.2.3", 4 | "description": "A Netlify plugin to fetch and cache content from remote feeds including RSS and JSON", 5 | "main": "index.js", 6 | "repository": { 7 | "type": "git", 8 | "url": "https://github.com/philhawksworth/netlify-plugin-fetch-feeds" 9 | }, 10 | "keywords": [ 11 | "netlify", 12 | "rss", 13 | "ssg", 14 | "feed", 15 | "data", 16 | "cache", 17 | "plugin", 18 | "jamstack" 19 | ], 20 | "author": "Phil Hawksworth", 21 | "license": "ISC", 22 | "dependencies": { 23 | "chalk": "^4.0.0", 24 | "node-fetch": "^2.6.0", 25 | "xml2json": "^0.12.0" 26 | } 27 | } 28 | --------------------------------------------------------------------------------