├── .prettierrc ├── .github ├── COMMIT_TEMPLATE.md ├── PULL_REQUEST_TEMPLATE.md ├── FUNDING.yml ├── CONTRIBUTING.md ├── ISSUE_TEMPLATE │ ├── feature_request.md │ └── bug_report.md ├── workflows │ └── release.yml └── CODE_OF_CONDUCT.md ├── .npmignore ├── .prettierignore ├── .vscode └── settings.json ├── .editorconfig ├── .gitattributes ├── .gitignore ├── eslint.config.js ├── LICENSE ├── package.json ├── eleventy-cache-webmentions.min.js ├── eleventy-cache-webmentions.min.cjs ├── eleventy-cache-webmentions.test.js ├── README.md ├── eleventy-cache-webmentions.js └── eleventy-cache-webmentions.cjs /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "tabWidth": 4, 3 | "trailingComma": "all" 4 | } 5 | -------------------------------------------------------------------------------- /.github/COMMIT_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | [refs #00000] Subject line 2 | 3 | Body (72 chars) 4 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | Fixes #. 2 | 3 | Changes proposed in this pull request: 4 | 5 | - 6 | - 7 | - 8 | 9 | @chrisburnell 10 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | .github 2 | .gitattributes 3 | .vscode 4 | .editorconfig 5 | .eslintrc.json 6 | .prettierrc 7 | .prettierignore 8 | eleventy-cache-webmentions.test.js 9 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | # Ignore generated/vendor files 2 | **/generated 3 | **/vendors 4 | 5 | # Ignore specific filetypes 6 | *.html 7 | *.json 8 | *.md 9 | *.njk 10 | *.yml 11 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # Docs for this file can be found here: 2 | # https://docs.github.com/en/github/administering-a-repository/displaying-a-sponsor-button-in-your-repository 3 | 4 | github: chrisburnell 5 | buy_me_a_coffee: chrisburnell 6 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "workbench.colorCustomizations": { 3 | "titleBar.activeBackground" : "#4f758e", 4 | "titleBar.activeForeground" : "#f9f9f9", 5 | "titleBar.inactiveBackground" : "#44657A", 6 | "titleBar.inactiveForeground" : "#f9f9f9", 7 | "statusBar.background" : "#4f758e", 8 | "statusBar.foreground" : "#f9f9f9" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | indent_style = tab 6 | indent_size = 4 7 | end_of_line = lf 8 | insert_final_newline = true 9 | trim_trailing_whitespace = true 10 | 11 | [*.{json,svg,yaml,yml}] 12 | indent_style = space 13 | indent_size = 2 14 | 15 | [*.md] 16 | indent_style = space 17 | indent_size = 4 18 | trim_trailing_whitespace = false 19 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # These files are text and should be normalized (convert crlf => lf) 2 | *.css text 3 | *.js text 4 | *.html text 5 | *.php text 6 | *.phtml text 7 | *.json text 8 | 9 | # Images should be treated as binary 10 | # (binary is a macro for -text -diff) 11 | *.png binary 12 | *.jpeg binary 13 | *.jpg binary 14 | *.gif binary 15 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Folder view configuration files 2 | _Store 3 | .DS_Store* 4 | Desktop.ini 5 | Icon 6 | 7 | # Thumbnail cache files 8 | ._* 9 | Thumbs.db 10 | 11 | # Files that might appear on external disks 12 | .Spotlight-V100 13 | .Trashes 14 | 15 | # Packages 16 | *.7z 17 | *.dmg 18 | *.gz 19 | *.iso 20 | *.jar 21 | *.rar 22 | *.tar 23 | *.zip 24 | 25 | # Logs and databases 26 | *.log 27 | *.sql 28 | *.sqlite 29 | 30 | # Eleventy 31 | .env 32 | _site/ 33 | 34 | # Sass 35 | .sass-cache 36 | 37 | # NPM 38 | node_modules/ 39 | 40 | # Git Config 41 | .gitconfig 42 | 43 | # Site Assets 44 | /css 45 | /fonts 46 | /static 47 | /.cache 48 | -------------------------------------------------------------------------------- /.github/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guide 2 | 3 | 4 | ## Commit messages 5 | 6 | Commits must follow the [commit template](COMMIT_TEMPLATE.md). 7 | 8 | 0. Separate subject from body with a blank line 9 | 0. Limit the subject line to 50 characters 10 | 0. Capitalize the subject line 11 | 0. Do not end the subject line with a period 12 | 0. Use the imperative mood in the subject line 13 | 0. Wrap the body at 72 characters 14 | 0. Use the body to explain what and why vs. how 15 | 16 | 17 | ## Pull Requests 18 | 19 | Pull requests must follow the [pull request template](PULL_REQUEST_TEMPLATE.md). 20 | 21 | 22 | ## Issues 23 | 24 | Issues must follow the [issue template](ISSUE_TEMPLATE.md). 25 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request! 3 | about: Enhancements. e.g. “I wish eleventy-cache-webmentions did this.” Suggest an idea! 4 | title: "" 5 | labels: enhancement 6 | assignees: chrisburnell 7 | --- 8 | 9 | **Is your feature request related to a problem? Please describe.** 10 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 11 | 12 | **Describe the solution you'd like** 13 | A clear and concise description of what you want to happen. 14 | 15 | **Describe alternatives you've considered** 16 | A clear and concise description of any alternative solutions or features you've considered. 17 | 18 | **Additional context** 19 | Add any other context or screenshots about the feature request here. 20 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: I’m having trouble with eleventy-cache-webmentions 3 | about: Encountering a problem? Create a bug report to help us improve. 4 | title: "" 5 | labels: needs-triage 6 | assignees: chrisburnell 7 | --- 8 | 9 | **Describe the bug** 10 | A clear and concise description of what the bug is. 11 | 12 | **To Reproduce** 13 | Steps to reproduce the behavior: 14 | 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Environment:** 27 | 28 | - OS and Version: [e.g. Windows/Mac/Linux] 29 | - Eleventy Version [via `eleventy --version` or `npx @11ty/eleventy --version`] 30 | 31 | **Additional context** 32 | Add any other context about the problem here. 33 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Publish release to npm 2 | on: 3 | release: 4 | types: [published] 5 | permissions: read-all 6 | jobs: 7 | release: 8 | runs-on: ubuntu-latest 9 | permissions: 10 | contents: read 11 | id-token: write 12 | steps: 13 | - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # 4.1.7 14 | - uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # 4.0.3 15 | with: 16 | node-version: "20" 17 | registry-url: "https://registry.npmjs.org" 18 | - run: npm install -g npm@latest 19 | - run: npm ci 20 | - run: node --test 21 | - if: ${{ github.event.release.tag_name != '' && env.NPM_PUBLISH_TAG != '' }} 22 | run: npm publish --provenance --access=public --tag=${{ env.NPM_PUBLISH_TAG }} 23 | env: 24 | NPM_PUBLISH_TAG: ${{ contains(github.event.release.tag_name, '-beta.') && 'beta' || contains(github.event.release.tag_name, '-alpha.') && 'canary' || 'latest' }} 25 | -------------------------------------------------------------------------------- /eslint.config.js: -------------------------------------------------------------------------------- 1 | import js from "@eslint/js"; 2 | import jsdoc from "eslint-plugin-jsdoc"; 3 | import globals from "globals"; 4 | 5 | export default [ 6 | js.configs.recommended, 7 | { 8 | files: ["**/*.js"], 9 | plugins: { jsdoc }, 10 | languageOptions: { 11 | ecmaVersion: "latest", 12 | globals: { 13 | ...globals.node, 14 | }, 15 | }, 16 | rules: { 17 | "no-process-env": 0, 18 | "jsdoc/check-access": "warn", 19 | "jsdoc/check-alignment": "warn", 20 | "jsdoc/check-indentation": "warn", 21 | "jsdoc/check-param-names": "warn", 22 | "jsdoc/check-tag-names": "warn", 23 | "jsdoc/check-types": "warn", 24 | "jsdoc/implements-on-classes": "warn", 25 | "jsdoc/no-undefined-types": "warn", 26 | "jsdoc/require-jsdoc": [ 27 | "warn", 28 | { 29 | require: { 30 | FunctionDeclaration: true, 31 | MethodDefinition: true, 32 | ClassDeclaration: true, 33 | }, 34 | }, 35 | ], 36 | "jsdoc/require-param": "warn", 37 | "jsdoc/require-param-type": "warn", 38 | "jsdoc/require-returns": "warn", 39 | "jsdoc/require-returns-type": "warn", 40 | }, 41 | }, 42 | ]; 43 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Chris Burnell 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@chrisburnell/eleventy-cache-webmentions", 3 | "version": "2.2.7", 4 | "description": "Cache webmentions using eleventy-fetch and make them available to use in collections, layouts, pages, etc. in Eleventy.", 5 | "license": "MIT", 6 | "repository": { 7 | "type": "git", 8 | "url": "git@github.com:chrisburnell/eleventy-cache-webmentions.git" 9 | }, 10 | "homepage": "https://chrisburnell.com/eleventy-cache-webmentions/", 11 | "bugs": { 12 | "url": "https://github.com/chrisburnell/eleventy-cache-webmentions/issues" 13 | }, 14 | "author": { 15 | "name": "Chris Burnell", 16 | "email": "me@chrisburnell.com", 17 | "url": "https://chrisburnell.com/" 18 | }, 19 | "contributors": [ 20 | { 21 | "name": "Chris Burnell", 22 | "email": "me@chrisburnell.com", 23 | "url": "https://chrisburnell.com" 24 | } 25 | ], 26 | "funding": [ 27 | { 28 | "type": "buymeacoffee", 29 | "url": "https://buymeacoffee.com/chrisburnell" 30 | }, 31 | { 32 | "type": "github", 33 | "url": "https://github.com/sponsors/chrisburnell" 34 | } 35 | ], 36 | "publishConfig": { 37 | "access": "public", 38 | "provenance": true 39 | }, 40 | "main": "eleventy-cache-webmentions.cjs", 41 | "exports": { 42 | ".": { 43 | "import": "./eleventy-cache-webmentions.js", 44 | "require": "./eleventy-cache-webmentions.cjs" 45 | } 46 | }, 47 | "scripts": { 48 | "build": "esbuild eleventy-cache-webmentions.js --minify-whitespace --minify-syntax --outfile=eleventy-cache-webmentions.min.js && esbuild eleventy-cache-webmentions.cjs --minify-whitespace --minify-syntax --outfile=eleventy-cache-webmentions.min.cjs", 49 | "lint": "eslint eleventy-cache-webmentions.js", 50 | "test": "node --test" 51 | }, 52 | "keywords": [ 53 | "eleventy", 54 | "eleventy-plugin", 55 | "indieweb", 56 | "javascript", 57 | "js", 58 | "webmention" 59 | ], 60 | "engines": { 61 | "node": ">=18" 62 | }, 63 | "dependencies": { 64 | "@11ty/eleventy-fetch": "^5.1.1", 65 | "sanitize-html": "^2.17.0" 66 | }, 67 | "devDependencies": { 68 | "esbuild": "^0.27.1", 69 | "eslint": "^9.39.1", 70 | "eslint-plugin-jsdoc": "^61.4.1", 71 | "globals": "^16.5.0", 72 | "nock": "^14.0.10" 73 | }, 74 | "type": "module" 75 | } 76 | -------------------------------------------------------------------------------- /.github/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. 6 | 7 | ## Our Standards 8 | 9 | Examples of behavior that contributes to creating a positive environment include: 10 | 11 | * Using welcoming and inclusive language 12 | * Being respectful of differing viewpoints and experiences 13 | * Gracefully accepting constructive criticism 14 | * Focusing on what is best for the community 15 | * Showing empathy towards other community members 16 | 17 | Examples of unacceptable behavior by participants include: 18 | 19 | * The use of sexualized language or imagery and unwelcome sexual attention or advances 20 | * Trolling, insulting/derogatory comments, and personal or political attacks 21 | * Public or private harassment 22 | * Publishing others' private information, such as a physical or electronic address, without explicit permission 23 | * Other conduct which could reasonably be considered inappropriate in a professional setting 24 | 25 | ## Our Responsibilities 26 | 27 | Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. 28 | 29 | Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. 30 | 31 | ## Scope 32 | 33 | This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. 34 | 35 | ## Enforcement 36 | 37 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at me@chrisburnell.com. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. 38 | 39 | Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. 40 | 41 | ## Attribution 42 | 43 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version] 44 | 45 | [homepage]: http://contributor-covenant.org 46 | [version]: http://contributor-covenant.org/version/1/4/ 47 | -------------------------------------------------------------------------------- /eleventy-cache-webmentions.min.js: -------------------------------------------------------------------------------- 1 | import{AssetCache}from"@11ty/eleventy-fetch";import{styleText}from"node:util";import sanitizeHTML from"sanitize-html";export const defaults={refresh:!1,duration:"1d",uniqueKey:"webmentions",cacheDirectory:void 0,allowedHTML:{allowedTags:["a","b","em","i","strong"],allowedAttributes:{a:["href"]}},allowlist:[],blocklist:[],urlReplacements:{},maximumHtmlLength:1e3,maximumHtmlText:"mentioned this in"};const absoluteURL=(url,domain)=>{try{return new URL(url,domain).toString()}catch(error){return console.error(`Trying to convert ${styleText("bold",url)} to be an absolute url with base ${styleText("bold",domain)} and failed.`,error),url}},baseURL=url=>url.split("#")[0].split("?")[0],fixURL=(url,urlReplacements)=>Object.entries(urlReplacements).reduce((accumulator,[key,value])=>{const regex=new RegExp(key,"g");return accumulator.replace(regex,value)},url),hostname=url=>typeof url=="string"&&url.includes("//")?new URL(url).hostname:url,epoch=date=>new Date(date).getTime(),removeDuplicates=webmentions=>[...webmentions.reduce((map,webmention)=>{const key=webmention==null?webmention:getSource(webmention);return map.has(key)||map.set(key,webmention),map},new Map).values()];export const getPublished=webmention=>webmention?.data?.published||webmention.published||webmention["wm-received"]||webmention.verified_date,getWebmentionPublished=getPublished,getReceived=webmention=>webmention["wm-received"]||webmention.verified_date||webmention.published||webmention?.data?.published,getWebmentionReceived=getReceived,getContent=webmention=>webmention?.contentSanitized||webmention?.content?.html||webmention?.content?.value||webmention?.content||webmention?.data?.content||"",getWebmentionContent=getContent,getSource=webmention=>webmention["wm-source"]||webmention.source||webmention?.data?.url||webmention.url,getWebmentionSource=getSource,getURL=webmention=>webmention?.data?.url||webmention.url||webmention["wm-source"]||webmention.source,getWebmentionURL=getURL,getTarget=webmention=>webmention["wm-target"]||webmention.target,getWebmentionTarget=getTarget,getType=webmention=>webmention["wm-property"]||webmention?.activity?.type||webmention.type,getWebmentionType=getType,getByTypes=(webmentions,types)=>webmentions.filter(webmention=>typeof types=="string"?types===getType(webmention):types.includes(getType(webmention))),getByType=getByTypes,getWebmentionsByTypes=getByTypes,getWebmentionsByType=getByTypes,processBlocklist=(webmentions,blocklist)=>webmentions.filter(webmention=>{let url=getSource(webmention),source=getSource(webmention);for(let blocklistURL of blocklist)if(url.includes(blocklistURL.replace(/\/?$/,"/"))||source.includes(blocklistURL.replace(/\/?$/,"/")))return!1;return!0}),processWebmentionBlocklist=processBlocklist,processWebmentionsBlocklist=processBlocklist,processAllowlist=(webmentions,allowlist)=>webmentions.filter(webmention=>{let url=getSource(webmention),source=getSource(webmention);for(let allowlistURL of allowlist)if(url.includes(allowlistURL.replace(/\/?$/,"/"))||source.includes(allowlistURL.replace(/\/?$/,"/")))return!0;return!1}),processWebmentionAllowlist=processAllowlist,processWebmentionsAllowlist=processAllowlist,fetchWebmentions=async(options,webmentions,url)=>await fetch(url).then(async response=>{if(!response.ok)return Promise.reject(response);const feed=await response.json();return options.key in feed?(webmentions=feed[options.key].concat(webmentions),webmentions=removeDuplicates(webmentions),options.blocklist.length&&(webmentions=processBlocklist(webmentions,options.blocklist)),options.allowlist.length&&(webmentions=processAllowlist(webmentions,options.allowlist)),webmentions=webmentions.sort((a,b)=>epoch(getReceived(b))-epoch(getReceived(a))),{found:feed[options.key].length,webmentions}):(console.log(`${styleText("grey",`[${hostname(options.domain)}]`)} ${options.key} was not found as a key in the response from ${styleText("bold",hostname(options.feed))}!`),Promise.reject(response))}).catch(error=>(console.warn(`${styleText("grey",`[${hostname(options.domain)}]`)} Something went wrong with your Webmention request to ${styleText("bold",hostname(options.feed))}!`),console.warn(error instanceof Error?error.message:error),{found:0,webmentions})),retrieveWebmentions=async options=>{if(!options.domain)throw new Error("`domain` is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.");if(!options.feed)throw new Error("`feed` is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.");if(!options.key)throw new Error("`key` is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.");let asset=new AssetCache(options.uniqueKey||`webmentions-${hostname(options.domain)}`,options.cacheDirectory),webmentions=[];asset.isCacheValid("9001y")&&!options.refresh&&(webmentions=await asset.getCachedValue());const webmentionsCachedLength=webmentions.length;if(!asset.isCacheValid(options.refresh?"0s":options.duration)){const performanceStart=process.hrtime(),since=webmentions.length?getReceived(webmentions[0]):!1,url=`${options.feed}${since?`${options.feed.includes("?")?"&":"?"}since=${since}`:""}`;if(url.includes("https://webmention.io")){const urlObject=new URL(url),perPage=Number(urlObject.searchParams.get("per-page"))||1e3;urlObject.searchParams.delete("per-page");let page=0;for(;;){const urlPaginated=urlObject.href+`&per-page=${perPage}&page=${page}`,fetched=await fetchWebmentions(options,webmentions,urlPaginated);if(!fetched&&!fetched.found&&!fetched.webmentions||fetched.found===0||(webmentions=fetched.webmentions,fetched.foundsetTimeout(resolve,1e3))}}else webmentions=(await fetchWebmentions(options,webmentions,url)).webmentions;options.blocklist.length&&(webmentions=processBlocklist(webmentions,options.blocklist)),options.allowlist.length&&(webmentions=processAllowlist(webmentions,options.allowlist)),await asset.save(webmentions,"json");const performance=process.hrtime(performanceStart);webmentionsCachedLength(Object.keys(WEBMENTIONS).length||(await retrieveWebmentions(options)).forEach(webmention=>{let url=baseURL(fixURL(getTarget(webmention).replace(/\/?$/,"/"),options.urlReplacements));WEBMENTIONS[url]||(WEBMENTIONS[url]=[]),WEBMENTIONS[url].push(webmention)}),WEBMENTIONS),webmentionsByUrl=webmentionsByURL,filteredWebmentions=webmentionsByURL,getWebmentions=async(options,url,types=[])=>{const webmentions=await webmentionsByURL(options);return url=absoluteURL(url,options.domain),!url||!webmentions||!webmentions[url]?[]:webmentions[url].filter(entry=>typeof types=="object"&&Object.keys(types).length?types.includes(getType(entry)):typeof types=="string"?types===getType(entry):!0).map(entry=>{const html=getContent(entry);return html.length&&(entry.contentSanitized=sanitizeHTML(html,options.allowedHTML),html.length>options.maximumHtmlLength&&(entry.contentSanitized=`${options.maximumHtmlText} ${getSource(entry)}`)),entry}).sort((a,b)=>epoch(getPublished(a))-epoch(getPublished(b)))},eleventyCacheWebmentions=async(eleventyConfig,options={})=>{options=Object.assign(defaults,options);const byURL=await webmentionsByURL(options),all=Object.values(byURL).reduce((array,webmentions)=>[...array,...webmentions],[]);eleventyConfig.addGlobalData("webmentionsDefaults",defaults),eleventyConfig.addGlobalData("webmentionsOptions",options),eleventyConfig.addGlobalData("webmentionsByURL",byURL),eleventyConfig.addGlobalData("webmentionsByUrl",byURL),eleventyConfig.addGlobalData("webmentionsAll",all),eleventyConfig.addLiquidFilter("getWebmentionsByType",getByTypes),eleventyConfig.addLiquidFilter("getWebmentionsByTypes",getByTypes),eleventyConfig.addLiquidFilter("getWebmentionPublished",getPublished),eleventyConfig.addLiquidFilter("getWebmentionReceived",getReceived),eleventyConfig.addLiquidFilter("getWebmentionContent",getContent),eleventyConfig.addLiquidFilter("getWebmentionSource",getSource),eleventyConfig.addLiquidFilter("getWebmentionURL",getURL),eleventyConfig.addLiquidFilter("getWebmentionTarget",getTarget),eleventyConfig.addLiquidFilter("getWebmentionType",getType),eleventyConfig.addNunjucksFilter("getWebmentionsByType",getByTypes),eleventyConfig.addNunjucksFilter("getWebmentionsByTypes",getByTypes),eleventyConfig.addNunjucksFilter("getWebmentionPublished",getPublished),eleventyConfig.addNunjucksFilter("getWebmentionReceived",getReceived),eleventyConfig.addNunjucksFilter("getWebmentionContent",getContent),eleventyConfig.addNunjucksFilter("getWebmentionSource",getSource),eleventyConfig.addNunjucksFilter("getWebmentionURL",getURL),eleventyConfig.addNunjucksFilter("getWebmentionTarget",getTarget),eleventyConfig.addNunjucksFilter("getWebmentionType",getType)};export default eleventyCacheWebmentions; 2 | -------------------------------------------------------------------------------- /eleventy-cache-webmentions.min.cjs: -------------------------------------------------------------------------------- 1 | const{AssetCache}=require("@11ty/eleventy-fetch"),{styleText}=require("node:util"),sanitizeHTML=require("sanitize-html"),defaults={refresh:!1,duration:"1d",uniqueKey:"webmentions",cacheDirectory:void 0,allowedHTML:{allowedTags:["a","b","em","i","strong"],allowedAttributes:{a:["href"]}},allowlist:[],blocklist:[],urlReplacements:{},maximumHtmlLength:1e3,maximumHtmlText:"mentioned this in"},absoluteURL=(url,domain)=>{try{return new URL(url,domain).toString()}catch(error){return console.error(`Trying to convert ${styleText("bold",url)} to be an absolute url with base ${styleText("bold",domain)} and failed.`,error),url}},baseURL=url=>url.split("#")[0].split("?")[0],fixURL=(url,urlReplacements)=>Object.entries(urlReplacements).reduce((accumulator,[key,value])=>{const regex=new RegExp(key,"g");return accumulator.replace(regex,value)},url),hostname=url=>typeof url=="string"&&url.includes("//")?new URL(url).hostname:url,epoch=date=>new Date(date).getTime(),removeDuplicates=webmentions=>[...webmentions.reduce((map,webmention)=>{const key=webmention==null?webmention:getSource(webmention);return map.has(key)||map.set(key,webmention),map},new Map).values()],getPublished=webmention=>webmention?.data?.published||webmention.published||webmention["wm-received"]||webmention.verified_date,getReceived=webmention=>webmention["wm-received"]||webmention.verified_date||webmention.published||webmention?.data?.published,getContent=webmention=>webmention?.contentSanitized||webmention?.content?.html||webmention?.content?.value||webmention?.content||webmention?.data?.content||"",getSource=webmention=>webmention["wm-source"]||webmention.source||webmention?.data?.url||webmention.url,getURL=webmention=>webmention?.data?.url||webmention.url||webmention["wm-source"]||webmention.source,getTarget=webmention=>webmention["wm-target"]||webmention.target,getType=webmention=>webmention["wm-property"]||webmention?.activity?.type||webmention.type,getByTypes=(webmentions,types)=>webmentions.filter(webmention=>typeof types=="string"?types===getType(webmention):types.includes(getType(webmention))),processBlocklist=(webmentions,blocklist)=>webmentions.filter(webmention=>{let url=getSource(webmention),source=getSource(webmention);for(let blocklistURL of blocklist)if(url.includes(blocklistURL.replace(/\/?$/,"/"))||source.includes(blocklistURL.replace(/\/?$/,"/")))return!1;return!0}),processAllowlist=(webmentions,allowlist)=>webmentions.filter(webmention=>{let url=getSource(webmention),source=getSource(webmention);for(let allowlistURL of allowlist)if(url.includes(allowlistURL.replace(/\/?$/,"/"))||source.includes(allowlistURL.replace(/\/?$/,"/")))return!0;return!1}),fetchWebmentions=async(options,webmentions,url)=>await fetch(url).then(async response=>{if(!response.ok)return Promise.reject(response);const feed=await response.json();return options.key in feed?(webmentions=feed[options.key].concat(webmentions),webmentions=removeDuplicates(webmentions),options.blocklist.length&&(webmentions=processBlocklist(webmentions,options.blocklist)),options.allowlist.length&&(webmentions=processAllowlist(webmentions,options.allowlist)),webmentions=webmentions.sort((a,b)=>epoch(getReceived(b))-epoch(getReceived(a))),{found:feed[options.key].length,webmentions}):(console.log(`${styleText("grey",`[${hostname(options.domain)}]`)} ${options.key} was not found as a key in the response from ${styleText("bold",hostname(options.feed))}!`),Promise.reject(response))}).catch(error=>(console.warn(`${styleText("grey",`[${hostname(options.domain)}]`)} Something went wrong with your Webmention request to ${styleText("bold",hostname(options.feed))}!`),console.warn(error instanceof Error?error.message:error),{found:0,webmentions})),retrieveWebmentions=async options=>{if(!options.domain)throw new Error("`domain` is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.");if(!options.feed)throw new Error("`feed` is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.");if(!options.key)throw new Error("`key` is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.");let asset=new AssetCache(options.uniqueKey||`webmentions-${hostname(options.domain)}`,options.cacheDirectory),webmentions=[];asset.isCacheValid("9001y")&&!options.refresh&&(webmentions=await asset.getCachedValue());const webmentionsCachedLength=webmentions.length;if(!asset.isCacheValid(options.refresh?"0s":options.duration)){const performanceStart=process.hrtime(),since=webmentions.length?getReceived(webmentions[0]):!1,url=`${options.feed}${since?`${options.feed.includes("?")?"&":"?"}since=${since}`:""}`;if(url.includes("https://webmention.io")){const urlObject=new URL(url),perPage=Number(urlObject.searchParams.get("per-page"))||1e3;urlObject.searchParams.delete("per-page");let page=0;for(;;){const urlPaginated=urlObject.href+`&per-page=${perPage}&page=${page}`,fetched=await fetchWebmentions(options,webmentions,urlPaginated);if(!fetched&&!fetched.found&&!fetched.webmentions||fetched.found===0||(webmentions=fetched.webmentions,fetched.foundsetTimeout(resolve,1e3))}}else webmentions=(await fetchWebmentions(options,webmentions,url)).webmentions;options.blocklist.length&&(webmentions=processBlocklist(webmentions,options.blocklist)),options.allowlist.length&&(webmentions=processAllowlist(webmentions,options.allowlist)),await asset.save(webmentions,"json");const performance=process.hrtime(performanceStart);webmentionsCachedLength(Object.keys(WEBMENTIONS).length||(await retrieveWebmentions(options)).forEach(webmention=>{let url=baseURL(fixURL(getTarget(webmention).replace(/\/?$/,"/"),options.urlReplacements));WEBMENTIONS[url]||(WEBMENTIONS[url]=[]),WEBMENTIONS[url].push(webmention)}),WEBMENTIONS),getWebmentions=async(options,url,types=[])=>{const webmentions=await webmentionsByURL(options);return url=absoluteURL(url,options.domain),!url||!webmentions||!webmentions[url]?[]:webmentions[url].filter(entry=>typeof types=="object"&&Object.keys(types).length?types.includes(getType(entry)):typeof types=="string"?types===getType(entry):!0).map(entry=>{const html=getContent(entry);return html.length&&(entry.contentSanitized=sanitizeHTML(html,options.allowedHTML),html.length>options.maximumHtmlLength&&(entry.contentSanitized=`${options.maximumHtmlText} ${getSource(entry)}`)),entry}).sort((a,b)=>epoch(getPublished(a))-epoch(getPublished(b)))},eleventyCacheWebmentions=async(eleventyConfig,options={})=>{options=Object.assign(defaults,options);const byURL=await webmentionsByURL(options),all=Object.values(byURL).reduce((array,webmentions)=>[...array,...webmentions],[]);eleventyConfig.addGlobalData("webmentionsDefaults",defaults),eleventyConfig.addGlobalData("webmentionsOptions",options),eleventyConfig.addGlobalData("webmentionsByURL",byURL),eleventyConfig.addGlobalData("webmentionsByUrl",byURL),eleventyConfig.addGlobalData("webmentionsAll",all),eleventyConfig.addLiquidFilter("getWebmentionsByType",getByTypes),eleventyConfig.addLiquidFilter("getWebmentionsByTypes",getByTypes),eleventyConfig.addLiquidFilter("getWebmentionPublished",getPublished),eleventyConfig.addLiquidFilter("getWebmentionReceived",getReceived),eleventyConfig.addLiquidFilter("getWebmentionContent",getContent),eleventyConfig.addLiquidFilter("getWebmentionSource",getSource),eleventyConfig.addLiquidFilter("getWebmentionURL",getURL),eleventyConfig.addLiquidFilter("getWebmentionTarget",getTarget),eleventyConfig.addLiquidFilter("getWebmentionType",getType),eleventyConfig.addNunjucksFilter("getWebmentionsByType",getByTypes),eleventyConfig.addNunjucksFilter("getWebmentionsByTypes",getByTypes),eleventyConfig.addNunjucksFilter("getWebmentionPublished",getPublished),eleventyConfig.addNunjucksFilter("getWebmentionReceived",getReceived),eleventyConfig.addNunjucksFilter("getWebmentionContent",getContent),eleventyConfig.addNunjucksFilter("getWebmentionSource",getSource),eleventyConfig.addNunjucksFilter("getWebmentionURL",getURL),eleventyConfig.addNunjucksFilter("getWebmentionTarget",getTarget),eleventyConfig.addNunjucksFilter("getWebmentionType",getType)};module.exports=eleventyCacheWebmentions,module.exports.defaults=defaults,module.exports.getPublished=getPublished,module.exports.getWebmentionPublished=getPublished,module.exports.getReceived=getReceived,module.exports.getWebmentionReceived=getReceived,module.exports.getContent=getContent,module.exports.getWebmentionContent=getContent,module.exports.getSource=getSource,module.exports.getWebmentionSource=getSource,module.exports.getURL=getURL,module.exports.getWebmentionURL=getURL,module.exports.getTarget=getTarget,module.exports.getWebmentionTarget=getTarget,module.exports.getType=getType,module.exports.getWebmentionType=getType,module.exports.getByTypes=getByTypes,module.exports.getByType=getByTypes,module.exports.getWebmentionsByTypes=getByTypes,module.exports.getWebmentionsByType=getByTypes,module.exports.processBlocklist=processBlocklist,module.exports.processWebmentionBlocklist=processBlocklist,module.exports.processWebmentionsBlocklist=processBlocklist,module.exports.processAllowlist=processAllowlist,module.exports.processWebmentionAllowlist=processAllowlist,module.exports.processWebmentionsAllowlist=processAllowlist,module.exports.fetchWebmentions=fetchWebmentions,module.exports.retrieveWebmentions=retrieveWebmentions,module.exports.webmentionsByURL=webmentionsByURL,module.exports.webmentionsByUrl=webmentionsByURL,module.exports.filteredWebmentions=webmentionsByURL,module.exports.getWebmentions=getWebmentions; 2 | -------------------------------------------------------------------------------- /eleventy-cache-webmentions.test.js: -------------------------------------------------------------------------------- 1 | import nock from "nock"; 2 | import assert from "node:assert/strict"; 3 | import { describe, it } from "node:test"; 4 | import { 5 | defaults, 6 | fetchWebmentions, 7 | filteredWebmentions, 8 | getByType, 9 | getByTypes, 10 | getContent, 11 | getPublished, 12 | getReceived, 13 | getSource, 14 | getTarget, 15 | getType, 16 | getWebmentions, 17 | } from "./eleventy-cache-webmentions.js"; 18 | 19 | const options = Object.assign({}, defaults, { 20 | refresh: true, 21 | domain: "https://example.com", 22 | feed: `https://example.com/mentions.json`, 23 | key: "children", 24 | }); 25 | 26 | const mentions = { 27 | type: "feed", 28 | name: "Webmentions", 29 | children: [ 30 | { 31 | type: "entry", 32 | author: { 33 | type: "card", 34 | name: "Jane Doe", 35 | url: "https://example.com", 36 | }, 37 | url: "https://example.com/post1/", 38 | published: "2024-01-01T12:00:00Z", 39 | "wm-received": "2024-01-01T12:00:00Z", 40 | "wm-id": 123456, 41 | "wm-source": "https://example.com/post1/", 42 | "wm-target": "https://example.com/page1/", 43 | "wm-protocol": "webmention", 44 | name: "Example Post", 45 | content: { 46 | "content-type": "text/html", 47 | value: "

Lorem ipsum dolor sit amet, consectetur adipiscing elit.

", 48 | html: "

Lorem ipsum dolor sit amet, consectetur adipiscing elit.

", 49 | text: "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", 50 | }, 51 | "mention-of": "https://example.com/page1/", 52 | "wm-property": "mention-of", 53 | "wm-private": false, 54 | }, 55 | { 56 | type: "entry", 57 | author: { 58 | type: "card", 59 | name: "Jane Doe", 60 | url: "https://example.com", 61 | }, 62 | url: "https://example.com/post2/", 63 | published: "2024-01-01T12:00:00Z", 64 | "wm-received": "2024-01-01T12:00:00Z", 65 | "wm-id": 234567, 66 | "wm-source": "https://example.com/post2/", 67 | "wm-target": "https://example.com/page2/", 68 | "wm-protocol": "webmention", 69 | name: "Example Post", 70 | content: { 71 | "content-type": "text/html", 72 | value: "

Lorem ipsum dolor sit amet, consectetur adipiscing elit.

", 73 | html: "

Lorem ipsum dolor sit amet, consectetur adipiscing elit.

", 74 | text: "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", 75 | }, 76 | "in-reply-to": "https://example.com/page2/", 77 | "wm-property": "in-reply-to", 78 | "wm-private": false, 79 | }, 80 | { 81 | type: "entry", 82 | author: { 83 | type: "card", 84 | name: "Jane Doe", 85 | url: "https://example.com", 86 | }, 87 | url: "https://example.com/post3/", 88 | published: "2024-01-01T12:00:00Z", 89 | "wm-received": "2024-01-01T12:00:00Z", 90 | "wm-id": 345678, 91 | "wm-source": "https://example.com/post3/", 92 | "wm-target": "https://example.com/page2/", 93 | "wm-protocol": "webmention", 94 | name: "Example Post", 95 | content: { 96 | "content-type": "text/html", 97 | value: "

Lorem ipsum dolor sit amet, consectetur adipiscing elit.

", 98 | html: "

Lorem ipsum dolor sit amet, consectetur adipiscing elit.

", 99 | text: "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", 100 | }, 101 | "mention-of": "https://example.com/page2/", 102 | "wm-property": "mention-of", 103 | "wm-private": false, 104 | }, 105 | ], 106 | }; 107 | 108 | describe("filteredWebmentions()", () => { 109 | const scope = nock("https://example.com") 110 | .get("/mentions.json") 111 | .reply(200, mentions); 112 | it("Should return an object of Key: URL, Value: Array of Webmentions", async () => { 113 | const webmentions = await filteredWebmentions(options); 114 | assert.strictEqual(Object.keys(webmentions).length, 2); 115 | }); 116 | }); 117 | 118 | describe("fetchWebmentions()", () => { 119 | const scope = nock("https://example.com") 120 | .get("/mentions.json") 121 | .reply(200, mentions); 122 | it("Should return an object of Key: URL, Value: Array of Webmentions`", async () => { 123 | const fetched = await fetchWebmentions( 124 | options, 125 | [], 126 | "https://example.com/mentions.json", 127 | ); 128 | assert.strictEqual(fetched.webmentions.length, 3); 129 | }); 130 | }); 131 | 132 | describe("getWebmentions()", () => { 133 | const scope = nock("https://example.com") 134 | .get("/mentions.json") 135 | .reply(200, mentions); 136 | it("Should return an object of Key: URL, Value: Array of Webmentions`", async () => { 137 | const fetched = await getWebmentions( 138 | options, 139 | "https://example.com/page2/", 140 | ); 141 | assert.strictEqual(fetched.length, 2); 142 | }); 143 | it("Should return an object of Key: URL, Value: Array of Webmentions of a specific type`", async () => { 144 | const fetched = await getWebmentions( 145 | options, 146 | "https://example.com/page2/", 147 | ["mention-of"], 148 | ); 149 | assert.strictEqual(fetched.length, 1); 150 | }); 151 | }); 152 | 153 | describe("getPublished()", () => { 154 | it("Should return a published date from `data.published`", async () => { 155 | const webmention = { 156 | data: { 157 | published: "2024-01-01T12:00:00Z", 158 | }, 159 | }; 160 | assert.strictEqual(getPublished(webmention), "2024-01-01T12:00:00Z"); 161 | }); 162 | 163 | it("Should return a published date from `published`", async () => { 164 | const webmention = { 165 | published: "2024-01-01T12:00:00Z", 166 | }; 167 | assert.strictEqual(getPublished(webmention), "2024-01-01T12:00:00Z"); 168 | }); 169 | 170 | it("Should return a published date from `wm-received`", async () => { 171 | const webmention = { 172 | "wm-received": "2024-01-01T12:00:00Z", 173 | }; 174 | assert.strictEqual(getPublished(webmention), "2024-01-01T12:00:00Z"); 175 | }); 176 | 177 | it("Should return a published date from `verified_date`", async () => { 178 | const webmention = { 179 | verified_date: "2024-01-01T12:00:00Z", 180 | }; 181 | assert.strictEqual(getPublished(webmention), "2024-01-01T12:00:00Z"); 182 | }); 183 | }); 184 | 185 | describe("getReceived()", () => { 186 | it("Should return a received date from `wm-received`", async () => { 187 | const webmention = { 188 | "wm-received": "2024-01-01T12:00:00Z", 189 | }; 190 | assert.strictEqual(getReceived(webmention), "2024-01-01T12:00:00Z"); 191 | }); 192 | 193 | it("Should return a received date from `verified_date`", async () => { 194 | const webmention = { 195 | verified_date: "2024-01-01T12:00:00Z", 196 | }; 197 | assert.strictEqual(getReceived(webmention), "2024-01-01T12:00:00Z"); 198 | }); 199 | 200 | it("Should return a received date from `published`", async () => { 201 | const webmention = { 202 | published: "2024-01-01T12:00:00Z", 203 | }; 204 | assert.strictEqual(getReceived(webmention), "2024-01-01T12:00:00Z"); 205 | }); 206 | 207 | it("Should return a received date from `data.published`", async () => { 208 | const webmention = { 209 | data: { 210 | published: "2024-01-01T12:00:00Z", 211 | }, 212 | }; 213 | assert.strictEqual(getReceived(webmention), "2024-01-01T12:00:00Z"); 214 | }); 215 | }); 216 | 217 | describe("getContent()", () => { 218 | it("Should return content from `contentSanitized`", async () => { 219 | const webmention = { 220 | contentSanitized: "Lorem ipsum", 221 | }; 222 | assert.strictEqual(getContent(webmention), "Lorem ipsum"); 223 | }); 224 | 225 | it("Should return content from `content.html`", async () => { 226 | const webmention = { 227 | content: { 228 | html: "

Lorem ipsum

", 229 | }, 230 | }; 231 | assert.strictEqual(getContent(webmention), "

Lorem ipsum

"); 232 | }); 233 | 234 | it("Should return content from `content.value`", async () => { 235 | const webmention = { 236 | content: { 237 | value: "Lorem ipsum", 238 | }, 239 | }; 240 | assert.strictEqual(getContent(webmention), "Lorem ipsum"); 241 | }); 242 | 243 | it("Should return content from `content`", async () => { 244 | const webmention = { 245 | content: "Lorem ipsum", 246 | }; 247 | assert.strictEqual(getContent(webmention), "Lorem ipsum"); 248 | }); 249 | 250 | it("Should return content from `data.content`", async () => { 251 | const webmention = { 252 | data: { 253 | content: "Lorem ipsum", 254 | }, 255 | }; 256 | assert.strictEqual(getContent(webmention), "Lorem ipsum"); 257 | }); 258 | }); 259 | 260 | describe("getSource()", () => { 261 | it("Should return a source URL from `wm-source`", async () => { 262 | const webmention = { 263 | "wm-source": "https://example.com", 264 | }; 265 | assert.strictEqual(getSource(webmention), "https://example.com"); 266 | }); 267 | 268 | it("Should return a source URL from `source`", async () => { 269 | const webmention = { 270 | source: "https://example.com", 271 | }; 272 | assert.strictEqual(getSource(webmention), "https://example.com"); 273 | }); 274 | 275 | it("Should return a source URL from `data.url`", async () => { 276 | const webmention = { 277 | data: { 278 | url: "https://example.com", 279 | }, 280 | }; 281 | assert.strictEqual(getSource(webmention), "https://example.com"); 282 | }); 283 | 284 | it("Should return a source URL from `url`", async () => { 285 | const webmention = { 286 | url: "https://example.com", 287 | }; 288 | assert.strictEqual(getSource(webmention), "https://example.com"); 289 | }); 290 | }); 291 | 292 | describe("getURL()", () => { 293 | it("Should return a origin URL from `data.url`", async () => { 294 | const webmention = { 295 | data: { 296 | url: "https://example.com", 297 | }, 298 | }; 299 | assert.strictEqual(getSource(webmention), "https://example.com"); 300 | }); 301 | 302 | it("Should return a origin URL from `url`", async () => { 303 | const webmention = { 304 | url: "https://example.com", 305 | }; 306 | assert.strictEqual(getSource(webmention), "https://example.com"); 307 | }); 308 | 309 | it("Should return a origin URL from `wm-source`", async () => { 310 | const webmention = { 311 | "wm-source": "https://example.com", 312 | }; 313 | assert.strictEqual(getSource(webmention), "https://example.com"); 314 | }); 315 | 316 | it("Should return a origin URL from `source`", async () => { 317 | const webmention = { 318 | source: "https://example.com", 319 | }; 320 | assert.strictEqual(getSource(webmention), "https://example.com"); 321 | }); 322 | }); 323 | 324 | describe("getTarget()", () => { 325 | it("Should return a target URL from `wm-target`", async () => { 326 | const webmention = { 327 | "wm-target": "https://example.com", 328 | }; 329 | assert.strictEqual(getTarget(webmention), "https://example.com"); 330 | }); 331 | 332 | it("Should return a target URL from `target`", async () => { 333 | const webmention = { 334 | target: "https://example.com", 335 | }; 336 | assert.strictEqual(getTarget(webmention), "https://example.com"); 337 | }); 338 | }); 339 | 340 | describe("getType()", () => { 341 | it("Should return a Webmention type from `wm-property`", async () => { 342 | const webmention = { 343 | "wm-property": "mention-of", 344 | }; 345 | assert.strictEqual(getType(webmention), "mention-of"); 346 | }); 347 | 348 | it("Should return a Webmention type from `activity.type`", async () => { 349 | const webmention = { 350 | activity: { 351 | type: "mention-of", 352 | }, 353 | }; 354 | assert.strictEqual(getType(webmention), "mention-of"); 355 | }); 356 | 357 | it("Should return a Webmention type from `type`", async () => { 358 | const webmention = { 359 | type: "mention-of", 360 | }; 361 | assert.strictEqual(getType(webmention), "mention-of"); 362 | }); 363 | }); 364 | 365 | describe("getByType()", () => { 366 | it("Should return an array of Webmentions based on a type", async () => { 367 | const webmentions = [ 368 | { 369 | type: "mention-of", 370 | }, 371 | { 372 | type: "in-reply-to", 373 | }, 374 | ]; 375 | assert.strictEqual(getByType(webmentions, "mention-of").length, 1); 376 | }); 377 | }); 378 | 379 | describe("getByTypes()", () => { 380 | it("Should return an array of Webmentions based on multiple types", async () => { 381 | const webmentions = [ 382 | { 383 | type: "bookmark-of", 384 | }, 385 | { 386 | type: "mention-of", 387 | }, 388 | { 389 | type: "in-reply-to", 390 | }, 391 | ]; 392 | assert.strictEqual( 393 | getByTypes(webmentions, ["in-reply-to", "mention-of"]).length, 394 | 2, 395 | ); 396 | }); 397 | }); 398 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # eleventy-cache-webmentions 2 | 3 | > Cache webmentions using eleventy-fetch and make them available to use in collections, layouts, pages, etc. in Eleventy. 4 | 5 | [![Deploy](https://img.shields.io/github/actions/workflow/status/chrisburnell/eleventy-cache-webmentions/npm-publish.yml?logo=github)](https://github.com/chrisburnell/eleventy-cache-webmentions/actions/workflows/npm-publish.yml) ​ [![NPM Downloads](https://img.shields.io/npm/dm/%40chrisburnell%2Feleventy-cache-webmentions?logo=npm&color=%235f8aa6)](https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions) 6 | 7 | ## Breaking change for v2.0.0 8 | 9 | Version 2.0.0 introduces a breaking change for those migrating from earlier versions of the plugin. This affects usage of the plugin from JavaScript files; specifically, you will need to make a small change to the way that you `require()` the plugin by removing an extra set of parentheses: 10 | 11 | **v1.2.5 and below** 12 | 13 | ```javascript 14 | require("@chrisburnell/eleventy-cache-webmentions")() 15 | ``` 16 | 17 | **v2.0.0 and above** 18 | 19 | ```javascript 20 | require("@chrisburnell/eleventy-cache-webmentions") 21 | ``` 22 | 23 | ## Quick Guide 24 | 25 | I wrote a quicker and simpler guide to getting this Eleventy plugin working that cuts out all the fluff and extra details. 26 | 27 | Check it out: [Webmention Setup for Eleventy](https://chrisburnell.com/article/webmention-eleventy-setup/). 28 | 29 | ## Installation 30 | 31 | - **With npm:** `npm install @chrisburnell/eleventy-cache-webmentions` 32 | - **Direct download:** [https://github.com/chrisburnell/eleventy-cache-webmentions/archive/master.zip](https://github.com/chrisburnell/eleventy-cache-webmentions/archive/master.zip) 33 | 34 | *Important Note: This plugin uses Node.js features only present in versions 18+. If you’re deploying your website somewhere, check to make sure that your Node.js version is set to 18 or greater. ([Cloudflare Pages](https://community.cloudflare.com/t/pages-node-js-version/295548/3), [GitHub Actions](https://github.com/actions/setup-node), [Netlify](https://answers.netlify.com/t/specifying-a-node-version/9701))* 35 | 36 | Inside your Eleventy config file, use `addPlugin()` to add it to your project: 37 | 38 | ```javascript 39 | const pluginWebmentions = require("@chrisburnell/eleventy-cache-webmentions") 40 | 41 | module.exports = function(eleventyConfig) { 42 | eleventyConfig.addPlugin(pluginWebmentions, { 43 | // These 3 fields are all required! 44 | domain: "https://example.com", 45 | feed: "https://webmentions.example.com?token=S3cr3tT0k3n", 46 | key: "array_of_webmentions" 47 | }) 48 | } 49 | ``` 50 | 51 | Make sure you get the correct values for this configuration. Check below for both Webmention.io configuration and go-jamming configuration. 52 | 53 |
54 | Full options list 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 |
optiondefault valuedescriptionversion added
domain
required		—The website you’re fetching Webmentions for.0.0.1
feed
required		—The URL of your Webmention server’s feed for your domain.0.2.0
key
required		—The key in the above feed whose value is an Array of Webmentions.0.0.1
directory".cache"See Eleventy Fetch’s Cache Directory for more information.1.1.2
refreshfalseForces fresh results from the Webmention endpoint every time.2.1.3
duration"1d" or 1 daySee Eleventy Fetch’s Cache Duration for more information.0.0.1
uniqueKey"webmentions"The name of the file generated by Eleventy Fetch.0.1.9
allowedHTMLSee code example belowSee the sanitize-html package for more information.0.0.1
allowlist[]An Array of root URLs from which Webmentions are kept.1.1.0
blocklist[]An Array of root URLs from which Webmentions are discarded.1.1.0
urlReplacements{}An Object of key-value string pairs containing from-to URL replacements on this domain.0.0.3
maximumHtmlLength2000Maximum number of characters in a Webmention’s HTML content, beyond which point a different message is shown, referring to the original source.0.0.1
maximumHtmlText"mentioned this in"The glue-y part of the message displayed when a Webmention content’s character count exceeds maximumHtmlLength.0.1.0
145 |
146 | 147 | ## Usage 148 | 149 | `eleventy-cache-webmentions` comes with a number of ways of accessing your Webmentions as [Global Data](https://www.11ty.dev/docs/data-global-custom/) in both JavaScript and Liquid/Nunjucks as well as a series of [Eleventy Filters](https://www.11ty.dev/docs/filters/) and JavaScript Functions for filtering, sorting, and reading properties about each Webmention: 150 | 151 | ### Global Data 152 | 153 |
154 | JavaScript 155 | 156 | ```javascript 157 | const { 158 | defaults, // default options for the plugin 159 | webmentionsByUrl, // Object containing Arrays of Webmentions by URL 160 | } = require("@chrisburnell/eleventy-cache-webmentions") 161 | ``` 162 | 163 |
164 | 165 |
166 | Liquid / Nunjucks 167 | 168 | ```twig 169 | {# default options for the plugin #} 170 | {{ webmentionsDefaults }} 171 | {# Object containing Arrays of Webmentions by URL #} 172 | {{ webmentionsByUrl }} 173 | ``` 174 | 175 |
176 | 177 | ### Filters 178 | 179 |
180 | JavaScript 181 | 182 | ```javascript 183 | const { 184 | getWebmentions, // get Array of Webmentions for a given URL 185 | getByTypes, // filter Webmentions by their response type 186 | getPublished, // get received/published time of a Webmention 187 | getContent, // get content of a Webmention 188 | getSource, // get source URL of a Webmention (where it's from) 189 | getTarget, // get target URL of a Webmention (where it's sent to) 190 | getType, // get response type of a Webmention 191 | } = require("@chrisburnell/eleventy-cache-webmentions") 192 | 193 | // This is NOT the best way to get Webmentions! 194 | // See "Attach Webmentions to Pages using Directory Data" below. 195 | const webmentions = getWebmentions({ 196 | domain: "https://example.com", 197 | feed: "https://webmentions.example.com?token=S3cr3tT0k3n", 198 | key: "array_of_webmentions" 199 | }, "https://example.com/specific-page/") 200 | 201 | const responsesOnly = getByTypes(webmentions, ['mention-of', 'in-reply-to']) 202 | 203 | webmentions.forEach((webmention) => { 204 | const published = getPublished(webmention) 205 | const content = getContent(webmention) 206 | const source = getSource(webmention) 207 | const target = getTarget(webmention) 208 | const type = getType(webmention) 209 | }) 210 | ``` 211 | 212 |
213 | 214 |
215 | Liquid / Nunjucks 216 | 217 | ```twig 218 | {# filter Webmentions by their response type #} 219 | {{ set responses = webmentions | getWebmentionsByTypes(['mention-of', 'in-reply-to']) }} 220 | 221 | {% for webmention in webmentions %} 222 | {# get received/published time of a Webmention #} 223 | {{ webmentions | getWebmentionPublished }} 224 | {# get content of a Webmention #} 225 | {{ webmentions | getWebmentionContent }} 226 | {# get source URL of a Webmention (where it's from) #} 227 | {{ webmentions | getWebmentionSource }} 228 | {# get target URL of a Webmention (where it's sent to) #} 229 | {{ webmentions | getWebmentionTarget }} 230 | {# get response type of a Webmention #} 231 | {{ webmentions | getWebmentionType }} 232 | {% endfor %} 233 | ``` 234 | 235 |
236 | 237 | ### Attach Webmentions to Pages using Directory Data 238 | 239 | Using [Eleventy’s Data Cascade](https://www.11ty.dev/docs/data-cascade/), you can attach Webmentions to each page by using [Directory Specific Data Files](https://www.11ty.dev/docs/data-template-dir/). 240 | 241 | For example, if you have a folder, `/pages/`, and want to attach Webmentions to each page, create or add the following to a `pages.11tydata.js` file within the folder: 242 | 243 | ```javascript 244 | const { getWebmentions, getPublished } = require("@chrisburnell/eleventy-cache-webmentions") 245 | 246 | module.exports = { 247 | eleventyComputed: { 248 | webmentions: (data) => { 249 | // Get this page's Webmentions as an Array (based on the URL) 250 | const webmentionsForUrl = getWebmentions({ 251 | domain: "https://example.com", 252 | feed: "https://webmentions.example.com?token=S3cr3tT0k3n", 253 | key: "array_of_webmentions" 254 | }, "https://example.com" + data.page.url) 255 | 256 | // If there are Webmentions for this page 257 | if (webmentionsForUrl.length) { 258 | // Sort them (based on when they were received/published) 259 | return webmentionsForUrl.sort((a, b) => { 260 | return getPublished(b) - getPublished(a) 261 | }) 262 | } 263 | // Otherwise, return an empty Array 264 | return [] 265 | }, 266 | }, 267 | } 268 | ``` 269 | 270 | This attaches an Array containing Webmentions to each page (based on its URL). You can then access this Array of Webmentions with the variable, webmentions, within a [Layout](https://www.11ty.dev/docs/layouts/), [Include](https://www.11ty.dev/docs/includes/), or from the page itself: 271 | 272 | ```twig 273 | {% for webmention in webmentions %} 274 | {# Do something with each Webmention #} 275 | {% endfor %} 276 | ``` 277 | 278 | These Arrays of Webmentions can even be accessed when building [Collections](https://www.11ty.dev/docs/collections/), allowing you to create a Collection of pages sorted by their number of Webmentions, for example: 279 | 280 | ```javascript 281 | module.exports = (eleventyConfig) => { 282 | eleventyConfig.addCollection("popular", (collection) => { 283 | return collection 284 | .sort((a, b) => { 285 | return b.data.webmentions.length - a.data.webmentions.length 286 | }) 287 | }) 288 | } 289 | ``` 290 | 291 | ### Get specific types of Webmentions 292 | 293 | Instead of getting all the Webmentions for a given page, you may want to grab only certain types of Webmentions. This is useful if you want to display different types of Webmentions separately, e.g.: 294 | 295 | ```twig 296 | {% set bookmarks = webmentions | getWebmentionsByTypes(['bookmark-of']) %} 297 | {% set likes = webmentions | getWebmentionsByTypes(['like-of']) %} 298 | {% set reposts = webmentions | getWebmentionsByTypes(['repost-of']) %} 299 | 300 | {% set replies = webmentions | getWebmentionsByTypes(['mention-of', 'in-reply-to']) %} 301 | ``` 302 | 303 | ### Get all Webmentions at once 304 | 305 | If you need it, the plugin also makes available an Object containing your cached Webmentions organised in key:value pairs, where each key is a full URL on your website and its value is an Array of Webmentions sent to that URL: 306 | 307 | ```twig 308 | {% set count = 0 %} 309 | {% for url, array in webmentionsByUrl %} 310 | {% set count = array.length + count %} 311 | {% endfor %} 312 |

This website has received {{ count }} Webmentions!

313 | ``` 314 | 315 | ## Webmention.io 316 | 317 | [Webmention.io](https://webmention.io) is a in-place Webmention receiver solution that you can use by authenticating yourself via [IndieAuth](https://indieauth.com/) (or host it yourself), and, like *so much* other publicly-available IndieWeb software, is built and hosted by [Aaron Parecki](https://aaronparecki.com/). 318 | 319 | ### Add your token 320 | 321 | Get set up on [Webmention.io](https://webmention.io) and add your **API Key** (found on your [settings page](https://webmention.io/settings)) to your project as an environment variable, i.e. in a `.env` file in the root of your project: 322 | 323 | ```text 324 | WEBMENTION_IO_TOKEN=njJql0lKXnotreal4x3Wmd 325 | ``` 326 | 327 | ### Set your feed and key config options 328 | 329 | The example below requests the [JF2](https://www.w3.org/TR/jf2/) file format, which I highly recommend using; although, there is a JSON format available from [Webmention.io](https://webmention.io) as well. The [official documentation](https://github.com/aaronpk/webmention.io) has more information on how to use these two formats. 330 | 331 | The key difference between the two feed formats is in the *naming* of the keys: the JF2 format holds the array of Webmentions in the `children` key, whereas the JSON format holds them in the `links` key. The JF2 format, however, provides keys and values that more tightly-align with [microformats](https://indieweb.org/microformats), the method I recommend the most for marking up HTML such that it can be consumed and understood by search engines, aggregators, and other tools across the Indieweb. 332 | 333 | ```javascript 334 | const pluginWebmentions = require("@chrisburnell/eleventy-cache-webmentions") 335 | 336 | module.exports = function(eleventyConfig) { 337 | eleventyConfig.addPlugin(pluginWebmentions, { 338 | domain: "https://example.com", 339 | feed: `https://webmention.io/api/mentions.jf2?domain=example.com&per-page=9001&token=${process.env.WEBMENTION_IO_TOKEN}`, 340 | key: "children" 341 | }) 342 | } 343 | ``` 344 | 345 | If you want to use the JSON format instead, make sure that you replace `mentions.jf2` in the URL with `mentions.json` and change the value of the key from `children` to `links`. 346 | 347 | ## go-jamming 348 | 349 | [go-jamming](https://git.brainbaking.com/wgroeneveld/go-jamming) is a self-hosted Webmention sender and receiver, built in Go by [Wouter Groeneveld](https://brainbaking.com) and available with more information on his [personal git instance](https://git.brainbaking.com/wgroeneveld/go-jamming). 350 | 351 | ### Add your token 352 | 353 | Once you’ve set up your *go-jamming* server and you’ve defined your token, you’ll need add it to your project as an environment variable, i.e. in a `.env` file in the root of your project: 354 | 355 | ```text 356 | GO_JAMMING_TOKEN=njJql0lKXnotreal4x3Wmd 357 | ``` 358 | 359 | ### Set your feed and key config options 360 | 361 | ```javascript 362 | const pluginWebmentions = require("@chrisburnell/eleventy-cache-webmentions") 363 | 364 | module.exports = function(eleventyConfig) { 365 | eleventyConfig.addPlugin(pluginWebmentions, { 366 | domain: "https://example.com", 367 | feed: `https://jam.example.com/webmention/example.com/${process.env.GO_JAMMING_TOKEN}`, 368 | key: "json" 369 | }) 370 | } 371 | ``` 372 | 373 | ## Contributing 374 | 375 | Contributions of all kinds are welcome! Please [submit an Issue on GitHub](https://github.com/chrisburnell/eleventy-cache-webmentions/issues) or [get in touch with me](https://chrisburnell.com/about/#contact) if you’d like to do so. 376 | 377 | ## License 378 | 379 | This project is licensed under an MIT license. 380 | -------------------------------------------------------------------------------- /eleventy-cache-webmentions.js: -------------------------------------------------------------------------------- 1 | import { AssetCache } from "@11ty/eleventy-fetch"; 2 | import { styleText } from "node:util"; 3 | import sanitizeHTML from "sanitize-html"; 4 | 5 | /** 6 | * @typedef {sanitizeHTML.IOptions} AllowedHTML 7 | */ 8 | 9 | /** 10 | * @typedef {object} OptionsDefaults 11 | * @property {boolean} refresh 12 | * @property {string} duration 13 | * @property {string} uniqueKey 14 | * @property {string} [cacheDirectory] 15 | * @property {AllowedHTML} allowedHTML 16 | * @property {Array} allowlist 17 | * @property {Array} blocklist 18 | * @property {{[key: string]: string}} urlReplacements 19 | * @property {number} maximumHtmlLength 20 | * @property {string} maximumHtmlText 21 | */ 22 | export const defaults = { 23 | refresh: false, 24 | duration: "1d", 25 | uniqueKey: "webmentions", 26 | cacheDirectory: undefined, 27 | allowedHTML: { 28 | allowedTags: ["a", "b", "em", "i", "strong"], 29 | allowedAttributes: { 30 | a: ["href"], 31 | }, 32 | }, 33 | allowlist: [], 34 | blocklist: [], 35 | urlReplacements: {}, 36 | maximumHtmlLength: 1000, 37 | maximumHtmlText: "mentioned this in", 38 | }; 39 | 40 | /** 41 | * @typedef {object} OptionsUserInput 42 | * @property {string} domain 43 | * @property {string} feed 44 | * @property {string} key 45 | */ 46 | 47 | /** 48 | * @typedef {OptionsDefaults & OptionsUserInput} Options 49 | */ 50 | 51 | /** 52 | * @typedef {object} Webmention 53 | * @property {string} [source] 54 | * @property {string} [url] 55 | * @property {string} [target] 56 | * @property {string} [published] 57 | * @property {string} [contentSanitized] 58 | * @property {object} [content] 59 | * @property {string} [content.html] 60 | * @property {string} [content.value] 61 | * @property {object} [data] 62 | * @property {string} [data.title] 63 | * @property {string} [data.url] 64 | * @property {string} [data.published] 65 | * @property {string} [data.content] 66 | * @property {string} [type] 67 | * @property {object} [activity] 68 | * @property {string} [activity.type] 69 | * @property {boolean} [verified] 70 | * @property {string} ["wm-property"] 71 | * @property {string} ["wm-received"] 72 | * @property {string} ["wm-source"] 73 | * @property {string} ["wm-target"] 74 | * @property {string} ["verified_date"] 75 | */ 76 | 77 | /** 78 | * @typedef {"bookmark-of"|"like-of"|"repost-of"|"mention-of"|"in-reply-to"} WebmentionType 79 | */ 80 | 81 | /** 82 | * @param {string} url 83 | * @param {string} domain 84 | * @returns {string} 85 | */ 86 | const absoluteURL = (url, domain) => { 87 | try { 88 | return new URL(url, domain).toString(); 89 | } catch (error) { 90 | console.error( 91 | `Trying to convert ${styleText( 92 | "bold", 93 | url, 94 | )} to be an absolute url with base ${styleText( 95 | "bold", 96 | domain, 97 | )} and failed.`, 98 | error, 99 | ); 100 | return url; 101 | } 102 | }; 103 | 104 | /** 105 | * @param {string} url 106 | * @returns {string} 107 | */ 108 | const baseURL = (url) => { 109 | let hashSplit = url.split("#"); 110 | let queryparamSplit = hashSplit[0].split("?"); 111 | return queryparamSplit[0]; 112 | }; 113 | 114 | /** 115 | * @param {string} url 116 | * @param {{[key: string]: string}} [urlReplacements] 117 | * @returns {string} 118 | */ 119 | const fixURL = (url, urlReplacements) => { 120 | return Object.entries(urlReplacements).reduce( 121 | (accumulator, [key, value]) => { 122 | const regex = new RegExp(key, "g"); 123 | return accumulator.replace(regex, value); 124 | }, 125 | url, 126 | ); 127 | }; 128 | 129 | /** 130 | * @param {string} url 131 | * @returns {string} 132 | */ 133 | const hostname = (url) => { 134 | if (typeof url === "string" && url.includes("//")) { 135 | const urlObject = new URL(url); 136 | return urlObject.hostname; 137 | } 138 | return url; 139 | }; 140 | 141 | /** 142 | * @param {string|number|Date} date 143 | * @returns {number} 144 | */ 145 | const epoch = (date) => { 146 | return new Date(date).getTime(); 147 | }; 148 | 149 | /** 150 | * @param {Array} webmentions 151 | * @returns {Array} 152 | */ 153 | const removeDuplicates = (webmentions) => { 154 | return [ 155 | ...webmentions 156 | .reduce((map, webmention) => { 157 | const key = 158 | webmention === null || webmention === undefined 159 | ? webmention 160 | : getSource(webmention); 161 | if (!map.has(key)) { 162 | map.set(key, webmention); 163 | } 164 | return map; 165 | }, new Map()) 166 | .values(), 167 | ]; 168 | }; 169 | 170 | /** 171 | * @param {Webmention} webmention 172 | * @returns {string|undefined} 173 | */ 174 | export const getPublished = (webmention) => { 175 | return ( 176 | webmention?.["data"]?.["published"] || 177 | webmention["published"] || 178 | webmention["wm-received"] || 179 | webmention["verified_date"] 180 | ); 181 | }; 182 | export const getWebmentionPublished = getPublished; 183 | 184 | /** 185 | * @param {Webmention} webmention 186 | * @returns {string|undefined} 187 | */ 188 | export const getReceived = (webmention) => { 189 | return ( 190 | webmention["wm-received"] || 191 | webmention["verified_date"] || 192 | webmention["published"] || 193 | webmention?.["data"]?.["published"] 194 | ); 195 | }; 196 | export const getWebmentionReceived = getReceived; 197 | 198 | /** 199 | * @param {Webmention} webmention 200 | * @returns {string} 201 | */ 202 | export const getContent = (webmention) => { 203 | return ( 204 | webmention?.["contentSanitized"] || 205 | webmention?.["content"]?.["html"] || 206 | webmention?.["content"]?.["value"] || 207 | webmention?.["content"] || 208 | webmention?.["data"]?.["content"] || 209 | "" 210 | ); 211 | }; 212 | export const getWebmentionContent = getContent; 213 | 214 | /** 215 | * @param {Webmention} webmention 216 | * @returns {string|undefined} 217 | */ 218 | export const getSource = (webmention) => { 219 | return ( 220 | webmention["wm-source"] || 221 | webmention["source"] || 222 | webmention?.["data"]?.["url"] || 223 | webmention["url"] 224 | ); 225 | }; 226 | export const getWebmentionSource = getSource; 227 | 228 | /** 229 | * @param {Webmention} webmention 230 | * @returns {string|undefined} 231 | */ 232 | export const getURL = (webmention) => { 233 | return ( 234 | webmention?.["data"]?.["url"] || 235 | webmention["url"] || 236 | webmention["wm-source"] || 237 | webmention["source"] 238 | ); 239 | }; 240 | export const getWebmentionURL = getURL; 241 | 242 | /** 243 | * @param {Webmention} webmention 244 | * @returns {string|undefined} 245 | */ 246 | export const getTarget = (webmention) => { 247 | return webmention["wm-target"] || webmention["target"]; 248 | }; 249 | export const getWebmentionTarget = getTarget; 250 | 251 | /** 252 | * @param {Webmention} webmention 253 | * @returns {string|undefined} 254 | */ 255 | export const getType = (webmention) => { 256 | return ( 257 | webmention["wm-property"] || 258 | webmention?.["activity"]?.["type"] || 259 | webmention["type"] 260 | ); 261 | }; 262 | export const getWebmentionType = getType; 263 | 264 | /** 265 | * @param {Array} webmentions 266 | * @param {WebmentionType|Array} types 267 | * @returns {Array} 268 | */ 269 | export const getByTypes = (webmentions, types) => { 270 | return webmentions.filter((webmention) => { 271 | if (typeof types === "string") { 272 | return types === getType(webmention); 273 | } 274 | return types.includes(getType(webmention)); 275 | }); 276 | }; 277 | export const getByType = getByTypes; 278 | export const getWebmentionsByTypes = getByTypes; 279 | export const getWebmentionsByType = getByTypes; 280 | 281 | /** 282 | * @param {Array} webmentions 283 | * @param {Array} blocklist 284 | * @returns {Array} 285 | */ 286 | export const processBlocklist = (webmentions, blocklist) => { 287 | return webmentions.filter((webmention) => { 288 | let url = getSource(webmention); 289 | let source = getSource(webmention); 290 | for (let blocklistURL of blocklist) { 291 | if ( 292 | url.includes(blocklistURL.replace(/\/?$/, "/")) || 293 | source.includes(blocklistURL.replace(/\/?$/, "/")) 294 | ) { 295 | return false; 296 | } 297 | } 298 | return true; 299 | }); 300 | }; 301 | export const processWebmentionBlocklist = processBlocklist; 302 | export const processWebmentionsBlocklist = processBlocklist; 303 | 304 | /** 305 | * @param {Array} webmentions 306 | * @param {Array} allowlist 307 | * @returns {Array} 308 | */ 309 | export const processAllowlist = (webmentions, allowlist) => { 310 | return webmentions.filter((webmention) => { 311 | let url = getSource(webmention); 312 | let source = getSource(webmention); 313 | for (let allowlistURL of allowlist) { 314 | if ( 315 | url.includes(allowlistURL.replace(/\/?$/, "/")) || 316 | source.includes(allowlistURL.replace(/\/?$/, "/")) 317 | ) { 318 | return true; 319 | } 320 | } 321 | return false; 322 | }); 323 | }; 324 | export const processWebmentionAllowlist = processAllowlist; 325 | export const processWebmentionsAllowlist = processAllowlist; 326 | 327 | /** 328 | * @param {Options} options 329 | * @param {Array} webmentions 330 | * @param {string} url 331 | * @returns {Promise<{found: number, webmentions: Array}>} 332 | */ 333 | export const fetchWebmentions = async (options, webmentions, url) => { 334 | return await fetch(url) 335 | .then(async (response) => { 336 | if (!response.ok) { 337 | return Promise.reject(response); 338 | } 339 | 340 | const feed = await response.json(); 341 | 342 | if (!(options.key in feed)) { 343 | console.log( 344 | `${styleText("grey", `[${hostname(options.domain)}]`)} ${ 345 | options.key 346 | } was not found as a key in the response from ${styleText( 347 | "bold", 348 | hostname(options.feed), 349 | )}!`, 350 | ); 351 | return Promise.reject(response); 352 | } 353 | 354 | // Combine newly-fetched Webmentions with cached Webmentions 355 | webmentions = feed[options.key].concat(webmentions); 356 | // Remove duplicates by source URL 357 | webmentions = removeDuplicates(webmentions); 358 | // Process the blocklist, if it has any entries 359 | if (options.blocklist.length) { 360 | webmentions = processBlocklist(webmentions, options.blocklist); 361 | } 362 | // Process the allowlist, if it has any entries 363 | if (options.allowlist.length) { 364 | webmentions = processAllowlist(webmentions, options.allowlist); 365 | } 366 | // Sort webmentions by received date for getting most recent Webmention on subsequent requests 367 | webmentions = webmentions.sort((a, b) => { 368 | return epoch(getReceived(b)) - epoch(getReceived(a)); 369 | }); 370 | 371 | return { 372 | found: feed[options.key].length, 373 | webmentions: webmentions, 374 | }; 375 | }) 376 | .catch((error) => { 377 | console.warn( 378 | `${styleText( 379 | "grey", 380 | `[${hostname(options.domain)}]`, 381 | )} Something went wrong with your Webmention request to ${styleText( 382 | "bold", 383 | hostname(options.feed), 384 | )}!`, 385 | ); 386 | console.warn(error instanceof Error ? error.message : error); 387 | 388 | return { 389 | found: 0, 390 | webmentions: webmentions, 391 | }; 392 | }); 393 | }; 394 | 395 | /** 396 | * @param {Options} options 397 | * @returns {Promise>} 398 | */ 399 | export const retrieveWebmentions = async (options) => { 400 | if (!options.domain) { 401 | throw new Error( 402 | "`domain` is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.", 403 | ); 404 | } 405 | 406 | if (!options.feed) { 407 | throw new Error( 408 | "`feed` is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.", 409 | ); 410 | } 411 | 412 | if (!options.key) { 413 | throw new Error( 414 | "`key` is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.", 415 | ); 416 | } 417 | 418 | let asset = new AssetCache( 419 | options.uniqueKey || `webmentions-${hostname(options.domain)}`, 420 | options.cacheDirectory, 421 | ); 422 | 423 | let webmentions = []; 424 | 425 | // Unless specifically getting fresh Webmentions, if there is a cached file 426 | // at all, grab its contents now 427 | if (asset.isCacheValid("9001y") && !options.refresh) { 428 | webmentions = await asset.getCachedValue(); 429 | } 430 | 431 | // Get the number of cached Webmentions for diffing against fetched 432 | // Webmentions later 433 | const webmentionsCachedLength = webmentions.length; 434 | 435 | // If there is a cached file but it is outside of expiry, fetch fresh 436 | // results since the most recent Webmention 437 | if (!asset.isCacheValid(options.refresh ? "0s" : options.duration)) { 438 | const performanceStart = process.hrtime(); 439 | // Get the received date of the most recent Webmention, if it exists 440 | const since = webmentions.length ? getReceived(webmentions[0]) : false; 441 | // Build the URL for the fetch request 442 | const url = `${options.feed}${ 443 | since 444 | ? `${options.feed.includes("?") ? "&" : "?"}since=${since}` 445 | : "" 446 | }`; 447 | 448 | // If using webmention.io, loop through pages until no results found 449 | if (url.includes("https://webmention.io")) { 450 | const urlObject = new URL(url); 451 | const perPage = 452 | Number(urlObject.searchParams.get("per-page")) || 1000; 453 | urlObject.searchParams.delete("per-page"); 454 | // Start on page 0, to increment per subsequent request 455 | let page = 0; 456 | // Loop until a break condition is hit 457 | while (true) { 458 | const urlPaginated = 459 | urlObject.href + `&per-page=${perPage}&page=${page}`; 460 | const fetched = await fetchWebmentions( 461 | options, 462 | webmentions, 463 | urlPaginated, 464 | ); 465 | 466 | // An error occurred during fetching paged results → break 467 | if (!fetched && !fetched.found && !fetched.webmentions) { 468 | break; 469 | } 470 | 471 | // Page has no Webmentions → break 472 | if (fetched.found === 0) { 473 | break; 474 | } 475 | 476 | webmentions = fetched.webmentions; 477 | 478 | // If there are less Webmentions found than should be in each 479 | // page → break 480 | if (fetched.found < perPage) { 481 | break; 482 | } 483 | 484 | // Increment page 485 | page += 1; 486 | // Throttle next request 487 | await new Promise((resolve) => setTimeout(resolve, 1000)); 488 | } 489 | } else { 490 | const fetched = await fetchWebmentions(options, webmentions, url); 491 | webmentions = fetched.webmentions; 492 | } 493 | 494 | // Process the blocklist, if it has any entries 495 | if (options.blocklist.length) { 496 | webmentions = processBlocklist(webmentions, options.blocklist); 497 | } 498 | 499 | // Process the allowlist, if it has any entries 500 | if (options.allowlist.length) { 501 | webmentions = processAllowlist(webmentions, options.allowlist); 502 | } 503 | 504 | await asset.save(webmentions, "json"); 505 | 506 | const performance = process.hrtime(performanceStart); 507 | 508 | // Add a console message with the number of fetched and processed Webmentions, if any 509 | if (webmentionsCachedLength < webmentions.length) { 510 | console.log( 511 | `${styleText( 512 | "grey", 513 | `[${hostname(options.domain)}]`, 514 | )} ${styleText( 515 | "bold", 516 | String(webmentions.length - webmentionsCachedLength), 517 | )} new Webmentions fetched into cache in ${styleText( 518 | "bold", 519 | (performance[0] + performance[1] / 1e9).toFixed(3) + 520 | " seconds", 521 | )}.`, 522 | ); 523 | } 524 | } 525 | 526 | return webmentions; 527 | }; 528 | 529 | /** @type {Array} */ 530 | const WEBMENTIONS = {}; 531 | 532 | /** 533 | * @param {Options} options 534 | * @returns {Promise<{[key: string]: Array}>} 535 | */ 536 | export const webmentionsByURL = async (options) => { 537 | if (Object.keys(WEBMENTIONS).length) { 538 | return WEBMENTIONS; 539 | } 540 | 541 | let rawWebmentions = await retrieveWebmentions(options); 542 | 543 | // Fix local URLs based on urlReplacements and sort Webmentions into groups 544 | // by target base URL 545 | rawWebmentions.forEach((webmention) => { 546 | let url = baseURL( 547 | fixURL( 548 | getTarget(webmention).replace(/\/?$/, "/"), 549 | options.urlReplacements, 550 | ), 551 | ); 552 | 553 | if (!WEBMENTIONS[url]) { 554 | WEBMENTIONS[url] = []; 555 | } 556 | 557 | WEBMENTIONS[url].push(webmention); 558 | }); 559 | 560 | return WEBMENTIONS; 561 | }; 562 | export const webmentionsByUrl = webmentionsByURL; 563 | export const filteredWebmentions = webmentionsByURL; 564 | 565 | /** 566 | * @param {Options} options 567 | * @param {string} url 568 | * @param {WebmentionType|Array} [types] 569 | * @returns {Promise>} 570 | */ 571 | export const getWebmentions = async (options, url, types = []) => { 572 | const webmentions = await webmentionsByURL(options); 573 | url = absoluteURL(url, options.domain); 574 | 575 | if (!url || !webmentions || !webmentions[url]) { 576 | return []; 577 | } 578 | 579 | return ( 580 | webmentions[url] 581 | // Filter webmentions by allowed response post types 582 | .filter((entry) => { 583 | return typeof types === "object" && Object.keys(types).length 584 | ? types.includes(getType(entry)) 585 | : typeof types === "string" 586 | ? types === getType(entry) 587 | : true; 588 | }) 589 | // Sanitize content of webmentions against HTML limit 590 | .map((entry) => { 591 | const html = getContent(entry); 592 | 593 | if (html.length) { 594 | entry.contentSanitized = sanitizeHTML( 595 | html, 596 | options.allowedHTML, 597 | ); 598 | if (html.length > options.maximumHtmlLength) { 599 | entry.contentSanitized = `${ 600 | options.maximumHtmlText 601 | } ${getSource( 602 | entry, 603 | )}`; 604 | } 605 | } 606 | 607 | return entry; 608 | }) 609 | // Sort by published 610 | .sort((a, b) => { 611 | return epoch(getPublished(a)) - epoch(getPublished(b)); 612 | }) 613 | ); 614 | }; 615 | 616 | /** 617 | * @param {object} eleventyConfig 618 | * @param {Options} [options] 619 | */ 620 | export const eleventyCacheWebmentions = async ( 621 | eleventyConfig, 622 | options = {}, 623 | ) => { 624 | options = Object.assign(defaults, options); 625 | 626 | const byURL = await webmentionsByURL(options); 627 | const all = Object.values(byURL).reduce( 628 | (array, webmentions) => [...array, ...webmentions], 629 | [], 630 | ); 631 | 632 | // Global Data 633 | eleventyConfig.addGlobalData("webmentionsDefaults", defaults); 634 | eleventyConfig.addGlobalData("webmentionsOptions", options); 635 | eleventyConfig.addGlobalData("webmentionsByURL", byURL); 636 | eleventyConfig.addGlobalData("webmentionsByUrl", byURL); 637 | eleventyConfig.addGlobalData("webmentionsAll", all); 638 | 639 | // Liquid Filters 640 | eleventyConfig.addLiquidFilter("getWebmentionsByType", getByTypes); 641 | eleventyConfig.addLiquidFilter("getWebmentionsByTypes", getByTypes); 642 | eleventyConfig.addLiquidFilter("getWebmentionPublished", getPublished); 643 | eleventyConfig.addLiquidFilter("getWebmentionReceived", getReceived); 644 | eleventyConfig.addLiquidFilter("getWebmentionContent", getContent); 645 | eleventyConfig.addLiquidFilter("getWebmentionSource", getSource); 646 | eleventyConfig.addLiquidFilter("getWebmentionURL", getURL); 647 | eleventyConfig.addLiquidFilter("getWebmentionTarget", getTarget); 648 | eleventyConfig.addLiquidFilter("getWebmentionType", getType); 649 | 650 | // Nunjucks Filters 651 | eleventyConfig.addNunjucksFilter("getWebmentionsByType", getByTypes); 652 | eleventyConfig.addNunjucksFilter("getWebmentionsByTypes", getByTypes); 653 | eleventyConfig.addNunjucksFilter("getWebmentionPublished", getPublished); 654 | eleventyConfig.addNunjucksFilter("getWebmentionReceived", getReceived); 655 | eleventyConfig.addNunjucksFilter("getWebmentionContent", getContent); 656 | eleventyConfig.addNunjucksFilter("getWebmentionSource", getSource); 657 | eleventyConfig.addNunjucksFilter("getWebmentionURL", getURL); 658 | eleventyConfig.addNunjucksFilter("getWebmentionTarget", getTarget); 659 | eleventyConfig.addNunjucksFilter("getWebmentionType", getType); 660 | }; 661 | 662 | export default eleventyCacheWebmentions; 663 | -------------------------------------------------------------------------------- /eleventy-cache-webmentions.cjs: -------------------------------------------------------------------------------- 1 | const { AssetCache } = require("@11ty/eleventy-fetch"); 2 | const { styleText } = require("node:util"); 3 | const sanitizeHTML = require("sanitize-html"); 4 | 5 | /** 6 | * @typedef {sanitizeHTML.IOptions} AllowedHTML 7 | */ 8 | 9 | /** 10 | * @typedef {object} OptionsDefaults 11 | * @property {boolean} refresh 12 | * @property {string} duration 13 | * @property {string} uniqueKey 14 | * @property {string} [cacheDirectory] 15 | * @property {AllowedHTML} allowedHTML 16 | * @property {Array} allowlist 17 | * @property {Array} blocklist 18 | * @property {{[key: string]: string}} urlReplacements 19 | * @property {number} maximumHtmlLength 20 | * @property {string} maximumHtmlText 21 | */ 22 | const defaults = { 23 | refresh: false, 24 | duration: "1d", 25 | uniqueKey: "webmentions", 26 | cacheDirectory: undefined, 27 | allowedHTML: { 28 | allowedTags: ["a", "b", "em", "i", "strong"], 29 | allowedAttributes: { 30 | a: ["href"], 31 | }, 32 | }, 33 | allowlist: [], 34 | blocklist: [], 35 | urlReplacements: {}, 36 | maximumHtmlLength: 1000, 37 | maximumHtmlText: "mentioned this in", 38 | }; 39 | 40 | /** 41 | * @typedef {object} OptionsUserInput 42 | * @property {string} domain 43 | * @property {string} feed 44 | * @property {string} key 45 | */ 46 | 47 | /** 48 | * @typedef {OptionsDefaults & OptionsUserInput} Options 49 | */ 50 | 51 | /** 52 | * @typedef {object} Webmention 53 | * @property {string} [source] 54 | * @property {string} [url] 55 | * @property {string} [target] 56 | * @property {string} [published] 57 | * @property {string} [contentSanitized] 58 | * @property {object} [content] 59 | * @property {string} [content.html] 60 | * @property {string} [content.value] 61 | * @property {object} [data] 62 | * @property {string} [data.title] 63 | * @property {string} [data.url] 64 | * @property {string} [data.published] 65 | * @property {string} [data.content] 66 | * @property {string} [type] 67 | * @property {object} [activity] 68 | * @property {string} [activity.type] 69 | * @property {boolean} [verified] 70 | * @property {string} ["wm-property"] 71 | * @property {string} ["wm-received"] 72 | * @property {string} ["wm-source"] 73 | * @property {string} ["wm-target"] 74 | * @property {string} ["verified_date"] 75 | */ 76 | 77 | /** 78 | * @typedef {"bookmark-of"|"like-of"|"repost-of"|"mention-of"|"in-reply-to"} WebmentionType 79 | */ 80 | 81 | /** 82 | * @param {string} url 83 | * @param {string} domain 84 | * @returns {string} 85 | */ 86 | const absoluteURL = (url, domain) => { 87 | try { 88 | return new URL(url, domain).toString(); 89 | } catch (error) { 90 | console.error( 91 | `Trying to convert ${styleText( 92 | "bold", 93 | url, 94 | )} to be an absolute url with base ${styleText( 95 | "bold", 96 | domain, 97 | )} and failed.`, 98 | error, 99 | ); 100 | return url; 101 | } 102 | }; 103 | 104 | /** 105 | * @param {string} url 106 | * @returns {string} 107 | */ 108 | const baseURL = (url) => { 109 | let hashSplit = url.split("#"); 110 | let queryparamSplit = hashSplit[0].split("?"); 111 | return queryparamSplit[0]; 112 | }; 113 | 114 | /** 115 | * @param {string} url 116 | * @param {{[key: string], string}} [urlReplacements] 117 | * @returns {string} 118 | */ 119 | const fixURL = (url, urlReplacements) => { 120 | return Object.entries(urlReplacements).reduce( 121 | (accumulator, [key, value]) => { 122 | const regex = new RegExp(key, "g"); 123 | return accumulator.replace(regex, value); 124 | }, 125 | url, 126 | ); 127 | }; 128 | 129 | /** 130 | * @param {string} url 131 | * @returns {string} 132 | */ 133 | const hostname = (url) => { 134 | if (typeof url === "string" && url.includes("//")) { 135 | const urlObject = new URL(url); 136 | return urlObject.hostname; 137 | } 138 | return url; 139 | }; 140 | 141 | /** 142 | * @param {string|number|Date} date 143 | * @returns {number} 144 | */ 145 | const epoch = (date) => { 146 | return new Date(date).getTime(); 147 | }; 148 | 149 | /** 150 | * @param {Array} webmentions 151 | * @returns {Array} 152 | */ 153 | const removeDuplicates = (webmentions) => { 154 | return [ 155 | ...webmentions 156 | .reduce((map, webmention) => { 157 | const key = 158 | webmention === null || webmention === undefined 159 | ? webmention 160 | : getSource(webmention); 161 | if (!map.has(key)) { 162 | map.set(key, webmention); 163 | } 164 | return map; 165 | }, new Map()) 166 | .values(), 167 | ]; 168 | }; 169 | 170 | /** 171 | * @param {Webmention} webmention 172 | * @returns {string|undefined} 173 | */ 174 | const getPublished = (webmention) => { 175 | return ( 176 | webmention?.["data"]?.["published"] || 177 | webmention["published"] || 178 | webmention["wm-received"] || 179 | webmention["verified_date"] 180 | ); 181 | }; 182 | 183 | /** 184 | * @param {Webmention} webmention 185 | * @returns {string|undefined} 186 | */ 187 | const getReceived = (webmention) => { 188 | return ( 189 | webmention["wm-received"] || 190 | webmention["verified_date"] || 191 | webmention["published"] || 192 | webmention?.["data"]?.["published"] 193 | ); 194 | }; 195 | 196 | /** 197 | * @param {Webmention} webmention 198 | * @returns {string} 199 | */ 200 | const getContent = (webmention) => { 201 | return ( 202 | webmention?.["contentSanitized"] || 203 | webmention?.["content"]?.["html"] || 204 | webmention?.["content"]?.["value"] || 205 | webmention?.["content"] || 206 | webmention?.["data"]?.["content"] || 207 | "" 208 | ); 209 | }; 210 | 211 | /** 212 | * @param {Webmention} webmention 213 | * @returns {string|undefined} 214 | */ 215 | const getSource = (webmention) => { 216 | return ( 217 | webmention["wm-source"] || 218 | webmention["source"] || 219 | webmention?.["data"]?.["url"] || 220 | webmention["url"] 221 | ); 222 | }; 223 | 224 | /** 225 | * @param {Webmention} webmention 226 | * @returns {string|undefined} 227 | */ 228 | const getURL = (webmention) => { 229 | return ( 230 | webmention?.["data"]?.["url"] || 231 | webmention["url"] || 232 | webmention["wm-source"] || 233 | webmention["source"] 234 | ); 235 | }; 236 | 237 | /** 238 | * @param {Webmention} webmention 239 | * @returns {string|undefined} 240 | */ 241 | const getTarget = (webmention) => { 242 | return webmention["wm-target"] || webmention["target"]; 243 | }; 244 | 245 | /** 246 | * @param {Webmention} webmention 247 | * @returns {string|undefined} 248 | */ 249 | const getType = (webmention) => { 250 | return ( 251 | webmention["wm-property"] || 252 | webmention?.["activity"]?.["type"] || 253 | webmention["type"] 254 | ); 255 | }; 256 | 257 | /** 258 | * @param {Array} webmentions 259 | * @param {WebmentionType|Array} types 260 | * @returns {Array} 261 | */ 262 | const getByTypes = (webmentions, types) => { 263 | return webmentions.filter((webmention) => { 264 | if (typeof types === "string") { 265 | return types === getType(webmention); 266 | } 267 | return types.includes(getType(webmention)); 268 | }); 269 | }; 270 | 271 | /** 272 | * @param {Array} webmentions 273 | * @param {Array} blocklist 274 | * @returns {Array} 275 | */ 276 | const processBlocklist = (webmentions, blocklist) => { 277 | return webmentions.filter((webmention) => { 278 | let url = getSource(webmention); 279 | let source = getSource(webmention); 280 | for (let blocklistURL of blocklist) { 281 | if ( 282 | url.includes(blocklistURL.replace(/\/?$/, "/")) || 283 | source.includes(blocklistURL.replace(/\/?$/, "/")) 284 | ) { 285 | return false; 286 | } 287 | } 288 | return true; 289 | }); 290 | }; 291 | 292 | /** 293 | * @param {Array} webmentions 294 | * @param {Array} allowlist 295 | * @returns {Array} 296 | */ 297 | const processAllowlist = (webmentions, allowlist) => { 298 | return webmentions.filter((webmention) => { 299 | let url = getSource(webmention); 300 | let source = getSource(webmention); 301 | for (let allowlistURL of allowlist) { 302 | if ( 303 | url.includes(allowlistURL.replace(/\/?$/, "/")) || 304 | source.includes(allowlistURL.replace(/\/?$/, "/")) 305 | ) { 306 | return true; 307 | } 308 | } 309 | return false; 310 | }); 311 | }; 312 | 313 | /** 314 | * @param {Options} options 315 | * @param {Array} webmentions 316 | * @param {string} url 317 | * @returns {Promise<{found: number, webmentions: Array}>} 318 | */ 319 | const fetchWebmentions = async (options, webmentions, url) => { 320 | return await fetch(url) 321 | .then(async (response) => { 322 | if (!response.ok) { 323 | return Promise.reject(response); 324 | } 325 | 326 | const feed = await response.json(); 327 | 328 | if (!(options.key in feed)) { 329 | console.log( 330 | `${styleText("grey", `[${hostname(options.domain)}]`)} ${ 331 | options.key 332 | } was not found as a key in the response from ${styleText( 333 | "bold", 334 | hostname(options.feed), 335 | )}!`, 336 | ); 337 | return Promise.reject(response); 338 | } 339 | 340 | // Combine newly-fetched Webmentions with cached Webmentions 341 | webmentions = feed[options.key].concat(webmentions); 342 | // Remove duplicates by source URL 343 | webmentions = removeDuplicates(webmentions); 344 | // Process the blocklist, if it has any entries 345 | if (options.blocklist.length) { 346 | webmentions = processBlocklist(webmentions, options.blocklist); 347 | } 348 | // Process the allowlist, if it has any entries 349 | if (options.allowlist.length) { 350 | webmentions = processAllowlist(webmentions, options.allowlist); 351 | } 352 | // Sort webmentions by received date for getting most recent Webmention on subsequent requests 353 | webmentions = webmentions.sort((a, b) => { 354 | return epoch(getReceived(b)) - epoch(getReceived(a)); 355 | }); 356 | 357 | return { 358 | found: feed[options.key].length, 359 | webmentions: webmentions, 360 | }; 361 | }) 362 | .catch((error) => { 363 | console.warn( 364 | `${styleText( 365 | "grey", 366 | `[${hostname(options.domain)}]`, 367 | )} Something went wrong with your Webmention request to ${styleText( 368 | "bold", 369 | hostname(options.feed), 370 | )}!`, 371 | ); 372 | console.warn(error instanceof Error ? error.message : error); 373 | 374 | return { 375 | found: 0, 376 | webmentions: webmentions, 377 | }; 378 | }); 379 | }; 380 | 381 | /** 382 | * @param {Options} options 383 | * @returns {Promise>} 384 | */ 385 | const retrieveWebmentions = async (options) => { 386 | if (!options.domain) { 387 | throw new Error( 388 | "`domain` is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.", 389 | ); 390 | } 391 | 392 | if (!options.feed) { 393 | throw new Error( 394 | "`feed` is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.", 395 | ); 396 | } 397 | 398 | if (!options.key) { 399 | throw new Error( 400 | "`key` is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.", 401 | ); 402 | } 403 | 404 | let asset = new AssetCache( 405 | options.uniqueKey || `webmentions-${hostname(options.domain)}`, 406 | options.cacheDirectory, 407 | ); 408 | 409 | let webmentions = []; 410 | 411 | // Unless specifically getting fresh Webmentions, if there is a cached file 412 | // at all, grab its contents now 413 | if (asset.isCacheValid("9001y") && !options.refresh) { 414 | webmentions = await asset.getCachedValue(); 415 | } 416 | 417 | // Get the number of cached Webmentions for diffing against fetched 418 | // Webmentions later 419 | const webmentionsCachedLength = webmentions.length; 420 | 421 | // If there is a cached file but it is outside of expiry, fetch fresh 422 | // results since the most recent Webmention 423 | if (!asset.isCacheValid(options.refresh ? "0s" : options.duration)) { 424 | const performanceStart = process.hrtime(); 425 | // Get the received date of the most recent Webmention, if it exists 426 | const since = webmentions.length ? getReceived(webmentions[0]) : false; 427 | // Build the URL for the fetch request 428 | const url = `${options.feed}${ 429 | since 430 | ? `${options.feed.includes("?") ? "&" : "?"}since=${since}` 431 | : "" 432 | }`; 433 | 434 | // If using webmention.io, loop through pages until no results found 435 | if (url.includes("https://webmention.io")) { 436 | const urlObject = new URL(url); 437 | const perPage = 438 | Number(urlObject.searchParams.get("per-page")) || 1000; 439 | urlObject.searchParams.delete("per-page"); 440 | // Start on page 0, to increment per subsequent request 441 | let page = 0; 442 | // Loop until a break condition is hit 443 | while (true) { 444 | const urlPaginated = 445 | urlObject.href + `&per-page=${perPage}&page=${page}`; 446 | const fetched = await fetchWebmentions( 447 | options, 448 | webmentions, 449 | urlPaginated, 450 | ); 451 | 452 | // An error occurred during fetching paged results → break 453 | if (!fetched && !fetched.found && !fetched.webmentions) { 454 | break; 455 | } 456 | 457 | // Page has no Webmentions → break 458 | if (fetched.found === 0) { 459 | break; 460 | } 461 | 462 | webmentions = fetched.webmentions; 463 | 464 | // If there are less Webmentions found than should be in each 465 | // page → break 466 | if (fetched.found < perPage) { 467 | break; 468 | } 469 | 470 | // Increment page 471 | page += 1; 472 | // Throttle next request 473 | await new Promise((resolve) => setTimeout(resolve, 1000)); 474 | } 475 | } else { 476 | const fetched = await fetchWebmentions(options, webmentions, url); 477 | webmentions = fetched.webmentions; 478 | } 479 | 480 | // Process the blocklist, if it has any entries 481 | if (options.blocklist.length) { 482 | webmentions = processBlocklist(webmentions, options.blocklist); 483 | } 484 | 485 | // Process the allowlist, if it has any entries 486 | if (options.allowlist.length) { 487 | webmentions = processAllowlist(webmentions, options.allowlist); 488 | } 489 | 490 | await asset.save(webmentions, "json"); 491 | 492 | const performance = process.hrtime(performanceStart); 493 | 494 | // Add a console message with the number of fetched and processed Webmentions, if any 495 | if (webmentionsCachedLength < webmentions.length) { 496 | console.log( 497 | `${styleText( 498 | "grey", 499 | `[${hostname(options.domain)}]`, 500 | )} ${styleText( 501 | "bold", 502 | String(webmentions.length - webmentionsCachedLength), 503 | )} new Webmentions fetched into cache in ${styleText( 504 | "bold", 505 | (performance[0] + performance[1] / 1e9).toFixed(3) + 506 | " seconds", 507 | )}.`, 508 | ); 509 | } 510 | } 511 | 512 | return webmentions; 513 | }; 514 | 515 | /** @type {Array} */ 516 | const WEBMENTIONS = {}; 517 | 518 | /** 519 | * @param {Options} options 520 | * @returns {Promise<{[key: string]: Array}>} 521 | */ 522 | const webmentionsByURL = async (options) => { 523 | if (Object.keys(WEBMENTIONS).length) { 524 | return WEBMENTIONS; 525 | } 526 | 527 | let rawWebmentions = await retrieveWebmentions(options); 528 | 529 | // Fix local URLs based on urlReplacements and sort Webmentions into groups 530 | // by target base URL 531 | rawWebmentions.forEach((webmention) => { 532 | let url = baseURL( 533 | fixURL( 534 | getTarget(webmention).replace(/\/?$/, "/"), 535 | options.urlReplacements, 536 | ), 537 | ); 538 | 539 | if (!WEBMENTIONS[url]) { 540 | WEBMENTIONS[url] = []; 541 | } 542 | 543 | WEBMENTIONS[url].push(webmention); 544 | }); 545 | 546 | return WEBMENTIONS; 547 | }; 548 | 549 | /** 550 | * @param {Options} options 551 | * @param {string} url 552 | * @param {WebmentionType|Array} [types] 553 | * @returns {Promise>} 554 | */ 555 | const getWebmentions = async (options, url, types = []) => { 556 | const webmentions = await webmentionsByURL(options); 557 | url = absoluteURL(url, options.domain); 558 | 559 | if (!url || !webmentions || !webmentions[url]) { 560 | return []; 561 | } 562 | 563 | return ( 564 | webmentions[url] 565 | // Filter webmentions by allowed response post types 566 | .filter((entry) => { 567 | return typeof types === "object" && Object.keys(types).length 568 | ? types.includes(getType(entry)) 569 | : typeof types === "string" 570 | ? types === getType(entry) 571 | : true; 572 | }) 573 | // Sanitize content of webmentions against HTML limit 574 | .map((entry) => { 575 | const html = getContent(entry); 576 | 577 | if (html.length) { 578 | entry.contentSanitized = sanitizeHTML( 579 | html, 580 | options.allowedHTML, 581 | ); 582 | if (html.length > options.maximumHtmlLength) { 583 | entry.contentSanitized = `${ 584 | options.maximumHtmlText 585 | } ${getSource( 586 | entry, 587 | )}`; 588 | } 589 | } 590 | 591 | return entry; 592 | }) 593 | // Sort by published 594 | .sort((a, b) => { 595 | return epoch(getPublished(a)) - epoch(getPublished(b)); 596 | }) 597 | ); 598 | }; 599 | 600 | /** 601 | * @param {object} eleventyConfig 602 | * @param {Options} [options] 603 | */ 604 | const eleventyCacheWebmentions = async (eleventyConfig, options = {}) => { 605 | options = Object.assign(defaults, options); 606 | 607 | const byURL = await webmentionsByURL(options); 608 | const all = Object.values(byURL).reduce( 609 | (array, webmentions) => [...array, ...webmentions], 610 | [], 611 | ); 612 | 613 | // Global Data 614 | eleventyConfig.addGlobalData("webmentionsDefaults", defaults); 615 | eleventyConfig.addGlobalData("webmentionsOptions", options); 616 | eleventyConfig.addGlobalData("webmentionsByURL", byURL); 617 | eleventyConfig.addGlobalData("webmentionsByUrl", byURL); 618 | eleventyConfig.addGlobalData("webmentionsAll", all); 619 | 620 | // Liquid Filters 621 | eleventyConfig.addLiquidFilter("getWebmentionsByType", getByTypes); 622 | eleventyConfig.addLiquidFilter("getWebmentionsByTypes", getByTypes); 623 | eleventyConfig.addLiquidFilter("getWebmentionPublished", getPublished); 624 | eleventyConfig.addLiquidFilter("getWebmentionReceived", getReceived); 625 | eleventyConfig.addLiquidFilter("getWebmentionContent", getContent); 626 | eleventyConfig.addLiquidFilter("getWebmentionSource", getSource); 627 | eleventyConfig.addLiquidFilter("getWebmentionURL", getURL); 628 | eleventyConfig.addLiquidFilter("getWebmentionTarget", getTarget); 629 | eleventyConfig.addLiquidFilter("getWebmentionType", getType); 630 | 631 | // Nunjucks Filters 632 | eleventyConfig.addNunjucksFilter("getWebmentionsByType", getByTypes); 633 | eleventyConfig.addNunjucksFilter("getWebmentionsByTypes", getByTypes); 634 | eleventyConfig.addNunjucksFilter("getWebmentionPublished", getPublished); 635 | eleventyConfig.addNunjucksFilter("getWebmentionReceived", getReceived); 636 | eleventyConfig.addNunjucksFilter("getWebmentionContent", getContent); 637 | eleventyConfig.addNunjucksFilter("getWebmentionSource", getSource); 638 | eleventyConfig.addNunjucksFilter("getWebmentionURL", getURL); 639 | eleventyConfig.addNunjucksFilter("getWebmentionTarget", getTarget); 640 | eleventyConfig.addNunjucksFilter("getWebmentionType", getType); 641 | }; 642 | 643 | module.exports = eleventyCacheWebmentions; 644 | module.exports.defaults = defaults; 645 | module.exports.getPublished = getPublished; 646 | module.exports.getWebmentionPublished = getPublished; 647 | module.exports.getReceived = getReceived; 648 | module.exports.getWebmentionReceived = getReceived; 649 | module.exports.getContent = getContent; 650 | module.exports.getWebmentionContent = getContent; 651 | module.exports.getSource = getSource; 652 | module.exports.getWebmentionSource = getSource; 653 | module.exports.getURL = getURL; 654 | module.exports.getWebmentionURL = getURL; 655 | module.exports.getTarget = getTarget; 656 | module.exports.getWebmentionTarget = getTarget; 657 | module.exports.getType = getType; 658 | module.exports.getWebmentionType = getType; 659 | module.exports.getByTypes = getByTypes; 660 | module.exports.getByType = getByTypes; 661 | module.exports.getWebmentionsByTypes = getByTypes; 662 | module.exports.getWebmentionsByType = getByTypes; 663 | module.exports.processBlocklist = processBlocklist; 664 | module.exports.processWebmentionBlocklist = processBlocklist; 665 | module.exports.processWebmentionsBlocklist = processBlocklist; 666 | module.exports.processAllowlist = processAllowlist; 667 | module.exports.processWebmentionAllowlist = processAllowlist; 668 | module.exports.processWebmentionsAllowlist = processAllowlist; 669 | module.exports.fetchWebmentions = fetchWebmentions; 670 | module.exports.retrieveWebmentions = retrieveWebmentions; 671 | module.exports.webmentionsByURL = webmentionsByURL; 672 | module.exports.webmentionsByUrl = webmentionsByURL; 673 | module.exports.filteredWebmentions = webmentionsByURL; 674 | module.exports.getWebmentions = getWebmentions; 675 | --------------------------------------------------------------------------------