├── .editorconfig ├── .github └── workflows │ ├── ci.yml │ └── release.yml ├── .gitignore ├── .husky └── pre-commit ├── .npmignore ├── LICENSE ├── README.md ├── cli.js ├── package-lock.json ├── package.json ├── src ├── DataSource.js ├── DataSource │ ├── Atom.js │ ├── BlueskyUser.js │ ├── FediverseUser.js │ ├── HostedWordPressApi.js │ ├── Rss.js │ ├── WordPressApi.js │ └── YouTubeUser.js ├── DirectoryManager.js ├── Fetcher.js ├── HtmlTransformer.js ├── Importer.js ├── Logger.js ├── MarkdownToHtml.js ├── Persist.js └── Utils.js └── test ├── html-entities-test.js ├── markdown-test.js ├── sources ├── blog-awesome-author.json ├── blog-awesome-categories.json ├── blog-awesome-posts.json ├── bluesky-test.xml └── youtube-user.xml └── test.js /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | indent_style = tab 5 | end_of_line = lf 6 | insert_final_newline = true 7 | trim_trailing_whitespace = true 8 | charset = utf-8 9 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Node Unit Tests 2 | on: 3 | push: 4 | branches-ignore: 5 | - "gh-pages" 6 | jobs: 7 | build: 8 | runs-on: ${{ matrix.os }} 9 | strategy: 10 | matrix: 11 | os: ["ubuntu-latest", "macos-latest", "windows-latest"] 12 | node: ["18", "20", "22"] 13 | name: Node.js ${{ matrix.node }} on ${{ matrix.os }} 14 | steps: 15 | - uses: actions/checkout@v4 16 | - name: Setup node 17 | uses: actions/setup-node@v4 18 | with: 19 | node-version: ${{ matrix.node }} 20 | # cache: npm 21 | - run: npm install 22 | - run: npm test 23 | env: 24 | YARN_GPG: no 25 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Publish Release to npm 2 | on: 3 | release: 4 | types: [published] 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | permissions: 9 | contents: read 10 | id-token: write 11 | steps: 12 | - uses: actions/checkout@v4 13 | - uses: actions/setup-node@v4 14 | with: 15 | node-version: "20" 16 | registry-url: 'https://registry.npmjs.org' 17 | - run: npm install 18 | - run: npm test 19 | - if: ${{ github.event.release.tag_name != '' && env.NPM_PUBLISH_TAG != '' }} 20 | run: npm publish --provenance --access=public --tag=${{ env.NPM_PUBLISH_TAG }} 21 | env: 22 | NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} 23 | NPM_PUBLISH_TAG: ${{ contains(github.event.release.tag_name, '-beta.') && 'beta' || 'latest' }} 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .cache 2 | node_modules 3 | .env 4 | 5 | # local 6 | dist* 7 | DEVLOG.md 8 | -------------------------------------------------------------------------------- /.husky/pre-commit: -------------------------------------------------------------------------------- 1 | npm test 2 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | test 2 | .github 3 | .cache 4 | .env 5 | .editorconfig 6 | dist* 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Zach Leatherman @zachleat 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # `@11ty/import` 2 | 3 | A small utility (and CLI) to import content files from various content sources. Requires Node 18 or newer. 4 | 5 | ## Features 6 | 7 | - **Compatible**: Works with a bunch of different data sources (see below) and more to come. 8 | - Export your entire WordPress site statically in a single command. Related [video on YouTube](https://www.youtube.com/watch?v=WuH5QYCdh6w) and [docs on 11ty.dev](https://www.11ty.dev/docs/migrate/wordpress/). 9 | - Show recent Bluesky or Mastodon posts on your own web site without an expensive third party embed component. 10 | - Make anything on the web into a CMS for your web site using [Indieweb PESOS](https://indieweb.org/PESOS). 11 | - **Clean**: Converts imported content to markdown files in your repository (`--format=html` to use raw HTML). 12 | - **Standalone**: downloads all referenced assets (images, videos, stylesheets, scripts, etc) in content and co-locates assets with the content. 13 | - **Resumable**: Can stop and resume a large import later, reusing a local cache (with configurable cache duration) 14 | - **Repeatable**: avoids overwriting existing content files (unless you opt-in with `--overwrite`). 15 | - This allows you to continue using an import source for new content while editing the already imported content. 16 | - Use `--dryrun` for testing without writing any files. 17 | 18 | ## Usage 19 | 20 | Published to [npm](https://www.npmjs.com/package/@11ty/import). These commands do not require separate installation. 21 | 22 | ```sh 23 | npx @11ty/import --help 24 | npx @11ty/import --version 25 | 26 | # Import content 27 | npx @11ty/import [type] [target] 28 | 29 | # Dry run (don’t write files) 30 | npx @11ty/import [type] [target] --dryrun 31 | 32 | # Quietly (limit console output) 33 | npx @11ty/import [type] [target] --quiet 34 | 35 | # Change the output folder (default: ".") 36 | npx @11ty/import [type] [target] --output=dist 37 | 38 | # Allow overwriting existing files 39 | npx @11ty/import [type] [target] --overwrite 40 | 41 | # Allow draft entries to overwrite existing files (bypasses --overwrite) 42 | npx @11ty/import [type] [target] --overwrite-allow=drafts 43 | 44 | # Change local fetch cache duration (default: 24h) 45 | npx @11ty/import [type] [target] --cacheduration=20m 46 | 47 | # Only import entries created (or updated) within a duration (default: *) 48 | # Same syntax as --cacheduration 49 | npx @11ty/import [type] [target] --within=7d 50 | 51 | # Change output format (default: markdown) 52 | npx @11ty/import [type] [target] --format=html 53 | 54 | # Change asset reference URLs: relative (default), absolute, colocate, disabled 55 | # slug.md and assets/asset.png with 56 | npx @11ty/import [type] [target] --assetrefs=relative 57 | # slug.md and assets/asset.png with 58 | npx @11ty/import [type] [target] --assetrefs=absolute 59 | # slug/index.md and slug/asset.png with 60 | npx @11ty/import [type] [target] --assetrefs=colocate 61 | # Don’t download any assets 62 | npx @11ty/import [type] [target] --assetrefs=disabled 63 | 64 | # EXPERIMENTAL: Persist *new* non-draft content 65 | # - `github` persist type requires a `GITHUB_TOKEN` environment variable. 66 | npx @11ty/import [type] [target] --persist=github:zachleat/wp-awesome 67 | ``` 68 | 69 | ### Service Types 70 | 71 | - `atom` (URL) 72 | - `bluesky` (username) 73 | - `fediverse` (username) 74 | - `rss` (URL) 75 | - `wordpress` (blog home page URL) 76 | - `youtubeuser` (user id) 77 | 78 | #### YouTube 79 | 80 | ```sh 81 | # Import recent YouTube Videos for one user 82 | npx @11ty/import youtubeuser UCskGTioqrMBcw8pd14_334A 83 | ``` 84 | 85 | #### WordPress 86 | 87 | ```sh 88 | # Import *all* posts from the WordPress API 89 | # Draft posts available when WORDPRESS_USERNAME and WORDPRESS_PASSWORD environment 90 | # variables are supplied, read more: https://www.11ty.dev/docs/environment-vars/ 91 | npx @11ty/import wordpress https://blog.fontawesome.com 92 | ``` 93 | 94 | #### Atom Feeds 95 | 96 | ```sh 97 | # Import Atom feed posts 98 | npx @11ty/import atom https://www.11ty.dev/blog/feed.xml 99 | 100 | # Import GitHub releases (via Atom) 101 | npx @11ty/import atom https://github.com/11ty/eleventy/releases.atom 102 | ``` 103 | 104 | #### RSS Feeds 105 | 106 | ```sh 107 | # Import RSS feed posts 108 | npx @11ty/import rss https://fosstodon.org/users/eleventy.rss 109 | ``` 110 | 111 | #### Fediverse 112 | 113 | ```sh 114 | # Import recent Mastodon posts (via RSS) 115 | npx @11ty/import fediverse eleventy@fosstodon.org 116 | ``` 117 | 118 | #### Bluesky 119 | 120 | ```sh 121 | # Import recent Bluesky posts (via RSS) 122 | npx @11ty/import bluesky @11ty.dev 123 | ``` 124 | 125 | ### Programmatic API 126 | 127 | Don’t forget to install this into your project: `npm install @11ty/import` 128 | 129 | ```js 130 | import { Importer } from "@11ty/import"; 131 | 132 | let importer = new Importer(); 133 | 134 | importer.setOutputFolder("."); // --output 135 | importer.setCacheDuration("24h"); // --cacheduration 136 | importer.setVerbose(true); // --quiet 137 | importer.setSafeMode(false); // --overwrite 138 | importer.setDryRun(false); // --dryrun 139 | importer.setDraftsFolder("drafts"); 140 | importer.setAssetsFolder("assets"); 141 | importer.setAssetReferenceType("relative"); // --assetrefs 142 | 143 | // Sources (one or more) 144 | importer.addSource("bluesky", "@11ty.dev"); 145 | 146 | // Simple CSS selector (class names only) for preserved elements in Markdown conversion 147 | importer.addPreserved(".save-this-class-name"); 148 | 149 | // Allow draft entries to overwrite (independent of Safe Mode value) 150 | importer.setOverwriteAllow("drafts"); 151 | 152 | let entries = await importer.getEntries({ 153 | contentType: "markdown", // --format 154 | within: "*", // date or last updated date must be within this recent duration (e.g. 24h, 7d, 1y) 155 | }); 156 | 157 | await importer.toFiles(entries); 158 | 159 | importer.logResults(); 160 | ``` 161 | -------------------------------------------------------------------------------- /cli.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | import { parseArgs } from "node:util"; 4 | 5 | import { Importer } from "./src/Importer.js"; 6 | import { Logger } from "./src/Logger.js"; 7 | import { createRequire } from "node:module"; 8 | 9 | // "string" or "boolean" types are supported by parseArgs 10 | // https://nodejs.org/docs/latest/api/util.html#utilparseargsconfig 11 | let { positionals, values } = parseArgs({ 12 | allowPositionals: true, 13 | strict: true, 14 | tokens: true, 15 | options: { 16 | type: { 17 | type: "string", 18 | }, 19 | target: { 20 | type: "string", 21 | }, 22 | output: { 23 | type: "string", 24 | default: ".", 25 | }, 26 | quiet: { 27 | type: "boolean", 28 | default: false, 29 | }, 30 | dryrun: { 31 | type: "boolean", 32 | default: false, 33 | }, 34 | help: { 35 | type: "boolean", 36 | default: false, 37 | }, 38 | version: { 39 | type: "boolean", 40 | default: false, 41 | }, 42 | overwrite: { 43 | type: "boolean", 44 | default: false, 45 | }, 46 | "overwrite-allow": { 47 | type: "string", 48 | default: "", 49 | }, 50 | cacheduration: { 51 | type: "string", 52 | default: "24h", 53 | }, 54 | format: { 55 | type: "string", 56 | default: "markdown", 57 | }, 58 | persist: { 59 | type: "string", 60 | default: "", 61 | }, 62 | assetrefs: { 63 | type: "string", 64 | default: "relative", 65 | }, 66 | within: { 67 | type: "string", 68 | default: "", 69 | }, 70 | preserve: { 71 | type: "string", 72 | default: "", 73 | } 74 | }, 75 | }); 76 | 77 | let [ type, target ] = positionals; 78 | let { quiet, dryrun, output, help, version, overwrite, cacheduration, format, persist, assetrefs, within, preserve } = values; 79 | 80 | if(version) { 81 | const require = createRequire(import.meta.url); 82 | let pkg = require("./package.json"); 83 | Logger.log(pkg.version); 84 | process.exit(); 85 | } 86 | 87 | // If you modify this, maybe also add the instructions to README.md too? 88 | if(help) { 89 | Logger.log(`Usage: 90 | 91 | npx @11ty/import --help 92 | npx @11ty/import --version 93 | 94 | # Import content 95 | npx @11ty/import [type] [target] 96 | 97 | # Dry run (don’t write files) 98 | npx @11ty/import [type] [target] --dryrun 99 | 100 | # Quietly (limit console output) 101 | npx @11ty/import [type] [target] --quiet 102 | 103 | # Change the output folder (default: ".") 104 | npx @11ty/import [type] [target] --output=dist 105 | 106 | # Allow overwriting existing files 107 | npx @11ty/import [type] [target] --overwrite 108 | 109 | # Allow draft entries to overwrite existing files (bypasses --overwrite) 110 | npx @11ty/import [type] [target] --overwrite-allow=drafts 111 | 112 | # Change local fetch cache duration (default: 24h) 113 | npx @11ty/import [type] [target] --cacheduration=20m 114 | 115 | # Change output format (default: markdown) 116 | npx @11ty/import [type] [target] --format=html 117 | 118 | # Change asset reference URLs: relative (default), absolute, colocate, disabled 119 | npx @11ty/import [type] [target] --assetrefs=relative 120 | npx @11ty/import [type] [target] --assetrefs=absolute 121 | npx @11ty/import [type] [target] --assetrefs=colocate 122 | npx @11ty/import [type] [target] --assetrefs=disabled 123 | `); 124 | 125 | process.exit(); 126 | } 127 | 128 | // Input checking 129 | if(!type || !target) { 130 | console.error("Expected usage: npx @11ty/import [type] [target]"); 131 | process.exit(1); 132 | } else if(format !== "markdown" && format !== "html") { 133 | console.error("Invalid --format, expected `markdown` or `html`"); 134 | process.exit(1); 135 | } 136 | 137 | let importer = new Importer(); 138 | 139 | importer.setOutputFolder(output); 140 | importer.setCacheDuration(cacheduration); 141 | importer.setVerbose(!quiet); 142 | importer.setSafeMode(!overwrite); 143 | importer.setDryRun(dryrun); 144 | importer.addSource(type, target); 145 | 146 | // TODO wire these up to CLI 147 | importer.setDraftsFolder("drafts"); 148 | importer.setAssetsFolder("assets"); 149 | importer.setAssetReferenceType(assetrefs); 150 | 151 | // CSS selectors to preserve on markdown conversion 152 | if(preserve) { 153 | importer.addPreserved(preserve); 154 | } 155 | 156 | if(persist) { 157 | importer.setPersistTarget(persist); 158 | } 159 | 160 | importer.setOverwriteAllow(values['overwrite-allow']); 161 | 162 | let entries = await importer.getEntries({ 163 | contentType: format, 164 | within, 165 | // Pperformance improvement to avoids parsing html and fetching assets for documents outside of --within (and overwrite rules) 166 | target: "fs", 167 | }); 168 | 169 | await importer.toFiles(entries); 170 | 171 | importer.logResults(); 172 | 173 | -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@11ty/import", 3 | "version": "1.0.15", 4 | "lockfileVersion": 3, 5 | "requires": true, 6 | "packages": { 7 | "": { 8 | "name": "@11ty/import", 9 | "version": "1.0.15", 10 | "license": "MIT", 11 | "dependencies": { 12 | "@11ty/eleventy-fetch": "^5.0.2", 13 | "@11ty/eleventy-utils": "^2.0.1", 14 | "@11ty/posthtml-urls": "^1.0.1", 15 | "@prettier/sync": "^0.5.5", 16 | "@sindresorhus/slugify": "^2.2.1", 17 | "dotenv": "^16.4.7", 18 | "entities": "^5.0.0", 19 | "fast-xml-parser": "^4.5.3", 20 | "filesize": "^10.1.6", 21 | "github-publish": "^6.0.0", 22 | "graceful-fs": "^4.2.11", 23 | "js-yaml": "^4.1.0", 24 | "kleur": "^4.1.5", 25 | "posthtml": "^0.16.6", 26 | "prettier": "^3.5.3", 27 | "striptags": "^3.2.0", 28 | "turndown": "github:zachleat/fork-turndown" 29 | }, 30 | "bin": { 31 | "eleventy-import": "cli.js" 32 | }, 33 | "devDependencies": { 34 | "husky": "^9.1.7" 35 | }, 36 | "engines": { 37 | "node": ">=18" 38 | }, 39 | "funding": { 40 | "type": "opencollective", 41 | "url": "https://opencollective.com/11ty" 42 | } 43 | }, 44 | "node_modules/@11ty/eleventy-fetch": { 45 | "version": "5.0.2", 46 | "resolved": "https://registry.npmjs.org/@11ty/eleventy-fetch/-/eleventy-fetch-5.0.2.tgz", 47 | "integrity": "sha512-yu7oZ5iv7zvFDawSYcN19cz7ddJB7OXPGZ47z/MzYmLa2LkpJm0KnZW2xGwpKvVrXd+tyb96ts6AqlkJT/ibwQ==", 48 | "license": "MIT", 49 | "dependencies": { 50 | "@rgrove/parse-xml": "^4.2.0", 51 | "debug": "^4.3.7", 52 | "flat-cache": "^6.1.1", 53 | "graceful-fs": "^4.2.11", 54 | "p-queue": "6.6.2" 55 | }, 56 | "engines": { 57 | "node": ">=18" 58 | }, 59 | "funding": { 60 | "type": "opencollective", 61 | "url": "https://opencollective.com/11ty" 62 | } 63 | }, 64 | "node_modules/@11ty/eleventy-utils": { 65 | "version": "2.0.1", 66 | "resolved": "https://registry.npmjs.org/@11ty/eleventy-utils/-/eleventy-utils-2.0.1.tgz", 67 | "integrity": "sha512-hicG0vPyqfLvgHJQLtoh3XAj6wUbLX4yY2se8bQLdhCIcxK46mt4zDpgcrYVP3Sjx4HPifQOdwRfOEECoUcyXQ==", 68 | "license": "MIT", 69 | "engines": { 70 | "node": ">=18" 71 | }, 72 | "funding": { 73 | "type": "opencollective", 74 | "url": "https://opencollective.com/11ty" 75 | } 76 | }, 77 | "node_modules/@11ty/posthtml-urls": { 78 | "version": "1.0.1", 79 | "resolved": "https://registry.npmjs.org/@11ty/posthtml-urls/-/posthtml-urls-1.0.1.tgz", 80 | "integrity": "sha512-6EFN/yYSxC/OzYXpq4gXDyDMlX/W+2MgCvvoxf11X1z76bqkqFJ8eep5RiBWfGT5j0323a1pwpelcJJdR46MCw==", 81 | "license": "MIT", 82 | "dependencies": { 83 | "evaluate-value": "^2.0.0", 84 | "http-equiv-refresh": "^2.0.1", 85 | "list-to-array": "^1.1.0", 86 | "parse-srcset": "^1.0.2" 87 | }, 88 | "engines": { 89 | "node": ">= 6" 90 | } 91 | }, 92 | "node_modules/@11ty/posthtml-urls/node_modules/http-equiv-refresh": { 93 | "version": "2.0.1", 94 | "resolved": "https://registry.npmjs.org/http-equiv-refresh/-/http-equiv-refresh-2.0.1.tgz", 95 | "integrity": "sha512-XJpDL/MLkV3dKwLzHwr2dY05dYNfBNlyPu4STQ8WvKCFdc6vC5tPXuq28of663+gHVg03C+16pHHs/+FmmDjcw==", 96 | "license": "MIT", 97 | "engines": { 98 | "node": ">= 6" 99 | } 100 | }, 101 | "node_modules/@keyv/serialize": { 102 | "version": "1.0.1", 103 | "resolved": "https://registry.npmjs.org/@keyv/serialize/-/serialize-1.0.1.tgz", 104 | "integrity": "sha512-kKXeynfORDGPUEEl2PvTExM2zs+IldC6ZD8jPcfvI351MDNtfMlw9V9s4XZXuJNDK2qR5gbEKxRyoYx3quHUVQ==", 105 | "license": "MIT", 106 | "dependencies": { 107 | "buffer": "^6.0.3" 108 | } 109 | }, 110 | "node_modules/@mixmark-io/domino": { 111 | "version": "2.2.0", 112 | "resolved": "https://registry.npmjs.org/@mixmark-io/domino/-/domino-2.2.0.tgz", 113 | "integrity": "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==", 114 | "license": "BSD-2-Clause" 115 | }, 116 | "node_modules/@prettier/sync": { 117 | "version": "0.5.5", 118 | "resolved": "https://registry.npmjs.org/@prettier/sync/-/sync-0.5.5.tgz", 119 | "integrity": "sha512-6BMtNr7aQhyNcGzmumkL0tgr1YQGfm9d7ZdmRpWqWuqpc9vZBind4xMe5NMiRECOhjuSiWHfBWLBnXkpeE90bw==", 120 | "license": "MIT", 121 | "dependencies": { 122 | "make-synchronized": "^0.4.2" 123 | }, 124 | "funding": { 125 | "url": "https://github.com/prettier/prettier-synchronized?sponsor=1" 126 | }, 127 | "peerDependencies": { 128 | "prettier": "*" 129 | } 130 | }, 131 | "node_modules/@rgrove/parse-xml": { 132 | "version": "4.2.0", 133 | "resolved": "https://registry.npmjs.org/@rgrove/parse-xml/-/parse-xml-4.2.0.tgz", 134 | "integrity": "sha512-UuBOt7BOsKVOkFXRe4Ypd/lADuNIfqJXv8GvHqtXaTYXPPKkj2nS2zPllVsrtRjcomDhIJVBnZwfmlI222WH8g==", 135 | "license": "ISC", 136 | "engines": { 137 | "node": ">=14.0.0" 138 | } 139 | }, 140 | "node_modules/@sindresorhus/slugify": { 141 | "version": "2.2.1", 142 | "resolved": "https://registry.npmjs.org/@sindresorhus/slugify/-/slugify-2.2.1.tgz", 143 | "integrity": "sha512-MkngSCRZ8JdSOCHRaYd+D01XhvU3Hjy6MGl06zhOk614hp9EOAp5gIkBeQg7wtmxpitU6eAL4kdiRMcJa2dlrw==", 144 | "license": "MIT", 145 | "dependencies": { 146 | "@sindresorhus/transliterate": "^1.0.0", 147 | "escape-string-regexp": "^5.0.0" 148 | }, 149 | "engines": { 150 | "node": ">=12" 151 | }, 152 | "funding": { 153 | "url": "https://github.com/sponsors/sindresorhus" 154 | } 155 | }, 156 | "node_modules/@sindresorhus/transliterate": { 157 | "version": "1.6.0", 158 | "resolved": "https://registry.npmjs.org/@sindresorhus/transliterate/-/transliterate-1.6.0.tgz", 159 | "integrity": "sha512-doH1gimEu3A46VX6aVxpHTeHrytJAG6HgdxntYnCFiIFHEM/ZGpG8KiZGBChchjQmG0XFIBL552kBTjVcMZXwQ==", 160 | "license": "MIT", 161 | "dependencies": { 162 | "escape-string-regexp": "^5.0.0" 163 | }, 164 | "engines": { 165 | "node": ">=12" 166 | }, 167 | "funding": { 168 | "url": "https://github.com/sponsors/sindresorhus" 169 | } 170 | }, 171 | "node_modules/argparse": { 172 | "version": "2.0.1", 173 | "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", 174 | "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", 175 | "license": "Python-2.0" 176 | }, 177 | "node_modules/base64-js": { 178 | "version": "1.5.1", 179 | "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", 180 | "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", 181 | "funding": [ 182 | { 183 | "type": "github", 184 | "url": "https://github.com/sponsors/feross" 185 | }, 186 | { 187 | "type": "patreon", 188 | "url": "https://www.patreon.com/feross" 189 | }, 190 | { 191 | "type": "consulting", 192 | "url": "https://feross.org/support" 193 | } 194 | ], 195 | "license": "MIT" 196 | }, 197 | "node_modules/buffer": { 198 | "version": "6.0.3", 199 | "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", 200 | "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", 201 | "funding": [ 202 | { 203 | "type": "github", 204 | "url": "https://github.com/sponsors/feross" 205 | }, 206 | { 207 | "type": "patreon", 208 | "url": "https://www.patreon.com/feross" 209 | }, 210 | { 211 | "type": "consulting", 212 | "url": "https://feross.org/support" 213 | } 214 | ], 215 | "license": "MIT", 216 | "dependencies": { 217 | "base64-js": "^1.3.1", 218 | "ieee754": "^1.2.1" 219 | } 220 | }, 221 | "node_modules/cacheable": { 222 | "version": "1.8.4", 223 | "resolved": "https://registry.npmjs.org/cacheable/-/cacheable-1.8.4.tgz", 224 | "integrity": "sha512-eqcPwJIM8hcx2mQIZtgrBQ7BmOf2pkL+1URswJaKRikCDw5of/lGpBTxODL1z1VuVVuxZHTuTejAMd9vyAUpLg==", 225 | "license": "MIT", 226 | "dependencies": { 227 | "hookified": "^1.5.0", 228 | "keyv": "^5.2.1" 229 | } 230 | }, 231 | "node_modules/debug": { 232 | "version": "4.3.7", 233 | "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", 234 | "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", 235 | "license": "MIT", 236 | "dependencies": { 237 | "ms": "^2.1.3" 238 | }, 239 | "engines": { 240 | "node": ">=6.0" 241 | }, 242 | "peerDependenciesMeta": { 243 | "supports-color": { 244 | "optional": true 245 | } 246 | } 247 | }, 248 | "node_modules/dom-serializer": { 249 | "version": "1.4.1", 250 | "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-1.4.1.tgz", 251 | "integrity": "sha512-VHwB3KfrcOOkelEG2ZOfxqLZdfkil8PtJi4P8N2MMXucZq2yLp75ClViUlOVwyoHEDjYU433Aq+5zWP61+RGag==", 252 | "license": "MIT", 253 | "dependencies": { 254 | "domelementtype": "^2.0.1", 255 | "domhandler": "^4.2.0", 256 | "entities": "^2.0.0" 257 | }, 258 | "funding": { 259 | "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" 260 | } 261 | }, 262 | "node_modules/dom-serializer/node_modules/entities": { 263 | "version": "2.2.0", 264 | "resolved": "https://registry.npmjs.org/entities/-/entities-2.2.0.tgz", 265 | "integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==", 266 | "license": "BSD-2-Clause", 267 | "funding": { 268 | "url": "https://github.com/fb55/entities?sponsor=1" 269 | } 270 | }, 271 | "node_modules/domelementtype": { 272 | "version": "2.3.0", 273 | "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", 274 | "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", 275 | "funding": [ 276 | { 277 | "type": "github", 278 | "url": "https://github.com/sponsors/fb55" 279 | } 280 | ], 281 | "license": "BSD-2-Clause" 282 | }, 283 | "node_modules/domhandler": { 284 | "version": "4.3.1", 285 | "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-4.3.1.tgz", 286 | "integrity": "sha512-GrwoxYN+uWlzO8uhUXRl0P+kHE4GtVPfYzVLcUxPL7KNdHKj66vvlhiweIHqYYXWlw+T8iLMp42Lm67ghw4WMQ==", 287 | "license": "BSD-2-Clause", 288 | "dependencies": { 289 | "domelementtype": "^2.2.0" 290 | }, 291 | "engines": { 292 | "node": ">= 4" 293 | }, 294 | "funding": { 295 | "url": "https://github.com/fb55/domhandler?sponsor=1" 296 | } 297 | }, 298 | "node_modules/domutils": { 299 | "version": "2.8.0", 300 | "resolved": "https://registry.npmjs.org/domutils/-/domutils-2.8.0.tgz", 301 | "integrity": "sha512-w96Cjofp72M5IIhpjgobBimYEfoPjx1Vx0BSX9P30WBdZW2WIKU0T1Bd0kz2eNZ9ikjKgHbEyKx8BB6H1L3h3A==", 302 | "license": "BSD-2-Clause", 303 | "dependencies": { 304 | "dom-serializer": "^1.0.1", 305 | "domelementtype": "^2.2.0", 306 | "domhandler": "^4.2.0" 307 | }, 308 | "funding": { 309 | "url": "https://github.com/fb55/domutils?sponsor=1" 310 | } 311 | }, 312 | "node_modules/dotenv": { 313 | "version": "16.4.7", 314 | "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.7.tgz", 315 | "integrity": "sha512-47qPchRCykZC03FhkYAhrvwU4xDBFIj1QPqaarj6mdM/hgUzfPHcpkHJOn3mJAufFeeAxAzeGsr5X0M4k6fLZQ==", 316 | "license": "BSD-2-Clause", 317 | "engines": { 318 | "node": ">=12" 319 | }, 320 | "funding": { 321 | "url": "https://dotenvx.com" 322 | } 323 | }, 324 | "node_modules/entities": { 325 | "version": "5.0.0", 326 | "resolved": "https://registry.npmjs.org/entities/-/entities-5.0.0.tgz", 327 | "integrity": "sha512-BeJFvFRJddxobhvEdm5GqHzRV/X+ACeuw0/BuuxsCh1EUZcAIz8+kYmBp/LrQuloy6K1f3a0M7+IhmZ7QnkISA==", 328 | "license": "BSD-2-Clause", 329 | "engines": { 330 | "node": ">=0.12" 331 | }, 332 | "funding": { 333 | "url": "https://github.com/fb55/entities?sponsor=1" 334 | } 335 | }, 336 | "node_modules/escape-string-regexp": { 337 | "version": "5.0.0", 338 | "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz", 339 | "integrity": "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==", 340 | "license": "MIT", 341 | "engines": { 342 | "node": ">=12" 343 | }, 344 | "funding": { 345 | "url": "https://github.com/sponsors/sindresorhus" 346 | } 347 | }, 348 | "node_modules/evaluate-value": { 349 | "version": "2.0.0", 350 | "resolved": "https://registry.npmjs.org/evaluate-value/-/evaluate-value-2.0.0.tgz", 351 | "integrity": "sha512-VonfiuDJc0z4sOO7W0Pd130VLsXN6vmBWZlrog1mCb/o7o/Nl5Lr25+Kj/nkCCAhG+zqeeGjxhkK9oHpkgTHhQ==", 352 | "license": "MIT", 353 | "engines": { 354 | "node": ">= 8" 355 | } 356 | }, 357 | "node_modules/eventemitter3": { 358 | "version": "4.0.7", 359 | "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", 360 | "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", 361 | "license": "MIT" 362 | }, 363 | "node_modules/fast-xml-parser": { 364 | "version": "4.5.3", 365 | "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.5.3.tgz", 366 | "integrity": "sha512-RKihhV+SHsIUGXObeVy9AXiBbFwkVk7Syp8XgwN5U3JV416+Gwp/GO9i0JYKmikykgz/UHRrrV4ROuZEo/T0ig==", 367 | "funding": [ 368 | { 369 | "type": "github", 370 | "url": "https://github.com/sponsors/NaturalIntelligence" 371 | } 372 | ], 373 | "license": "MIT", 374 | "dependencies": { 375 | "strnum": "^1.1.1" 376 | }, 377 | "bin": { 378 | "fxparser": "src/cli/cli.js" 379 | } 380 | }, 381 | "node_modules/filesize": { 382 | "version": "10.1.6", 383 | "resolved": "https://registry.npmjs.org/filesize/-/filesize-10.1.6.tgz", 384 | "integrity": "sha512-sJslQKU2uM33qH5nqewAwVB2QgR6w1aMNsYUp3aN5rMRyXEwJGmZvaWzeJFNTOXWlHQyBFCWrdj3fV/fsTOX8w==", 385 | "license": "BSD-3-Clause", 386 | "engines": { 387 | "node": ">= 10.4.0" 388 | } 389 | }, 390 | "node_modules/flat-cache": { 391 | "version": "6.1.2", 392 | "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-6.1.2.tgz", 393 | "integrity": "sha512-WakhGOkx886u7DJGpgMpUU81VUYHyQlXuqPDI53g6lIVHf7Shepr/XGo7Qa0yYOPwyMItQs34dG7X0KgnHwWtQ==", 394 | "license": "MIT", 395 | "dependencies": { 396 | "cacheable": "^1.8.1", 397 | "flatted": "^3.3.1", 398 | "hookified": "^1.4.0" 399 | } 400 | }, 401 | "node_modules/flatted": { 402 | "version": "3.3.1", 403 | "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.1.tgz", 404 | "integrity": "sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw==", 405 | "license": "ISC" 406 | }, 407 | "node_modules/github-publish": { 408 | "version": "6.0.0", 409 | "resolved": "https://registry.npmjs.org/github-publish/-/github-publish-6.0.0.tgz", 410 | "integrity": "sha512-l2qCwt0QoAUH0Fgq05VQ2UZKCBOq/v3gogl0XViMf25Cn+7A9206uJ0cfnLUxVBq74IsziGc/jJFkiUDpJYVUg==", 411 | "license": "MIT", 412 | "dependencies": { 413 | "pony-cause": "^2.1.11", 414 | "undici": "^6.19.2" 415 | }, 416 | "engines": { 417 | "node": ">=18.17.0" 418 | } 419 | }, 420 | "node_modules/graceful-fs": { 421 | "version": "4.2.11", 422 | "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", 423 | "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", 424 | "license": "ISC" 425 | }, 426 | "node_modules/hookified": { 427 | "version": "1.5.0", 428 | "resolved": "https://registry.npmjs.org/hookified/-/hookified-1.5.0.tgz", 429 | "integrity": "sha512-4U0zw2ibOws7kfGdNCIL6oRg+t6ITxkgi9kUaJ71IDp0ZATHjvY6o7l90RBa/R8H2qOKl47SZISA5a3hNnei1g==", 430 | "license": "MIT" 431 | }, 432 | "node_modules/htmlparser2": { 433 | "version": "7.2.0", 434 | "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-7.2.0.tgz", 435 | "integrity": "sha512-H7MImA4MS6cw7nbyURtLPO1Tms7C5H602LRETv95z1MxO/7CP7rDVROehUYeYBUYEON94NXXDEPmZuq+hX4sog==", 436 | "funding": [ 437 | "https://github.com/fb55/htmlparser2?sponsor=1", 438 | { 439 | "type": "github", 440 | "url": "https://github.com/sponsors/fb55" 441 | } 442 | ], 443 | "license": "MIT", 444 | "dependencies": { 445 | "domelementtype": "^2.0.1", 446 | "domhandler": "^4.2.2", 447 | "domutils": "^2.8.0", 448 | "entities": "^3.0.1" 449 | } 450 | }, 451 | "node_modules/htmlparser2/node_modules/entities": { 452 | "version": "3.0.1", 453 | "resolved": "https://registry.npmjs.org/entities/-/entities-3.0.1.tgz", 454 | "integrity": "sha512-WiyBqoomrwMdFG1e0kqvASYfnlb0lp8M5o5Fw2OFq1hNZxxcNk8Ik0Xm7LxzBhuidnZB/UtBqVCgUz3kBOP51Q==", 455 | "license": "BSD-2-Clause", 456 | "engines": { 457 | "node": ">=0.12" 458 | }, 459 | "funding": { 460 | "url": "https://github.com/fb55/entities?sponsor=1" 461 | } 462 | }, 463 | "node_modules/husky": { 464 | "version": "9.1.7", 465 | "resolved": "https://registry.npmjs.org/husky/-/husky-9.1.7.tgz", 466 | "integrity": "sha512-5gs5ytaNjBrh5Ow3zrvdUUY+0VxIuWVL4i9irt6friV+BqdCfmV11CQTWMiBYWHbXhco+J1kHfTOUkePhCDvMA==", 467 | "dev": true, 468 | "license": "MIT", 469 | "bin": { 470 | "husky": "bin.js" 471 | }, 472 | "engines": { 473 | "node": ">=18" 474 | }, 475 | "funding": { 476 | "url": "https://github.com/sponsors/typicode" 477 | } 478 | }, 479 | "node_modules/ieee754": { 480 | "version": "1.2.1", 481 | "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", 482 | "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", 483 | "funding": [ 484 | { 485 | "type": "github", 486 | "url": "https://github.com/sponsors/feross" 487 | }, 488 | { 489 | "type": "patreon", 490 | "url": "https://www.patreon.com/feross" 491 | }, 492 | { 493 | "type": "consulting", 494 | "url": "https://feross.org/support" 495 | } 496 | ], 497 | "license": "BSD-3-Clause" 498 | }, 499 | "node_modules/is-json": { 500 | "version": "2.0.1", 501 | "resolved": "https://registry.npmjs.org/is-json/-/is-json-2.0.1.tgz", 502 | "integrity": "sha512-6BEnpVn1rcf3ngfmViLM6vjUjGErbdrL4rwlv+u1NO1XO8kqT4YGL8+19Q+Z/bas8tY90BTWMk2+fW1g6hQjbA==", 503 | "license": "ISC" 504 | }, 505 | "node_modules/js-yaml": { 506 | "version": "4.1.0", 507 | "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", 508 | "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", 509 | "license": "MIT", 510 | "dependencies": { 511 | "argparse": "^2.0.1" 512 | }, 513 | "bin": { 514 | "js-yaml": "bin/js-yaml.js" 515 | } 516 | }, 517 | "node_modules/keyv": { 518 | "version": "5.2.1", 519 | "resolved": "https://registry.npmjs.org/keyv/-/keyv-5.2.1.tgz", 520 | "integrity": "sha512-tpIgCaY02VCW2Pz0zAn4guyct+IeH6Mb5wZdOvpe4oqXeQOJO0C3Wo8fTnf7P3ZD83Vr9kghbkNmzG3lTOhy/A==", 521 | "license": "MIT", 522 | "dependencies": { 523 | "@keyv/serialize": "*" 524 | } 525 | }, 526 | "node_modules/kleur": { 527 | "version": "4.1.5", 528 | "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", 529 | "integrity": "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==", 530 | "license": "MIT", 531 | "engines": { 532 | "node": ">=6" 533 | } 534 | }, 535 | "node_modules/list-to-array": { 536 | "version": "1.1.0", 537 | "resolved": "https://registry.npmjs.org/list-to-array/-/list-to-array-1.1.0.tgz", 538 | "integrity": "sha512-+dAZZ2mM+/m+vY9ezfoueVvrgnHIGi5FvgSymbIgJOFwiznWyA59mav95L+Mc6xPtL3s9gm5eNTlNtxJLbNM1g==", 539 | "license": "MIT" 540 | }, 541 | "node_modules/make-synchronized": { 542 | "version": "0.4.2", 543 | "resolved": "https://registry.npmjs.org/make-synchronized/-/make-synchronized-0.4.2.tgz", 544 | "integrity": "sha512-EwEJSg8gSGLicKXp/VzNi1tvzhdmNBxOzslkkJSoNUCQFZKH/NIUIp7xlfN+noaHrz4BJDN73gne8IHnjl/F/A==", 545 | "license": "MIT", 546 | "funding": { 547 | "url": "https://github.com/fisker/make-synchronized?sponsor=1" 548 | } 549 | }, 550 | "node_modules/ms": { 551 | "version": "2.1.3", 552 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", 553 | "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", 554 | "license": "MIT" 555 | }, 556 | "node_modules/p-finally": { 557 | "version": "1.0.0", 558 | "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", 559 | "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", 560 | "license": "MIT", 561 | "engines": { 562 | "node": ">=4" 563 | } 564 | }, 565 | "node_modules/p-queue": { 566 | "version": "6.6.2", 567 | "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz", 568 | "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==", 569 | "license": "MIT", 570 | "dependencies": { 571 | "eventemitter3": "^4.0.4", 572 | "p-timeout": "^3.2.0" 573 | }, 574 | "engines": { 575 | "node": ">=8" 576 | }, 577 | "funding": { 578 | "url": "https://github.com/sponsors/sindresorhus" 579 | } 580 | }, 581 | "node_modules/p-timeout": { 582 | "version": "3.2.0", 583 | "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", 584 | "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", 585 | "license": "MIT", 586 | "dependencies": { 587 | "p-finally": "^1.0.0" 588 | }, 589 | "engines": { 590 | "node": ">=8" 591 | } 592 | }, 593 | "node_modules/parse-srcset": { 594 | "version": "1.0.2", 595 | "resolved": "https://registry.npmjs.org/parse-srcset/-/parse-srcset-1.0.2.tgz", 596 | "integrity": "sha512-/2qh0lav6CmI15FzA3i/2Bzk2zCgQhGMkvhOhKNcBVQ1ldgpbfiNTVslmooUmWJcADi1f1kIeynbDRVzNlfR6Q==", 597 | "license": "MIT" 598 | }, 599 | "node_modules/pony-cause": { 600 | "version": "2.1.11", 601 | "resolved": "https://registry.npmjs.org/pony-cause/-/pony-cause-2.1.11.tgz", 602 | "integrity": "sha512-M7LhCsdNbNgiLYiP4WjsfLUuFmCfnjdF6jKe2R9NKl4WFN+HZPGHJZ9lnLP7f9ZnKe3U9nuWD0szirmj+migUg==", 603 | "license": "0BSD", 604 | "engines": { 605 | "node": ">=12.0.0" 606 | } 607 | }, 608 | "node_modules/posthtml": { 609 | "version": "0.16.6", 610 | "resolved": "https://registry.npmjs.org/posthtml/-/posthtml-0.16.6.tgz", 611 | "integrity": "sha512-JcEmHlyLK/o0uGAlj65vgg+7LIms0xKXe60lcDOTU7oVX/3LuEuLwrQpW3VJ7de5TaFKiW4kWkaIpJL42FEgxQ==", 612 | "license": "MIT", 613 | "dependencies": { 614 | "posthtml-parser": "^0.11.0", 615 | "posthtml-render": "^3.0.0" 616 | }, 617 | "engines": { 618 | "node": ">=12.0.0" 619 | } 620 | }, 621 | "node_modules/posthtml-parser": { 622 | "version": "0.11.0", 623 | "resolved": "https://registry.npmjs.org/posthtml-parser/-/posthtml-parser-0.11.0.tgz", 624 | "integrity": "sha512-QecJtfLekJbWVo/dMAA+OSwY79wpRmbqS5TeXvXSX+f0c6pW4/SE6inzZ2qkU7oAMCPqIDkZDvd/bQsSFUnKyw==", 625 | "license": "MIT", 626 | "dependencies": { 627 | "htmlparser2": "^7.1.1" 628 | }, 629 | "engines": { 630 | "node": ">=12" 631 | } 632 | }, 633 | "node_modules/posthtml-render": { 634 | "version": "3.0.0", 635 | "resolved": "https://registry.npmjs.org/posthtml-render/-/posthtml-render-3.0.0.tgz", 636 | "integrity": "sha512-z+16RoxK3fUPgwaIgH9NGnK1HKY9XIDpydky5eQGgAFVXTCSezalv9U2jQuNV+Z9qV1fDWNzldcw4eK0SSbqKA==", 637 | "license": "MIT", 638 | "dependencies": { 639 | "is-json": "^2.0.1" 640 | }, 641 | "engines": { 642 | "node": ">=12" 643 | } 644 | }, 645 | "node_modules/prettier": { 646 | "version": "3.5.3", 647 | "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.5.3.tgz", 648 | "integrity": "sha512-QQtaxnoDJeAkDvDKWCLiwIXkTgRhwYDEQCghU9Z6q03iyek/rxRh/2lC3HB7P8sWT2xC/y5JDctPLBIGzHKbhw==", 649 | "license": "MIT", 650 | "bin": { 651 | "prettier": "bin/prettier.cjs" 652 | }, 653 | "engines": { 654 | "node": ">=14" 655 | }, 656 | "funding": { 657 | "url": "https://github.com/prettier/prettier?sponsor=1" 658 | } 659 | }, 660 | "node_modules/striptags": { 661 | "version": "3.2.0", 662 | "resolved": "https://registry.npmjs.org/striptags/-/striptags-3.2.0.tgz", 663 | "integrity": "sha512-g45ZOGzHDMe2bdYMdIvdAfCQkCTDMGBazSw1ypMowwGIee7ZQ5dU0rBJ8Jqgl+jAKIv4dbeE1jscZq9wid1Tkw==", 664 | "license": "MIT" 665 | }, 666 | "node_modules/strnum": { 667 | "version": "1.1.2", 668 | "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.1.2.tgz", 669 | "integrity": "sha512-vrN+B7DBIoTTZjnPNewwhx6cBA/H+IS7rfW68n7XxC1y7uoiGQBxaKzqucGUgavX15dJgiGztLJ8vxuEzwqBdA==", 670 | "funding": [ 671 | { 672 | "type": "github", 673 | "url": "https://github.com/sponsors/NaturalIntelligence" 674 | } 675 | ], 676 | "license": "MIT" 677 | }, 678 | "node_modules/turndown": { 679 | "version": "7.2.0", 680 | "resolved": "git+ssh://git@github.com/zachleat/fork-turndown.git#7c2b27875713d37d3eb079c001ecc0b2502539d2", 681 | "license": "MIT", 682 | "dependencies": { 683 | "@mixmark-io/domino": "^2.2.0" 684 | } 685 | }, 686 | "node_modules/undici": { 687 | "version": "6.21.0", 688 | "resolved": "https://registry.npmjs.org/undici/-/undici-6.21.0.tgz", 689 | "integrity": "sha512-BUgJXc752Kou3oOIuU1i+yZZypyZRqNPW0vqoMPl8VaoalSfeR0D8/t4iAS3yirs79SSMTxTag+ZC86uswv+Cw==", 690 | "license": "MIT", 691 | "engines": { 692 | "node": ">=18.17" 693 | } 694 | } 695 | } 696 | } 697 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@11ty/import", 3 | "version": "1.0.15", 4 | "description": "Utility to import content from multiple services (and a CLI, too)", 5 | "type": "module", 6 | "main": "./src/Importer.js", 7 | "exports": { 8 | ".": "./src/Importer.js", 9 | "./DataSource": "./src/DataSource.js" 10 | }, 11 | "engines": { 12 | "node": ">=18" 13 | }, 14 | "bin": { 15 | "eleventy-import": "cli.js" 16 | }, 17 | "scripts": { 18 | "test": "node --test", 19 | "prepare": "husky" 20 | }, 21 | "publishConfig": { 22 | "access": "public" 23 | }, 24 | "funding": { 25 | "type": "opencollective", 26 | "url": "https://opencollective.com/11ty" 27 | }, 28 | "author": "Zach Leatherman (https://zachleat.com/)", 29 | "repository": { 30 | "type": "git", 31 | "url": "git://github.com/11ty/eleventy-import.git" 32 | }, 33 | "bugs": "https://github.com/11ty/eleventy-import/issues", 34 | "homepage": "https://www.11ty.dev/", 35 | "license": "MIT", 36 | "dependencies": { 37 | "@11ty/eleventy-fetch": "^5.0.2", 38 | "@11ty/eleventy-utils": "^2.0.1", 39 | "@11ty/posthtml-urls": "^1.0.1", 40 | "@prettier/sync": "^0.5.5", 41 | "@sindresorhus/slugify": "^2.2.1", 42 | "dotenv": "^16.4.7", 43 | "entities": "^5.0.0", 44 | "fast-xml-parser": "^4.5.3", 45 | "filesize": "^10.1.6", 46 | "github-publish": "^6.0.0", 47 | "graceful-fs": "^4.2.11", 48 | "js-yaml": "^4.1.0", 49 | "kleur": "^4.1.5", 50 | "posthtml": "^0.16.6", 51 | "prettier": "^3.5.3", 52 | "striptags": "^3.2.0", 53 | "turndown": "github:zachleat/fork-turndown" 54 | }, 55 | "devDependencies": { 56 | "husky": "^9.1.7" 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/DataSource.js: -------------------------------------------------------------------------------- 1 | import kleur from 'kleur'; 2 | import { DateCompare } from "@11ty/eleventy-utils"; 3 | 4 | import { Logger } from "./Logger.js"; 5 | 6 | class DataSource { 7 | static UUID_PREFIX = "11ty/import"; 8 | 9 | #fetcher; 10 | #fetchDataOverrides = {}; 11 | #outputFolder = "."; 12 | 13 | constructor() { 14 | this.isVerbose = true; 15 | this.within = ""; 16 | } 17 | 18 | setWithin(within) { 19 | this.within = within; 20 | } 21 | 22 | setVerbose(isVerbose) { 23 | this.isVerbose = isVerbose; 24 | } 25 | 26 | setFetcher(fetcher) { 27 | this.#fetcher = fetcher; 28 | } 29 | 30 | get fetcher() { 31 | if(!this.#fetcher) { 32 | throw new Error("Missing Fetcher instance."); 33 | } 34 | return this.#fetcher; 35 | } 36 | 37 | // For testing 38 | setDataOverride(url, data) { 39 | this.#fetchDataOverrides[url] = data; 40 | } 41 | 42 | setLabel(label) { 43 | this.label = label; 44 | } 45 | 46 | setFilepathFormatFunction(format) { 47 | if(typeof format !== "function") { 48 | throw new Error("filepathFormat option expected to be a function."); 49 | } 50 | this.filepathFormat = format; 51 | } 52 | 53 | getFilepathFormatFunction() { 54 | return this.filepathFormat; 55 | } 56 | 57 | isValidHttpUrl(url) { 58 | try { 59 | new URL(url); 60 | return url.startsWith("https://") || url.startsWith("http://"); 61 | } catch(e) { 62 | // invalid url OR local path 63 | return false; 64 | } 65 | } 66 | 67 | setOutputFolder(dir) { 68 | this.#outputFolder = dir; 69 | } 70 | 71 | get outputFolder() { 72 | return this.#outputFolder; 73 | } 74 | 75 | toIsoDate(dateStr) { 76 | return (new Date(Date.parse(dateStr))).toISOString(); 77 | } 78 | 79 | toReadableDate(dateStr, locale = 'en-US', options = {}) { 80 | options = Object.assign({ 81 | year: "numeric", 82 | month: "long", 83 | day: "numeric", 84 | hour: "numeric", 85 | minute: "numeric", 86 | second: "numeric", 87 | timeZoneName: "short", 88 | }, options); 89 | 90 | let date = (new Date(Date.parse(dateStr))); 91 | return new Intl.DateTimeFormat(locale, options).format(date) 92 | } 93 | 94 | getHeaders() { 95 | return {}; 96 | } 97 | 98 | getUniqueIdFromEntry() { 99 | return ""; 100 | } 101 | 102 | // Thanks to https://stackoverflow.com/questions/7467840/nl2br-equivalent-in-javascript/7467863#7467863 103 | static nl2br(str) { 104 | if (typeof str === 'undefined' || str === null) { 105 | return ""; 106 | } 107 | return (str + '').replace(/([^>\r\n]?)(\r\n|\n\r|\r|\n)/g, '$1
$2'); 108 | } 109 | 110 | async getData(url, type, showErrors = true) { 111 | // For testing, all urls must be stubbed 112 | if(Object.keys(this.#fetchDataOverrides).length > 0) { 113 | if(this.#fetchDataOverrides[url]) { 114 | return this.#fetchDataOverrides[url]; 115 | } 116 | 117 | throw new Error("Testing error, missing data override url: " + url); 118 | } 119 | 120 | return this.fetcher.fetch(url, { 121 | type, 122 | fetchOptions: { 123 | headers: this.getHeaders(), 124 | }, 125 | }, { 126 | verbose: true, 127 | showErrors 128 | }); 129 | } 130 | 131 | async getCleanedEntries(data) { 132 | // data should be iterable 133 | let dataEntries = data; 134 | if(typeof this.getEntriesFromData === "function") { 135 | dataEntries = this.getEntriesFromData(data) || []; 136 | } 137 | 138 | let entries = []; 139 | for(let rawEntry of dataEntries) { 140 | if(this.isWithin(rawEntry)) { 141 | if(typeof this.cleanEntry === "function") { 142 | let cleaned = await this.cleanEntry(rawEntry, data); 143 | entries.push(cleaned); 144 | } else { 145 | entries.push(rawEntry); 146 | } 147 | } 148 | } 149 | 150 | return entries; 151 | } 152 | 153 | toDateObj(dateVal) { 154 | if(dateVal instanceof Date) { 155 | return dateVal; 156 | } 157 | if(dateVal) { 158 | return new Date(Date.parse(dateVal)); 159 | } 160 | } 161 | 162 | async getEntries() { 163 | let entries = []; 164 | if(typeof this.getUrl === "function") { 165 | let url = this.getUrl(); 166 | if(typeof url === "function") { 167 | let pageNumber = 1; 168 | let pagedUrl; 169 | 170 | try { 171 | while(pagedUrl = url(pageNumber)) { 172 | let found = 0; 173 | let data = await this.getData(pagedUrl, this.getType(), false); 174 | let cleanedData = await this.getCleanedEntries(data); 175 | 176 | for(let entry of cleanedData) { 177 | entries.push(entry); 178 | 179 | // careful here, if an entry was updated out of your `within` window, it will be ignored 180 | found++; 181 | } 182 | 183 | if(found === 0) { 184 | break; 185 | } 186 | 187 | pageNumber++; 188 | } 189 | } catch(e) { 190 | let shouldWorry = await this.isErrorWorthWorryingAbout(e); 191 | if(shouldWorry) { 192 | Logger.error(kleur.red(`Error: ${e.message}`), e); 193 | throw e; 194 | } 195 | } 196 | } else if(typeof url === "string" || url instanceof URL) { 197 | let data = await this.getData(url, this.getType(), true); 198 | for(let entry of await this.getCleanedEntries(data) || []) { 199 | entries.push(entry); 200 | } 201 | } 202 | } else if(typeof this.getData === "function") { 203 | let data = this.getData() || {}; 204 | for(let entry of await this.getCleanedEntries(data) || []) { 205 | entries.push(entry); 206 | } 207 | } 208 | 209 | return entries.map(entry => { 210 | // TODO check uuid uniqueness 211 | 212 | if(this.label) { 213 | entry.sourceLabel = this.label; 214 | } 215 | 216 | // create Date objects 217 | if(entry.date && !(entry.date instanceof Date)) { 218 | entry.date = this.toDateObj(entry.date); 219 | } 220 | 221 | if(entry.dateUpdated && !(entry.dateUpdated instanceof Date)) { 222 | entry.dateUpdated = this.toDateObj(entry.date); 223 | } 224 | 225 | Object.defineProperty(entry, "source", { 226 | enumerable: false, 227 | value: this, 228 | }); 229 | 230 | return entry; 231 | }); 232 | } 233 | 234 | cleanStatus(status) { 235 | // WordPress has draft/publish 236 | // For future use 237 | return status; 238 | } 239 | 240 | isWithin(rawEntry) { 241 | if(!this.within || typeof this.getRawEntryDates !== "function") { 242 | return true; 243 | } 244 | 245 | let dates = this.getRawEntryDates(rawEntry); 246 | 247 | if(dates.created) { 248 | if(DateCompare.isTimestampWithinDuration(dates.created.getTime(), this.within)) { 249 | return true; 250 | } 251 | } 252 | 253 | if(dates.updated) { 254 | if(DateCompare.isTimestampWithinDuration(dates.updated.getTime(), this.within)) { 255 | return true; 256 | } 257 | } 258 | 259 | return false; 260 | } 261 | } 262 | 263 | export { DataSource }; 264 | -------------------------------------------------------------------------------- /src/DataSource/Atom.js: -------------------------------------------------------------------------------- 1 | import { DataSource } from "../DataSource.js"; 2 | 3 | class Atom extends DataSource { 4 | static TYPE = "atom"; 5 | static TYPE_FRIENDLY = "Atom"; 6 | 7 | constructor(url) { 8 | super(); 9 | this.url = url; 10 | } 11 | 12 | getType() { 13 | return "xml"; 14 | } 15 | 16 | getUrl() { 17 | return this.url; 18 | } 19 | 20 | getEntriesFromData(data) { 21 | if(Array.isArray(data.feed?.entry)) { 22 | return data.feed.entry; 23 | } 24 | 25 | if(data.feed?.entry) { 26 | return [data.feed.entry]; 27 | } 28 | 29 | return []; 30 | } 31 | 32 | getUrlFromEntry(entry) { 33 | if(this.isValidHttpUrl(entry.id)) { 34 | return entry.id; 35 | } 36 | if(entry.link && entry.link["@_rel"] === "alternate" && entry.link["@_href"] && this.isValidHttpUrl(entry.link["@_href"])) { 37 | return entry.link["@_href"]; 38 | } 39 | return entry.id; 40 | } 41 | 42 | getUniqueIdFromEntry(entry) { 43 | // id is a unique URL 44 | return `${DataSource.UUID_PREFIX}::${Atom.TYPE}::${entry.id}`; 45 | } 46 | 47 | getRawEntryDates(rawEntry) { 48 | return { 49 | created: this.toDateObj(rawEntry.published || rawEntry.updated), 50 | updated: this.toDateObj(rawEntry.updated) 51 | }; 52 | } 53 | 54 | cleanEntry(rawEntry, data) { 55 | let authors = []; 56 | if(Array.isArray(rawEntry?.author)) { 57 | authors = rawEntry.author.map(author => ({ name: author })); 58 | } else { 59 | authors.push({ 60 | name: rawEntry?.author?.name || data.feed?.author?.name, 61 | }); 62 | } 63 | 64 | let { created, updated } = this.getRawEntryDates(rawEntry); 65 | 66 | return { 67 | uuid: this.getUniqueIdFromEntry(rawEntry), 68 | type: Atom.TYPE, 69 | title: rawEntry.title, 70 | url: this.getUrlFromEntry(rawEntry), 71 | authors, 72 | date: created, 73 | dateUpdated: updated, 74 | content: rawEntry.content["#text"], 75 | contentType: rawEntry.content["@_type"], 76 | } 77 | } 78 | } 79 | 80 | export {Atom}; 81 | -------------------------------------------------------------------------------- /src/DataSource/BlueskyUser.js: -------------------------------------------------------------------------------- 1 | import path from "node:path"; 2 | import { Rss } from "./Rss.js"; 3 | 4 | class BlueskyUser extends Rss { 5 | static TYPE = "bluesky"; 6 | static TYPE_FRIENDLY = "Bluesky"; 7 | 8 | static normalizeUsername(username) { 9 | if(username.startsWith("@")) { 10 | return username.slice(1); 11 | } 12 | return username; 13 | } 14 | 15 | constructor(username) { 16 | super(`https://bsky.app/profile/${BlueskyUser.normalizeUsername(username)}/rss`); 17 | } 18 | 19 | static getFilePath(url) { 20 | let {pathname} = new URL(url); 21 | let [empty, profile, username, post, id] = pathname.split("/"); 22 | return path.join(username, id); 23 | } 24 | 25 | cleanEntry(entry, data) { 26 | let obj = super.cleanEntry(entry, data); 27 | obj.type = BlueskyUser.TYPE; 28 | obj.contentType = "text"; 29 | 30 | return obj; 31 | } 32 | } 33 | 34 | export { BlueskyUser }; 35 | -------------------------------------------------------------------------------- /src/DataSource/FediverseUser.js: -------------------------------------------------------------------------------- 1 | import path from "node:path"; 2 | import { DataSource } from "../DataSource.js"; 3 | import { Rss } from "./Rss.js"; 4 | 5 | class FediverseUser extends Rss { 6 | static TYPE = "fediverse"; 7 | static TYPE_FRIENDLY = "Fediverse"; 8 | 9 | constructor(fullUsername) { 10 | let { username, hostname } = FediverseUser.parseUsername(fullUsername); 11 | super(`https://${hostname}/users/${username}.rss`); 12 | 13 | this.username = username; 14 | this.hostname = hostname; 15 | } 16 | 17 | static parseUsername(fullUsername) { 18 | if(fullUsername.startsWith("@")) { 19 | fullUsername = fullUsername.slice(1); 20 | } 21 | 22 | let [ username, hostname ]= fullUsername.split("@"); 23 | 24 | return { 25 | username, 26 | hostname 27 | } 28 | } 29 | 30 | static parseFromUrl(url) { 31 | let { hostname, pathname } = new URL(url); 32 | let [empty, username, postId] = pathname.split("/"); 33 | 34 | return { 35 | username: username.startsWith("@") ? username.slice(1) : username, 36 | hostname, 37 | postId, 38 | } 39 | } 40 | 41 | static getFilePath(url) { 42 | let { hostname, username, postId } = FediverseUser.parseFromUrl(url); 43 | return path.join(`${username}@${hostname}`, postId); 44 | } 45 | 46 | cleanEntry(entry, data) { 47 | let obj = super.cleanEntry(entry, data); 48 | obj.type = FediverseUser.TYPE; 49 | obj.contentType = "html"; 50 | 51 | return obj; 52 | } 53 | } 54 | 55 | export { FediverseUser }; 56 | -------------------------------------------------------------------------------- /src/DataSource/HostedWordPressApi.js: -------------------------------------------------------------------------------- 1 | import { DataSource } from "../DataSource.js"; 2 | 3 | class HostedWordPressApi extends DataSource { 4 | static TYPE = "wordpressapi-hosted"; 5 | static TYPE_FRIENDLY = "WordPress.com"; 6 | 7 | static #getHostname(url) { 8 | try { 9 | let u = new URL(url); 10 | return u.hostname; 11 | } catch(e) {} 12 | return ""; 13 | } 14 | 15 | static isValid(url) { 16 | let hostname = this.#getHostname(url); 17 | return hostname.endsWith(".wordpress.com"); 18 | } 19 | 20 | constructor(url) { 21 | super(); 22 | this.url = url; 23 | 24 | if(!HostedWordPressApi.isValid(url)) { 25 | throw new Error("HostedWordPressApi expects a .wordpress.com URL, if you’re looking to use a self-hosted WordPress API please use the `wordpress` type (`WordPressApi` class)."); 26 | } 27 | 28 | this.hostname = HostedWordPressApi.#getHostname(url); 29 | } 30 | 31 | getType() { 32 | return "json"; 33 | } 34 | 35 | getUrl() { 36 | // return function for paging 37 | return (pageNumber = 1) => { 38 | // DRAFTS NOT SUPPORTED 39 | return `https://public-api.wordpress.com/rest/v1.1/sites/${this.hostname}/posts/?page=${pageNumber}&per_page=100`; 40 | }; 41 | } 42 | 43 | getEntriesFromData(data) { 44 | return data.posts || []; 45 | } 46 | 47 | getUrlFromEntry(entry) { 48 | return entry.URL; 49 | } 50 | 51 | getUniqueIdFromEntry(entry) { 52 | return `${DataSource.UUID_PREFIX}::${HostedWordPressApi.TYPE}::${entry.guid}`; 53 | } 54 | 55 | // stock WordPress is single-author 56 | #getAuthorData(author) { 57 | return [ 58 | { 59 | name: author.name, 60 | url: author.profile_URL, 61 | avatarUrl: author.avatar_URL, 62 | } 63 | ]; 64 | } 65 | 66 | getRawEntryDates(rawEntry) { 67 | return { 68 | created: this.toDateObj(rawEntry.date), 69 | updated: this.toDateObj(rawEntry.modified), 70 | }; 71 | } 72 | 73 | async cleanEntry(rawEntry, data) { 74 | let metadata = { 75 | categories: Object.keys(rawEntry.categories), 76 | tags: Object.keys(rawEntry.tags), 77 | }; 78 | 79 | if(rawEntry.featured_image) { 80 | metadata.media = { 81 | featuredImage: rawEntry.featured_image, 82 | }; 83 | 84 | // backwards compatibility (not downloaded or optimized) 85 | metadata.featuredImage = rawEntry.featured_image; 86 | } 87 | 88 | let { created, updated } = this.getRawEntryDates(rawEntry); 89 | 90 | return { 91 | uuid: this.getUniqueIdFromEntry(rawEntry), 92 | type: HostedWordPressApi.TYPE, 93 | title: rawEntry.title, 94 | url: this.getUrlFromEntry(rawEntry), 95 | authors: this.#getAuthorData(rawEntry.author), 96 | date: created, 97 | dateUpdated: updated, 98 | content: rawEntry.content, 99 | contentType: "html", 100 | status: this.cleanStatus(rawEntry.status), 101 | metadata, 102 | } 103 | } 104 | } 105 | 106 | export { HostedWordPressApi }; 107 | -------------------------------------------------------------------------------- /src/DataSource/Rss.js: -------------------------------------------------------------------------------- 1 | import { DataSource } from "../DataSource.js"; 2 | 3 | class Rss extends DataSource { 4 | static TYPE = "rss"; 5 | static TYPE_FRIENDLY = "RSS"; 6 | 7 | constructor(url) { 8 | super(); 9 | this.url = url; 10 | } 11 | 12 | getType() { 13 | return "xml"; 14 | } 15 | 16 | getUrl() { 17 | return this.url; 18 | } 19 | 20 | getEntriesFromData(data) { 21 | if(Array.isArray(data.rss?.channel?.item)) { 22 | return data.rss.channel.item; 23 | } 24 | 25 | if(data.rss?.channel?.item) { 26 | return [data.rss.channel.item]; 27 | } 28 | 29 | return []; 30 | } 31 | 32 | getUniqueIdFromEntry(entry) { 33 | return `${DataSource.UUID_PREFIX}::${Rss.TYPE}::${entry.guid["#text"]}`; 34 | } 35 | 36 | getHtmlFromMediaEntry(mediaSources) { 37 | if(!Array.isArray(mediaSources)) { 38 | mediaSources = [mediaSources]; 39 | } 40 | /* { 41 | 'media:rating': { '#text': 'nonadult', '@_scheme': 'urn:simple' }, 42 | 'media:description': { 43 | '#text': 'A fake blue sky wall sits behind a body of water. A man climbs a flight of stairs meant to blend in with the wall. From the Truman Show', 44 | '@_type': 'plain' 45 | }, 46 | '@_url': 'https://cdn.masto.host/fediversezachleatcom/media_attachments/files/113/487/344/514/939/049/original/ff062be4c5eaf642.png', 47 | '@_type': 'image/png', 48 | '@_fileSize': 879593, 49 | '@_medium': 'image' 50 | } */ 51 | 52 | return mediaSources.filter(source => { 53 | // Only supporting images for now 54 | return !source["@_medium"] || source["@_medium"] === "image"; 55 | }).map(source => { 56 | return `${source[`; 57 | }).join("\n"); 58 | } 59 | 60 | getRawEntryDates(rawEntry) { 61 | return { 62 | created: this.toDateObj(this.toIsoDate(rawEntry.pubDate)), 63 | // updated: this.toDateObj(rawEntry.updated), 64 | }; 65 | } 66 | 67 | cleanEntry(rawEntry, data) { 68 | let authors = []; 69 | // https://www.rssboard.org/rss-profile#namespace-elements-dublin-creator 70 | if(Array.isArray(rawEntry['dc:creator'])) { 71 | for(let name of rawEntry['dc:creator']) { 72 | authors.push({ name }); 73 | } 74 | } else if(rawEntry['dc:creator']) { 75 | authors.push({ name: rawEntry['dc:creator'] }); 76 | } else { 77 | authors.push({ 78 | name: data.rss.channel.title, 79 | url: data.rss.channel.link, 80 | }); 81 | } 82 | 83 | let content = rawEntry["content:encoded"] || rawEntry.content || rawEntry.description; 84 | 85 | if(rawEntry["media:content"]) { 86 | content += `\n${this.getHtmlFromMediaEntry(rawEntry["media:content"])}`; 87 | } 88 | 89 | let { created } = this.getRawEntryDates(rawEntry); 90 | 91 | return { 92 | uuid: this.getUniqueIdFromEntry(rawEntry), 93 | type: Rss.TYPE, 94 | title: rawEntry.title || this.toReadableDate(rawEntry.pubDate), 95 | url: rawEntry.link, 96 | authors, 97 | date: created, 98 | // dateUpdated: rawEntry.updated, 99 | content, 100 | // contentType: "", // unknown 101 | } 102 | } 103 | } 104 | 105 | export {Rss}; 106 | -------------------------------------------------------------------------------- /src/DataSource/WordPressApi.js: -------------------------------------------------------------------------------- 1 | import "dotenv/config" 2 | import { DateCompare } from "@11ty/eleventy-utils"; 3 | 4 | import { DataSource } from "../DataSource.js"; 5 | import { HostedWordPressApi } from "./HostedWordPressApi.js" 6 | 7 | class WordPressApi extends DataSource { 8 | static TYPE = "wordpress"; 9 | static TYPE_FRIENDLY = "WordPress"; 10 | static IGNORED_CATEGORIES = ["Uncategorized"]; 11 | 12 | constructor(url) { 13 | if(HostedWordPressApi.isValid(url)) { 14 | return new HostedWordPressApi(url); 15 | } 16 | 17 | super(); 18 | this.url = url; 19 | } 20 | 21 | // some pagination errors just mean there are no more pages 22 | async isErrorWorthWorryingAbout(e) { 23 | if(e?.cause instanceof Response) { 24 | let errorData = await e.cause.json(); 25 | if(errorData?.code === "rest_post_invalid_page_number") { 26 | return false; 27 | } 28 | } 29 | 30 | return true; 31 | } 32 | 33 | getType() { 34 | return "json"; 35 | } 36 | 37 | #getSubtypeUrl(subtype, suffix = "") { 38 | let {pathname} = new URL(this.url); 39 | return (new URL(pathname + `wp-json/wp/v2/${subtype}/${suffix}`, this.url)).toString(); 40 | } 41 | 42 | #getAuthorUrl(id) { 43 | return this.#getSubtypeUrl("users", id); 44 | } 45 | 46 | #getCategoryUrl(id) { 47 | return this.#getSubtypeUrl("categories", id); 48 | } 49 | 50 | #getTagsUrl(id) { 51 | return this.#getSubtypeUrl("tags", id); 52 | } 53 | 54 | getUrl() { 55 | // return function for paging 56 | return (pageNumber = 1) => { 57 | // status=publish,future,draft,pending,private 58 | // status=any 59 | 60 | let withinStr = ""; 61 | if(this.within) { 62 | let ms = DateCompare.getDurationMs(this.within); 63 | let d = this.toIsoDate(new Date(Date.now() - ms)); 64 | withinStr = `&after=${d}&modified_after=${d}` 65 | } 66 | 67 | let statusStr = ""; 68 | // Only request Drafts if auth’d 69 | if(process.env.WORDPRESS_USERNAME && process.env.WORDPRESS_PASSWORD) { 70 | // Commas are encoded 71 | statusStr = `&status=${encodeURIComponent("publish,draft")}`; 72 | } 73 | 74 | return this.#getSubtypeUrl("posts", `?page=${pageNumber}&per_page=100${statusStr}${withinStr}`); 75 | }; 76 | } 77 | 78 | getHeaders() { 79 | if(process.env.WORDPRESS_USERNAME && process.env.WORDPRESS_PASSWORD) { 80 | return { 81 | "Content-Type": "application/json", 82 | "Authorization": "Basic " + btoa(`${process.env.WORDPRESS_USERNAME}:${process.env.WORDPRESS_PASSWORD}`), 83 | } 84 | } 85 | 86 | return {}; 87 | } 88 | 89 | getEntriesFromData(data) { 90 | if(Array.isArray(data)) { 91 | return data; 92 | } 93 | 94 | return []; 95 | } 96 | 97 | getUrlFromEntry(entry) { 98 | return entry.link; 99 | } 100 | 101 | getUniqueIdFromEntry(entry) { 102 | return `${DataSource.UUID_PREFIX}::${WordPressApi.TYPE}::${entry.guid.rendered}`; 103 | } 104 | 105 | // stock WordPress is single-author 106 | async #getAuthors(authorId) { 107 | try { 108 | // Warning: extra API call 109 | let authorData = await this.getData(this.#getAuthorUrl(authorId), this.getType()); 110 | 111 | return [ 112 | { 113 | // _wordpress_author_id: entry.author, 114 | name: authorData.name, 115 | url: authorData.url || authorData.link, 116 | avatarUrl: authorData.avatar_urls[Object.keys(authorData.avatar_urls).pop()], 117 | } 118 | ]; 119 | } catch(e) { 120 | // Fetch logs the error upstream 121 | return []; 122 | } 123 | } 124 | 125 | async #getTags(ids) { 126 | return Promise.all(ids.map(tagId => { 127 | // Warning: extra API call 128 | return this.getData(this.#getTagsUrl(tagId), this.getType()).then(tagData => { 129 | return tagData.name; 130 | }); 131 | })); 132 | } 133 | 134 | async #getCategories(ids) { 135 | let categoryNames = await Promise.all(ids.map(categoryId => { 136 | // Warning: extra API call 137 | return this.getData(this.#getCategoryUrl(categoryId), this.getType()).then(categoryData => { 138 | return categoryData.name; 139 | }); 140 | })); 141 | 142 | return categoryNames.filter(name => { 143 | return !WordPressApi.IGNORED_CATEGORIES.includes(name); 144 | }); 145 | } 146 | 147 | getRawEntryDates(rawEntry) { 148 | return { 149 | created: this.toDateObj(rawEntry.date_gmt), 150 | updated: this.toDateObj(rawEntry.modified_gmt), 151 | }; 152 | } 153 | 154 | // Supports: Title, Author, Published/Updated Dates 155 | async cleanEntry(rawEntry, data) { 156 | let url = this.getUrlFromEntry(rawEntry); 157 | let status = this.cleanStatus(rawEntry.status) 158 | 159 | let metadata = {}; 160 | if(rawEntry.jetpack_featured_media_url || rawEntry.og_image) { 161 | let media = {}; 162 | if(rawEntry.og_image) { 163 | media.opengraphImage = rawEntry.og_image?.url; 164 | } 165 | if(rawEntry.jetpack_featured_media_url) { 166 | media.featuredImage = rawEntry.jetpack_featured_media_url; 167 | 168 | // backwards compatibility (not downloaded or optimized) 169 | metadata.featuredImage = rawEntry.jetpack_featured_media_url; 170 | } 171 | metadata.media = media; 172 | } 173 | 174 | let categories = await this.#getCategories(rawEntry.categories); 175 | if(categories.length) { 176 | metadata.categories = categories; 177 | } 178 | 179 | let tags = await this.#getTags(rawEntry.tags); 180 | if(tags.length) { 181 | metadata.tags = tags; 182 | } 183 | 184 | let { created, updated } = this.getRawEntryDates(rawEntry); 185 | 186 | let cleanEntry = { 187 | uuid: this.getUniqueIdFromEntry(rawEntry), 188 | type: WordPressApi.TYPE, 189 | title: rawEntry.title?.rendered, 190 | url, 191 | authors: await this.#getAuthors(rawEntry.author), 192 | date: created, 193 | dateUpdated: updated, 194 | content: rawEntry.content.rendered, 195 | contentType: "html", 196 | status, 197 | metadata, 198 | }; 199 | 200 | if(metadata.categories) { 201 | // map WordPress categories for use in Eleventy tags (not WordPress metadata tags, which are different) 202 | cleanEntry.tags = metadata.categories; 203 | } 204 | 205 | return cleanEntry; 206 | } 207 | } 208 | 209 | export { WordPressApi }; 210 | -------------------------------------------------------------------------------- /src/DataSource/YouTubeUser.js: -------------------------------------------------------------------------------- 1 | import { DataSource } from "../DataSource.js"; 2 | 3 | class YouTubeUser extends DataSource { 4 | static TYPE = "youtube"; 5 | static TYPE_FRIENDLY = "YouTube"; 6 | 7 | constructor(channelId) { 8 | super(); 9 | this.channelId = channelId; 10 | } 11 | 12 | getType() { 13 | return "xml"; 14 | } 15 | 16 | getUrl() { 17 | return `https://www.youtube.com/feeds/videos.xml?channel_id=${this.channelId}` 18 | } 19 | 20 | getEntriesFromData(data) { 21 | return data.feed?.entry || []; 22 | } 23 | 24 | getUniqueIdFromEntry(entry) { 25 | return `${DataSource.UUID_PREFIX}::${YouTubeUser.TYPE}::${entry['yt:videoId']}`; 26 | } 27 | 28 | static getFilePath(url) { 29 | let { searchParams } = new URL(url); 30 | return searchParams.get("v"); 31 | } 32 | 33 | getRawEntryDates(rawEntry) { 34 | return { 35 | created: this.toDateObj(rawEntry.published), 36 | updated: this.toDateObj(rawEntry.updated), 37 | }; 38 | } 39 | 40 | cleanEntry(rawEntry) { 41 | let { created, updated } = this.getRawEntryDates(rawEntry); 42 | 43 | return { 44 | uuid: this.getUniqueIdFromEntry(rawEntry), 45 | type: YouTubeUser.TYPE, 46 | title: rawEntry.title, 47 | url: `https://www.youtube.com/watch?v=${rawEntry['yt:videoId']}`, 48 | authors: [ 49 | { 50 | name: rawEntry.author.name, 51 | url: rawEntry.author.uri, 52 | } 53 | ], 54 | date: created, 55 | dateUpdated: updated, 56 | // TODO linkify, nl2br 57 | content: rawEntry['media:group']['media:description'], 58 | contentType: "text", 59 | } 60 | } 61 | } 62 | 63 | export {YouTubeUser}; 64 | -------------------------------------------------------------------------------- /src/DirectoryManager.js: -------------------------------------------------------------------------------- 1 | import fs from "graceful-fs"; 2 | 3 | class DirectoryManager { 4 | static getDirectory(pathname) { 5 | let dirs = pathname.split("/"); 6 | dirs.pop(); 7 | return dirs.join("/"); 8 | } 9 | 10 | constructor() { 11 | this.created = new Set(); 12 | this.dryRun = false; 13 | } 14 | 15 | setDryRun(isDryRun) { 16 | this.dryRun = Boolean(isDryRun); 17 | } 18 | 19 | createDirectoryForPath(pathname) { 20 | if(this.dryRun) { 21 | return; 22 | } 23 | 24 | let dir = DirectoryManager.getDirectory(pathname); 25 | if(dir && !this.created.has(dir)) { 26 | fs.mkdirSync(dir, { recursive: true }) 27 | 28 | this.created.add(dir); 29 | } 30 | } 31 | } 32 | 33 | export { DirectoryManager }; 34 | -------------------------------------------------------------------------------- /src/Fetcher.js: -------------------------------------------------------------------------------- 1 | import fs from "graceful-fs"; 2 | import { createHash } from "node:crypto"; 3 | import kleur from "kleur"; 4 | import { XMLParser } from "fast-xml-parser"; 5 | 6 | import EleventyFetch from "@11ty/eleventy-fetch"; 7 | import { DirectoryManager } from "./DirectoryManager.js"; 8 | import { Logger } from "./Logger.js"; 9 | import { Utils } from "./Utils.js"; 10 | 11 | // 255 total (hash + url + extension) 12 | const HASH_FILENAME_MAXLENGTH = 12; 13 | const MAXIMUM_URL_FILENAME_SIZE = 30; 14 | 15 | // TODO use `type: "parsed-xml" type from Eleventy Fetch 16 | const xmlParser = new XMLParser({ 17 | attributeNamePrefix : "@_", 18 | ignoreAttributes: false, 19 | allowBooleanAttributes: true, 20 | parseAttributeValue: true, 21 | processEntities: false, // disable this, was causing inconsistencies in Bluesky entries 22 | // htmlEntities: true, 23 | }); 24 | 25 | class Fetcher { 26 | #cacheDuration = "0s"; 27 | #directoryManager; 28 | #assetsFolder = "assets"; 29 | #persistManager; 30 | #outputFolder = "."; 31 | #downloadAssets = true; 32 | 33 | static USER_AGENT = "Eleventy Import v1.0.0"; 34 | 35 | static getContextPathname(url) { 36 | if(url) { 37 | let u = (new URL(url)).pathname.split("/").filter(entry => Boolean(entry)); 38 | // pop off the top folder 39 | u.pop(); 40 | return u.join("/"); 41 | } 42 | return ""; 43 | } 44 | 45 | static getFilenameFromSrc(src, contentType = "") { 46 | let {pathname} = new URL(src); 47 | let hash = this.createHash(src); 48 | 49 | let filename = decodeURIComponent(pathname.split("/").pop()); 50 | let lastDot = filename.lastIndexOf("."); 51 | 52 | if(lastDot > -1) { 53 | let filenameWithoutExtension = filename.slice(0, Math.min(lastDot, MAXIMUM_URL_FILENAME_SIZE)); 54 | let extension = filename.slice(lastDot + 1); 55 | return `${filenameWithoutExtension}-${hash}.${extension}`; 56 | } 57 | 58 | let [, fileExtensionFallback] = contentType.split("/"); 59 | 60 | // No known file extension 61 | return `${filename.slice(0, MAXIMUM_URL_FILENAME_SIZE)}-${hash}${fileExtensionFallback ? `.${fileExtensionFallback}` : ""}`; 62 | } 63 | 64 | static createHash(str) { 65 | let base64Hash = createHash("sha256").update(str).digest("base64"); 66 | 67 | return base64Hash.replace(/[^A-Z0-9]/gi, "").slice(0, HASH_FILENAME_MAXLENGTH); 68 | } 69 | 70 | static parseXml(content) { 71 | return xmlParser.parse(content); 72 | } 73 | 74 | constructor() { 75 | this.fetchedUrls = new Set(); 76 | this.writtenAssetFiles = new Set(); 77 | this.errors = new Set(); 78 | this.isVerbose = true; 79 | this.dryRun = false; 80 | this.safeMode = true; 81 | this.useRelativeAssets = true; 82 | this.counts = { 83 | assets: 0, 84 | }; 85 | } 86 | 87 | setVerbose(isVerbose) { 88 | this.isVerbose = Boolean(isVerbose); 89 | } 90 | 91 | setDryRun(isDryRun) { 92 | this.dryRun = Boolean(isDryRun); 93 | } 94 | 95 | setSafeMode(safeMode) { 96 | this.safeMode = Boolean(safeMode); 97 | } 98 | 99 | setAssetsFolder(folder) { 100 | this.#assetsFolder = folder; 101 | } 102 | 103 | setDownloadAssets(download) { 104 | this.#downloadAssets = Boolean(download); 105 | } 106 | 107 | setUseRelativeAssetPaths(use) { 108 | this.useRelativeAssets = Boolean(use); 109 | } 110 | 111 | setOutputFolder(dir) { 112 | this.#outputFolder = dir; 113 | } 114 | 115 | getCounts() { 116 | return { 117 | assets: this.counts.assets, 118 | errors: this.errors.size, 119 | } 120 | } 121 | 122 | setCacheDuration(duration) { 123 | this.#cacheDuration = duration; 124 | } 125 | 126 | setDirectoryManager(manager) { 127 | this.#directoryManager = manager; 128 | } 129 | 130 | setPersistManager(manager) { 131 | this.#persistManager = manager; 132 | } 133 | 134 | getAssetLocation(assetUrl, assetContentType, contextEntry) { 135 | let filename = Fetcher.getFilenameFromSrc(assetUrl, assetContentType); 136 | let assetUrlLocation = Utils.pathJoin(this.#assetsFolder, filename); 137 | // root /assets folder 138 | if(!this.useRelativeAssets) { 139 | return { 140 | url: `/${assetUrlLocation}`, 141 | filePath: Utils.pathJoin(this.#outputFolder, assetUrlLocation), 142 | }; 143 | } 144 | 145 | let contextPathname; 146 | if(contextEntry.filePath) { 147 | contextPathname = DirectoryManager.getDirectory(contextEntry.filePath); 148 | } else { 149 | // backwards compatibility 150 | contextPathname = Fetcher.getContextPathname(contextEntry.url); 151 | } 152 | 153 | return { 154 | url: assetUrlLocation, 155 | // filePath: Utils.pathJoin(this.#outputFolder, contextPathname, assetUrlLocation), 156 | filePath: Utils.pathJoin(contextPathname, assetUrlLocation), 157 | } 158 | } 159 | 160 | async fetchAsset(assetUrl, contextEntry) { 161 | if(!this.#downloadAssets) { 162 | return assetUrl; 163 | } 164 | 165 | // Adds protocol from original page URL if a protocol relative URL 166 | if(assetUrl.startsWith("//") && contextEntry.url) { 167 | let contextUrl = new URL(contextEntry.url); 168 | if(contextUrl.protocol) { 169 | assetUrl = `${contextUrl.protocol}${assetUrl}`; 170 | } 171 | } 172 | 173 | // TODO move this upstream as a Fetch `alias` feature. 174 | return this.fetch(assetUrl, { 175 | type: "buffer", 176 | returnType: "response", 177 | }, 178 | { 179 | verbose: true, 180 | showErrors: true, 181 | }).then(result => { 182 | let { url: urlValue, filePath: fullOutputLocation } = this.getAssetLocation(assetUrl, result.headers?.["content-type"], contextEntry); 183 | 184 | if(this.writtenAssetFiles.has(fullOutputLocation)) { 185 | return urlValue; 186 | } 187 | 188 | this.writtenAssetFiles.add(fullOutputLocation); 189 | 190 | // TODO compare file contents and skip 191 | if(this.safeMode && fs.existsSync(fullOutputLocation)) { 192 | if(this.isVerbose) { 193 | Logger.skipping("asset", fullOutputLocation, assetUrl); 194 | } 195 | return urlValue; 196 | } 197 | 198 | if(this.#directoryManager) { 199 | this.#directoryManager.createDirectoryForPath(fullOutputLocation); 200 | } 201 | 202 | if(this.isVerbose) { 203 | Logger.importing("asset", fullOutputLocation, assetUrl, { 204 | size: result.body.length, 205 | dryRun: this.dryRun 206 | }); 207 | } 208 | 209 | if(!this.dryRun) { 210 | this.counts.assets++; 211 | 212 | fs.writeFileSync(fullOutputLocation, result.body); 213 | } 214 | 215 | // Don’t persist (e.g. back to GitHub) assets if upstream post is a draft 216 | if(contextEntry.status !== "draft" && this.#persistManager.canPersist()) { 217 | this.#persistManager.persistFile(fullOutputLocation, result.body, { 218 | assetUrl, 219 | type: "asset", 220 | }); 221 | } 222 | 223 | return urlValue; 224 | }, error => { 225 | // Error logging happens in .fetch() upstream 226 | // Fetching the asset failed but we don’t want to fail the upstream document promise 227 | return assetUrl; 228 | }); 229 | } 230 | 231 | async fetch(url, options = {}, verbosity = {}) { 232 | let { verbose, showErrors } = Object.assign({ 233 | verbose: true, // whether to log the initial fetch request 234 | showErrors: true, // whether to show if a request has an error. 235 | }, verbosity); 236 | 237 | let opts = Object.assign({ 238 | duration: this.#cacheDuration, 239 | type: "text", 240 | verbose: false, // don’t use Fetch logging—we’re handling it ourself 241 | fetchOptions: {}, 242 | }, options); 243 | 244 | if(!opts.fetchOptions.headers) { 245 | opts.fetchOptions.headers = {}; 246 | } 247 | Object.assign(opts.fetchOptions.headers, { 248 | "user-agent": Fetcher.USER_AGENT 249 | }); 250 | 251 | if(!this.fetchedUrls.has(url) && this.isVerbose && verbose) { 252 | let logAdds = []; 253 | if(Boolean(options?.fetchOptions?.headers?.Authorization)) { 254 | logAdds.push(kleur.blue("Auth")); 255 | } 256 | if(opts.duration) { 257 | logAdds.push(kleur.green(`(${opts.duration} cache)`)); 258 | } 259 | 260 | Logger.log(kleur.gray("Fetching"), url, logAdds.join(" ") ); 261 | } 262 | 263 | this.fetchedUrls.add(url); 264 | 265 | return EleventyFetch(url, opts).then(result => { 266 | if(opts.type === "xml") { 267 | return Fetcher.parseXml(result); 268 | } 269 | 270 | return result; 271 | }, error => { 272 | if(!this.errors.has(url)) { 273 | this.errors.add(url); 274 | 275 | if(this.isVerbose && showErrors) { 276 | Logger.log(kleur.red(`Error fetching`), url, kleur.red(error.message)); 277 | } 278 | } 279 | 280 | return Promise.reject(error); 281 | }); 282 | } 283 | } 284 | 285 | export { Fetcher }; 286 | -------------------------------------------------------------------------------- /src/HtmlTransformer.js: -------------------------------------------------------------------------------- 1 | import posthtml from "posthtml"; 2 | import urls from "@11ty/posthtml-urls"; 3 | 4 | class HtmlTransformer { 5 | #fetcher; 6 | 7 | setFetcher(fetcher) { 8 | this.#fetcher = fetcher; 9 | } 10 | 11 | async transform(content, entry) { 12 | let options = { 13 | eachURL: async (rawUrl, attr, tagName) => { 14 | // See https://github.com/11ty/eleventy-posthtml-urls/blob/main/lib/defaultOptions.js 15 | if(tagName === "img" || tagName === "video" || tagName === "source" || tagName === "link" || tagName === "script" || tagName === "track") { 16 | return this.#fetcher.fetchAsset(rawUrl, entry); 17 | } 18 | 19 | return rawUrl; 20 | } 21 | }; 22 | 23 | let result = await posthtml() 24 | .use(urls(options)) 25 | .process(content); 26 | 27 | return result.html; 28 | } 29 | } 30 | 31 | export { HtmlTransformer } 32 | -------------------------------------------------------------------------------- /src/Importer.js: -------------------------------------------------------------------------------- 1 | import path from "node:path"; 2 | import fs from "graceful-fs"; 3 | import yaml from "js-yaml"; 4 | import kleur from "kleur"; 5 | import slugify from '@sindresorhus/slugify'; 6 | import * as entities from "entities"; 7 | 8 | import { Logger } from "./Logger.js"; 9 | import { Fetcher } from "./Fetcher.js"; 10 | import { DirectoryManager } from "./DirectoryManager.js"; 11 | import { MarkdownToHtml } from "./MarkdownToHtml.js"; 12 | import { HtmlTransformer } from "./HtmlTransformer.js"; 13 | import { Persist } from "./Persist.js"; 14 | 15 | // Data Sources 16 | import { DataSource } from "./DataSource.js"; 17 | import { YouTubeUser } from "./DataSource/YouTubeUser.js"; 18 | import { Atom } from "./DataSource/Atom.js"; 19 | import { Rss } from "./DataSource/Rss.js"; 20 | import { WordPressApi } from "./DataSource/WordPressApi.js"; 21 | import { BlueskyUser } from "./DataSource/BlueskyUser.js"; 22 | import { FediverseUser } from "./DataSource/FediverseUser.js"; 23 | 24 | import pkg from "../package.json" with { type: "json" }; 25 | 26 | // For testing 27 | const MAX_IMPORT_SIZE = 0; 28 | 29 | class Importer { 30 | #draftsFolder = "drafts"; 31 | #outputFolder = "."; 32 | #assetReferenceType; 33 | 34 | constructor() { 35 | this.startTime = new Date(); 36 | this.sources = []; 37 | this.isVerbose = true; 38 | this.dryRun = false; 39 | this.safeMode = true; 40 | this.allowDraftsToOverwrite = false; 41 | this.counts = { 42 | files: 0 43 | }; 44 | 45 | this.markdownService = new MarkdownToHtml(); 46 | this.htmlTransformer = new HtmlTransformer(); 47 | this.directoryManager = new DirectoryManager(); 48 | this.persistManager = new Persist(); 49 | this.fetcher = new Fetcher(); 50 | 51 | this.htmlTransformer.setFetcher(this.fetcher); 52 | 53 | this.fetcher.setDirectoryManager(this.directoryManager); 54 | this.fetcher.setPersistManager(this.persistManager); 55 | } 56 | 57 | // CSS selectors to preserve on markdown conversion 58 | addPreserved(selectors) { 59 | for(let sel of (selectors || "").split(",")) { 60 | this.markdownService.addPreservedSelector(sel); 61 | } 62 | } 63 | 64 | getCounts() { 65 | return { 66 | ...this.counts, 67 | ...this.fetcher.getCounts(), 68 | ...this.markdownService.getCounts(), 69 | ...this.persistManager.getCounts() 70 | } 71 | } 72 | 73 | // --overwrite--allow (independent of and bypasses --overwrite) 74 | setOverwriteAllow(overwrite = "") { 75 | let s = overwrite.split(","); 76 | if(s.includes("drafts")) { 77 | this.allowDraftsToOverwrite = true; 78 | } 79 | } 80 | 81 | setSafeMode(safeMode) { 82 | this.safeMode = Boolean(safeMode); 83 | 84 | this.fetcher.setSafeMode(safeMode); 85 | } 86 | 87 | setDryRun(isDryRun) { 88 | this.dryRun = Boolean(isDryRun); 89 | 90 | this.fetcher.setDryRun(isDryRun); 91 | this.directoryManager.setDryRun(isDryRun); 92 | this.persistManager.setDryRun(isDryRun); 93 | } 94 | 95 | setVerbose(isVerbose) { 96 | this.isVerbose = Boolean(isVerbose); 97 | 98 | this.fetcher.setVerbose(isVerbose); 99 | this.markdownService.setVerbose(isVerbose); 100 | this.persistManager.setVerbose(isVerbose); 101 | 102 | for(let source of this.sources) { 103 | source.setVerbose(isVerbose); 104 | } 105 | } 106 | 107 | setAssetsFolder(folder) { 108 | this.fetcher.setAssetsFolder(folder); 109 | } 110 | 111 | shouldDownloadAssets() { 112 | return this.#assetReferenceType !== "disabled"; 113 | } 114 | 115 | isAssetsColocated() { 116 | return this.#assetReferenceType === "colocate"; 117 | } 118 | 119 | setAssetReferenceType(refType) { 120 | if(refType === "colocate") { 121 | // no assets subfolder 122 | this.setAssetsFolder(""); 123 | } 124 | 125 | if(refType === "disabled") { 126 | this.fetcher.setDownloadAssets(false); 127 | } else if(refType === "absolute") { 128 | this.fetcher.setUseRelativeAssetPaths(false); 129 | } else if(refType === "relative" || refType === "colocate") { 130 | this.fetcher.setUseRelativeAssetPaths(true); 131 | } else { 132 | throw new Error(`Invalid value for --assetrefs, must be one of: relative, colocate, absolute, or disabled. Received: ${refType} (${typeof refType})`); 133 | } 134 | 135 | this.#assetReferenceType = refType; 136 | } 137 | 138 | setDraftsFolder(dir) { 139 | this.#draftsFolder = dir; 140 | } 141 | 142 | setOutputFolder(dir) { 143 | this.#outputFolder = dir; 144 | this.fetcher.setOutputFolder(dir); 145 | } 146 | 147 | setCacheDuration(duration) { 148 | if(duration) { 149 | this.fetcher.setCacheDuration(duration); 150 | } 151 | } 152 | 153 | setPersistTarget(persistTarget) { 154 | this.persistManager.setTarget(persistTarget); 155 | } 156 | 157 | addSource(type, options = {}) { 158 | let cls; 159 | if(typeof type === "string") { 160 | type = type?.toLowerCase(); 161 | 162 | if(type === "youtubeuser") { 163 | cls = YouTubeUser; 164 | } else if(type === "atom") { 165 | cls = Atom; 166 | } else if(type === "rss") { 167 | cls = Rss; 168 | } else if(type === "wordpress") { 169 | cls = WordPressApi; 170 | } else if(type === "bluesky") { 171 | cls = BlueskyUser; // RSS 172 | } else if(type === "fediverse") { 173 | cls = FediverseUser; // RSS 174 | } 175 | } else if(typeof type === "function") { 176 | cls = type; 177 | } 178 | 179 | if(!cls) { 180 | throw new Error(`${type} is not a supported type for addSource(). Requires a string type or a DataSource class.`); 181 | } 182 | 183 | let identifier; 184 | let label; 185 | let filepathFormat; 186 | 187 | if(typeof options === "string") { 188 | identifier = options; 189 | } else { 190 | identifier = options.url || options.id; 191 | label = options.label; 192 | filepathFormat = options.filepathFormat; 193 | } 194 | 195 | let source = new cls(identifier); 196 | 197 | if(!(source instanceof DataSource)) { 198 | throw new Error(`${cls?.name} is not a supported type for addSource(). Requires a string type or a DataSource class.`); 199 | } 200 | 201 | source.setFetcher(this.fetcher); 202 | source.setVerbose(this.isVerbose); 203 | 204 | if(this.#outputFolder) { 205 | source.setOutputFolder(this.#outputFolder); 206 | } 207 | 208 | if(label) { 209 | source.setLabel(label); 210 | } 211 | 212 | if(filepathFormat) { 213 | source.setFilepathFormatFunction(filepathFormat); 214 | } 215 | 216 | this.sources.push(source); 217 | } 218 | 219 | getSources() { 220 | return this.sources; 221 | } 222 | 223 | getSourcesForType(type) { 224 | return this.sources.filter(entry => entry.constructor.TYPE === type); 225 | } 226 | 227 | addDataOverride(type, url, data) { 228 | let found = false; 229 | for(let source of this.getSourcesForType(type)) { 230 | source.setDataOverride(url, data); 231 | found = true; 232 | } 233 | 234 | if(!found) { 235 | throw new Error("addDataOverride(type) not found: " + type) 236 | } 237 | } 238 | 239 | static shouldUseMarkdownFileExtension(entry) { 240 | return this.isText(entry) || this.isHtml(entry); 241 | } 242 | 243 | static shouldConvertToMarkdown(entry) { 244 | return this.isHtml(entry); 245 | } 246 | 247 | static isText(entry) { 248 | return entry.contentType === "text"; 249 | } 250 | 251 | static isHtml(entry) { 252 | // TODO add a CLI override for --importContentType? 253 | // TODO add another path to guess if content is HTML https://mimesniff.spec.whatwg.org/#identifying-a-resource-with-an-unknown-mime-type 254 | return entry.contentType === "html"; 255 | } 256 | 257 | async fetchRelatedMedia(cleanEntry) { 258 | let relatedMedia = cleanEntry?.metadata?.media; 259 | if(!relatedMedia) { 260 | return; 261 | } 262 | 263 | for(let mediaType in relatedMedia || {}) { 264 | let rawUrl = relatedMedia[mediaType]; 265 | let localUrl = await this.fetcher.fetchAsset(rawUrl, cleanEntry); 266 | 267 | // TODO parallel 268 | cleanEntry.metadata.media[mediaType] = localUrl; 269 | } 270 | } 271 | 272 | async getTransformedContent(entry, isWritingToMarkdown) { 273 | let content = entry.content; 274 | 275 | if(Importer.isHtml(entry)) { 276 | let transformedHtml = content; 277 | if(!isWritingToMarkdown) { 278 | // decoding built-in with Markdown 279 | transformedHtml = entities.decodeHTML(content); 280 | } 281 | 282 | if(!this.shouldDownloadAssets()) { 283 | content = transformedHtml; 284 | } else { 285 | content = await this.htmlTransformer.transform(transformedHtml, entry); 286 | } 287 | } 288 | 289 | if(isWritingToMarkdown) { 290 | if(Importer.isText(entry)) { 291 | // _only_ decode newlines 292 | content = content.split(" ").join("\n"); 293 | } 294 | 295 | if(Importer.shouldConvertToMarkdown(entry)) { 296 | await this.markdownService.asyncInit(); 297 | 298 | content = await this.markdownService.toMarkdown(content, entry); 299 | } 300 | } 301 | 302 | return content; 303 | } 304 | 305 | 306 | // Is used to filter getEntries and in toFiles (which also checks conflicts) 307 | shouldSkipEntry(entry) { 308 | if(entry.filePath === false) { 309 | return true; 310 | } 311 | 312 | // File system operations 313 | // TODO use https://www.npmjs.com/package/diff to compare file contents and skip 314 | if(this.safeMode && fs.existsSync(entry.filePath)) { 315 | // Not a draft or drafts are skipped (via --overwrite-allow) 316 | if(entry.status !== "draft" || !this.allowDraftsToOverwrite) { 317 | return true; 318 | } 319 | } 320 | 321 | return false; 322 | } 323 | 324 | async getEntries(options = {}) { 325 | let isWritingToMarkdown = options.contentType === "markdown"; 326 | 327 | for(let source of this.sources) { 328 | source.setWithin(options.within); 329 | } 330 | 331 | let entries = []; 332 | for(let source of this.sources) { 333 | for(let entry of await source.getEntries()) { 334 | let contentType = entry.contentType; 335 | if(Importer.shouldUseMarkdownFileExtension(entry) && isWritingToMarkdown) { 336 | contentType = "markdown"; 337 | } 338 | 339 | entry.filePath = this.getFilePath(entry, contentType); 340 | 341 | // to prevent fetching assets and transforming contents on entries that won’t get written 342 | if(options.target === "fs" && this.shouldSkipEntry(entry)) { 343 | // do nothing 344 | } else { 345 | entries.push(entry); 346 | } 347 | } 348 | } 349 | 350 | // purely for internals testing 351 | if(MAX_IMPORT_SIZE) { 352 | entries = entries.slice(0, MAX_IMPORT_SIZE); 353 | } 354 | 355 | let promises = await Promise.allSettled(entries.map(async entry => { 356 | await this.fetchRelatedMedia(entry); 357 | 358 | entry.content = await this.getTransformedContent(entry, isWritingToMarkdown); 359 | 360 | if(isWritingToMarkdown && Importer.shouldConvertToMarkdown(entry)) { 361 | entry.contentType = "markdown"; 362 | } 363 | 364 | return entry; 365 | })); 366 | 367 | if(!this.dryRun) { 368 | this.markdownService.cleanup(); 369 | } 370 | 371 | return promises.filter(entry => { 372 | // Documents with errors 373 | return entry.status !== "rejected"; 374 | }).map(entry => { 375 | return entry.value; 376 | }).sort((a, b) => { 377 | if(a.date < b.date) { 378 | return 1; 379 | } 380 | if(a.date > b.date) { 381 | return -1; 382 | } 383 | return 0; 384 | }); 385 | } 386 | 387 | getFilePath(entry, contentType) { 388 | let { url } = entry; 389 | 390 | let source = entry.source; 391 | 392 | // prefer addSource specific override, then fallback to DataSource type default 393 | let fallbackPath; 394 | let hasFilePathFallback = typeof source?.constructor?.getFilePath === "function"; 395 | if(hasFilePathFallback) { 396 | fallbackPath = source?.constructor?.getFilePath(url); 397 | } else { 398 | fallbackPath = (new URL(url)).pathname; 399 | } 400 | 401 | // Data source specific override 402 | let outputOverrideFn = source?.getFilepathFormatFunction(); 403 | if(outputOverrideFn && typeof outputOverrideFn === "function") { 404 | let pathname = outputOverrideFn(url, fallbackPath); 405 | if(pathname === false) { 406 | return false; 407 | } 408 | 409 | // does method does *not* add a file extension for you, you must supply one in `filepathFormat` function 410 | return path.join(this.#outputFolder, pathname); 411 | } 412 | 413 | // WordPress draft posts only have a `p` query param e.g. ?p=ID_NUMBER 414 | if(fallbackPath === "/") { 415 | fallbackPath = Fetcher.createHash(entry.url); 416 | } 417 | 418 | let subdirs = []; 419 | if(this.#outputFolder) { 420 | subdirs.push(this.#outputFolder); 421 | } 422 | if(this.#draftsFolder && entry.status === "draft") { 423 | subdirs.push(this.#draftsFolder); 424 | } 425 | 426 | let pathname = path.join(".", ...subdirs, path.normalize(fallbackPath)); 427 | let extension = contentType === "markdown" ? ".md" : ".html"; 428 | 429 | if(pathname.endsWith("/")) { 430 | if(this.isAssetsColocated()) { 431 | return `${pathname}index${extension}`; 432 | } 433 | return `${pathname.slice(0, -1)}${extension}`; 434 | } 435 | 436 | if(this.isAssetsColocated()) { 437 | return `${pathname}/index${extension}`; 438 | } 439 | return `${pathname}${extension}`; 440 | } 441 | 442 | static convertEntryToYaml(entry) { 443 | let data = {}; 444 | data.title = entry.title; 445 | data.authors = entry.authors; 446 | data.date = entry.date; 447 | data.metadata = entry.metadata || {}; 448 | data.metadata.uuid = entry.uuid; 449 | data.metadata.type = entry.type; 450 | data.metadata.url = entry.url; 451 | 452 | // Eleventy specific options 453 | if(entry.status === "draft") { 454 | data.draft = true; 455 | } 456 | 457 | if(entry.tags) { 458 | if(!Array.isArray(entry.tags)) { 459 | entry.tags = [entry.tags]; 460 | } 461 | 462 | // slugify the tags 463 | data.tags = entry.tags.map(tag => slugify(tag)); 464 | } 465 | 466 | // https://www.npmjs.com/package/js-yaml#dump-object---options- 467 | let frontMatter = yaml.dump(data, { 468 | // sortKeys: true, 469 | noCompatMode: true, 470 | }); 471 | 472 | return frontMatter; 473 | } 474 | 475 | // TODO options.pathPrefix 476 | async toFiles(entries = []) { 477 | let filepathConflicts = {}; 478 | 479 | for(let entry of entries) { 480 | let pathname = entry.filePath; 481 | if(pathname === false) { 482 | continue; 483 | } 484 | 485 | if(filepathConflicts[pathname]) { 486 | throw new Error(`Multiple entries attempted to write to the same place: ${pathname} (originally via ${filepathConflicts[pathname]})`); 487 | } 488 | filepathConflicts[pathname] = entry.url || true; 489 | 490 | let frontMatter = Importer.convertEntryToYaml(entry); 491 | let content = `--- 492 | ${frontMatter}--- 493 | ${entry.content}`; 494 | 495 | if(this.shouldSkipEntry(entry)) { 496 | if(this.isVerbose) { 497 | Logger.skipping("post", pathname, entry.url); 498 | } 499 | continue; 500 | } 501 | 502 | if(this.isVerbose) { 503 | Logger.importing("post", pathname, entry.url, { 504 | size: content.length, 505 | dryRun: this.dryRun 506 | }); 507 | } 508 | 509 | if(!this.dryRun) { 510 | this.counts.files++; 511 | 512 | this.directoryManager.createDirectoryForPath(pathname); 513 | 514 | fs.writeFileSync(pathname, content, { encoding: "utf8" }); 515 | } 516 | 517 | // Happens independent of file system (--dryrun or --overwrite) 518 | // Don’t persist if post is a draft 519 | if(entry.status !== "draft" && this.persistManager.canPersist()) { 520 | await this.persistManager.persistFile(pathname, content, { 521 | url: entry.url, 522 | type: "post", 523 | }); 524 | } 525 | } 526 | } 527 | 528 | logResults() { 529 | let counts = this.getCounts(); 530 | let sourcesDisplay = this.getSources().map(source => source.constructor.TYPE_FRIENDLY || source.constructor.TYPE).join(", "); 531 | let content = []; 532 | content.push(kleur.green("Wrote")); 533 | content.push(kleur.green(`${counts.files} ${Logger.plural(counts.files, "document")}`)); 534 | content.push(kleur.green("and")); 535 | content.push(kleur.green(`${counts.assets - counts.cleaned} ${Logger.plural(counts.assets - counts.cleaned, "asset")}`)); 536 | if(counts.cleaned) { 537 | content.push(kleur.gray(`(${counts.cleaned} cleaned, unused)`)); 538 | } 539 | content.push(kleur.green(`from ${sourcesDisplay}`)); 540 | if(counts.persist) { 541 | content.push(kleur.blue(`(${counts.persist} persisted)`)); 542 | } 543 | content.push(kleur[counts.errors > 0 ? "red" : "gray"](`(${counts.errors} ${Logger.plural(counts.errors, "error")})`)); 544 | if(this.startTime) { 545 | content.push(`in ${Logger.time(Date.now() - this.startTime)}`); 546 | } 547 | 548 | content.push(`(v${pkg.version})`); 549 | 550 | Logger.log(content.join(" ")); 551 | } 552 | } 553 | 554 | export { Importer }; 555 | -------------------------------------------------------------------------------- /src/Logger.js: -------------------------------------------------------------------------------- 1 | import kleur from "kleur"; 2 | import { filesize } from "filesize"; 3 | 4 | class Logger { 5 | static log(...messages) { 6 | console.log(...messages); 7 | } 8 | 9 | static _logFsOperation(label, type, local, remote, options = {}) { 10 | let { size, dryRun } = options; 11 | 12 | let extras = []; 13 | let prefix = ""; 14 | if(label === "Skipping") { 15 | prefix = " (no --overwrite)"; 16 | } else { 17 | if(size) { 18 | extras.push(filesize(size, { 19 | spacer: "" 20 | })); 21 | } 22 | 23 | if(dryRun) { 24 | prefix = " (dry run)"; 25 | } 26 | } 27 | 28 | let extrasStr = extras.length ? `(${extras.join(", ")}) ` : ""; 29 | if(remote) { 30 | this.log(kleur.gray(`${label} ${type}${prefix}`), local, kleur.gray(`${extrasStr}from`), remote); 31 | } else { 32 | this.log(kleur.gray(`${label} ${type}${prefix}`), local, kleur.gray(extrasStr)); 33 | } 34 | } 35 | 36 | static importing(type, local, remote, options = {}) { 37 | this._logFsOperation("Importing", type, local, remote, options); 38 | } 39 | 40 | static persisting(type, local, remote, options = {}) { 41 | this._logFsOperation("Persisting", type, local, remote, options); 42 | } 43 | 44 | static skipping(type, local, remote, options = {}) { 45 | this._logFsOperation("Skipping", type, local, remote, options); 46 | } 47 | 48 | static cleanup(type, local, options = {}) { 49 | this._logFsOperation("Cleaning", type, local, undefined, options); 50 | } 51 | 52 | // alias for log 53 | static message(...messages) { 54 | this.log(...messages); 55 | } 56 | 57 | static warning(...messages) { 58 | this.message(...(messages.map(msg => kleur.yellow(msg)))); 59 | } 60 | 61 | static error(...messages) { 62 | this.message(...(messages.map(msg => kleur.red(msg)))); 63 | } 64 | 65 | static time(ms) { 66 | if(ms > 1000) { 67 | let v = ms/1000; 68 | return `${v.toFixed(2)} ${this.plural(v, "second")}`; 69 | } 70 | return `${ms} ${this.plural(ms, "millisecond")}`; 71 | } 72 | 73 | static plural(num, singular, plural) { 74 | if(!plural) { 75 | plural = singular + "s"; 76 | } 77 | return num !== 1 ? plural : singular; 78 | } 79 | } 80 | 81 | export { Logger } 82 | -------------------------------------------------------------------------------- /src/MarkdownToHtml.js: -------------------------------------------------------------------------------- 1 | import path from "node:path"; 2 | import fs from "graceful-fs"; 3 | import TurndownService from "turndown"; 4 | import * as prettier from "prettier"; 5 | import prettierSync from "@prettier/sync"; 6 | import striptags from "striptags"; 7 | import * as entities from "entities"; 8 | 9 | import { Logger } from "./Logger.js"; 10 | import { DirectoryManager } from "./DirectoryManager.js"; 11 | import { WordPressApi } from "./DataSource/WordPressApi.js"; 12 | import { HostedWordPressApi } from "./DataSource/HostedWordPressApi.js"; 13 | 14 | const WORDPRESS_TO_PRISM_LANGUAGE_TRANSLATION = { 15 | jscript: "js", 16 | markup: "html", 17 | }; 18 | 19 | const TAGS_TO_KEEP = [ 20 | "abbr", 21 | "address", 22 | "audio", 23 | "cite", 24 | "dd", 25 | "del", 26 | "details", 27 | // "dialog", 28 | "dfn", 29 | // "figure", 30 | "form", 31 | "iframe", 32 | "ins", 33 | "kbd", 34 | "object", 35 | "q", 36 | "sub", 37 | "s", 38 | "samp", 39 | "svg", 40 | "table", 41 | "time", 42 | "var", 43 | "video", 44 | "wbr", 45 | ]; 46 | 47 | class MarkdownToHtml { 48 | #prettierLanguages; 49 | #initStarted; 50 | 51 | constructor() { 52 | this.assetsToKeep = new Set(); 53 | this.assetsToDelete = new Set(); 54 | this.preservedSelectors = new Set(); 55 | this.isVerbose = true; 56 | this.counts = { 57 | cleaned: 0 58 | } 59 | } 60 | 61 | addPreservedSelector(selector) { 62 | if(!selector.startsWith(".")) { 63 | throw new Error("Invalid preserved selector. Only class names are supported."); 64 | } 65 | this.preservedSelectors.add(selector); 66 | } 67 | 68 | async asyncInit() { 69 | if(this.#initStarted) { 70 | return; 71 | } 72 | 73 | this.#initStarted = true; 74 | 75 | /* Sample output language 76 | { 77 | language: { 78 | linguistLanguageId: 50, 79 | name: 'CSS', 80 | type: 'markup', 81 | tmScope: 'source.css', 82 | aceMode: 'css', 83 | codemirrorMode: 'css', 84 | codemirrorMimeType: 'text/css', 85 | color: '#563d7c', 86 | extensions: [ '.css', '.wxss' ], 87 | parsers: [ 'css' ], 88 | vscodeLanguageIds: [ 'css' ] 89 | } 90 | } 91 | */ 92 | 93 | let map = { 94 | // extension without dot => array of parser types 95 | }; 96 | 97 | let supportInfo = await prettier.getSupportInfo(); 98 | for(let language of supportInfo.languages) { 99 | for(let ext of language.extensions) { 100 | if(language.parsers.length > 0) { 101 | map[ext.slice(1)] = language.parsers; 102 | } 103 | } 104 | } 105 | 106 | this.#prettierLanguages = map; 107 | } 108 | 109 | getCounts() { 110 | return this.counts; 111 | } 112 | 113 | setVerbose(isVerbose) { 114 | this.isVerbose = Boolean(isVerbose); 115 | } 116 | 117 | recontextifyRelativeAssetPath(assetPath, filePath) { 118 | if(path.isAbsolute(assetPath) || assetPath.startsWith("https:") || assetPath.startsWith("http:")) { 119 | return false; 120 | } 121 | 122 | let dir = DirectoryManager.getDirectory(filePath); 123 | return path.join(dir, assetPath); 124 | } 125 | 126 | // /small/jpeg/ 375w, /medium/jpeg/ 650w 127 | static getSrcsetUrls(srcsetAttr) { 128 | return (srcsetAttr || "").split(",").map(entry => { 129 | let [url, size] = entry.trim().split(" "); 130 | return url.trim(); 131 | }).filter(url => Boolean(url)).reverse() 132 | } 133 | 134 | static getImageSrcUrls(srcsetAttr, srcAttr) { 135 | let s = new Set(); 136 | for(let srcsetUrl of this.getSrcsetUrls(srcsetAttr) || []) { 137 | s.add(srcsetUrl); 138 | } 139 | if(srcAttr) { 140 | s.add(srcAttr); 141 | } 142 | return Array.from(s); 143 | } 144 | 145 | get prettierLanguages() { 146 | if(!this.#prettierLanguages) { 147 | throw new Error("Internal error: missing this.prettierLanguages—did you call asyncInit()?"); 148 | } 149 | 150 | return this.#prettierLanguages; 151 | } 152 | 153 | static outputMarkdownCodeBlock(content, language) { 154 | return `\`\`\`${language || ""}\n${content.trim()}\n\`\`\`\n\n` 155 | } 156 | 157 | // Supports .className selectors 158 | static hasClass(node, className) { 159 | if(className.startsWith(".")) { 160 | className = className.slice(1); 161 | } 162 | return this.hasAttribute(node, "class", className); 163 | } 164 | 165 | static matchAttributeEntry(value, expected) { 166 | // https://developer.mozilla.org/en-US/docs/Web/CSS/Attribute_selectors#attrvalue_3 167 | if(expected.startsWith("|=")) { 168 | let actual = expected.slice(2); 169 | // |= is equal to or starts with (and a hyphen) 170 | return value === actual || value.startsWith(`${actual}-`); 171 | } 172 | 173 | return value === expected; 174 | } 175 | 176 | static hasAttribute(node, attrName, attrValueMatch) { 177 | if(node._attrKeys?.includes(`|${attrName}`)) { 178 | let attrValue = node._attrsByQName?.[attrName]?.data; 179 | // [class] is special, space separated values 180 | if(attrName === "class") { 181 | return attrValue.split(" ").find(entry => { 182 | return this.matchAttributeEntry(entry, attrValueMatch); 183 | }); 184 | } 185 | 186 | // not [class] 187 | return attrValue === attrValueMatch; 188 | } 189 | 190 | return false; 191 | } 192 | 193 | getTurndownService(options = {}) { 194 | let { filePath, type } = options; 195 | let isFromWordPress = type === WordPressApi.TYPE || type === HostedWordPressApi.TYPE; 196 | 197 | let ts = new TurndownService({ 198 | headingStyle: "atx", 199 | bulletListMarker: "-", 200 | codeBlockStyle: "fenced", 201 | 202 | // Workaround to keep icon elements 203 | blankReplacement(content, node) { 204 | if(node.localName === "i") { 205 | if(MarkdownToHtml.hasClass(node, "|=fa")) { 206 | return node.outerHTML; 207 | } 208 | } 209 | 210 | if(node.localName === "svg") { 211 | let iconName = node._attrsByQName?.["data-icon"]?.data; 212 | let iconPrefix = node._attrsByQName?.["data-prefix"]?.data; 213 | 214 | if(MarkdownToHtml.hasClass(node, "svg-inline--fa") && iconName && iconPrefix) { 215 | return `` 216 | } 217 | } 218 | 219 | // content will be empty unless it has a preserved child, e.g.

220 | return node.isBlock ? `\n\n${content}\n\n` : content; 221 | }, 222 | 223 | // Intentionally opt-out 224 | // preformattedCode: true, 225 | }); 226 | 227 | ts.keep(TAGS_TO_KEEP); // tags run through `keepReplacement` function if match 228 | 229 | if(this.preservedSelectors.size > 0) { 230 | let preserved = Array.from(this.preservedSelectors); 231 | ts.addRule("keep-via-classes", { 232 | filter: function(node) { 233 | return preserved.find(cls => MarkdownToHtml.hasClass(node, cls)); 234 | }, 235 | replacement: (content, node) => { 236 | return node.outerHTML; 237 | } 238 | }); 239 | } 240 | 241 | ts.addRule("pre-without-code-to-fenced-codeblock", { 242 | filter: ["pre"], 243 | replacement: (content, node) => { 244 | try { 245 | let cls = node.getAttribute("class") || ""; 246 | let clsSplit = cls.split(" "); 247 | let isPreformattedWordPressBlock = clsSplit.includes("wp-block-preformatted"); 248 | if(isPreformattedWordPressBlock && isFromWordPress) { 249 | return content; 250 | } 251 | 252 | let languageClass = clsSplit.find(className => className.startsWith("language-")); 253 | let language; 254 | if(languageClass) { 255 | language = languageClass.slice("language-".length).trim(); 256 | } else if(isFromWordPress) { 257 | // WordPress specific 258 | let brush = cls.split(";").filter(entry => entry.startsWith("brush:")); 259 | language = (brush[0] || ":").split(":")[1].trim(); 260 | } 261 | 262 | let finalLanguage = language; 263 | 264 | // WordPress-only options 265 | if(isFromWordPress) { 266 | finalLanguage = WORDPRESS_TO_PRISM_LANGUAGE_TRANSLATION[language] || language; 267 | 268 | // TODO customizable 269 | // Questionable default: for code blocks unnecessarily bookended with ` 270 | let trimmed = content.trim(); 271 | if(trimmed.startsWith("`") && trimmed.endsWith("`")) { 272 | content = trimmed.slice(1, -1); 273 | } 274 | } 275 | 276 | try { 277 | if(isFromWordPress && language === "markup" && !content.trimStart().startsWith("<")) { 278 | // This code block was mislabeled as "markup" (hi WordPress), so we do nothing 279 | } else if(this.prettierLanguages[finalLanguage]) { 280 | // Attempt to format the code with Prettier 281 | let parserName = this.prettierLanguages[finalLanguage][0]; 282 | content = prettierSync.format(content, { parser: parserName }); 283 | } else { 284 | // preserve \n 285 | content = entities.decodeHTML(striptags(""+node.innerHTML)); 286 | } 287 | } catch(e) { 288 | console.error(`Error running code formatting on code block from ${filePath}${language ? ` (${language})` : ""}. Returning unformatted code:\n\n${content}`, e); 289 | } 290 | 291 | return MarkdownToHtml.outputMarkdownCodeBlock(content, finalLanguage); 292 | } catch(e) { 293 | // Otherwise errors get swallowed without feedback by Turndown 294 | console.error(`Error processing code block from ${filePath}`, e); 295 | 296 | return MarkdownToHtml.outputMarkdownCodeBlock(content); 297 | } 298 | } 299 | }); 300 | 301 | // ts.addRule("picture-unsupported", { 302 | // filter: ["picture"], 303 | // replacement: (content, node) => { 304 | // Logger.warning( ` node found, but not yet supported in markdown import.` ); 305 | // return ""; 306 | // } 307 | // }); 308 | 309 | ts.addRule("source-cleanup", { 310 | filter: ["source"], 311 | replacement: (content, node) => { 312 | try { 313 | let srcset = node.getAttribute("srcset"); 314 | if(node.parentNode.localName === "picture" && srcset) { 315 | let urls = MarkdownToHtml.getImageSrcUrls(srcset); 316 | for(let asset of urls) { 317 | this.assetsToDelete.add(this.recontextifyRelativeAssetPath(asset, filePath)); 318 | } 319 | } 320 | return content; 321 | } catch(e) { 322 | // Otherwise errors get swallowed without feedback by Turndown 323 | console.error(`Error processing on ${filePath}`, e); 324 | return content; 325 | } 326 | } 327 | }); 328 | 329 | ts.addRule("prefer-highest-resolution-images", { 330 | filter: ["img"], 331 | replacement: (content, node, options) => { 332 | try { 333 | // prefer highest-resolution (first) srcset 334 | let [src, ...remainingUrls] = MarkdownToHtml.getImageSrcUrls(node.getAttribute("srcset"), node.getAttribute("src")); 335 | 336 | this.assetsToKeep.add(this.recontextifyRelativeAssetPath(src, filePath)); 337 | 338 | for(let asset of remainingUrls) { 339 | this.assetsToDelete.add(this.recontextifyRelativeAssetPath(asset, filePath)); 340 | } 341 | 342 | // New lines are stripped by markdown-it anyway when encoding back to HTML 343 | let altString = (node.getAttribute("alt") || "").replace(/\n+/gi, " "); 344 | return `![${entities.escapeAttribute(altString)}](${src})`; 345 | } catch(e) { 346 | // Otherwise errors get swallowed without feedback by Turndown 347 | console.error(`Error processing high-resolution images on ${filePath}`, e); 348 | return content; 349 | } 350 | } 351 | }); 352 | 353 | return ts; 354 | } 355 | 356 | // Removes unnecessarily downloaded and `srcset` assets that didn’t end up in the markdown simplification 357 | cleanup() { 358 | // Don’t delete assets that are in both Sets 359 | for(let asset of this.assetsToKeep) { 360 | if(asset) { 361 | this.assetsToDelete.delete(asset); 362 | } 363 | } 364 | 365 | for(let asset of this.assetsToDelete) { 366 | if(!asset) { 367 | continue; 368 | } 369 | 370 | if(fs.existsSync(asset)) { 371 | if(this.isVerbose) { 372 | Logger.cleanup("unused asset", asset); 373 | } 374 | 375 | this.counts.cleaned++; 376 | fs.unlinkSync(asset); 377 | } 378 | } 379 | } 380 | 381 | async toMarkdown(html, entry) { 382 | let ts = this.getTurndownService({ 383 | type: entry.type, 384 | filePath: entry.filePath, 385 | }); 386 | 387 | return ts.turndown(html); 388 | } 389 | } 390 | 391 | export { MarkdownToHtml } 392 | -------------------------------------------------------------------------------- /src/Persist.js: -------------------------------------------------------------------------------- 1 | import { GitHubPublisher } from "github-publish"; 2 | 3 | import { Logger } from "./Logger.js"; 4 | 5 | class Persist { 6 | static SUPPORTED_TYPES = ["github"]; 7 | static READABLE_TYPES = { 8 | github: "GitHub" 9 | }; 10 | 11 | static parseTarget(target = "") { 12 | let [type, remainder] = target.split(":"); 13 | let [username, repository] = remainder.split("/"); 14 | let [repositoryName, repositoryBranch] = repository.split("#"); 15 | 16 | return { 17 | type, 18 | username, 19 | repository: repositoryName, 20 | branch: repositoryBranch || undefined, 21 | } 22 | } 23 | 24 | #publisher; 25 | #verboseMode = true; 26 | #dryRun = false; 27 | 28 | constructor() { 29 | this.counts = { 30 | persist: 0 31 | }; 32 | } 33 | 34 | getCounts() { 35 | return this.counts; 36 | } 37 | 38 | setVerbose(isVerbose) { 39 | this.#verboseMode = Boolean(isVerbose); 40 | } 41 | 42 | setDryRun(dryRun) { 43 | this.#dryRun = Boolean(dryRun); 44 | } 45 | 46 | // has setTarget been successful? 47 | canPersist() { 48 | return Boolean(this.type && this.username && this.repository); 49 | } 50 | 51 | setTarget(target) { 52 | // Must have a token to use this feature 53 | if(!process.env.GITHUB_TOKEN) { 54 | throw new Error("Missing GITHUB_TOKEN environment variable."); 55 | } 56 | 57 | let { type, username, repository, branch } = Persist.parseTarget(target); 58 | if(!Persist.SUPPORTED_TYPES.includes(type)) { 59 | throw new Error("Invalid persist type: " + type); 60 | } 61 | 62 | this.type = type; 63 | this.username = username; 64 | this.repository = repository; 65 | this.branch = branch; 66 | } 67 | 68 | get publisher() { 69 | if(!this.canPersist()) { 70 | throw new Error("Missing Persist target. Have you called setTarget()?"); 71 | } 72 | 73 | if(!this.#publisher) { 74 | this.#publisher = new GitHubPublisher(process.env.GITHUB_TOKEN, this.username, this.repository, this.branch); 75 | } 76 | 77 | return this.#publisher; 78 | } 79 | 80 | persistFile(filePath, content, metadata = {}) { 81 | // safeMode is handled upstream, otherwise the file will always exist on the file system (because writes happen before persistence) 82 | if(this.#dryRun) { 83 | // Skipping, don’t log the skip 84 | return; 85 | } 86 | 87 | let options = { 88 | // Persist should not happen if safe mode is enabled and the file already exists 89 | force: true, 90 | message: `@11ty/import ${metadata.url ? `via ${metadata.url}` : ""}`, 91 | // sha: undefined // required for updating 92 | } 93 | 94 | this.counts.persist++; 95 | 96 | if(this.#verboseMode) { 97 | let readableType = Persist.READABLE_TYPES[this.type] || this.type; 98 | Logger.persisting(`${metadata.type ? `${metadata.type} ` : ""}to ${readableType}`, filePath, metadata.url, { 99 | size: content.length, 100 | }); 101 | } 102 | 103 | return this.publisher.publish(filePath, content, options); 104 | } 105 | } 106 | 107 | export { Persist }; 108 | -------------------------------------------------------------------------------- /src/Utils.js: -------------------------------------------------------------------------------- 1 | import path from "node:path"; 2 | 3 | class Utils { 4 | static pathJoin(...refs) { 5 | return path.join(...refs).split(path.sep).join("/"); 6 | } 7 | } 8 | 9 | export { Utils } 10 | -------------------------------------------------------------------------------- /test/html-entities-test.js: -------------------------------------------------------------------------------- 1 | import test from 'node:test'; 2 | import assert from "node:assert/strict"; 3 | import * as entities from "entities"; 4 | 5 | test("Escape an attribute", async (t) => { 6 | assert.equal(entities.escapeAttribute("test"), `test`); 7 | assert.equal(entities.escapeAttribute("test\ntest"), `test\ntest`); 8 | }); 9 | -------------------------------------------------------------------------------- /test/markdown-test.js: -------------------------------------------------------------------------------- 1 | import "dotenv/config"; 2 | import test from 'node:test'; 3 | import assert from "node:assert/strict"; 4 | 5 | import { MarkdownToHtml } from "../src/MarkdownToHtml.js"; 6 | 7 | const sampleEntry = { 8 | filePath: "/index.md" 9 | }; 10 | 11 | test("Markdown Code", async (t) => { 12 | let md = new MarkdownToHtml(); 13 | 14 | assert.equal(await md.toMarkdown(`<div>`, sampleEntry), `\\
`); 15 | assert.equal(await md.toMarkdown(`This is a <div>`, sampleEntry), `This is a \\
`); 16 | assert.equal(await md.toMarkdown(`
This is a test
`, sampleEntry), `This is a test`); 17 | }); 18 | 19 | test("Markdown HTML", async (t) => { 20 | let md = new MarkdownToHtml(); 21 | 22 | assert.equal(await md.toMarkdown(`This is a test`, sampleEntry), `This is a test`); 23 | assert.equal(await md.toMarkdown(`This is a test`, sampleEntry), `This is a test`); 24 | assert.equal(await md.toMarkdown(`
`, sampleEntry), `
`); 25 | }); 26 | 27 | test("Keep elements with `fa-` classes", async (t) => { 28 | let md = new MarkdownToHtml(); 29 | 30 | assert.equal(await md.toMarkdown(`This is an icon `, sampleEntry), `This is an icon`); 31 | assert.equal(await md.toMarkdown(`This is an icon `, sampleEntry), `This is an icon`); 32 | assert.equal(await md.toMarkdown(`This is an icon`, sampleEntry), `This is an icon`); 33 | assert.equal(await md.toMarkdown(` This is an icon`, sampleEntry), `This is an icon`); 34 | assert.equal(await md.toMarkdown(` This is an icon`, sampleEntry), `This is an icon`); 35 | assert.equal(await md.toMarkdown(`This is an icon`, sampleEntry), `This is an icon`); 36 | }); 37 | 38 | test("Keep elements with `fa-` classes (nested) in an empty parent", async (t) => { 39 | let md = new MarkdownToHtml(); 40 | 41 | assert.equal(await md.toMarkdown(`

`, sampleEntry), ``); 42 | 43 | assert.equal(await md.toMarkdown(`

`, sampleEntry), ``); 44 | }); 45 | 46 | test("If the has content, italics takes precedence", async (t) => { 47 | let md = new MarkdownToHtml(); 48 | assert.equal(await md.toMarkdown(`Testing`, sampleEntry), `_Testing_`); 49 | }); 50 | 51 | test("Preserve other classes", async (t) => { 52 | let md = new MarkdownToHtml(); 53 | md.addPreservedSelector(".c-button--primary"); 54 | 55 | assert.equal(await md.toMarkdown(`Listen to the Full Episode!`, sampleEntry), `Listen to the Full Episode!`); 56 | }); 57 | 58 | test("newlines in ", async (t) => { 59 | let md = new MarkdownToHtml(); 60 | 61 | assert.equal(await md.toMarkdown(`Graphic of new Pro+ plans:\n\nPro Lite+: Everything in our online-only Pro Lite plan plus all our Pro+ icons and more custom icons, Kits,\nand pageviews.\n\nPro+: Everything in our Pro plan plus all our Pro+ icons and more custom icons, Kits, pageviews,\nand bandwidth.\n\nPro Max+: Everything in our Pro Max plan plus all our Pro+ icons, and even more pageviews and bandwidth.`, sampleEntry), `![Graphic of new Pro+ plans: Pro Lite+: Everything in our online-only Pro Lite plan plus all our Pro+ icons and more custom icons, Kits, and pageviews. Pro+: Everything in our Pro plan plus all our Pro+ icons and more custom icons, Kits, pageviews, and bandwidth. Pro Max+: Everything in our Pro Max plan plus all our Pro+ icons, and even more pageviews and bandwidth.](https://i0.wp.com/blog.fontawesome.com/wp-content/uploads/2025/03/v7-announce-plans-1.png?w=1440&ssl=1)`); 62 | }); 63 | 64 | test("Keep icons too", async (t) => { 65 | let md = new MarkdownToHtml(); 66 | 67 | assert.equal(await md.toMarkdown(``, sampleEntry), ``); 68 | }); 69 | 70 | -------------------------------------------------------------------------------- /test/sources/blog-awesome-author.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": 155431370, 3 | "name": "Matt Johnson", 4 | "url": "", 5 | "description": "", 6 | "link": "https://blog.fontawesome.com/author/iamthedayofcurrenttaste/", 7 | "slug": "iamthedayofcurrenttaste", 8 | "avatar_urls": { 9 | "24": "https://secure.gravatar.com/avatar/b548253c6adbbd36ce84487da009e1aa?s=24&r=g", 10 | "48": "https://secure.gravatar.com/avatar/b548253c6adbbd36ce84487da009e1aa?s=48&r=g", 11 | "96": "https://secure.gravatar.com/avatar/b548253c6adbbd36ce84487da009e1aa?s=96&r=g" 12 | }, 13 | "meta": [], 14 | "yoast_head": "\nMatt Johnson, Author at Blog Awesome\n\n\n\n\n\n\n\n\n\n\n\n\n", 15 | "yoast_head_json": { 16 | "title": "Matt Johnson, Author at Blog Awesome", 17 | "robots": { 18 | "index": "index", 19 | "follow": "follow", 20 | "max-snippet": "max-snippet:-1", 21 | "max-image-preview": "max-image-preview:large", 22 | "max-video-preview": "max-video-preview:-1" 23 | }, 24 | "canonical": "https://blog.fontawesome.com/author/iamthedayofcurrenttaste/", 25 | "og_locale": "en_US", 26 | "og_type": "profile", 27 | "og_title": "Matt Johnson, Author at Blog Awesome", 28 | "og_url": "https://blog.fontawesome.com/author/iamthedayofcurrenttaste/", 29 | "og_site_name": "Blog Awesome", 30 | "og_image": [ 31 | { 32 | "url": "https://secure.gravatar.com/avatar/b548253c6adbbd36ce84487da009e1aa?s=500&r=g" 33 | } 34 | ], 35 | "twitter_card": "summary_large_image", 36 | "twitter_site": "@fontawesome", 37 | "schema": { 38 | "@context": "https://schema.org", 39 | "@graph": [ 40 | { 41 | "@type": "ProfilePage", 42 | "@id": "https://blog.fontawesome.com/author/iamthedayofcurrenttaste/", 43 | "url": "https://blog.fontawesome.com/author/iamthedayofcurrenttaste/", 44 | "name": "Matt Johnson, Author at Blog Awesome", 45 | "isPartOf": { "@id": "https://blog.fontawesome.com/#website" }, 46 | "breadcrumb": { 47 | "@id": "https://blog.fontawesome.com/author/iamthedayofcurrenttaste/#breadcrumb" 48 | }, 49 | "inLanguage": "en-US", 50 | "potentialAction": [ 51 | { 52 | "@type": "ReadAction", 53 | "target": [ 54 | "https://blog.fontawesome.com/author/iamthedayofcurrenttaste/" 55 | ] 56 | } 57 | ] 58 | }, 59 | { 60 | "@type": "BreadcrumbList", 61 | "@id": "https://blog.fontawesome.com/author/iamthedayofcurrenttaste/#breadcrumb", 62 | "itemListElement": [ 63 | { 64 | "@type": "ListItem", 65 | "position": 1, 66 | "name": "Home", 67 | "item": "https://blog.fontawesome.com/" 68 | }, 69 | { 70 | "@type": "ListItem", 71 | "position": 2, 72 | "name": "Archives for Matt Johnson" 73 | } 74 | ] 75 | }, 76 | { 77 | "@type": "WebSite", 78 | "@id": "https://blog.fontawesome.com/#website", 79 | "url": "https://blog.fontawesome.com/", 80 | "name": "Blog Awesome", 81 | "description": "News and information from Font Awesome – the internet's favorite icon set; mixed with musings and nerdery from the team behind it.", 82 | "potentialAction": [ 83 | { 84 | "@type": "SearchAction", 85 | "target": { 86 | "@type": "EntryPoint", 87 | "urlTemplate": "https://blog.fontawesome.com/?s={search_term_string}" 88 | }, 89 | "query-input": { 90 | "@type": "PropertyValueSpecification", 91 | "valueRequired": true, 92 | "valueName": "search_term_string" 93 | } 94 | } 95 | ], 96 | "inLanguage": "en-US" 97 | }, 98 | { 99 | "@type": "Person", 100 | "@id": "https://blog.fontawesome.com/#/schema/person/7bc6a686f3fdf796f6e069c674ea7dbe", 101 | "name": "Matt Johnson", 102 | "image": { 103 | "@type": "ImageObject", 104 | "inLanguage": "en-US", 105 | "@id": "https://blog.fontawesome.com/#/schema/person/image/", 106 | "url": "https://secure.gravatar.com/avatar/b548253c6adbbd36ce84487da009e1aa?s=96&r=g", 107 | "contentUrl": "https://secure.gravatar.com/avatar/b548253c6adbbd36ce84487da009e1aa?s=96&r=g", 108 | "caption": "Matt Johnson" 109 | }, 110 | "mainEntityOfPage": { 111 | "@id": "https://blog.fontawesome.com/author/iamthedayofcurrenttaste/" 112 | } 113 | } 114 | ] 115 | } 116 | }, 117 | "amp_review_panel_dismissed_for_template_mode": "", 118 | "amp_dev_tools_enabled": false, 119 | "_links": { 120 | "self": [ 121 | { 122 | "href": "https://blog.fontawesome.com/wp-json/wp/v2/users/155431370", 123 | "targetHints": { "allow": ["GET", "POST", "PUT", "PATCH", "DELETE"] } 124 | } 125 | ], 126 | "collection": [ 127 | { "href": "https://blog.fontawesome.com/wp-json/wp/v2/users" } 128 | ] 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /test/sources/blog-awesome-categories.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": 1, 3 | "count": 91, 4 | "description": "", 5 | "link": "https://blog.fontawesome.com/category/uncategorized/", 6 | "name": "Uncategorized", 7 | "slug": "uncategorized", 8 | "taxonomy": "category", 9 | "parent": 0, 10 | "meta": [], 11 | "yoast_head": "\u003C!-- This site is optimized with the Yoast SEO plugin v23.8 - https://yoast.com/wordpress/plugins/seo/ --\u003E\n\u003Ctitle\u003EUncategorized Archives - Blog Awesome\u003C/title\u003E\n\u003Cmeta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" /\u003E\n\u003Clink rel=\"canonical\" href=\"https://blog.fontawesome.com/category/uncategorized/\" /\u003E\n\u003Cmeta property=\"og:locale\" content=\"en_US\" /\u003E\n\u003Cmeta property=\"og:type\" content=\"article\" /\u003E\n\u003Cmeta property=\"og:title\" content=\"Uncategorized Archives - Blog Awesome\" /\u003E\n\u003Cmeta property=\"og:url\" content=\"https://blog.fontawesome.com/category/uncategorized/\" /\u003E\n\u003Cmeta property=\"og:site_name\" content=\"Blog Awesome\" /\u003E\n\u003Cmeta name=\"twitter:card\" content=\"summary_large_image\" /\u003E\n\u003Cmeta name=\"twitter:site\" content=\"@fontawesome\" /\u003E\n\u003Cscript type=\"application/ld+json\" class=\"yoast-schema-graph\"\u003E{\"@context\":\"https://schema.org\",\"@graph\":[{\"@type\":\"CollectionPage\",\"@id\":\"https://blog.fontawesome.com/category/uncategorized/\",\"url\":\"https://blog.fontawesome.com/category/uncategorized/\",\"name\":\"Uncategorized Archives - Blog Awesome\",\"isPartOf\":{\"@id\":\"https://blog.fontawesome.com/#website\"},\"breadcrumb\":{\"@id\":\"https://blog.fontawesome.com/category/uncategorized/#breadcrumb\"},\"inLanguage\":\"en-US\"},{\"@type\":\"BreadcrumbList\",\"@id\":\"https://blog.fontawesome.com/category/uncategorized/#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"Home\",\"item\":\"https://blog.fontawesome.com/\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"Uncategorized\"}]},{\"@type\":\"WebSite\",\"@id\":\"https://blog.fontawesome.com/#website\",\"url\":\"https://blog.fontawesome.com/\",\"name\":\"Blog Awesome\",\"description\":\"News and information from Font Awesome – the internet's favorite icon set; mixed with musings and nerdery from the team behind it.\",\"potentialAction\":[{\"@type\":\"SearchAction\",\"target\":{\"@type\":\"EntryPoint\",\"urlTemplate\":\"https://blog.fontawesome.com/?s={search_term_string}\"},\"query-input\":{\"@type\":\"PropertyValueSpecification\",\"valueRequired\":true,\"valueName\":\"search_term_string\"}}],\"inLanguage\":\"en-US\"}]}\u003C/script\u003E\n\u003C!-- / Yoast SEO plugin. --\u003E", 12 | "yoast_head_json": { 13 | "title": "Uncategorized Archives - Blog Awesome", 14 | "robots": { 15 | "index": "index", 16 | "follow": "follow", 17 | "max-snippet": "max-snippet:-1", 18 | "max-image-preview": "max-image-preview:large", 19 | "max-video-preview": "max-video-preview:-1" 20 | }, 21 | "canonical": "https://blog.fontawesome.com/category/uncategorized/", 22 | "og_locale": "en_US", 23 | "og_type": "article", 24 | "og_title": "Uncategorized Archives - Blog Awesome", 25 | "og_url": "https://blog.fontawesome.com/category/uncategorized/", 26 | "og_site_name": "Blog Awesome", 27 | "twitter_card": "summary_large_image", 28 | "twitter_site": "@fontawesome", 29 | "schema": { 30 | "@context": "https://schema.org", 31 | "@graph": [ 32 | { 33 | "@type": "CollectionPage", 34 | "@id": "https://blog.fontawesome.com/category/uncategorized/", 35 | "url": "https://blog.fontawesome.com/category/uncategorized/", 36 | "name": "Uncategorized Archives - Blog Awesome", 37 | "isPartOf": { 38 | "@id": "https://blog.fontawesome.com/#website" 39 | }, 40 | "breadcrumb": { 41 | "@id": "https://blog.fontawesome.com/category/uncategorized/#breadcrumb" 42 | }, 43 | "inLanguage": "en-US" 44 | }, 45 | { 46 | "@type": "BreadcrumbList", 47 | "@id": "https://blog.fontawesome.com/category/uncategorized/#breadcrumb", 48 | "itemListElement": [ 49 | { 50 | "@type": "ListItem", 51 | "position": 1, 52 | "name": "Home", 53 | "item": "https://blog.fontawesome.com/" 54 | }, 55 | { 56 | "@type": "ListItem", 57 | "position": 2, 58 | "name": "Uncategorized" 59 | } 60 | ] 61 | }, 62 | { 63 | "@type": "WebSite", 64 | "@id": "https://blog.fontawesome.com/#website", 65 | "url": "https://blog.fontawesome.com/", 66 | "name": "Blog Awesome", 67 | "description": "News and information from Font Awesome – the internet's favorite icon set; mixed with musings and nerdery from the team behind it.", 68 | "potentialAction": [ 69 | { 70 | "@type": "SearchAction", 71 | "target": { 72 | "@type": "EntryPoint", 73 | "urlTemplate": "https://blog.fontawesome.com/?s={search_term_string}" 74 | }, 75 | "query-input": { 76 | "@type": "PropertyValueSpecification", 77 | "valueRequired": true, 78 | "valueName": "search_term_string" 79 | } 80 | } 81 | ], 82 | "inLanguage": "en-US" 83 | } 84 | ] 85 | } 86 | }, 87 | "_links": { 88 | "self": [ 89 | { 90 | "href": "https://blog.fontawesome.com/wp-json/wp/v2/categories/1", 91 | "targetHints": { 92 | "allow": [ 93 | "GET" 94 | ] 95 | } 96 | } 97 | ], 98 | "collection": [ 99 | { 100 | "href": "https://blog.fontawesome.com/wp-json/wp/v2/categories" 101 | } 102 | ], 103 | "about": [ 104 | { 105 | "href": "https://blog.fontawesome.com/wp-json/wp/v2/taxonomies/category" 106 | } 107 | ], 108 | "wp:post_type": [ 109 | { 110 | "href": "https://blog.fontawesome.com/wp-json/wp/v2/posts?categories=1" 111 | } 112 | ], 113 | "curies": [ 114 | { 115 | "name": "wp", 116 | "href": "https://api.w.org/{rel}", 117 | "templated": true 118 | } 119 | ] 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /test/sources/blog-awesome-posts.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "id": 3829, 4 | "date": "2021-12-07T11:35:48", 5 | "date_gmt": "2021-12-07T17:35:48", 6 | "guid": { "rendered": "https://blog.fontawesome.com/?p=3829" }, 7 | "modified": "2021-12-07T11:36:05", 8 | "modified_gmt": "2021-12-07T17:36:05", 9 | "slug": "font-awesome-6", 10 | "status": "publish", 11 | "type": "post", 12 | "link": "https://blog.fontawesome.com/font-awesome-6/", 13 | "title": { 14 | "rendered": "It’s Official. Font Awesome 6 is Coming in February 2022! " 15 | }, 16 | "content": { 17 | "rendered": "\n

We’re so close to launching version 6, and we figured it was high time to make an official announcement. So, save the date for February. Font Awesome 6 will go beyond pure icon-imagination! 

\n\n\n\n
\"\"
Save the date! February 2022 is just around the corner!
\n\n\n\n

So, what’s new? 

\n\n\n\n
\n\n\n\n

More Icons

\n\n\n\n

Font Awesome 6 contains over 7,000 new icons, so you’re sure to find what you need for your project. Plus, we’ve redesigned most of our icons from scratch, so they’re more consistent and easier to use.

\n\n\n\n
\"\"
\n\n\n\n
\n\n\n\n

More Styles

\n\n\n\n

Font Awesome 6 includes five icons styles: solid, regular, light, duotone, and the new THIN style — not to mention all of our brand icons. And coming later in 2022 is the entirely new SHARP family of styles.

\n\n\n\n
\"\"
\n\n\n\n
\n\n\n\n

More Ways to Use 

\n\n\n\n

Font Awesome 6 makes it even easier to use icons where you want to. More plugins and packages to match your stack. Less time wrestling browser rendering.

\n\n\n\n
\"\"
\n\n\n\n
\n\n\n\n

We’ll keep fine-tuning that sweet, sweet recipe until February. Believe us; the web’s going to have a new scrumpdillyicious secret ingredient!

\n\n\n\nCheck Out the Beta!\n\n\n\n

\n", 18 | "protected": false 19 | }, 20 | "excerpt": { 21 | "rendered": "

We’re so close to launching version 6, and we figured it was high time to make an official announcement. So, save the date for February. Font Awesome 6 will go beyond pure icon-imagination!  So, what’s new?  More Icons Font Awesome 6 contains over 7,000 new icons, so you’re sure to find what you need for […]

\n", 22 | "protected": false 23 | }, 24 | "author": 155431370, 25 | "featured_media": 3850, 26 | "comment_status": "open", 27 | "ping_status": "open", 28 | "sticky": false, 29 | "template": "", 30 | "format": "standard", 31 | "meta": { 32 | "jetpack_post_was_ever_published": false, 33 | "_jetpack_newsletter_access": "", 34 | "_jetpack_dont_email_post_to_subs": false, 35 | "_jetpack_newsletter_tier_id": 0, 36 | "_jetpack_memberships_contains_paywalled_content": false, 37 | "_jetpack_memberships_contains_paid_content": false, 38 | "footnotes": "", 39 | "jetpack_publicize_message": "", 40 | "jetpack_publicize_feature_enabled": true, 41 | "jetpack_social_post_already_shared": true, 42 | "jetpack_social_options": { 43 | "image_generator_settings": { "template": "highway", "enabled": false }, 44 | "version": 2 45 | } 46 | }, 47 | "categories": [1], 48 | "tags": [], 49 | "class_list": [ 50 | "post-3829", 51 | "post", 52 | "type-post", 53 | "status-publish", 54 | "format-standard", 55 | "has-post-thumbnail", 56 | "hentry", 57 | "category-uncategorized" 58 | ], 59 | "jetpack_publicize_connections": [], 60 | "yoast_head": "\nIt’s Official. Font Awesome 6 is Coming in February 2022!  - Blog Awesome\n\n\n\n\n\n\n\n\n\n\n\n\n\t\n\t\n\t\n\n\n\n\n\n\t\n\t\n\t\n\n", 61 | "yoast_head_json": { 62 | "title": "It’s Official. Font Awesome 6 is Coming in February 2022!  - Blog Awesome", 63 | "robots": { 64 | "index": "index", 65 | "follow": "follow", 66 | "max-snippet": "max-snippet:-1", 67 | "max-image-preview": "max-image-preview:large", 68 | "max-video-preview": "max-video-preview:-1" 69 | }, 70 | "canonical": "https://blog.fontawesome.com/font-awesome-6/", 71 | "og_locale": "en_US", 72 | "og_type": "article", 73 | "og_title": "It’s Official. Font Awesome 6 is Coming in February 2022!  - Blog Awesome", 74 | "og_description": "We’re so close to launching version 6, and we figured it was high time to make an official announcement. So, save the date for February. Font Awesome 6 will go beyond pure icon-imagination!  So, what’s new?  More Icons Font Awesome 6 contains over 7,000 new icons, so you’re sure to find what you need for […]", 75 | "og_url": "https://blog.fontawesome.com/font-awesome-6/", 76 | "og_site_name": "Blog Awesome", 77 | "article_published_time": "2021-12-07T17:35:48+00:00", 78 | "article_modified_time": "2021-12-07T17:36:05+00:00", 79 | "og_image": [ 80 | { 81 | "width": 1920, 82 | "height": 1080, 83 | "url": "https://blog.fontawesome.com/wp-content/uploads/2021/12/image-save-the-date.png", 84 | "type": "image/png" 85 | } 86 | ], 87 | "author": "Matt Johnson", 88 | "twitter_card": "summary_large_image", 89 | "twitter_creator": "@fontawesome", 90 | "twitter_site": "@fontawesome", 91 | "twitter_misc": { 92 | "Written by": "Matt Johnson", 93 | "Est. reading time": "2 minutes" 94 | }, 95 | "schema": { 96 | "@context": "https://schema.org", 97 | "@graph": [ 98 | { 99 | "@type": "WebPage", 100 | "@id": "https://blog.fontawesome.com/font-awesome-6/", 101 | "url": "https://blog.fontawesome.com/font-awesome-6/", 102 | "name": "It’s Official. Font Awesome 6 is Coming in February 2022!  - Blog Awesome", 103 | "isPartOf": { "@id": "https://blog.fontawesome.com/#website" }, 104 | "primaryImageOfPage": { 105 | "@id": "https://blog.fontawesome.com/font-awesome-6/#primaryimage" 106 | }, 107 | "image": { 108 | "@id": "https://blog.fontawesome.com/font-awesome-6/#primaryimage" 109 | }, 110 | "thumbnailUrl": "https://i0.wp.com/blog.fontawesome.com/wp-content/uploads/2021/12/image-save-the-date.png?fit=1920%2C1080&ssl=1", 111 | "datePublished": "2021-12-07T17:35:48+00:00", 112 | "dateModified": "2021-12-07T17:36:05+00:00", 113 | "author": { 114 | "@id": "https://blog.fontawesome.com/#/schema/person/7bc6a686f3fdf796f6e069c674ea7dbe" 115 | }, 116 | "breadcrumb": { 117 | "@id": "https://blog.fontawesome.com/font-awesome-6/#breadcrumb" 118 | }, 119 | "inLanguage": "en-US", 120 | "potentialAction": [ 121 | { 122 | "@type": "ReadAction", 123 | "target": ["https://blog.fontawesome.com/font-awesome-6/"] 124 | } 125 | ] 126 | }, 127 | { 128 | "@type": "ImageObject", 129 | "inLanguage": "en-US", 130 | "@id": "https://blog.fontawesome.com/font-awesome-6/#primaryimage", 131 | "url": "https://i0.wp.com/blog.fontawesome.com/wp-content/uploads/2021/12/image-save-the-date.png?fit=1920%2C1080&ssl=1", 132 | "contentUrl": "https://i0.wp.com/blog.fontawesome.com/wp-content/uploads/2021/12/image-save-the-date.png?fit=1920%2C1080&ssl=1", 133 | "width": 1920, 134 | "height": 1080 135 | }, 136 | { 137 | "@type": "BreadcrumbList", 138 | "@id": "https://blog.fontawesome.com/font-awesome-6/#breadcrumb", 139 | "itemListElement": [ 140 | { 141 | "@type": "ListItem", 142 | "position": 1, 143 | "name": "Home", 144 | "item": "https://blog.fontawesome.com/" 145 | }, 146 | { 147 | "@type": "ListItem", 148 | "position": 2, 149 | "name": "It’s Official. Font Awesome 6 is Coming in February 2022! " 150 | } 151 | ] 152 | }, 153 | { 154 | "@type": "WebSite", 155 | "@id": "https://blog.fontawesome.com/#website", 156 | "url": "https://blog.fontawesome.com/", 157 | "name": "Blog Awesome", 158 | "description": "News and information from Font Awesome – the internet's favorite icon set; mixed with musings and nerdery from the team behind it.", 159 | "potentialAction": [ 160 | { 161 | "@type": "SearchAction", 162 | "target": { 163 | "@type": "EntryPoint", 164 | "urlTemplate": "https://blog.fontawesome.com/?s={search_term_string}" 165 | }, 166 | "query-input": { 167 | "@type": "PropertyValueSpecification", 168 | "valueRequired": true, 169 | "valueName": "search_term_string" 170 | } 171 | } 172 | ], 173 | "inLanguage": "en-US" 174 | }, 175 | { 176 | "@type": "Person", 177 | "@id": "https://blog.fontawesome.com/#/schema/person/7bc6a686f3fdf796f6e069c674ea7dbe", 178 | "name": "Matt Johnson", 179 | "image": { 180 | "@type": "ImageObject", 181 | "inLanguage": "en-US", 182 | "@id": "https://blog.fontawesome.com/#/schema/person/image/", 183 | "url": "https://secure.gravatar.com/avatar/b548253c6adbbd36ce84487da009e1aa?s=96&r=g", 184 | "contentUrl": "https://secure.gravatar.com/avatar/b548253c6adbbd36ce84487da009e1aa?s=96&r=g", 185 | "caption": "Matt Johnson" 186 | }, 187 | "url": "https://blog.fontawesome.com/author/iamthedayofcurrenttaste/" 188 | } 189 | ] 190 | } 191 | }, 192 | "jetpack_featured_media_url": "https://i0.wp.com/blog.fontawesome.com/wp-content/uploads/2021/12/image-save-the-date.png?fit=1920%2C1080&ssl=1", 193 | "jetpack_likes_enabled": true, 194 | "jetpack_sharing_enabled": true, 195 | "jetpack_shortlink": "https://wp.me/pasa0D-ZL", 196 | "amp_enabled": false, 197 | "_links": { 198 | "self": [ 199 | { 200 | "href": "https://blog.fontawesome.com/wp-json/wp/v2/posts/3829", 201 | "targetHints": { "allow": ["GET", "POST", "PUT", "PATCH", "DELETE"] } 202 | } 203 | ], 204 | "collection": [ 205 | { "href": "https://blog.fontawesome.com/wp-json/wp/v2/posts" } 206 | ], 207 | "about": [ 208 | { "href": "https://blog.fontawesome.com/wp-json/wp/v2/types/post" } 209 | ], 210 | "author": [ 211 | { 212 | "embeddable": true, 213 | "href": "https://blog.fontawesome.com/wp-json/wp/v2/users/155431370" 214 | } 215 | ], 216 | "replies": [ 217 | { 218 | "embeddable": true, 219 | "href": "https://blog.fontawesome.com/wp-json/wp/v2/comments?post=3829" 220 | } 221 | ], 222 | "version-history": [ 223 | { 224 | "count": 12, 225 | "href": "https://blog.fontawesome.com/wp-json/wp/v2/posts/3829/revisions" 226 | } 227 | ], 228 | "predecessor-version": [ 229 | { 230 | "id": 3859, 231 | "href": "https://blog.fontawesome.com/wp-json/wp/v2/posts/3829/revisions/3859" 232 | } 233 | ], 234 | "wp:featuredmedia": [ 235 | { 236 | "embeddable": true, 237 | "href": "https://blog.fontawesome.com/wp-json/wp/v2/media/3850" 238 | } 239 | ], 240 | "wp:attachment": [ 241 | { 242 | "href": "https://blog.fontawesome.com/wp-json/wp/v2/media?parent=3829" 243 | } 244 | ], 245 | "wp:term": [ 246 | { 247 | "taxonomy": "category", 248 | "embeddable": true, 249 | "href": "https://blog.fontawesome.com/wp-json/wp/v2/categories?post=3829" 250 | }, 251 | { 252 | "taxonomy": "post_tag", 253 | "embeddable": true, 254 | "href": "https://blog.fontawesome.com/wp-json/wp/v2/tags?post=3829" 255 | } 256 | ], 257 | "curies": [ 258 | { "name": "wp", "href": "https://api.w.org/{rel}", "templated": true } 259 | ] 260 | } 261 | } 262 | ] 263 | -------------------------------------------------------------------------------- /test/sources/bluesky-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 🌍 https://zachleat.com 🎈🐀 Creator/Maintainer of @11ty.dev 🎉 Builder at Font Awesome 🏳️‍⚧️ Listen to Trans Folks 👋🏻 He/him/they 🐘 Mastodon https://zachleat.com/@zachleat ✅ Front of the Front-end ✅ Static Sites ✅ Web Components ✅ Web Performancehttps://bsky.app/profile/zachleat.com@zachleat.com - Zach Leathermanhttps://bsky.app/profile/zachleat.com/post/3lckusgtkuk2rtime to review my HTML wrapped 2024 Most used: <a> Doing work to reduce infrastructure bills: <picture> Underrated: <output> Misunderstood: <details> Tame but a small win: <search> Hope the design never calls for it: <dialog> Not today Satan: <canvas> Pure vibes: <noscript>05 Dec 2024 14:26 +0000at://did:plc:xpchjovbk6sxl3bv74z7cs54/app.bsky.feed.post/3lckusgtkuk2r 3 | -------------------------------------------------------------------------------- /test/sources/youtube-user.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | yt:channel:skGTioqrMBcw8pd14_334A 5 | skGTioqrMBcw8pd14_334A 6 | Eleventy 7 | 8 | 9 | Eleventy 10 | https://www.youtube.com/channel/UCskGTioqrMBcw8pd14_334A 11 | 12 | 2022-02-14T17:18:45+00:00 13 | 14 | yt:video:kNI_LcbvtNk 15 | kNI_LcbvtNk 16 | UCskGTioqrMBcw8pd14_334A 17 | Managing content management (with no vendor lock-in) — David Large, Liam Bigelow (11ty Conf 2024) 18 | 19 | 20 | Eleventy 21 | https://www.youtube.com/channel/UCskGTioqrMBcw8pd14_334A 22 | 23 | 2024-05-17T18:27:22+00:00 24 | 2024-09-24T01:49:49+00:00 25 | 26 | Managing content management (with no vendor lock-in) — David Large, Liam Bigelow (11ty Conf 2024) 27 | 28 | 29 | CloudCannon is the Recommended CMS Partner of 11ty: 30 | 31 | https://cloudcannon.com/11tyconf/ 32 | https://cloudcannon.com/blog/how-to-manage-hundreds-of-connected-websites-with-a-git-based-headless-cms/ 33 | 34 | This was a talk given at the 11ty International Symposium on Making Web Sites Real Good (2024): https://conf.11ty.dev/2024/managing-content-management/ 35 | 36 | If Jamstack has taught us anything, it’s that websites work best when they’re generated from folders full of flat files. Even massively interconnected websites! 37 | 38 | We talk through a classically Jamstacky approach to content management for large organizations: mounting shared layout and component repositories, creating a central content lake to aggregate content like news articles, and automating site builds and deployments when your content or dependencies change. 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | yt:video:-Pl0ibl47Rc 47 | -Pl0ibl47Rc 48 | UCskGTioqrMBcw8pd14_334A 49 | 11ty and Large-Project Tooling — Paul Everitt (11ty Conf 2024) 50 | 51 | 52 | Eleventy 53 | https://www.youtube.com/channel/UCskGTioqrMBcw8pd14_334A 54 | 55 | 2024-05-17T18:27:18+00:00 56 | 2024-08-19T19:05:07+00:00 57 | 58 | 11ty and Large-Project Tooling — Paul Everitt (11ty Conf 2024) 59 | 60 | 61 | This was a talk given at the 11ty International Symposium on Making Web Sites Real Good (2024): https://conf.11ty.dev/2024/11ty-and-large-project-tooling/ 62 | 63 | JetBrains has a knowledge base with lots of content, customizing, authors, developers… so we built it in Gatsby. Then switched to 11ty. But we wanted to keep some of the large-project tooling, so we added it. In this talk, we'll cover component driven development through TypeScript and TSX, unit testing with Vitest, in-editor validation of Markdown frontmatter for authors, content queries, and more. 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | yt:video:P-I5D6BlejM 72 | P-I5D6BlejM 73 | UCskGTioqrMBcw8pd14_334A 74 | Digital Frontiers, IndieWeb Cowboys, and A Place Online To Call… — Henry Desroches (11ty Conf 2024) 75 | 76 | 77 | Eleventy 78 | https://www.youtube.com/channel/UCskGTioqrMBcw8pd14_334A 79 | 80 | 2024-05-17T18:27:14+00:00 81 | 2024-09-23T21:12:23+00:00 82 | 83 | Digital Frontiers, IndieWeb Cowboys, and A Place Online To Call… — Henry Desroches (11ty Conf 2024) 84 | 85 | 86 | This was a talk given at the 11ty International Symposium on Making Web Sites Real Good (2024): https://conf.11ty.dev/2024/digital-frontiers-indieweb-cowboys-and-a-place-online-to-call-your-own/ 87 | 88 | The IndieWeb is a community of developers and designers and web-surfing wraiths connected by a belief in personal websites/domains as identity, digital self-publishing, and perhaps most chiefly owning one's content. In this talk, Henry will explore the key principles of the IndieWeb, and specifically focus on the concept of POSSE (or "Publish (on your) Own Site, Syndicate Elsewhere"), its benefits, and how it empowers folks to reclaim ownership of their online content or data. They will also discuss concrete and actionable strategies for implementing POSSE in one's own online presence (PERHAPS USING ELEVENTY?!?!?) and the potential impact it can have on the wider web 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | yt:video:1vrPtBCLoro 97 | 1vrPtBCLoro 98 | UCskGTioqrMBcw8pd14_334A 99 | You're Probably Doing Web Performance Wrong —  Sia Karamalegos (11ty Conf 2024) 100 | 101 | 102 | Eleventy 103 | https://www.youtube.com/channel/UCskGTioqrMBcw8pd14_334A 104 | 105 | 2024-05-17T18:27:10+00:00 106 | 2024-09-24T02:43:20+00:00 107 | 108 | You're Probably Doing Web Performance Wrong —  Sia Karamalegos (11ty Conf 2024) 109 | 110 | 111 | This was a talk given at the 11ty International Symposium on Making Web Sites Real Good (2024): https://conf.11ty.dev/2024/you-re-probably-doing-web-performance-wrong/ 112 | 113 | Lighthouse 100 is a common goal for developers seeking to make their sites fast, but does this really center user experience? In this talk, Sia will cover how to get a broader perspective on how real users are experiencing your website performance and better strategies for improving it. 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | yt:video:VNIy2MaXFNM 122 | VNIy2MaXFNM 123 | UCskGTioqrMBcw8pd14_334A 124 | Building a Town That Doesn't Exist — Dan Sinker (11ty Conf 2024) 125 | 126 | 127 | Eleventy 128 | https://www.youtube.com/channel/UCskGTioqrMBcw8pd14_334A 129 | 130 | 2024-05-17T18:27:06+00:00 131 | 2024-08-06T15:00:32+00:00 132 | 133 | Building a Town That Doesn't Exist — Dan Sinker (11ty Conf 2024) 134 | 135 | 136 | This was a talk given at the 11ty International Symposium on Making Web Sites Real Good (2024): https://conf.11ty.dev/2024/building-a-town-that-doesnt-exist/ 137 | 138 | The town of Question Mark, Ohio has a city hall, a newspaper, a library, a chicken joint, an ice cream store, a dump, a haunted factory, two cults, a sordid past, and a mysterious glowing void in the woods. 139 | 140 | None of it is real, of course, but all of it exists on the web, thanks to Eleventy, Tailwind CSS, and Alpine.js, the crucial building blocks for creating a year-long immersive novel told not in a book but across the entire internet. This talk looks under the hood at how one person built 40 websites to tell the story of Question Mark, Ohio, a town that disappears into time. 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | yt:video:8Z8H2NEbLtE 149 | 8Z8H2NEbLtE 150 | UCskGTioqrMBcw8pd14_334A 151 | Don't Fear the Cascade — Mayank (11ty Conf 2024) 152 | 153 | 154 | Eleventy 155 | https://www.youtube.com/channel/UCskGTioqrMBcw8pd14_334A 156 | 157 | 2024-05-17T18:27:02+00:00 158 | 2024-08-24T14:18:56+00:00 159 | 160 | Don't Fear the Cascade — Mayank (11ty Conf 2024) 161 | 162 | 163 | This was a talk given at the 11ty International Symposium on Making Web Sites Real Good (2024): https://conf.11ty.dev/2024/dont-fear-the-cascade/ 164 | 165 | In this talk, Mayank will teach the audience how to make sense of the cascade, clear up some common misconceptions (e.g. shadow DOM vs host context styles), and ultimately offer practical advice on how to make best use of the cascade. 166 | 167 | They will show off some newer game-changing CSS features (like :where, cascade layers, and @scope) that enable us to write styles with confidence and intention. They may also sprinkle in some tips on how these new features interact with current workflows, including frameworks, build tools, browser devtools, and IDEs. 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | yt:video:kwHVpcwsJOE 176 | kwHVpcwsJOE 177 | UCskGTioqrMBcw8pd14_334A 178 | 11ty for “Non-Developers” — Adrianna Tan (11ty Conf 2024) 179 | 180 | 181 | Eleventy 182 | https://www.youtube.com/channel/UCskGTioqrMBcw8pd14_334A 183 | 184 | 2024-05-17T18:26:58+00:00 185 | 2024-08-06T20:21:33+00:00 186 | 187 | 11ty for “Non-Developers” — Adrianna Tan (11ty Conf 2024) 188 | 189 | 190 | This was a talk given at the 11ty International Symposium on Making Web Sites Real Good (2024): https://conf.11ty.dev/2024/11ty-sites-for-people-who-dont-think-they-are-web-developers/ 191 | 192 | For a long time, we've thought of web development in two separate streams. WYSWYG builders for non-techical people, code for the rest. What if 'non-technical' isn't real, ‘web dev’ doesn't have to be just something you do for money, and building sites on the open web can be a fun hobby, especially with 11ty? 193 | 194 | In this talk, Adrianna will share how 11ty helped them finally break down the ’I’m not a web developer’ thoughts so they could build cool sites. Just like everyone can draw without being an artist, everyone can cook without being a chef. 195 | 196 | Adrianna will also share a framework they've developed for talking about and teaching 11ty to people who think they are not web developers, but who really have things to share on the web. 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | yt:video:uIuf5LlA6KQ 205 | uIuf5LlA6KQ 206 | UCskGTioqrMBcw8pd14_334A 207 | Come to the light side: HTML Web Components — Chris Ferdinandi (11ty Conf 2024) 208 | 209 | 210 | Eleventy 211 | https://www.youtube.com/channel/UCskGTioqrMBcw8pd14_334A 212 | 213 | 2024-05-17T18:26:54+00:00 214 | 2024-09-25T04:58:00+00:00 215 | 216 | Come to the light side: HTML Web Components — Chris Ferdinandi (11ty Conf 2024) 217 | 218 | 219 | This was a talk given at the 11ty International Symposium on Making Web Sites Real Good (2024): https://conf.11ty.dev/2024/come-to-the-light-side-html-web-components/ 220 | 221 | While Web Components have been around for years, they're seeing a bit of a renaissance thanks to an emerging approach for authoring them: ditch the shadow DOM and progressively enhance existing HTML. 222 | 223 | In this talk, we'll look at what Web Components, how the "HTML Web Component" approach works, why it's awesome, and some tips, tricks, and gotchas when working with them. 224 | 225 | 226 | 227 | 228 | 229 | 230 | 231 | yt:video:Pkg-Brizep8 232 | Pkg-Brizep8 233 | UCskGTioqrMBcw8pd14_334A 234 | Chinese Type Systems — ivan zhao (11ty Conf 2024) 235 | 236 | 237 | Eleventy 238 | https://www.youtube.com/channel/UCskGTioqrMBcw8pd14_334A 239 | 240 | 2024-05-17T18:26:50+00:00 241 | 2024-08-29T07:32:37+00:00 242 | 243 | Chinese Type Systems — ivan zhao (11ty Conf 2024) 244 | 245 | 246 | This was a talk given at the 11ty International Symposium on Making Web Sites Real Good (2024): https://conf.11ty.dev/2024/chinese-type-systems/ 247 | 248 | In this talk, ivan will educate listeners about understanding Chinese type systems, pairing Chinese fonts, and why its so hard to put your Chinese font in the @font-face in CSS. Typesetting on the web with Chinese fonts is extremely difficult and ivan will give examples of pages that do this beautifully, and recommendations on how to find these. 249 | 250 | They will also talk about the history of the Chinese language, the differences in character based vs glyph based, and the difficulties of pairing the two types together. They’ll dive into the various classifications of systems that they use and talk about how to notice and pair Chinese fonts together, as well as actionably implementing them in CSS and the bundle size of the file. 251 | 252 | 253 | 254 | 255 | 256 | 257 | 258 | yt:video:iZ4XjLeaKH4 259 | iZ4XjLeaKH4 260 | UCskGTioqrMBcw8pd14_334A 261 | Light mode versus Dark mode, why not both? — Sara Joy (11ty Conf 2024) 262 | 263 | 264 | Eleventy 265 | https://www.youtube.com/channel/UCskGTioqrMBcw8pd14_334A 266 | 267 | 2024-05-17T18:26:47+00:00 268 | 2024-09-24T14:17:03+00:00 269 | 270 | Light mode versus Dark mode, why not both? — Sara Joy (11ty Conf 2024) 271 | 272 | 273 | This was a talk given at the 11ty International Symposium on Making Web Sites Real Good (2024): https://conf.11ty.dev/2024/light-mode-versus-dark-mode/ 274 | 275 | It doesn’t have to be this way. Some people have very real physical reasons to need one mode over another. The rest of us are also allowed to have our favourites, or might prefer light mode in some situations and dark in others. 276 | 277 | What if Sara said it was super easy to code up both dark and light modes at the same time? Despite Sara’s evangelising efforts, not enough people know of the CSS property color-scheme and how powerful it is. 278 | 279 | 280 | 281 | 282 | 283 | 284 | 285 | yt:video:tXNsWsEE7S0 286 | tXNsWsEE7S0 287 | UCskGTioqrMBcw8pd14_334A 288 | The Future of 11ty — Zach Leatherman (11ty Conf 2024) 289 | 290 | 291 | Eleventy 292 | https://www.youtube.com/channel/UCskGTioqrMBcw8pd14_334A 293 | 294 | 2024-05-17T18:26:42+00:00 295 | 2024-09-24T15:54:12+00:00 296 | 297 | The Future of 11ty — Zach Leatherman (11ty Conf 2024) 298 | 299 | 300 | This was a talk given at the 11ty International Symposium on Making Web Sites Real Good (2024): https://conf.11ty.dev/2024/the-future-of-11ty/ 301 | 302 | Learn more about CloudCannon and 11ty: https://cloudcannon.com/11tyconf/ 303 | 304 | A talk about the current state of the 11ty project, new tricks and releases, and where we’re going next. 305 | 306 | 307 | 308 | 309 | 310 | 311 | 312 | yt:video:uaN9kY8lKPU 313 | uaN9kY8lKPU 314 | UCskGTioqrMBcw8pd14_334A 315 | Hints and Suggestions — Miriam Suzanne (11ty Conf 2024) 316 | 317 | 318 | Eleventy 319 | https://www.youtube.com/channel/UCskGTioqrMBcw8pd14_334A 320 | 321 | 2024-05-17T18:04:53+00:00 322 | 2024-08-29T14:28:06+00:00 323 | 324 | Hints and Suggestions — Miriam Suzanne (11ty Conf 2024) 325 | 326 | 327 | This was a talk given at the 11ty International Symposium on Making Web Sites Real Good (2024): https://conf.11ty.dev/2024/hints-and-suggestions-first-do-no-harm/ 328 | 329 | The web is fundamentally different from other platforms, built around a radically political vision for resilience and user-control. CSS takes that to another level, attempting the almost absurd task of collaborative styling across devices and interfaces and languages. This is a quick dive into the origins of the web, and CSS in particular—the design constraints, and the range of strange proposals, and how we got where we are. By the end, we see the CSS is Awesome meme in a whole new light. 330 | 331 | 332 | 333 | 334 | 335 | 336 | 337 | yt:video:b-aQ7_yD44s 338 | b-aQ7_yD44s 339 | UCskGTioqrMBcw8pd14_334A 340 | Break Slideshow: Personal Sites Montage (11ty Conf 2024) 341 | 342 | 343 | Eleventy 344 | https://www.youtube.com/channel/UCskGTioqrMBcw8pd14_334A 345 | 346 | 2024-05-15T16:01:31+00:00 347 | 2024-08-26T19:42:34+00:00 348 | 349 | Break Slideshow: Personal Sites Montage (11ty Conf 2024) 350 | 351 | 352 | This slideshow appeared during the 11ty Conference (2024) during the breaks to highlight community personal sites. The sites were gathered from the 11ty Leaderboard https://www.11ty.dev/speedlify/ and our Authors pages https://www.11ty.dev/authors/ 353 | 354 | 355 | 356 | 357 | 358 | 359 | 360 | yt:video:iLxJ6PtuF9M 361 | iLxJ6PtuF9M 362 | UCskGTioqrMBcw8pd14_334A 363 | The 11ty International Symposium on Making Web Sites Real Good (Live stream) 364 | 365 | 366 | Eleventy 367 | https://www.youtube.com/channel/UCskGTioqrMBcw8pd14_334A 368 | 369 | 2024-05-09T21:10:46+00:00 370 | 2024-08-24T19:51:44+00:00 371 | 372 | The 11ty International Symposium on Making Web Sites Real Good (Live stream) 373 | 374 | 375 | https://conf.11ty.dev/ 376 | 377 | 00:00:00 Countdown 378 | 00:20:05 Kickoff! 379 | 00:35:56 Future of 11ty by Zach Leatherman 380 | 01:09:50 Hints & Suggestions (First, Do No Harm) by Miriam Suzanne 381 | 01:34:00 11ty and Large-Project Tooling by Paul Everitt 382 | 01:55:41 Break 383 | 02:22:22 Digital Frontiers, IndieWeb Cowboys, and A Place Online To Call Your Own by Henry Desroches 384 | 02:39:40 You're Probably Doing Web Performance Wrong by Sia Karamalegos 385 | 03:00:30 Building a Town That Doesn't Exist by Dan Sinker 386 | 03:14:43 Break 387 | 03:35:00 11ty Sites for People Who Don't Think they are Web Developers by Adrianna Tan 388 | 03:50:50 Don't Fear the Cascade by Mayank 389 | 04:16:50 Managing content management (with no vendor lock-in): Git CMS and static API generation, together at last! by Liam Bigelow and David Large 390 | 04:31:00 Break 391 | 04:50:35 Come to the light side: HTML Web Components by Chris Ferdinandi 392 | 05:15:55 Chinese Type Systems by Ivan Zhao 393 | 05:31:55 Light mode versus Dark mode by Sara Joy 394 | 05:54:59 Recap 395 | 396 | 397 | 398 | 399 | 400 | 401 | 402 | yt:video:nypsmn70ipI 403 | nypsmn70ipI 404 | UCskGTioqrMBcw8pd14_334A 405 | Big Announcement: Eleventy and @CloudCannon! 406 | 407 | 408 | Eleventy 409 | https://www.youtube.com/channel/UCskGTioqrMBcw8pd14_334A 410 | 411 | 2023-07-25T15:00:46+00:00 412 | 2024-08-24T07:51:29+00:00 413 | 414 | Big Announcement: Eleventy and @CloudCannon! 415 | 416 | 417 | https://cloudcannon.com/blog/cloudcannon-the-official-cms-partner-of-eleventy/ 418 | 419 | and at https://www.11ty.dev/blog/cloudcannon/ 420 | 421 | 422 | 423 | 424 | 425 | 426 | 427 | -------------------------------------------------------------------------------- /test/test.js: -------------------------------------------------------------------------------- 1 | import "dotenv/config"; 2 | import test from 'node:test'; 3 | import assert from "node:assert/strict"; 4 | import fs from "node:fs"; 5 | import { createRequire } from "node:module"; 6 | 7 | import { Importer } from "../src/Importer.js"; 8 | import { DataSource } from "../src/DataSource.js"; 9 | import { Persist } from "../src/Persist.js"; 10 | import { Fetcher } from "../src/Fetcher.js"; 11 | 12 | function cleanContent(content) { 13 | // trim extra whitespace (dirty workaround for trailing whitespace) 14 | return content.split("\n").map(line => line.trim()).join("\n"); 15 | } 16 | 17 | const require = createRequire(import.meta.url); 18 | 19 | test("YouTube user", async (t) => { 20 | let importer = new Importer(); 21 | 22 | importer.setVerbose(false); 23 | importer.setDryRun(true); 24 | 25 | importer.addSource("youtubeUser", "UCskGTioqrMBcw8pd14_334A"); 26 | 27 | let stubContent = fs.readFileSync("./test/sources/youtube-user.xml"); 28 | importer.addDataOverride("youtube", "https://www.youtube.com/feeds/videos.xml?channel_id=UCskGTioqrMBcw8pd14_334A", Fetcher.parseXml(stubContent.toString("utf8"))); 29 | 30 | let entries = await importer.getEntries({ contentType: "markdown" }); 31 | assert.equal(entries.length, 15); 32 | 33 | let [post] = entries; 34 | 35 | assert.deepEqual(Object.keys(post).sort(), ["authors", "content", "contentType", "date", "dateUpdated", "filePath", "title", "type", "url", "uuid"]); 36 | assert.equal(post.content.length, 812); 37 | assert.equal(post.content, `CloudCannon is the Recommended CMS Partner of 11ty: 38 | 39 | https://cloudcannon.com/11tyconf/ 40 | https://cloudcannon.com/blog/how-to-manage-hundreds-of-connected-websites-with-a-git-based-headless-cms/ 41 | 42 | This was a talk given at the 11ty International Symposium on Making Web Sites Real Good (2024): https://conf.11ty.dev/2024/managing-content-management/ 43 | 44 | If Jamstack has taught us anything, it’s that websites work best when they’re generated from folders full of flat files. Even massively interconnected websites! 45 | 46 | We talk through a classically Jamstacky approach to content management for large organizations: mounting shared layout and component repositories, creating a central content lake to aggregate content like news articles, and automating site builds and deployments when your content or dependencies change.`); 47 | 48 | assert.equal(post.authors[0].name, "Eleventy"); 49 | }); 50 | 51 | test("Bluesky posts", async (t) => { 52 | let importer = new Importer(); 53 | 54 | importer.setVerbose(false); 55 | importer.setDryRun(true); 56 | 57 | importer.addSource("bluesky", "zachleat.com"); 58 | 59 | let stubContent = fs.readFileSync("./test/sources/bluesky-test.xml"); 60 | 61 | importer.addDataOverride("bluesky", "https://bsky.app/profile/zachleat.com/rss", Fetcher.parseXml(stubContent.toString("utf8"))); 62 | 63 | let entries = await importer.getEntries({ contentType: "markdown" }); 64 | assert.equal(entries.length, 1); 65 | 66 | let [post] = entries; 67 | 68 | assert.deepEqual(Object.keys(post).sort(), ["authors", "content", "contentType", "date", "filePath", "title", "type", "url", "uuid"]); 69 | assert.equal(post.content.length, 323); 70 | assert.equal(post.content, `time to review my HTML wrapped 2024 71 | 72 | Most used: <a> 73 | Doing work to reduce infrastructure bills: <picture> 74 | Underrated: <output> 75 | Misunderstood: <details> 76 | Tame but a small win: <search> 77 | Hope the design never calls for it: <dialog> 78 | Not today Satan: <canvas> 79 | Pure vibes: <noscript>`); 80 | 81 | assert.equal(post.authors[0].name, "@zachleat.com - Zach Leatherman"); 82 | }); 83 | 84 | test("WordPress import", async (t) => { 85 | 86 | let importer = new Importer(); 87 | 88 | importer.setVerbose(false); 89 | importer.setDryRun(true); 90 | importer.setAssetReferenceType("disabled"); 91 | 92 | importer.addSource("wordpress", "https://blog.fontawesome.com/"); 93 | 94 | if(process.env.WORDPRESS_USERNAME) { 95 | importer.addDataOverride("wordpress", "https://blog.fontawesome.com/wp-json/wp/v2/posts/?page=1&per_page=100&status=publish%2Cdraft", require("./sources/blog-awesome-posts.json")); 96 | importer.addDataOverride("wordpress", "https://blog.fontawesome.com/wp-json/wp/v2/posts/?page=2&per_page=100&status=publish%2Cdraft", []); 97 | } else { 98 | importer.addDataOverride("wordpress", "https://blog.fontawesome.com/wp-json/wp/v2/posts/?page=1&per_page=100", require("./sources/blog-awesome-posts.json")); 99 | importer.addDataOverride("wordpress", "https://blog.fontawesome.com/wp-json/wp/v2/posts/?page=2&per_page=100", []); 100 | } 101 | 102 | importer.addDataOverride("wordpress", "https://blog.fontawesome.com/wp-json/wp/v2/categories/1", require("./sources/blog-awesome-categories.json")); 103 | importer.addDataOverride("wordpress", "https://blog.fontawesome.com/wp-json/wp/v2/users/155431370", require("./sources/blog-awesome-author.json")); 104 | 105 | importer.addPreserved(".c-button--primary"); 106 | 107 | let entries = await importer.getEntries({ contentType: "markdown" }); 108 | assert.equal(entries.length, 1); 109 | 110 | let [post] = entries; 111 | assert.deepEqual(Object.keys(post).sort(), ["authors", "content", "contentType", "date", "dateUpdated", "filePath", "metadata", "status", "title", "type", "url", "uuid"]); 112 | 113 | assert.equal(cleanContent(post.content), `We’re so close to launching version 6, and we figured it was high time to make an official announcement. So, save the date for February. Font Awesome 6 will go beyond pure icon-imagination! 114 | 115 | ![](https://i0.wp.com/blog.fontawesome.com/wp-content/uploads/2021/12/image-calendar-exclamation-2.png?w=1440&ssl=1) 116 | 117 | Save the date! February 2022 is just around the corner! 118 | 119 | So, what’s new? 120 | 121 | * * * 122 | 123 | ## More Icons 124 | 125 | Font Awesome 6 contains over 7,000 new icons, so you’re sure to find what you need for your project. Plus, we’ve redesigned most of our icons from scratch, so they’re more consistent and easier to use. 126 | 127 | ![](https://i0.wp.com/blog.fontawesome.com/wp-content/uploads/2021/12/image-icons-2.png?w=1440&ssl=1) 128 | 129 | * * * 130 | 131 | ## More Styles 132 | 133 | Font Awesome 6 includes five icons styles: solid, regular, light, duotone, and the new THIN style — not to mention all of our brand icons. And coming later in 2022 is the entirely new SHARP family of styles. 134 | 135 | ![](https://i0.wp.com/blog.fontawesome.com/wp-content/uploads/2021/12/image-styles-2.png?w=1440&ssl=1) 136 | 137 | * * * 138 | 139 | ## More Ways to Use 140 | 141 | Font Awesome 6 makes it even easier to use icons where you want to. More plugins and packages to match your stack. Less time wrestling browser rendering. 142 | 143 | ![](https://i0.wp.com/blog.fontawesome.com/wp-content/uploads/2021/12/image-awesome-2.png?w=720&ssl=1) 144 | 145 | * * * 146 | 147 | We’ll keep fine-tuning that sweet, sweet recipe until February. Believe us; the web’s going to have a new scrumpdillyicious secret ingredient! 148 | 149 | Check Out the Beta!`); 150 | 151 | assert.equal(post.content.length, 1634); 152 | assert.equal(post.authors[0].name, "Matt Johnson"); 153 | }); 154 | 155 | test("addSource using DataSource", async (t) => { 156 | let importer = new Importer(); 157 | 158 | importer.setVerbose(false); 159 | importer.setDryRun(true); 160 | 161 | class MySource extends DataSource { 162 | static TYPE = "arbitrary"; 163 | static TYPE_FRIENDLY = "Arbitrary"; 164 | 165 | getData() { 166 | return [{ 167 | lol: "hi", 168 | url: "https://example.com/test/" 169 | }]; 170 | } 171 | } 172 | 173 | importer.addSource(MySource); 174 | 175 | let entries = await importer.getEntries(); 176 | assert.equal(entries.length, 1); 177 | }); 178 | 179 | test("addSource needs to use DataSource", async (t) => { 180 | let importer = new Importer(); 181 | 182 | importer.setVerbose(false); 183 | importer.setDryRun(true); 184 | 185 | assert.throws(() => { 186 | importer.addSource(class MySource {}); 187 | }, { 188 | message: "MySource is not a supported type for addSource(). Requires a string type or a DataSource class." 189 | }) 190 | }); 191 | 192 | test("Persist parseTarget", async (t) => { 193 | assert.deepEqual(Persist.parseTarget("github:11ty/eleventy"), { 194 | type: "github", 195 | username: "11ty", 196 | repository: "eleventy", 197 | branch: undefined, 198 | }); 199 | 200 | assert.deepEqual(Persist.parseTarget("github:11ty/eleventy#main"), { 201 | type: "github", 202 | username: "11ty", 203 | repository: "eleventy", 204 | branch: "main", 205 | }); 206 | }); 207 | 208 | test("Persist constructor (no token)", async (t) => { 209 | let p = new Persist(); 210 | 211 | assert.throws(() => p.setTarget("gitlab:11ty/eleventy"), { 212 | // message: "Invalid persist type: gitlab" 213 | message: "Missing GITHUB_TOKEN environment variable." 214 | }); 215 | }); 216 | 217 | test("Persist constructor (gitlab)", async (t) => { 218 | let p = new Persist(); 219 | process.env.GITHUB_TOKEN = "FAKE_TOKEN"; 220 | 221 | assert.throws(() => p.setTarget("gitlab:11ty/eleventy"), { 222 | message: "Invalid persist type: gitlab" 223 | }); 224 | }); 225 | 226 | test("Fetcher asset location tests (relative)", async (t) => { 227 | let f = new Fetcher(); 228 | 229 | let relative1 = f.getAssetLocation("https://example.com/test.png", "image/png", { filePath: "/test.html" }); 230 | assert.deepEqual(relative1, { 231 | filePath: "assets/test-NzhbK6MSYu2g.png", 232 | url: "assets/test-NzhbK6MSYu2g.png", 233 | }); 234 | 235 | let relativeNoExt = f.getAssetLocation("https://example.com/test", "image/png", { filePath: "/test.html" }); 236 | assert.deepEqual(relativeNoExt, { 237 | filePath: "assets/test-m4HI5oTdgEt4.png", 238 | url: "assets/test-m4HI5oTdgEt4.png", 239 | }); 240 | 241 | let relative2 = f.getAssetLocation("https://example.com/subdir/test.png", "image/png", { filePath: "localsubdirectory/test.html" }); 242 | assert.deepEqual(relative2, { 243 | filePath: "localsubdirectory/assets/test-slaK8pecO8QR.png", 244 | url: "assets/test-slaK8pecO8QR.png", 245 | }); 246 | }); 247 | 248 | test("Fetcher asset location tests (absolute)", async (t) => { 249 | let f = new Fetcher(); 250 | f.setUseRelativeAssetPaths(false); 251 | 252 | let abs1 = f.getAssetLocation("https://example.com/test.png", "image/png"); 253 | assert.deepEqual(abs1, { 254 | filePath: "assets/test-NzhbK6MSYu2g.png", 255 | url: "/assets/test-NzhbK6MSYu2g.png", 256 | }); 257 | }); 258 | 259 | 260 | test("Markdown cleanup code blocks strip tags", async (t) => { 261 | let importer = new Importer(); 262 | 263 | importer.setVerbose(false); 264 | importer.setDryRun(true); 265 | 266 | let content = await importer.getTransformedContent({ 267 | content: `
$ npm config set “@fortawesome:registry” https://npm.fontawesome.com/
`, 268 | contentType: "html" 269 | }, true); 270 | 271 | assert.equal(content.trim(), "```\n$ npm config set “@fortawesome:registry” https://npm.fontawesome.com/\n```"); 272 | }); 273 | 274 | test("Markdown cleanup code blocks with nested ", async (t) => { 275 | let importer = new Importer(); 276 | 277 | importer.setVerbose(false); 278 | importer.setDryRun(true); 279 | 280 | let content2 = await importer.getTransformedContent({ 281 | content: `
Authorization: Bearer DEAD-BEEF
`, 282 | contentType: "html" 283 | }, true); 284 | 285 | assert.equal(content2.trim(), "```\nAuthorization: Bearer DEAD-BEEF\n```"); 286 | }); 287 | 288 | test("within yes", async (t) => { 289 | let importer = new Importer(); 290 | 291 | importer.setVerbose(false); 292 | importer.setDryRun(true); 293 | 294 | class MySource extends DataSource { 295 | static TYPE = "arbitrary"; 296 | static TYPE_FRIENDLY = "Arbitrary"; 297 | 298 | getRawEntryDates(rawEntry) { 299 | return { 300 | created: rawEntry.date, 301 | } 302 | } 303 | 304 | getData() { 305 | return [{ 306 | lol: "hi", 307 | url: "https://example.com/test/", 308 | date: new Date(), 309 | }]; 310 | } 311 | } 312 | 313 | importer.addSource(MySource); 314 | 315 | let entries = await importer.getEntries({ 316 | within: "1m" 317 | }); 318 | assert.equal(entries.length, 1); 319 | }); 320 | 321 | test("within no", async (t) => { 322 | let importer = new Importer(); 323 | 324 | importer.setVerbose(false); 325 | importer.setDryRun(true); 326 | 327 | class MySource extends DataSource { 328 | static TYPE = "arbitrary"; 329 | static TYPE_FRIENDLY = "Arbitrary"; 330 | 331 | getRawEntryDates(rawEntry) { 332 | return { 333 | created: rawEntry.date, 334 | } 335 | } 336 | 337 | getData() { 338 | return [{ 339 | lol: "hi", 340 | url: "https://example.com/test/", 341 | date: new Date(2010, 0,1), 342 | }]; 343 | } 344 | } 345 | 346 | importer.addSource(MySource); 347 | 348 | let entries = await importer.getEntries({ 349 | within: "1m" 350 | }); 351 | assert.equal(entries.length, 0); 352 | }); 353 | --------------------------------------------------------------------------------