├── .changeset ├── README.md └── config.json ├── .eslintrc.cjs ├── .github └── workflows │ ├── main.yml │ └── release.yml ├── .gitignore ├── .npmignore ├── .nvmrc ├── CHANGELOG.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── package-lock.json ├── package.json ├── public ├── subtle_gltch_logo.png └── welcome_gltch_rounded.png ├── rollup.config.mjs ├── src ├── index.ts ├── notion-blocks-html-parser.ts ├── notion-blocks-md-parser.ts ├── notion-blocks-parser.ts ├── notion-blocks-plaintext-parser.ts ├── notion-cms.ts ├── notion-logger.ts ├── plugins │ ├── head.ts │ ├── images.ts │ ├── linker.ts │ └── render.ts ├── tests │ ├── custom-render.spec.ts │ ├── limiter.spec.ts │ ├── notion-api-mock.spec.ts │ ├── notion-cms-caching.spec.ts │ ├── notion-cms.spec.ts │ └── test.ts ├── types.ts └── utilities.ts └── tsconfig.json /.changeset/README.md: -------------------------------------------------------------------------------- 1 | # Changesets 2 | 3 | Hello and welcome! This folder has been automatically generated by `@changesets/cli`, a build tool that works 4 | with multi-package repos, or single-package repos to help you version and publish your code. You can 5 | find the full documentation for it [in our repository](https://github.com/changesets/changesets) 6 | 7 | We have a quick list of common questions to get you started engaging with this project in 8 | [our documentation](https://github.com/changesets/changesets/blob/main/docs/common-questions.md) 9 | -------------------------------------------------------------------------------- /.changeset/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://unpkg.com/@changesets/config@2.3.0/schema.json", 3 | "changelog": "@changesets/cli/changelog", 4 | "commit": false, 5 | "fixed": [], 6 | "linked": [], 7 | "access": "public", 8 | "baseBranch": "main", 9 | "updateInternalDependencies": "patch", 10 | "ignore": [] 11 | } -------------------------------------------------------------------------------- /.eslintrc.cjs: -------------------------------------------------------------------------------- 1 | // .eslintrc.js 2 | process.env.ESLINT_TSCONFIG = 'tsconfig.json' 3 | 4 | module.exports = { 5 | extends: '@antfu', 6 | rules: { 7 | 'no-console': 'warn', 8 | }, 9 | } 10 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: Node.js CI 2 | 3 | on: 4 | pull_request: 5 | branches: [main] 6 | 7 | jobs: 8 | build: 9 | 10 | runs-on: ubuntu-latest 11 | 12 | steps: 13 | - uses: actions/checkout@v3 14 | - name: Use Node.js 15 | uses: actions/setup-node@v3 16 | with: 17 | node-version: 19.x 18 | - run: npm ci --include=dev 19 | - run: npm run build --if-present 20 | - run: npm test 21 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | concurrency: ${{ github.workflow }}-${{ github.ref }} 9 | 10 | jobs: 11 | release: 12 | name: Release 13 | runs-on: ubuntu-latest 14 | steps: 15 | - name: Checkout Repo 16 | uses: actions/checkout@v3 17 | 18 | - name: Setup Node.js 18.x 19 | uses: actions/setup-node@v3 20 | with: 21 | node-version: 18.x 22 | 23 | - name: Install Dependencies 24 | run: npm i 25 | 26 | - name: Build & Bundle 27 | run: npm run build:dts 28 | 29 | - name: Test 30 | run: npm test 31 | 32 | - name: Create Release Pull Request or Publish to npm 33 | id: changesets 34 | uses: changesets/action@v1 35 | with: 36 | # This expects you to have a script called release which does a build for your packages and calls changeset publish 37 | publish: npm run release 38 | env: 39 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 40 | NPM_TOKEN: ${{ secrets.NPM_TOKEN }} 41 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .env 3 | debug/* 4 | dist 5 | index.d.ts 6 | coverage/ 7 | .vscode/ 8 | my.secrets 9 | .tmp/ 10 | test/ 11 | *.tgz 12 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | test/ 2 | debug/ 3 | src/ 4 | coverage/ 5 | .changeset/ 6 | .env 7 | tsconfig.json 8 | rollup.config.js 9 | dist/.notion-cms 10 | .github/ 11 | public/ 12 | rollup.config.mjs 13 | .eslintrc.cjs 14 | my.secrets 15 | .vscode/ 16 | .tmp/ 17 | *.tgz 18 | -------------------------------------------------------------------------------- /.nvmrc: -------------------------------------------------------------------------------- 1 | 19 2 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # @agency-kit/notion-cms 2 | 3 | ## 0.10.0 4 | 5 | ### Minor Changes 6 | 7 | - 8862d40: Fix: image cache plugin will now only run when a page needs updated (not using cache). 8 | 9 | New: cache PDFs using the image plugin. Long term this will likely be a unified plugin for fetching/caching all AWS S3 hosted Notion assets. 10 | 11 | ### Patch Changes 12 | 13 | - bef414e: Fix: Keep terminal logging mechanism from swallowing important error messages. 14 | 15 | Fix: Throw an error when `rootAlias` is non-existant in Notion database. 16 | 17 | Fix: Show warning when there are multiple pages at the root level AND a `rootAlias` is set (you probably don't want that). 18 | 19 | Fix: Keep from failing when in quiet mode. 20 | 21 | Upd: Improve bundle size a tiny amount by switching terminal colors to picocolors. 22 | 23 | - 26dccd0: Fix: maintain proper spacing in codeblock renders. 24 | 25 | ## 0.9.1 26 | 27 | ### Patch Changes 28 | 29 | - bd28226: Upd: improve terminal output and add quiet flag. 30 | 31 | Fix: [images plugin] make sure existing images don't get overwritten. 32 | 33 | Fix: [linker plugin] proper uuid detection in anchor tags. 34 | 35 | ## 0.9.0 36 | 37 | ### Minor Changes 38 | 39 | - bc57a66: New: optimized content fetching only when content changes. This lets us achieve fastest possible builds 🚄. This comes with an `autoUpdate` flag for turning it off, but you probably shouldn't. 40 | 41 | New: added an every hook \*, recommended for testing only. 42 | 43 | New: added `purgeCache` function that clears the local cache. 44 | 45 | Fix: fixed a bug in linker plugin where non-uuids would get picked up. 46 | 47 | Fix: fixed bug in terminal spinner that was causing ctrl-c not to close the terminal in some cases. 48 | 49 | Fix: fixed typo bug where `rootUrl` was being set to a bogus string. 50 | 51 | ## 0.8.1 52 | 53 | ### Patch Changes 54 | 55 | - 45151f6: New: add core plugins to repo: Images, Linker, Head. 56 | 57 | New: add support for a root alias option that gets its key set to '/' route for convenience. 58 | 59 | ## 0.8.0 60 | 61 | ### Minor Changes 62 | 63 | - d60bd35: Upd: No longer support cjs. 64 | 65 | New: Support rendering link preview blocks. 66 | 67 | New: Add a loading spinner that reports to the terminal for some feedback. 68 | 69 | Upd: Alias `fetch` at `pull` which will match future `push` commands better. 70 | 71 | ## 0.7.2 72 | 73 | ### Patch Changes 74 | 75 | - f493c16: Fix: Caching was broken by a typo in the options checking mechanism in last release. 76 | 77 | Fix: Make sure that the refresh timeout gets set properly whether a number or string is used. 78 | 79 | ## 0.7.1 80 | 81 | ### Patch Changes 82 | 83 | - 521cc9e: Upd: Support mermaid as a highlight js lang. 84 | 85 | Upd: Support detection of changes in options to trigger a re-fetch, not including functions. 86 | 87 | ## 0.7.0 88 | 89 | ### Minor Changes 90 | 91 | - 1575bff: New: Support Notion Link-to-page blocks including with custom renderers. 92 | 93 | New: Support multiple instances of NotionCMS in a project. Before this was limited because they would share a cache which would break things. Fixed by using unique identifiers (last 4 digits of the Notion database id) in the cache name. 94 | 95 | New: Add `_createCMSWalker` static utility class for quickly creating tree walker (walkjs) functions with the appropriate settings for when developing plugins that utilize the `post-tree` hook. 96 | 97 | Upd: Add more API and page stats in CMS metadata and update `duration` to `durationSeconds` for clarity. 98 | 99 | Fix: Make sure that the page content filter accounts for the `post-tree` case where the parent exists. 100 | 101 | ### Patch Changes 102 | 103 | - e4423de: Upd: convert all tests to typescript. 104 | 105 | ## 0.6.0 106 | 107 | ### Minor Changes 108 | 109 | - a3fa1f3: New: support markdown and plaintext in content object. 110 | New: add support for unlimited database pages and unlimited blocks per page. 111 | New: add support for human readable strings in cache timeout option. 112 | New: add run duration stats. 113 | Fix: use Marked options instead of `used` for extensions as they bleed to all instances of Marked (uses a singleton under the hood). 114 | Fix: update mis-typed Notion Block Object Responses. 115 | Fix: proper access kitchen sink test object to fix render test. 116 | Fix: ensure route array is populated. 117 | 118 | ## 0.5.8 119 | 120 | ### Patch Changes 121 | 122 | - 03c9c9a: Fix: Update CI to run tests and publish. Remove vscode settings from publish. 123 | 124 | ## 0.5.7 125 | 126 | ### Patch Changes 127 | 128 | - 4a2b7d3: Upd: use manual npmrc creation in action. 129 | 130 | ## 0.5.6 131 | 132 | ### Patch Changes 133 | 134 | - b4a71fd: Upd: new logo and minor readme updates. 135 | 136 | ## 0.5.5 137 | 138 | ### Patch Changes 139 | 140 | - 09208f8: Fix: Add build to release action but don't test. We test for every PR so this is fine. 141 | 142 | ## 0.5.4 143 | 144 | ### Patch Changes 145 | 146 | - 4e9f98c: Fix: Remove debug console logs from tests 147 | 148 | ## 0.5.3 149 | 150 | ### Patch Changes 151 | 152 | - fc92ad0: Fix: make sure that debug=true doesn't write to files (bug in last build) and remove indents so Marked doesn't wrap them in code tags. 153 | 154 | ## 0.5.2 155 | 156 | ### Patch Changes 157 | 158 | - Upd: add p (and other) tags back into render by taking a parsing pass at the full markdown file generated by marked. 159 | 160 | ## 0.5.1 161 | 162 | ### Patch Changes 163 | 164 | - 60efae2: Fix: Address deeply nested rendering bugs by pulling in the Notion-Blocks-HTML-parser (Thanks @notion-stuff) and reworking some of the internals. 165 | Fix: Remove deprecated `headerIds` option in `marked` and instead use the header-id extension. 166 | 167 | ## 0.5.0 168 | 169 | ### Minor Changes 170 | 171 | - 3d0d5d2: New: Add support for recursive extraction of child blocks from Notion API. Unlocks existing functionality in the Notion markdown parser. 172 | 173 | Upd: Add official Notion debug logs in `debug` mode as well as error handling for API failure cases. 174 | 175 | ## 0.4.1 176 | 177 | ### Patch Changes 178 | 179 | - 65ab3c1: Add debug, local cache path, and notion client to each plugin exec function 180 | 181 | ## 0.4.0 182 | 183 | ### Minor Changes 184 | 185 | - 534abe5: New: Add ability to leverage custom Notion block renderers using a core plugin. 186 | Fix: Import now capable of parsing non-flatted (which is used for caching) format files. 187 | Fix: Proper error handling and fallback to using fresh Notion API call when cache fails. 188 | Upd: Improve test coverage to 95+, remove unused code. 189 | Upd: Expose tree walking filter as static utility class for use in plugins. 190 | 191 | ### Patch Changes 192 | 193 | - 78be33a: If a coverImage property exists, don't overwrite it when we look for the backup cover image. 194 | - 5866586: Add working sourcemaps to the build. 195 | 196 | ## 0.3.0 197 | 198 | ### Minor Changes 199 | 200 | - 9157243: Fix: ensure that tree walkers only operate on page content nodes. 201 | Upd: Remove ancestor nodes from public API. 202 | Upd: Add support for starting a walk/async walk from a partial path. 203 | New: Add helper method to capture only page data and remove that pages children. Useful for sending individual page data to client without sending its sub tree. 204 | - 8ae6713: New: Add import plugin hook and remove previous state importing in constructor. Now you have to manually import anytime you want to revive a CMS. 205 | 206 | ### Patch Changes 207 | 208 | - 84433cb: Upd: remove otherProps to keep from polluting final tree structure. Guide users down the path of building plugins to extract props at an earlier stage. 209 | - 8ae6713: New: add support for a plugin function returning an array with multiple plugin hook executors. 210 | - 41015e2: Fix: increase page size to maximum number of blocks when pulling page content. 211 | 212 | ## 0.2.0 213 | 214 | ### Minor Changes 215 | 216 | - 93cc1e0: Improved the internal CMS tree building algorithm and added public API for walking the tree. 217 | 218 | This fixed a bug where deeply nested pages didn't get inserted into the tree as intended, another bug 219 | where searching for pages by their key didn't allow for duplicated keys at different places in the structure. 220 | 221 | Rewrote tests to be more maintainable and test all public API surfaces. 222 | 223 | Added references to ancestor nodes in the CMS tree for usage in plugins. This required proper serialization/deserialization 224 | of objects with circular references (the ancestors) but also functions, which are required in the templating plugin and are generally useful to store in the cache. 225 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | 2 | PRs welcome. 3 | 4 | If you make changes, use `npx changeset`. Versioning and publishing is done via GitHub Actions. 5 | 6 | Please make tests. We use `nock` to simulate the Notion API so remember that when you are making changes to your Notion test page and the API is returning stale results... you probably forgot to run with Nock turned off. 7 | 8 | Or better yet, use `Insomnia` and copy/paste the API results into `notion-api-mock.spec.mjs`. 9 | 10 | 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Jacob Milhorn 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | 3 | 4 | 5 |

The Notion CMS

6 | 7 |

8 | 9 |

Leverage the power of Notion for your website's content organization and creation,

10 |

with the simplicity of Ghost's Content API.

11 | 12 |

13 | 14 | NPM Version 15 | 16 | 17 | workflow 18 | 19 | 20 | NPM 21 | 22 | 23 | commit activity 24 | 25 |

26 | 27 | ![notion -_ notionCMS -_ prod](https://github.com/agency-kit/notion-cms/assets/68669571/b5990c57-d3a9-49ce-ba67-02b67f3b545c) 28 | 29 |

30 | docs | 31 | quickstart 32 |

33 | 34 | ## Features 35 | 36 | 🏗️  Framework agnostic - it’s just JS. 37 | 🌲 Build a collection-based CMS tree from your Notion database. 38 | 🎚️ Leverage database structure to control your routing structure. 39 | ⚙️ Geared for Static Site Generation. 40 | 📑  Transform Notion blocks → Markdown, plaintext, and (customizable) html. 41 | 🗃️ Optimized Content Caching for super fast builds. 42 | 🧩 Plugin ready with some powerful core plugins on the way. 43 | 🦾 Tagging, filtering, path queries, and tree-walking utilities. 44 | ⌨️ Totally Typesafe. 45 | 46 | ## Install 47 | 48 | ``` npm install @agency-kit/notion-cms ``` 49 | ```pnpm add @agency-kit/notion-cms``` 50 | ```yarn add @agency-kit/notion-cms``` 51 | 52 | ## Why. 53 | 54 | Notion excels at managing content. It also has a great API and SDK. But why is it so challenging to _actually_ leverage Notion as a CMS (Content Management System) in production? 55 | 56 | Until recently there wasn't support for sub-pages (sub-items in a database) in Notion. So most existing Notion-as-a-cms solutions don't provide a way to leverage this new feature, which turns out to be crucial for building a collection-based CMS. Another barrier is that pulling content from Notion can be time consuming. Responses can take a few seconds at times, the API provides a lot of data to sift through, and multiple calls to different endpoints are required in order to go from CMS database to the content for each page. All of this together results in a less than excellent developer experience when using a static site generator that often makes requests on each build. 57 | 58 | NotionCMS exists to address each of these issues and provide an excellent developer experience while using Notion as your Content Management System. 59 | 60 | ## The Database Structure 61 | 62 | In order to make use of NotionCMS, you have to subscribe to a specific database structure. Its an extremely generic design that gives you all the things you need for basic sites but lends flexibility for types of content other than the standard web page, blog post etc. 63 | 64 | See the structure in this template [this template](https://cooked-shovel-3c3.notion.site/NotionCMS-Quickstart-Database-Template-719f1f9d1547465d96bcd7e80333c831?pvs=4). 65 | 66 | ## Basic Usage 67 | 68 | ```javascript 69 | 70 | // initialize 71 | const myCoolCMS = new NotionCMS({ 72 | databaseId: 'e4fcd5b3-1d6a-4afd-b951-10d56ce436ad', 73 | notionAPIKey: process.env.NOTION, 74 | // Other options 75 | }) 76 | 77 | // Pull down all Notion content 78 | await myCoolCMS.pull() 79 | 80 | // Access the routes here: 81 | console.log(myCoolCMS.routes) 82 | 83 | // Access the page content here: 84 | console.log(myCoolCMS.data) 85 | 86 | // Access paths like this: 87 | const postA = myCoolCMS.data['/posts']['/how-to-build-a-blog-with-notion'] 88 | const postB = myCoolCMS.data['/posts']['/how-to-use-notion-cms'] 89 | 90 | ``` 91 | 92 | ## Advanced Usage 93 | 94 | ```javascript 95 | 96 | const customPlugin = () => { 97 | return { 98 | name: 'my-custom-plugin', 99 | hook: 'pre-parse', // other option is 'post-parse' 100 | // list of blocks to transform. 101 | // If hook is post parse, its the string of html parsed from blocks 102 | exec: (blocksOrHtml) => { 103 | // do some transformations, 104 | const transformedBlocksOrHtml = someXform(blocksOrHtml) 105 | return transformedBlocksOrHtml 106 | } 107 | } 108 | } 109 | 110 | const myAdvancedCMS = new NotionCMS({ 111 | databaseId: 'e4fcd5b3-1d6a-4afd-b951-10d56ce436ad', 112 | notionAPIKey: process.env.NOTION, 113 | rootUrl: 'https://mycoolsite.com', 114 | localCacheDirectory: `${process.cwd()}/localcache/`, 115 | refreshTimeout: '1 hour', 116 | plugins: [customPlugin()], 117 | }) 118 | 119 | await myAdvancedCMS.pull() 120 | 121 | ``` 122 | 123 | See the full [API reference](https://www.agencykit.so/notion-cms/guide/api/). 124 | 125 | ## Some Helper methods 126 | 127 | ```javascript 128 | // returns page reference 129 | myCMS.queryByPath('/full/path/to/page') 130 | 131 | // returns an array of only child pages of a page looked up using the key. 132 | // This runs queryByPath under the hood so you can save a step 133 | myCMS.filterSubPages('/full/path/to/page' /* or Page reference*/) 134 | 135 | // returns page object without any children - just the content. Useful for serializing and sending a 136 | // single pages data to the client. 137 | myCoolCMS.rejectSubPages('/full/path/to/page' /* or Page reference*/) 138 | 139 | // Get tagged collections this way or by passing a single tag: 140 | const tagged = myCoolCMS.getTaggedCollection(['blog', 'programming']) 141 | 142 | // walk the CMS from the root node and perform some operation for each node. 143 | // the node parameter will be of type PageContent so you have access to all of the page data 144 | myCoolCMS.walk(node => console.log(node), '/partial/path/to/start') 145 | 146 | // for async callbacks 147 | await myCoolCMS.asyncWalk(async node => await asynchronousFunc()) 148 | ``` 149 | 150 | ## Project Stats 151 | 152 | ![Alt](https://repobeats.axiom.co/api/embed/c2535fbb6cbc2731c212c332d5bc6cd695c32e0a.svg "Repobeats analytics image") 153 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@agency-kit/notion-cms", 3 | "type": "module", 4 | "version": "0.10.0", 5 | "description": "NotionCMS. Turn Notion into a full-fledged headless CMS (content management system).", 6 | "author": "Jacob Milhorn", 7 | "license": "MIT", 8 | "homepage": "https://www.agencykit.so/notion-cms/guide/", 9 | "repository": { 10 | "type": "git", 11 | "url": "git+https://github.com/agency-kit/notion-cms.git" 12 | }, 13 | "bugs": { 14 | "url": "https://github.com/agency-kit/notion-cms/issues" 15 | }, 16 | "keywords": [ 17 | "notion", 18 | "content", 19 | "content management", 20 | "cms", 21 | "notion-cms", 22 | "notion cms", 23 | "NotionCMS", 24 | "notion content management", 25 | "static site generation", 26 | "ssg", 27 | "ssr", 28 | "server side rendered", 29 | "server rendered", 30 | "node", 31 | "11ty", 32 | "eleventy", 33 | "headless cms", 34 | "headless content management system", 35 | "framework agnostic", 36 | "agnostic" 37 | ], 38 | "main": "./dist/index.js", 39 | "types": "./index.d.ts", 40 | "scripts": { 41 | "dev": "tsc --watch", 42 | "watch": "NODE_OPTIONS=--experimental-vm-modules NODE_NO_WARNINGS=1 chokidar \"src/**/*.ts\" -c \"rollup -c && tsc --project ./tsconfig.json && npx uvu test\"", 43 | "test:no-mock": "node test/notion-cms.spec.js true", 44 | "build": "rm -rf dist && tsc --project ./tsconfig.json && rollup -c", 45 | "build:dts": "rm -rf dist && tsc --project ./tsconfig.json && rollup -c && npm run dts", 46 | "dts": "dts-bundle-generator -o index.d.ts src/index.ts --no-check && rm -rf .tmp", 47 | "test": "uvu test", 48 | "build:test": "npm run build:dts && c8 npm test", 49 | "lint": "eslint .", 50 | "lint:fix": "eslint . --fix", 51 | "release": "changeset publish" 52 | }, 53 | "dependencies": { 54 | "@clack/prompts": "^0.6.3", 55 | "@notionhq/client": "^2.2.3", 56 | "file-type": "^18.5.0", 57 | "flatted": "^3.2.7", 58 | "highlight.js": "^11.7.0", 59 | "human-interval": "^2.0.1", 60 | "lodash": "^4.17.21", 61 | "marked": "^5.0.2", 62 | "marked-gfm-heading-id": "^3.0.3", 63 | "nanoid": "^4.0.2", 64 | "notion-utils": "^6.16.0", 65 | "picocolors": "^1.0.0", 66 | "serialize-javascript": "^6.0.1", 67 | "sharp": "^0.32.4", 68 | "walkjs": "^4.0.5", 69 | "zod": "^3.21.4" 70 | }, 71 | "devDependencies": { 72 | "@antfu/eslint-config": "^0.38.6", 73 | "@changesets/cli": "^2.26.1", 74 | "@notion-stuff/v4-types": "^6.0.0", 75 | "@rollup/plugin-terser": "^0.4.0", 76 | "@types/lodash": "^4.14.191", 77 | "@types/marked": "^4.0.8", 78 | "@types/node": "^20.1.5", 79 | "@types/serialize-javascript": "^5.0.2", 80 | "bottleneck": "^2.19.5", 81 | "c8": "^7.13.0", 82 | "chokidar": "^3.5.3", 83 | "chokidar-cli": "^3.0.0", 84 | "dotenv": "^16.0.3", 85 | "dts-bundle-generator": "^8.0.0", 86 | "esbuild": "^0.17.11", 87 | "nock": "^13.3.0", 88 | "rollup": "^3.19.1", 89 | "rollup-plugin-esbuild": "^5.0.0", 90 | "simple-git-hooks": "^2.8.1", 91 | "terser": "^5.16.6", 92 | "tslib": "^2.5.0", 93 | "typescript": "^4.9.5", 94 | "uvu": "^0.5.6" 95 | }, 96 | "simple-git-hooks": { 97 | "pre-commit": "npx lint-staged" 98 | }, 99 | "lint-staged": { 100 | "*": "eslint --fix" 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /public/subtle_gltch_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agency-kit/notion-cms/6a32a0fc3fbca5655cbe4716f26bc6fa7fa2ee5d/public/subtle_gltch_logo.png -------------------------------------------------------------------------------- /public/welcome_gltch_rounded.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agency-kit/notion-cms/6a32a0fc3fbca5655cbe4716f26bc6fa7fa2ee5d/public/welcome_gltch_rounded.png -------------------------------------------------------------------------------- /rollup.config.mjs: -------------------------------------------------------------------------------- 1 | import esbuild from 'rollup-plugin-esbuild' 2 | import terser from '@rollup/plugin-terser' 3 | 4 | export default [{ 5 | input: 'src/index.ts', 6 | plugins: [esbuild()], 7 | output: [ 8 | { format: 'esm', file: './dist/index.js' }, 9 | ], 10 | }, 11 | { 12 | input: 'src/index.ts', 13 | plugins: [esbuild({ sourceMap: true }), terser()], 14 | output: [ 15 | { format: 'esm', file: './dist/index.min.js', sourcemap: 'inline' }, 16 | ], 17 | }, 18 | { 19 | input: 'src/tests/test.ts', 20 | plugins: [esbuild()], 21 | output: [ 22 | { format: 'esm', file: './test/test.mjs' }, 23 | ], 24 | }] 25 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | export type { 2 | Properties, 3 | Options, 4 | CMS, 5 | Route, 6 | Page, 7 | PageObjectTitle, 8 | PageObjectRelation, 9 | PageObjectUser, 10 | PageMultiSelect, 11 | PageRichText, 12 | PageSelect, 13 | Cover, 14 | RouteObject, 15 | Plugin, 16 | PluginPassthrough, 17 | } from './types' 18 | 19 | export type { BlockRenderers } from './notion-blocks-parser' 20 | export { default as NotionBlocksParser } from './notion-blocks-parser' 21 | export { default } from './notion-cms' 22 | export { default as blocksRenderPlugin } from './plugins/render' 23 | export { default as Linker } from './plugins/linker' 24 | export { default as Images } from './plugins/images' 25 | export { default as Head } from './plugins/head' 26 | -------------------------------------------------------------------------------- /src/notion-blocks-html-parser.ts: -------------------------------------------------------------------------------- 1 | import type { Renderer as MarkedRenderer } from 'marked' 2 | import { marked } from 'marked' 3 | import hljs from 'highlight.js' 4 | import type { Blocks } from '@notion-stuff/v4-types' 5 | import _ from 'lodash' 6 | 7 | // @ts-expect-error module 8 | import { gfmHeadingId } from 'marked-gfm-heading-id' 9 | import type NotionBlocksMarkdownParser from './notion-blocks-md-parser' 10 | 11 | export default class NotionBlocksHtmlParser { 12 | markdownParser: NotionBlocksMarkdownParser 13 | renderer: MarkedRenderer 14 | markedOptions 15 | debug: boolean 16 | 17 | constructor(parser: NotionBlocksMarkdownParser, debug?: boolean) { 18 | this.markdownParser = parser 19 | this.debug = debug || false 20 | 21 | this.renderer = new marked.Renderer() 22 | this.renderer.code = this._highlight.bind(this) 23 | this.markedOptions = { 24 | renderer: this.renderer, 25 | pedantic: false, 26 | mangle: false, 27 | gfm: true, 28 | headerIds: true, 29 | breaks: false, 30 | sanitize: false, 31 | smartypants: false, 32 | xhtml: false, 33 | // eslint-disable-next-line @typescript-eslint/no-unsafe-call 34 | extensions: [gfmHeadingId({ prefix: '' })], 35 | } 36 | marked.use({ silent: true }) 37 | // This is a workaround so that hljs doesn't complain about mermaid not being a registered lang. 38 | hljs.registerAliases('mermaid', { languageName: 'plaintext' }) 39 | } 40 | 41 | marked(md: string): string { 42 | return marked(md, this.markedOptions) 43 | } 44 | 45 | parse(blocks: Blocks) { 46 | let markdown = this.markdownParser.parse(blocks) 47 | // if (this.debug) 48 | // fs.writeFileSync('./debug/parsed.md', markdown) 49 | // Take another pass to wrap any deeply nested mixed HTML content's inner text in p tags 50 | markdown = this._mixedHTML(markdown) 51 | // if (this.debug) 52 | // fs.appendFileSync('./debug/parsed.md', `--------ALTERED----------**\n\n\n${markdown}`) 53 | return marked(markdown, this.markedOptions) 54 | } 55 | 56 | _highlight(code: string, lang: string | undefined): string { 57 | let language 58 | if (lang === 'mermaid') 59 | language = 'mermaid' 60 | else 61 | language = (lang && hljs.getLanguage(lang)) ? lang : 'plaintext' 62 | const higlighted = hljs.highlight(code, { language }) 63 | const langClass = `language-${ 64 | (!language || language.includes('plain')) ? 'none' : language}` 65 | return `
${higlighted.value}
` 66 | } 67 | 68 | _mixedHTML(mixedHtml: string) { 69 | return mixedHtml.replaceAll(/[^<>]+?(?=<\/)/g, (match) => { 70 | // Must trim or Marked classifies text preceded by tabs as indented code. 71 | const tokens = marked.lexer(_.trim(match), this.markedOptions) 72 | return marked.parser(tokens, this.markedOptions) 73 | }) 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/notion-blocks-md-parser.ts: -------------------------------------------------------------------------------- 1 | import type { 2 | Annotations, 3 | AudioBlock, 4 | Blocks, 5 | BulletedListItemBlock, 6 | CalloutBlock, 7 | CalloutIconEmoji, 8 | CalloutIconExternal, 9 | CalloutIconFile, 10 | CodeBlock, 11 | EmbedBlock, 12 | ExternalFileWithCaption, 13 | FileBlock, 14 | FileWithCaption, 15 | HeadingBlock, 16 | ImageBlock, 17 | LinkPreviewBlock, 18 | LinkToPageBlock, 19 | NumberedListItemBlock, 20 | PDFBlock, 21 | ParagraphBlock, 22 | QuoteBlock, 23 | RichText, 24 | RichTextEquation, 25 | RichTextMention, 26 | RichTextText, 27 | ToDoBlock, 28 | ToggleBlock, 29 | VideoBlock, 30 | } from '@notion-stuff/v4-types' 31 | import { uuidToId } from 'notion-utils' 32 | 33 | const EOL_MD = '\n' 34 | 35 | export default class NotionBlocksMarkdownParser { 36 | parse(blocks: Blocks, depth = 0): string { 37 | return blocks 38 | .reduce((markdown, childBlock) => { 39 | let childBlockString = '' 40 | // @ts-expect-error children 41 | // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access 42 | if (childBlock.has_children && childBlock[childBlock.type].children) { 43 | childBlockString = ' ' 44 | .repeat(depth) 45 | .concat( 46 | childBlockString, 47 | // @ts-expect-error children 48 | // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-argument 49 | this.parse(childBlock[childBlock.type].children, depth + 2), 50 | ) 51 | } 52 | 53 | if (childBlock.type === 'unsupported') { 54 | markdown += 'NotionAPI Unsupported'.concat( 55 | EOL_MD.repeat(2), 56 | childBlockString, 57 | ) 58 | } 59 | 60 | if (childBlock.type === 'paragraph') 61 | markdown += this.parseParagraph(childBlock).concat(childBlockString) 62 | 63 | if (childBlock.type === 'code') 64 | markdown += this.parseCodeBlock(childBlock).concat(childBlockString) 65 | 66 | if (childBlock.type === 'quote') 67 | markdown += this.parseQuoteBlock(childBlock).concat(childBlockString) 68 | 69 | if (childBlock.type === 'callout') { 70 | markdown 71 | += this.parseCalloutBlock(childBlock).concat(childBlockString) 72 | } 73 | 74 | if (childBlock.type.startsWith('heading_')) { 75 | const headingLevel = Number(childBlock.type.split('_')[1]) 76 | markdown += this.parseHeading( 77 | childBlock as HeadingBlock, 78 | headingLevel, 79 | ).concat(childBlockString) 80 | } 81 | 82 | if (childBlock.type === 'bulleted_list_item') { 83 | markdown 84 | += this.parseBulletedListItems(childBlock).concat(childBlockString) 85 | } 86 | 87 | if (childBlock.type === 'numbered_list_item') { 88 | markdown 89 | += this.parseNumberedListItems(childBlock).concat(childBlockString) 90 | } 91 | 92 | if (childBlock.type === 'to_do') 93 | markdown += this.parseTodoBlock(childBlock).concat(childBlockString) 94 | 95 | if (childBlock.type === 'toggle') { 96 | markdown += this.parseToggleBlock(childBlock).replace( 97 | '{{childBlock}}', 98 | childBlockString, 99 | ) 100 | } 101 | 102 | if (childBlock.type === 'image') 103 | markdown += this.parseImageBlock(childBlock).concat(childBlockString) 104 | 105 | if (childBlock.type === 'embed') 106 | markdown += this.parseEmbedBlock(childBlock).concat(childBlockString) 107 | 108 | if (childBlock.type === 'audio') 109 | markdown += this.parseAudioBlock(childBlock).concat(childBlockString) 110 | 111 | if (childBlock.type === 'video') 112 | markdown += this.parseVideoBlock(childBlock).concat(childBlockString) 113 | 114 | if (childBlock.type === 'file') 115 | markdown += this.parseFileBlock(childBlock).concat(childBlockString) 116 | 117 | if (childBlock.type === 'pdf') 118 | markdown += this.parsePdfBlock(childBlock).concat(childBlockString) 119 | 120 | if (childBlock.type === 'link_to_page') 121 | markdown += this.parseLinkToPageBlock(childBlock).concat(childBlockString) 122 | 123 | if (childBlock.type === 'divider') 124 | markdown += EOL_MD.concat('---', EOL_MD, childBlockString) 125 | 126 | if (childBlock.type === 'link_preview') 127 | markdown += this.parseLinkPreview(childBlock).concat(childBlockString) 128 | 129 | return markdown 130 | }, '') 131 | .concat(EOL_MD).replace(/\t/g, ' ') 132 | } 133 | 134 | parseLinkPreview(linkPreviewBlock: LinkPreviewBlock): string { 135 | return `
136 | ${linkPreviewBlock.link_preview.url} 137 |
\n\n` 138 | } 139 | 140 | parseParagraph(paragraphBlock: ParagraphBlock): string { 141 | const text: string = this.parseRichTexts(paragraphBlock.paragraph.rich_text) 142 | return EOL_MD.concat(text, EOL_MD) 143 | } 144 | 145 | parseCodeBlock(codeBlock: CodeBlock): string { 146 | return `\n\n\`\`\`${codeBlock.code.language.toLowerCase() || ''} 147 | ${(codeBlock.code.rich_text[0] as RichTextText).text.content} 148 | \`\`\`\n\n`.concat(EOL_MD) 149 | } 150 | 151 | parseQuoteBlock(quoteBlock: QuoteBlock): string { 152 | return EOL_MD.concat( 153 | `> ${this.parseRichTexts(quoteBlock.quote.rich_text)}`, 154 | EOL_MD, 155 | ) 156 | } 157 | 158 | parseCalloutBlock(calloutBlock: CalloutBlock) { 159 | const callout = `
160 | {{icon}} 161 | 162 | ${this.parseRichTexts(calloutBlock.callout.rich_text)} 163 | 164 |
` 165 | 166 | function getCalloutIcon( 167 | icon: CalloutIconEmoji | CalloutIconExternal | CalloutIconFile, 168 | ) { 169 | switch (icon.type) { 170 | case 'emoji': 171 | return `${icon.emoji}` 172 | case 'external': 173 | return `notion-callout-external-link` 174 | case 'file': 175 | // TODO: add support for Callout File 176 | return 'notion-callout-file' 177 | } 178 | } 179 | 180 | return EOL_MD.concat( 181 | // @ts-expect-error callout 182 | callout.replace('{{icon}}', getCalloutIcon(calloutBlock.callout.icon)), 183 | EOL_MD, 184 | ) 185 | } 186 | 187 | parseHeading(headingBlock: HeadingBlock, headingLevel: number): string { 188 | return EOL_MD.concat( 189 | '#'.repeat(headingLevel), 190 | ' ', 191 | // @ts-expect-error heading 192 | // eslint-disable-next-line @typescript-eslint/no-unsafe-argument, @typescript-eslint/no-unsafe-member-access 193 | this.parseRichTexts(headingBlock[headingBlock.type].rich_text), 194 | EOL_MD, 195 | ) 196 | } 197 | 198 | parseBulletedListItems(bulletedListItemBlock: BulletedListItemBlock): string { 199 | return '* '.concat( 200 | this.parseRichTexts(bulletedListItemBlock.bulleted_list_item.rich_text), 201 | EOL_MD, 202 | ) 203 | } 204 | 205 | parseNumberedListItems(numberedListItemBlock: NumberedListItemBlock): string { 206 | return '1. '.concat( 207 | this.parseRichTexts(numberedListItemBlock.numbered_list_item.rich_text), 208 | EOL_MD, 209 | ) 210 | } 211 | 212 | parseTodoBlock(todoBlock: ToDoBlock): string { 213 | return `- [${todoBlock.to_do.checked ? 'x' : ' '}] `.concat( 214 | this.parseRichTexts(todoBlock.to_do.rich_text), 215 | EOL_MD, 216 | ) 217 | } 218 | 219 | parseToggleBlock(toggleBlock: ToggleBlock): string { 220 | return `
${this.parseRichTexts( 221 | toggleBlock.toggle.rich_text, 222 | )}{{childBlock}}
` 223 | } 224 | 225 | parseImageBlock(imageBlock: ImageBlock): string { 226 | const { url, caption } = this.parseFile(imageBlock.image) 227 | return ` 228 |
229 | ${caption} 230 |
${caption}
231 |
232 | `.concat(EOL_MD) 233 | } 234 | 235 | parseAudioBlock(audioBlock: AudioBlock): string { 236 | const { url, caption } = this.parseFile(audioBlock.audio) 237 | return `![${caption}](${url})` 238 | } 239 | 240 | parseVideoBlock(videoBlock: VideoBlock): string { 241 | const { url, caption } = this.parseFile(videoBlock.video) 242 | 243 | const [processed, iframeOrUrl] = processExternalVideoUrl(url) 244 | 245 | if (processed) 246 | return EOL_MD.concat(iframeOrUrl, EOL_MD) 247 | 248 | return `To be supported: ${url} with ${caption}`.concat(EOL_MD) 249 | } 250 | 251 | parseFileBlock(fileBlock: FileBlock): string { 252 | const { url, caption } = this.parseFile(fileBlock.file) 253 | return `To be supported: ${url} with ${caption}`.concat(EOL_MD) 254 | } 255 | 256 | parsePdfBlock(pdfBlock: PDFBlock): string { 257 | const { url, caption } = this.parseFile(pdfBlock.pdf) 258 | return ` 259 |
260 | 261 |
${caption}
262 |
263 | `.concat(EOL_MD) 264 | } 265 | 266 | parseEmbedBlock(embedBlock: EmbedBlock): string { 267 | const embedded = `` 268 | 269 | if (embedBlock.embed.caption) { 270 | return ` 271 |
272 | ${embedded} 273 |
${this.parseRichTexts(embedBlock.embed.caption)}
274 |
`.concat(EOL_MD) 275 | } 276 | 277 | return embedded.concat(EOL_MD) 278 | } 279 | 280 | parseRichTexts(richTexts: RichText[]): string { 281 | return richTexts.reduce((parsedContent, richText) => { 282 | switch (richText.type) { 283 | case 'text': 284 | parsedContent += this.parseText(richText) 285 | break 286 | case 'mention': 287 | parsedContent += this.parseMention(richText) 288 | break 289 | case 'equation': 290 | parsedContent += this.parseEquation(richText) 291 | break 292 | } 293 | 294 | return parsedContent 295 | }, '') 296 | } 297 | 298 | parseText(richText: RichTextText): string { 299 | const content = this.annotate(richText.annotations, richText.text.content) 300 | 301 | return richText.text.link 302 | ? this.annotateLink(richText.text, content) 303 | : content 304 | } 305 | 306 | // TODO: support mention when we know what it actually means 307 | 308 | parseMention(mention: RichTextMention): string { 309 | switch (mention.mention.type) { 310 | case 'user': 311 | break 312 | case 'page': 313 | break 314 | case 'database': 315 | break 316 | case 'date': 317 | break 318 | } 319 | return this.annotate(mention.annotations, mention.plain_text) 320 | } 321 | 322 | parseEquation(equation: RichTextEquation): string { 323 | return this.annotate( 324 | equation.annotations, 325 | `$${equation.equation.expression}$`, 326 | ) 327 | } 328 | 329 | parseLinkToPageBlock(linkToPage: LinkToPageBlock) { 330 | const link = linkToPage.link_to_page as { 331 | type: 'page_id' 332 | page_id: string 333 | } 334 | const id = uuidToId(link.page_id) 335 | // These will get replaced if ncms-plugin-linker is used. 336 | return `/${id}` 337 | } 338 | 339 | parseFile(file: ExternalFileWithCaption | FileWithCaption): { 340 | caption: string 341 | url: string 342 | } { 343 | const fileContent = { 344 | caption: '', 345 | url: '', 346 | } 347 | 348 | switch (file.type) { 349 | case 'external': 350 | fileContent.url = file.external.url 351 | break 352 | case 'file': 353 | fileContent.url = file.file.url 354 | break 355 | } 356 | 357 | fileContent.caption = file.caption 358 | ? this.parseRichTexts(file.caption) 359 | : fileContent.url 360 | 361 | return fileContent 362 | } 363 | 364 | private annotate(annotations: Annotations, originalContent: string): string { 365 | // @ts-expect-error reduce 366 | return Object.entries(annotations).reduce( 367 | // @ts-expect-error reduce 368 | ( 369 | annotatedContent, 370 | [modifier, isOnOrColor]: [ 371 | keyof Annotations, 372 | boolean | Annotations['color'], 373 | ], 374 | ) => 375 | isOnOrColor 376 | ? this.annotateModifier( 377 | modifier, 378 | annotatedContent, 379 | isOnOrColor as Annotations['color'], 380 | ) 381 | : annotatedContent, 382 | originalContent, 383 | ) 384 | } 385 | 386 | private annotateLink( 387 | text: RichTextText['text'], 388 | annotatedContent: string, 389 | ): string { 390 | return `[${annotatedContent}](${ 391 | // eslint-disable-next-line @typescript-eslint/restrict-template-expressions 392 | text?.link?.url ? text?.link.url : text.link 393 | })` 394 | } 395 | 396 | private annotateModifier( 397 | modifier: keyof Annotations, 398 | originalContent: string, 399 | color?: Annotations['color'], 400 | ): string { 401 | switch (modifier) { 402 | case 'bold': 403 | return `**${originalContent}**` 404 | case 'italic': 405 | return `_${originalContent}_` 406 | case 'strikethrough': 407 | return `~~${originalContent}~~` 408 | case 'underline': 409 | return `${originalContent}` 410 | case 'code': 411 | return `\`${originalContent}\`` 412 | case 'color': 413 | if (color !== 'default') 414 | // eslint-disable-next-line @typescript-eslint/restrict-template-expressions 415 | return `${originalContent}` 416 | return originalContent 417 | } 418 | } 419 | } 420 | 421 | function processExternalVideoUrl(url: string): [boolean, string] { 422 | if (url.includes('youtu')) 423 | return processYoutubeUrl(url) 424 | 425 | return [false, url] 426 | } 427 | 428 | function processYoutubeUrl(youtubeUrl: string): [boolean, string] { 429 | const lastPart = youtubeUrl.split('/').pop() 430 | 431 | const youtubeIframe = '' 432 | if (!lastPart) 433 | return [true, youtubeIframe] 434 | if (lastPart.includes('watch')) { 435 | const [, queryString] = lastPart.split('watch') 436 | const queryParams = new URLSearchParams(queryString) 437 | if (queryParams.has('v')) 438 | return [true, youtubeIframe.replace('{{videoId}}', queryParams.get('v') || '')] 439 | 440 | return [false, youtubeUrl] 441 | } 442 | 443 | return [true, youtubeIframe.replace('{{videoId}}', lastPart)] 444 | } 445 | -------------------------------------------------------------------------------- /src/notion-blocks-parser.ts: -------------------------------------------------------------------------------- 1 | import type { 2 | AudioBlock, 3 | Block, 4 | Blocks, 5 | BulletedListItemBlock, 6 | CalloutBlock, 7 | CodeBlock, 8 | EmbedBlock, 9 | FileBlock, 10 | HeadingBlock, 11 | ImageBlock, 12 | LinkPreviewBlock, 13 | LinkToPageBlock, 14 | NumberedListItemBlock, 15 | PDFBlock, 16 | ParagraphBlock, 17 | QuoteBlock, 18 | RichText, 19 | ToDoBlock, 20 | VideoBlock, 21 | } from '@notion-stuff/v4-types' 22 | import { z } from 'zod' 23 | import NotionBlocksMarkdownParser from './notion-blocks-md-parser' 24 | import NotionBlocksHtmlParser from './notion-blocks-html-parser' 25 | import NotionBlocksPlaintextParser from './notion-blocks-plaintext-parser' 26 | 27 | const blockRenderers = z.object({ 28 | AudioBlock: z.function().returns(z.string()), 29 | BulletedListItemBlock: z.function().returns(z.string()), 30 | CalloutBlock: z.function().returns(z.string()), 31 | CodeBlock: z.function().returns(z.string()), 32 | EmbedBlock: z.function().returns(z.string()), 33 | FileBlock: z.function().returns(z.string()), 34 | HeadingBlock: z.function().returns(z.string()), 35 | ImageBlock: z.function().returns(z.string()), 36 | LinkToPageBlock: z.function().returns(z.string()), 37 | NumberedListItemBlock: z.function().returns(z.string()), 38 | ParagraphBlock: z.function().returns(z.string()), 39 | PDFBlock: z.function().returns(z.string()), 40 | QuoteBlock: z.function().returns(z.string()), 41 | RichText: z.function().returns(z.string()), 42 | RichTextEquation: z.function().returns(z.string()), 43 | RichTextMention: z.function().returns(z.string()), 44 | RichTextText: z.function().returns(z.string()), 45 | ToDoBlock: z.function().returns(z.string()), 46 | ToggleBlock: z.function().returns(z.string()), 47 | VideoBlock: z.function().returns(z.string()), 48 | LinkPreviewBlock: z.function().returns(z.string()), 49 | }).partial() 50 | 51 | export type BlockRenderers = z.infer 52 | 53 | type Renderer = (block: Block | RichText[], ...args: unknown[]) => string 54 | type CustomRenderer = (block: Block | RichText[], ...args: unknown[]) => string | null 55 | 56 | function modularize( 57 | custom: CustomRenderer | undefined, 58 | def: Renderer): Renderer { 59 | return function render(block: Block | RichText[], ...args: unknown[]) { 60 | if (custom) { 61 | const customRender = custom(block, ...args) 62 | if (customRender !== null) 63 | return customRender 64 | } 65 | return def(block, ...args) 66 | } 67 | } 68 | 69 | export default class NotionBlocksParser { 70 | mdParser: NotionBlocksMarkdownParser 71 | htmlParser: NotionBlocksHtmlParser 72 | plainTextParser: NotionBlocksPlaintextParser 73 | debug: boolean 74 | 75 | constructor({ blockRenderers, debug }: { blockRenderers?: BlockRenderers; debug?: boolean }) { 76 | this.mdParser = new NotionBlocksMarkdownParser() 77 | this.plainTextParser = new NotionBlocksPlaintextParser() 78 | this.debug = debug || false 79 | 80 | this.mdParser.parseParagraph = modularize( 81 | blockRenderers?.ParagraphBlock, 82 | this.mdParser.parseParagraph.bind(this.mdParser) as Renderer, 83 | ) as (block: ParagraphBlock) => string 84 | 85 | this.mdParser.parseCodeBlock = modularize( 86 | blockRenderers?.CodeBlock, 87 | this.mdParser.parseCodeBlock.bind(this.mdParser) as Renderer, 88 | ) as (block: CodeBlock) => string 89 | 90 | this.mdParser.parseQuoteBlock = modularize( 91 | blockRenderers?.QuoteBlock, 92 | this.mdParser.parseQuoteBlock.bind(this.mdParser) as Renderer, 93 | ) as (block: QuoteBlock) => string 94 | 95 | this.mdParser.parseCalloutBlock = modularize( 96 | blockRenderers?.CalloutBlock, 97 | this.mdParser.parseCalloutBlock.bind(this.mdParser) as Renderer, 98 | ) as (block: CalloutBlock) => string 99 | 100 | this.mdParser.parseHeading = modularize( 101 | blockRenderers?.HeadingBlock, 102 | this.mdParser.parseHeading.bind(this.mdParser) as Renderer, 103 | ) as (block: HeadingBlock) => string 104 | 105 | this.mdParser.parseBulletedListItems = modularize( 106 | blockRenderers?.BulletedListItemBlock, 107 | this.mdParser.parseBulletedListItems.bind(this.mdParser) as Renderer, 108 | ) as (block: BulletedListItemBlock) => string 109 | 110 | this.mdParser.parseLinkToPageBlock = modularize( 111 | blockRenderers?.LinkToPageBlock, 112 | this.mdParser.parseLinkToPageBlock.bind(this.mdParser) as Renderer, 113 | ) as (block: LinkToPageBlock) => string 114 | 115 | this.mdParser.parseNumberedListItems = modularize( 116 | blockRenderers?.NumberedListItemBlock, 117 | this.mdParser.parseNumberedListItems.bind(this.mdParser) as Renderer, 118 | ) as (block: NumberedListItemBlock) => string 119 | 120 | this.mdParser.parseTodoBlock = modularize( 121 | blockRenderers?.ToDoBlock, 122 | this.mdParser.parseTodoBlock.bind(this.mdParser) as Renderer, 123 | ) as (block: ToDoBlock) => string 124 | 125 | this.mdParser.parseImageBlock = modularize( 126 | blockRenderers?.ImageBlock, 127 | this.mdParser.parseImageBlock.bind(this.mdParser) as Renderer, 128 | ) as (block: ImageBlock) => string 129 | 130 | this.mdParser.parseEmbedBlock = modularize( 131 | blockRenderers?.EmbedBlock, 132 | this.mdParser.parseEmbedBlock.bind(this.mdParser) as Renderer, 133 | ) as (block: EmbedBlock) => string 134 | 135 | this.mdParser.parseAudioBlock = modularize( 136 | blockRenderers?.AudioBlock, 137 | this.mdParser.parseAudioBlock.bind(this.mdParser) as Renderer, 138 | ) as (block: AudioBlock) => string 139 | 140 | this.mdParser.parseVideoBlock = modularize( 141 | blockRenderers?.VideoBlock, 142 | this.mdParser.parseVideoBlock.bind(this.mdParser) as Renderer, 143 | ) as (block: VideoBlock) => string 144 | 145 | this.mdParser.parseFileBlock = modularize( 146 | blockRenderers?.FileBlock, 147 | this.mdParser.parseFileBlock.bind(this.mdParser) as Renderer, 148 | ) as (block: FileBlock) => string 149 | 150 | this.mdParser.parsePdfBlock = modularize( 151 | blockRenderers?.PDFBlock, 152 | this.mdParser.parsePdfBlock.bind(this.mdParser) as Renderer, 153 | ) as (block: PDFBlock) => string 154 | 155 | this.mdParser.parseLinkPreview = modularize( 156 | blockRenderers?.LinkPreviewBlock, 157 | this.mdParser.parseLinkPreview.bind(this.mdParser) as Renderer, 158 | ) as (block: LinkPreviewBlock) => string 159 | 160 | // Warning: this parser is used in many of the other parsers internally. 161 | // Modding it could affect the others unexpectedly. 162 | this.mdParser.parseRichTexts = modularize( 163 | blockRenderers?.RichText, 164 | this.mdParser.parseRichTexts.bind(this.mdParser) as Renderer, 165 | ) as (block: RichText[]) => string 166 | 167 | this.htmlParser = new NotionBlocksHtmlParser(this.mdParser, this.debug) 168 | } 169 | 170 | markdownToPlainText(markdown: string): string { 171 | return this.plainTextParser.parse(markdown) 172 | } 173 | 174 | blocksToPlainText(blocks: Blocks, depth?: number): string { 175 | return this.plainTextParser.parse( 176 | this.blocksToMarkdown(blocks, depth)) 177 | } 178 | 179 | blocksToMarkdown(blocks: Blocks, depth?: number): string { 180 | return this.mdParser.parse(blocks, depth) 181 | } 182 | 183 | blocksToHtml(blocks: Blocks): string { 184 | return this.htmlParser.parse(blocks) 185 | } 186 | 187 | static parseRichText(richTexts: RichText[]) { 188 | const tempParser = new NotionBlocksMarkdownParser() 189 | return tempParser.parseRichTexts(richTexts) 190 | } 191 | } 192 | -------------------------------------------------------------------------------- /src/notion-blocks-plaintext-parser.ts: -------------------------------------------------------------------------------- 1 | import type { Renderer as MarkedRenderer } from 'marked' 2 | import { marked } from 'marked' 3 | import _ from 'lodash' 4 | 5 | const block = (text: string) => `${text}\n\n` 6 | const escapeBlock = (text: string) => `${_.escape(text)}\n\n` 7 | const line = (text: string) => `${text}\n` 8 | const inline = (text: string) => text 9 | const newline = () => '\n' 10 | const empty = () => '' 11 | 12 | export default class NotionBlocksPlaintextParser { 13 | renderer: MarkedRenderer 14 | markedOptions 15 | 16 | constructor() { 17 | this.renderer = { 18 | // Block elements 19 | code: escapeBlock, 20 | blockquote: block, 21 | html: empty, 22 | heading: block, 23 | hr: newline, 24 | list: text => block(text.trim()), 25 | listitem: line, 26 | checkbox: empty, 27 | paragraph: block, 28 | table: (header, body) => line(header + body), 29 | tablerow: text => line(text.trim()), 30 | tablecell: text => `${text} `, 31 | // Inline elements 32 | strong: inline, 33 | em: inline, 34 | codespan: inline, 35 | br: newline, 36 | del: inline, 37 | link: (_0, _1, text) => text, 38 | image: (_0, _1, text) => text, 39 | text: inline, 40 | // etc. 41 | options: {}, 42 | } 43 | this.markedOptions = { 44 | renderer: this.renderer, 45 | pedantic: false, 46 | mangle: false, 47 | gfm: false, 48 | extensions: undefined, 49 | breaks: false, 50 | headerIds: false, 51 | sanitize: false, 52 | smartypants: false, 53 | xhtml: false, 54 | } 55 | } 56 | 57 | parse(markdown: string) { 58 | const unmarked = marked(markdown, this.markedOptions) 59 | const unescaped = _.unescape(unmarked) 60 | const trimmed = _.trim(unescaped) 61 | return trimmed 62 | } 63 | } 64 | 65 | /* Much thanks to Eric Buss for making: https://github.com/ejrbuss/markdown-to-txt 66 | From which the Marked Renderer and utilities were copied. 67 | */ 68 | -------------------------------------------------------------------------------- /src/notion-cms.ts: -------------------------------------------------------------------------------- 1 | import fs from 'node:fs' 2 | import path from 'node:path' 3 | import { fileURLToPath } from 'node:url' 4 | import { createHash } from 'node:crypto' 5 | import { Client, LogLevel, collectPaginatedAPI, isFullBlock, isFullPage } from '@notionhq/client' 6 | import type { 7 | BlockObjectResponse, 8 | PageObjectResponse, 9 | SelectPropertyItemObjectResponse, 10 | } from '@notionhq/client/build/src/api-endpoints' 11 | import type { Blocks } from '@notion-stuff/v4-types' 12 | import _ from 'lodash' 13 | import type { AsyncCallbackFn, WalkNode } from 'walkjs' 14 | import { AsyncWalkBuilder, WalkBuilder } from 'walkjs' 15 | import humanInterval from 'human-interval' 16 | import { log, spinner } from '@clack/prompts' 17 | import color from 'picocolors' 18 | import type { 19 | CMS, 20 | Content, 21 | Cover, 22 | ExtendedPageContent, 23 | FlatListItem, 24 | Options, 25 | Page, 26 | PageContent, 27 | PageMultiSelect, 28 | PageObjectRelation, 29 | PageObjectTitle, 30 | PageObjectUser, 31 | Plugin, 32 | PluginPassthrough, 33 | UnsafePlugin, 34 | } from './types' 35 | import { 36 | JSONParseWithFunctions, 37 | JSONStringifyWithFunctions, 38 | filterAncestors, 39 | routify, 40 | slugify, 41 | writeFile, 42 | } from './utilities' 43 | import NotionLogger from './notion-logger' 44 | import renderer from './plugins/render' 45 | 46 | const __filename = fileURLToPath(import.meta.url) 47 | const __dirname = path.dirname(__filename) 48 | 49 | const COVER_IMAGE_REGEX = /
[\s\S]+]*src=['|"](https?:\/\/[^'|"]+)(?:['|"])/ 50 | 51 | const STEADY_PROPS = [ 52 | 'name', 53 | 'Author', 54 | 'Published', 55 | 'Tags', 56 | 'publishDate', 57 | 'parent-page', 58 | 'sub-page', 59 | ] 60 | 61 | const clackSpinner = spinner() 62 | 63 | export default class NotionCMS { 64 | cms: CMS 65 | cmsId: string 66 | notionClient: Client 67 | autoUpdate: boolean 68 | refreshTimeout: number | string 69 | draftMode: boolean 70 | defaultCacheFilename: string 71 | localCacheDirectory: string 72 | localCacheUrl: string 73 | debug: boolean | undefined 74 | limiter: { schedule: Function } 75 | plugins: Array | undefined 76 | options: Options 77 | private timer: number 78 | private coreRenderer: UnsafePlugin 79 | private logger: NotionLogger 80 | pull: () => Promise 81 | rootAlias: string 82 | withinRefreshTimeout: boolean 83 | quietMode: boolean 84 | 85 | constructor({ 86 | databaseId, 87 | notionAPIKey, 88 | debug = false, 89 | draftMode = false, 90 | refreshTimeout = 0, 91 | autoUpdate = true, 92 | localCacheDirectory = './.notion-cms/', 93 | rootAlias = '', 94 | rootUrl = '', 95 | quiet = false, 96 | limiter = { 97 | schedule: (func: Function) => { 98 | const result = func() as unknown 99 | return Promise.resolve(result) 100 | }, 101 | }, 102 | plugins = [], 103 | }: Options = { databaseId: '', notionAPIKey: '' }) { 104 | this.timer = Date.now() 105 | this.options = { 106 | databaseId, 107 | notionAPIKey, 108 | debug, 109 | draftMode, 110 | refreshTimeout, 111 | autoUpdate, 112 | localCacheDirectory, 113 | rootUrl, 114 | rootAlias, 115 | limiter, 116 | plugins, 117 | quiet, 118 | } 119 | this.cms = { 120 | metadata: { 121 | options: this._documentOptions(this.options), 122 | databaseId, 123 | rootUrl: rootUrl || '', 124 | stats: { 125 | durationSeconds: 0, 126 | totalPages: 0, 127 | }, 128 | }, 129 | stages: [], 130 | routes: [], 131 | tags: [], 132 | tagGroups: {}, 133 | siteData: {}, 134 | } 135 | this.cmsId = this._produceCMSIdentifier(databaseId) // Can't have multiple instances that reference the same db. 136 | this.debug = debug 137 | this.logger = new NotionLogger({ debug: this.debug }) 138 | 139 | this.notionClient = new Client({ 140 | auth: notionAPIKey, 141 | logLevel: LogLevel.DEBUG, 142 | logger: this.logger.log.bind(this.logger), 143 | }) 144 | this.autoUpdate = autoUpdate 145 | this.refreshTimeout 146 | = (refreshTimeout && _.isString(refreshTimeout)) 147 | ? (humanInterval(refreshTimeout) || refreshTimeout) 148 | : (refreshTimeout || 0) 149 | this.draftMode = draftMode || false 150 | this.localCacheDirectory = localCacheDirectory 151 | this.defaultCacheFilename = `ncms-cache-${this.cmsId}.json` 152 | this.localCacheUrl = path.resolve(__dirname, this.localCacheDirectory + this.defaultCacheFilename) 153 | this.limiter = limiter 154 | this.limiter.schedule.bind(limiter) 155 | 156 | this.coreRenderer = renderer({ blockRenderers: {}, debug }) 157 | this.coreRenderer.name = 'core-renderer' 158 | this.plugins = this._dedupePlugins([...plugins, this.coreRenderer]) 159 | this.pull = this.fetch.bind(this) 160 | this.rootAlias = rootAlias 161 | this.withinRefreshTimeout = false 162 | this.quietMode = quiet 163 | } 164 | 165 | get data() { 166 | if (_.isEmpty(this.cms.siteData)) 167 | return 168 | return this.cms.siteData 169 | } 170 | 171 | get routes(): Array { 172 | const routes = [] as Array 173 | this.walk((node: PageContent) => { 174 | if (node.path) 175 | routes.push(node.path) 176 | }) 177 | return routes 178 | } 179 | 180 | _documentOptions(options: Options): string { 181 | let hex 182 | try { 183 | const tempOptions = _.cloneDeep(options) 184 | delete tempOptions.limiter 185 | const hasher = createHash('md5') 186 | // Detecting Function string differences needs 187 | // to be implemented using the JSONStringifyWithFunctions util. 188 | const json = JSON.stringify(tempOptions) 189 | hasher.update(json) 190 | hex = hasher.digest('hex') 191 | } 192 | catch (e) { 193 | // eslint-disable-next-line @typescript-eslint/restrict-template-expressions 194 | throw new Error(`Failed to document options object: ${e}`) 195 | } 196 | return hex 197 | } 198 | 199 | _produceCMSIdentifier(id: string): string { 200 | return id.slice(-4) 201 | } 202 | 203 | _dedupePlugins(plugins: Array): Array { 204 | const numParsePlugins = _.filter(plugins, { hook: 'parse' }) 205 | if (numParsePlugins.length > 1) 206 | return _.initial(plugins) 207 | 208 | return plugins 209 | } 210 | 211 | _checkDuplicateParsePlugins(pluginsList: Array): boolean { 212 | return _(pluginsList).filter(plugin => plugin.hook === 'parse').uniq().value().length > 1 213 | } 214 | 215 | async _runPlugins( 216 | context: PluginPassthrough, 217 | hook: Plugin['hook'] | 'parse'): Promise { 218 | if (!this.plugins?.length) 219 | return context 220 | if (this._checkDuplicateParsePlugins(this.plugins)) 221 | throw new Error('Only one parse-capable plugin must be used. Use the default NotionCMS render plugin.') 222 | 223 | let val = context 224 | for (const plugin of this.plugins.flat()) { 225 | if (plugin.hook === hook || plugin.hook === '*') { 226 | // eslint-disable-next-line @typescript-eslint/await-thenable 227 | val = await plugin.exec(val, { 228 | debug: !!this.debug, 229 | localCacheDirectory: this.localCacheDirectory, 230 | notion: this.notionClient, 231 | }) 232 | } 233 | } 234 | return val 235 | } 236 | 237 | _flatListToTree = ( 238 | flatList: Partial[], 239 | idPath: keyof FlatListItem, 240 | parentIdPath: keyof FlatListItem, 241 | isRoot: (t: Partial) => boolean, 242 | ): Record => { 243 | const rootParents: Partial[] = [] 244 | const map = {} as { [x: string]: Partial } 245 | const tree = {} 246 | for (const item of flatList) 247 | map[item[idPath] as string] = item 248 | 249 | for (const item of flatList) { 250 | const parentId = item[parentIdPath] 251 | if (isRoot(item)) { 252 | if (this.rootAlias) { 253 | if (item._key === this.rootAlias) 254 | item._key = '/' 255 | else 256 | throw new Error(`Incorrect rootAlias. "${this.rootAlias}" not found in Notion data.`) 257 | } 258 | rootParents.push(item) 259 | } 260 | else { 261 | const parentItem = map[parentId as string] 262 | parentItem[item._key as string] = item 263 | } 264 | } 265 | _.forEach(rootParents, (page) => { 266 | _.assign(tree, { [page._key as string]: page }) 267 | }) 268 | return tree 269 | } 270 | 271 | _notionListToTree(list: Partial[]): Record { 272 | const flatTree = this._flatListToTree(list, 'id', 'pid', (node: Partial) => !node.pid) 273 | const topLevelKeys = _.keys(flatTree) 274 | if (topLevelKeys.length > 1 && this.rootAlias) 275 | log.warn(color.yellow('[NotionCMS][Warning]: You have set a root alias but there is more than one node at the top level of the CMS tree.')) 276 | return flatTree 277 | } 278 | 279 | static _isPageContentObject(node: WalkNode): boolean { 280 | return typeof node.key === 'string' && node?.key?.startsWith('/') 281 | && ((typeof node?.parent?.key === 'string' && node?.parent?.key?.startsWith('/')) 282 | || !node?.parent?.key || node?.parent?.key === 'siteData') 283 | } 284 | 285 | static _createCMSWalker(cb: (node: ExtendedPageContent) => void): WalkBuilder { 286 | return new WalkBuilder() 287 | .withCallback({ 288 | filters: [(node: WalkNode) => NotionCMS._isPageContentObject(node)], 289 | nodeTypeFilters: ['object'], 290 | positionFilter: 'postWalk', 291 | callback: (node: WalkNode) => cb(node.val as ExtendedPageContent), 292 | }) 293 | } 294 | 295 | _getParentPageId(response: PageObjectResponse): string { 296 | const parentPage = response?.properties['parent-page'] as PageObjectRelation 297 | return parentPage?.relation[0]?.id 298 | } 299 | 300 | _getBlockName(response: PageObjectResponse): string { 301 | const nameProp = response?.properties.name as PageObjectTitle 302 | return nameProp.title[0]?.plain_text 303 | } 304 | 305 | _extractTags(response: PageObjectResponse): Array { 306 | const tagProp = response?.properties?.Tags as PageMultiSelect 307 | return tagProp.multi_select ? tagProp.multi_select.map(multiselect => multiselect.name) : [] 308 | } 309 | 310 | _assignTagGroup(tag: string, path: string, cms: CMS): void { 311 | if (!cms.tagGroups[tag]) 312 | cms.tagGroups[tag] = [] 313 | cms.tagGroups[tag].push(path) 314 | } 315 | 316 | _buildTagGroups(tags: Array, path: string, cms: CMS): void { 317 | _.forEach(tags, (tag) => { 318 | if (!_.includes(cms.tags, tag)) 319 | cms.tags.push(tag) 320 | this._assignTagGroup(tag, path, cms) 321 | }) 322 | } 323 | 324 | _getCoverImage(page: PageObjectResponse): string | undefined { 325 | const pageCoverProp = (page)?.cover as Cover 326 | let coverImage 327 | if (pageCoverProp && 'external' in pageCoverProp) 328 | coverImage = pageCoverProp?.external?.url 329 | 330 | else if (pageCoverProp?.file) 331 | coverImage = pageCoverProp?.file.url 332 | 333 | return coverImage 334 | } 335 | 336 | async _pullPageContent(id: string): Promise { 337 | let pageContent 338 | try { 339 | pageContent = await this.limiter.schedule( 340 | async () => await collectPaginatedAPI(this.notionClient.blocks.children.list, { 341 | block_id: id, 342 | }), 343 | ) as BlockObjectResponse[] 344 | } 345 | catch (e) { 346 | if (this.debug) 347 | // eslint-disable-next-line @typescript-eslint/restrict-template-expressions 348 | console.error(`NotionCMS Error: ${e}`) 349 | } 350 | if (!pageContent) 351 | return 352 | for (const block of pageContent) { 353 | if (isFullBlock(block) && block.has_children) 354 | // @ts-expect-error children 355 | // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access 356 | block[block.type].children = (await this._pullPageContent(block.id)) 357 | } 358 | return pageContent 359 | } 360 | 361 | async _parsePageContent(pageContent: BlockObjectResponse[]): Promise { 362 | const results = await this._runPlugins(pageContent as Blocks, 'pre-parse') 363 | const markdown: string = this.coreRenderer.parser.blocksToMarkdown(pageContent as Blocks) 364 | const plaintext: string = this.coreRenderer.parser.markdownToPlainText(markdown) 365 | const parsedBlocks = await this._runPlugins(results as Blocks, 'parse') 366 | const html = await this._runPlugins(parsedBlocks, 'post-parse') as string 367 | return { 368 | plaintext, 369 | markdown, 370 | html, 371 | } 372 | } 373 | 374 | async _getPageContent(state: CMS, cachedState?: CMS): Promise { 375 | let stateWithContent = _.cloneDeep(state) 376 | 377 | await new AsyncWalkBuilder() 378 | .withCallback({ 379 | filters: [(node: WalkNode) => NotionCMS._isPageContentObject(node)], 380 | nodeTypeFilters: ['object'], 381 | positionFilter: 'postWalk', 382 | callback: async (node: WalkNode) => { 383 | const pageContent = node.val as PageContent 384 | if (!pageContent || !pageContent?._notion?.id) 385 | return 386 | if (!this.quietMode && pageContent.path) 387 | clackSpinner.start(`[NotionCMS][handling page]: ${pageContent.path}`) 388 | // Definitely grab content if there is no cache. 389 | if (pageContent._updateNeeded || !cachedState) { 390 | if (!this.quietMode && pageContent.path) 391 | clackSpinner.stop(`[NotionCMS][using API]: ${pageContent.path}`) 392 | const blocks = await this._pullPageContent(pageContent._notion.id) 393 | if (!blocks) 394 | return 395 | const content = await this._parsePageContent(blocks) 396 | _.assign(pageContent, { 397 | content, 398 | ...(!pageContent.coverImage && { coverImage: content.html.match(COVER_IMAGE_REGEX)?.[1] }), 399 | _ancestors: this._gatherNodeAncestors(node), 400 | }) 401 | } 402 | else if (cachedState && pageContent.path) { 403 | const cachedPage = this._queryByPath(pageContent.path, cachedState?.siteData) 404 | if (cachedPage) { 405 | _.assign(pageContent, { 406 | content: cachedPage.content, 407 | ...(!cachedPage.coverImage && { coverImage: cachedPage.content?.html.match(COVER_IMAGE_REGEX)?.[1] }), 408 | _ancestors: this._gatherNodeAncestors(node), 409 | }) 410 | if (!this.quietMode && pageContent.path) 411 | clackSpinner.stop(color.italic(`[NotionCMS][using cache]: ${pageContent.path}`)) 412 | } 413 | } 414 | else { 415 | clackSpinner.stop() 416 | throw new Error(`Unable to update content. No page found for ${node.key || 'undetermined node key'}`) 417 | } 418 | 419 | _.assign( 420 | pageContent, 421 | await this._runPlugins(pageContent, 'during-tree') as Page) 422 | delete pageContent.otherProps 423 | // We only want access to ancestors for plugins, otherwise it creates circular ref headaches. 424 | delete pageContent._ancestors 425 | delete pageContent._updateNeeded 426 | }, 427 | }) 428 | .withRootObjectCallbacks(false) 429 | .withParallelizeAsyncCallbacks(true) 430 | .walk(stateWithContent.siteData) 431 | 432 | stateWithContent.stages.push('content') 433 | stateWithContent = await this._runPlugins(stateWithContent, 'post-tree') as CMS 434 | return stateWithContent 435 | } 436 | 437 | _extractUnsteadyProps(properties: PageObjectResponse['properties']): PageObjectResponse['properties'] { 438 | return _(properties) 439 | .entries() 440 | .reject(([key]) => _.includes(STEADY_PROPS, key)) 441 | .fromPairs().value() 442 | } 443 | 444 | _getPageUpdate(entry: PageObjectResponse): Page { 445 | const tags = [] as Array 446 | if (isFullPage(entry)) { 447 | const name = this._getBlockName(entry) 448 | const authorProp = entry.properties?.Author as PageObjectUser 449 | const authors = authorProp.people.map(authorId => authorId.name as string) 450 | 451 | const coverImage = this._getCoverImage(entry) 452 | const extractedTags = this._extractTags(entry) 453 | extractedTags.forEach(tag => tags.push(tag)) 454 | const otherProps = this._extractUnsteadyProps(entry.properties) 455 | 456 | return { 457 | name, 458 | otherProps, 459 | slug: slugify(name), 460 | authors, 461 | tags, 462 | coverImage, 463 | _notion: { 464 | id: entry.id, 465 | last_edited_time: entry.last_edited_time, 466 | }, 467 | } 468 | } 469 | return {} 470 | } 471 | 472 | _publishedFilter = (e: PageObjectResponse) => { 473 | const publishProp = e.properties.Published as SelectPropertyItemObjectResponse 474 | return this.draftMode ? true : (publishProp.select && publishProp.select.name === 'Published') 475 | } 476 | 477 | _gatherNodeAncestors(node: WalkNode): Array { 478 | return _(node.ancestors).map((ancestor) => { 479 | if ((ancestor.val as PageContent)._notion) 480 | return ancestor.val as PageContent 481 | return false 482 | }).compact().value() 483 | } 484 | 485 | async _getDb(state: CMS, cachedState?: CMS): Promise { 486 | let stateWithDb = _.cloneDeep(state) 487 | let db 488 | try { 489 | db = await this.limiter.schedule( 490 | async () => (await collectPaginatedAPI( 491 | this.notionClient.databases.query, { database_id: state.metadata.databaseId }, 492 | )), 493 | ) as PageObjectResponse[] 494 | if (!db) 495 | return 496 | 497 | stateWithDb.metadata.stats.totalPages = db.length 498 | stateWithDb.siteData = this._notionListToTree( 499 | _(db) 500 | // @ts-expect-error filter 501 | .filter(this._publishedFilter) 502 | .map(page => _.assign({}, { 503 | _key: routify(this._getBlockName(page)), 504 | id: page.id, 505 | pid: this._getParentPageId(page), 506 | _notion: page, // this property is recycled to eventually house metadata. 507 | })) 508 | .value(), 509 | ) 510 | 511 | if (_.isEmpty(stateWithDb.siteData)) 512 | throw new Error('NotionCMS is empty. Did you mean to set `draftMode: true`?') 513 | 514 | new WalkBuilder() 515 | .withCallback({ 516 | nodeTypeFilters: ['object'], 517 | callback: (node: WalkNode) => { 518 | const pageContent = node.val as PageContent 519 | if (!pageContent?._notion) 520 | return 521 | pageContent._updateNeeded = !this.withinRefreshTimeout 522 | const update = this._getPageUpdate(pageContent._notion as PageObjectResponse) 523 | _.assign(pageContent, update) 524 | _.assign(pageContent, { 525 | // Replace double // so that root aliasing works properly 526 | path: node.getPath(node => `${node.key as string}`) 527 | .replace('siteData', '') 528 | .replace('//', '/'), 529 | url: (stateWithDb.metadata.rootUrl && pageContent.path) 530 | ? (`${stateWithDb.metadata.rootUrl as string}${pageContent.path}`) 531 | : '', 532 | }) 533 | if (cachedState && pageContent.path && !this.withinRefreshTimeout) { 534 | const cachedPage = this._queryByPath(pageContent.path, cachedState?.siteData) 535 | pageContent._updateNeeded = this.autoUpdate 536 | && (update._notion?.last_edited_time !== cachedPage?._notion?.last_edited_time) 537 | } 538 | if (node.key && typeof node.key === 'string' && pageContent.tags && pageContent.path) 539 | this._buildTagGroups(pageContent.tags, pageContent.path, stateWithDb) 540 | }, 541 | }) 542 | .withRootObjectCallbacks(false) 543 | .walk(stateWithDb) 544 | } 545 | catch (e: unknown) { 546 | if (e instanceof Error) 547 | throw new Error(e.message) 548 | } 549 | stateWithDb.stages.push('db') 550 | stateWithDb = await this._runPlugins(stateWithDb, 'pre-tree') as CMS 551 | return stateWithDb 552 | } 553 | 554 | async fetch(): Promise { 555 | let cachedCMS, optionsHaveChanged 556 | if (fs.existsSync(this.localCacheUrl)) { 557 | try { 558 | cachedCMS = JSONParseWithFunctions(fs.readFileSync(this.localCacheUrl, 'utf-8')) as CMS 559 | optionsHaveChanged = cachedCMS.metadata.options !== this.cms.metadata.options 560 | } 561 | catch (e) { 562 | if (this.debug) 563 | console.error('Parsing cached CMS failed. Using API instead.') 564 | } 565 | } 566 | this.withinRefreshTimeout = Boolean(cachedCMS && ((cachedCMS.lastUpdateTimestamp 567 | && (Date.now() < cachedCMS.lastUpdateTimestamp + _.toNumber(this.refreshTimeout))) 568 | && !optionsHaveChanged)) 569 | 570 | if (!this.quietMode) 571 | log.step('[NotionCMS]: pulling your content from Notion...🛸') 572 | try { 573 | const cmsOutput = await this._getDb(this.cms, cachedCMS) 574 | if (!cmsOutput) 575 | throw new Error('Notion db fetch unsuccessful.') 576 | this.cms = cmsOutput 577 | 578 | this.cms = await this._getPageContent(this.cms, cachedCMS) 579 | this.cms.stages.push('complete') 580 | this.export() 581 | if (!this.quietMode) 582 | log.step('[NotionCMS]: mission complete! 👽') 583 | } 584 | catch (e: unknown) { 585 | // All errors must flow through Clack or they will get swallowed i.e. Clack prints over them 586 | if (e instanceof Error) 587 | log.error(color.red(`[NotionCMS]: Mission aborted. Reason: ${e.message}`)) 588 | } 589 | 590 | this.cms.routes = this.routes 591 | this.cms.metadata.stats = { 592 | ...this.logger.stats, 593 | totalPages: this.cms.metadata.stats.totalPages, 594 | durationSeconds: (Date.now() - this.timer) / 1000, 595 | } 596 | if (this.debug) 597 | writeFile('debug/site-data.json', JSONStringifyWithFunctions(this.cms)) 598 | return this.cms 599 | } 600 | 601 | async asyncWalk(cb: Function, path?: string) { 602 | const startPoint = path ? this.queryByPath(path) : this.cms.siteData 603 | await new AsyncWalkBuilder() 604 | .withCallback({ 605 | nodeTypeFilters: ['object'], 606 | filters: [(node: WalkNode) => NotionCMS._isPageContentObject(node)], 607 | callback: (node: WalkNode) => cb(node.val) as AsyncCallbackFn, 608 | }) 609 | .withRootObjectCallbacks(false) 610 | .withParallelizeAsyncCallbacks(true) 611 | .walk(startPoint) 612 | } 613 | 614 | walk(cb: Function, path?: string) { 615 | const startPoint = path ? this.queryByPath(path) : this.cms.siteData 616 | new WalkBuilder() 617 | .withCallback({ 618 | nodeTypeFilters: ['object'], 619 | filters: [(node: WalkNode) => NotionCMS._isPageContentObject(node)], 620 | callback: (node: WalkNode) => cb(node.val) as unknown, 621 | }) 622 | .withRootObjectCallbacks(false) 623 | .walk(startPoint) 624 | } 625 | 626 | getTaggedCollection(tags: string | Array): Array { 627 | if (!_.isArray(tags)) 628 | tags = [tags] 629 | const taggedPages = [] as Array 630 | for (const tag of tags) 631 | taggedPages.push(...this.cms.tagGroups[tag]) 632 | 633 | if (typeof this.cms.siteData !== 'string') 634 | return _(taggedPages).map(path => this.queryByPath(path)).uniq().value() 635 | 636 | return [] 637 | } 638 | 639 | filterSubPages(pathOrPage: string | Page): Array { 640 | if (typeof pathOrPage === 'string') 641 | pathOrPage = this.queryByPath(pathOrPage) 642 | 643 | return _(pathOrPage) 644 | .entries() 645 | .filter(([key]) => key.startsWith('/')) 646 | .map(e => e[1]).value() as Page[] 647 | } 648 | 649 | rejectSubPages(pathOrPage: string | Page): Page { 650 | if (typeof pathOrPage === 'string') 651 | pathOrPage = this.queryByPath(pathOrPage) 652 | 653 | return _(pathOrPage) 654 | .entries() 655 | .reject(([key]) => key.startsWith('/')) 656 | .fromPairs().value() as Page 657 | } 658 | 659 | _queryByPath(path: string, siteData: Record): Page { 660 | const segments = path.split('/').slice(1) 661 | if (this.rootAlias && path !== '/') 662 | segments.unshift('') // This lets us access the root aliased page. 663 | let access: Page = siteData 664 | for (const segment of segments) { 665 | // @ts-expect-error-next-line 666 | access = access[`/${segment}`] as Page 667 | } 668 | return access 669 | } 670 | 671 | queryByPath(path: string): Page { 672 | return this._queryByPath(path, this.cms.siteData) 673 | } 674 | 675 | export({ pretty = false, path = this.localCacheUrl }: 676 | { pretty?: boolean; path?: string } = {}) { 677 | this.cms.lastUpdateTimestamp = Date.now() 678 | if (pretty) { 679 | // This drops Functions too, so only use for inspection 680 | writeFile(path, JSON.stringify(this.cms, filterAncestors)) 681 | } 682 | else { 683 | writeFile(path, JSONStringifyWithFunctions(this.cms)) 684 | } 685 | } 686 | 687 | async import(previousState: string, flatted?: boolean): Promise { 688 | let parsedPreviousState 689 | try { 690 | if (flatted) 691 | parsedPreviousState = JSONParseWithFunctions(previousState) as CMS 692 | 693 | else 694 | parsedPreviousState = JSON.parse(previousState) as CMS 695 | } 696 | catch (e) { 697 | throw new Error('Parsing input CMS failed. Ensure the input follows the NotionCMS spec.') 698 | } 699 | const transformedPreviousState = await this._runPlugins(parsedPreviousState, 'import') as CMS 700 | return this.cms = transformedPreviousState 701 | } 702 | 703 | purgeCache(): boolean { 704 | try { 705 | fs.rmSync(this.localCacheUrl) 706 | } 707 | catch (e) { 708 | if (this.debug) 709 | // eslint-disable-next-line @typescript-eslint/restrict-template-expressions 710 | log.error(`[NotionCMS][error]: error when attempting to clear cache: ${e}`) 711 | return false 712 | } 713 | if (this.debug) 714 | log.info('[NotionCMS][info]: cache has been successfully cleared.') 715 | return true 716 | } 717 | } 718 | -------------------------------------------------------------------------------- /src/notion-logger.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-console */ 2 | import type { Stats } from './types' 3 | 4 | const START = 'request start' 5 | const SUCCESS = 'request success' 6 | const FAILURE = 'request fail' 7 | 8 | type ExtraInfo = Record 9 | 10 | export default class NotionLogger { 11 | debug: boolean 12 | stats: Pick 16 | 17 | constructor({ debug = false }: { debug: boolean }) { 18 | this.debug = debug 19 | this.stats = { 20 | totalAPICalls: 0, 21 | succeededCalls: 0, 22 | failedCalls: 0, 23 | } 24 | } 25 | 26 | log(logLevel: string, message: string, extraInfo: ExtraInfo): void { 27 | if (this.debug) { 28 | console.log('logLevel:', logLevel) 29 | console.log('message:', message) 30 | console.log('extraInfo:', extraInfo) 31 | } 32 | this.handleMessage(message) 33 | } 34 | 35 | handleMessage(message: string): void { 36 | switch (message) { 37 | case START: 38 | this.stats.totalAPICalls++ 39 | break 40 | case SUCCESS: 41 | this.stats.succeededCalls++ 42 | break 43 | case FAILURE: 44 | this.stats.failedCalls++ 45 | break 46 | } 47 | } 48 | 49 | handleLogLevel(logLevel: string): void {} 50 | 51 | handleExtraInfo(extraInfo: ExtraInfo): void {} 52 | 53 | resetStats(): void { 54 | this.stats = { 55 | totalAPICalls: 0, 56 | failedCalls: 0, 57 | succeededCalls: 0, 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/plugins/head.ts: -------------------------------------------------------------------------------- 1 | import type { PageContent } from '../types' 2 | 3 | export default function () { 4 | return { 5 | name: 'ncms-plugin-head', 6 | hook: 'during-tree', 7 | core: true, 8 | exec: (context: PageContent) => { 9 | const copyOfContext = structuredClone(context) as PageContentWithMeta 10 | copyOfContext.meta = { 11 | title: '', 12 | description: '', 13 | } 14 | if (copyOfContext.otherProps?.metaTitle 15 | && copyOfContext.otherProps?.metaTitle.type === 'rich_text') 16 | copyOfContext.meta.title = copyOfContext.otherProps?.metaTitle?.rich_text[0]?.plain_text 17 | 18 | if (copyOfContext.otherProps?.metaDescription 19 | && copyOfContext.otherProps?.metaDescription.type === 'rich_text') 20 | copyOfContext.meta.description = copyOfContext.otherProps?.metaDescription?.rich_text[0]?.plain_text 21 | 22 | return copyOfContext 23 | }, 24 | } 25 | } 26 | 27 | export interface PageContentWithMeta extends PageContent { 28 | meta: Meta 29 | } 30 | 31 | interface Meta { 32 | title: string 33 | description: string 34 | } 35 | -------------------------------------------------------------------------------- /src/plugins/images.ts: -------------------------------------------------------------------------------- 1 | import fs from 'node:fs' 2 | import { Buffer } from 'node:buffer' 3 | import { fileTypeFromBuffer } from 'file-type' 4 | import { nanoid } from 'nanoid' 5 | import sharp from 'sharp' 6 | import type { Content, PageContent, PluginExecOptions } from '../types' 7 | 8 | interface ImageCacheEntry { 9 | filename?: string 10 | location?: string 11 | url?: string 12 | } 13 | 14 | interface ImageCache { [key: string]: Array } 15 | 16 | const IMAGE_FILE_MATCH_REGEX = /(.*)X-Amz-Algorithm/g 17 | const IMAGE_CACHE_FILENAME = 'ncms-image-cache.json' 18 | const GENERIC_MATCH = /\b(https?:\/\/[\w_#&?.\/-]*?\.(?:png|jpe?g|svg|ico))(?=[`'")\]])/ig 19 | const IMAGE_SOURCE_MATCH = /]*src=['|"](https?:\/\/[^'|"]+)(?:['|"])/ig 20 | const OBJECT_PDF_MATCH = /]+data=(?:'|")(https?:\/\/[^'"\s]+\.pdf\?[^'"\s]*)?/ig 21 | 22 | function multiStringMatch(stringA: unknown, stringB: unknown): Boolean { 23 | if (typeof stringA !== 'string' || typeof stringB !== 'string' || !stringA || !stringB) 24 | return false 25 | const matchA = stringA.match(IMAGE_FILE_MATCH_REGEX) 26 | const matchB = stringB.match(IMAGE_FILE_MATCH_REGEX) 27 | return Boolean(matchA && matchB && (matchA[0] === matchB[0])) 28 | } 29 | 30 | export default function ({ 31 | globalExtension = 'webp', 32 | compression = 80, 33 | imageCacheDirectory = './public', 34 | customMatchers = [], 35 | }: { 36 | globalExtension?: 'webp' | 'png' | 'jpeg' 37 | compression?: number 38 | imageCacheDirectory?: string 39 | customMatchers?: RegExp[] 40 | } = {}) { 41 | let imageCache: ImageCache 42 | 43 | try { 44 | // Pull existing imageCache 45 | if (fs.existsSync(`${imageCacheDirectory}/remote/${IMAGE_CACHE_FILENAME}`)) { 46 | imageCache = JSON.parse( 47 | fs.readFileSync(`${imageCacheDirectory}/remote/${IMAGE_CACHE_FILENAME}`, 'utf-8')) as ImageCache 48 | } 49 | else { 50 | imageCache = {} 51 | } 52 | } 53 | catch (e) { 54 | console.warn(e, 'ncms-plugin-images: error attempting to read image cache.') 55 | imageCache = {} 56 | } 57 | 58 | async function writeOutAsset(imageUrl: string, existingImageFile: ImageCacheEntry, isPdf: boolean): Promise { 59 | let filename = '' 60 | if (existingImageFile) 61 | return existingImageFile.filename as string 62 | const response = await fetch(imageUrl) 63 | const arrayBuffer = await response.arrayBuffer() 64 | const buffer = Buffer.from(arrayBuffer) 65 | const fileType = await fileTypeFromBuffer(buffer) 66 | 67 | if (isPdf) { 68 | filename = `pdf-${nanoid(6)}.remote.pdf` 69 | const outputPath = `${imageCacheDirectory}/remote/${filename}` 70 | fs.writeFileSync(outputPath, buffer) 71 | } 72 | else { 73 | if (fileType?.ext) { 74 | const id = nanoid(6) 75 | filename = `${id}.remote.${globalExtension}` 76 | const outputFilePath = `${imageCacheDirectory}/remote/${filename}` 77 | const imageBuffer = sharp(buffer) 78 | const webPBuffer = await imageBuffer[globalExtension]({ 79 | quality: compression, 80 | nearLossless: true, 81 | effort: 6, 82 | }).toBuffer() 83 | const writeStream = fs.createWriteStream(outputFilePath) 84 | // eslint-disable-next-line @typescript-eslint/restrict-template-expressions 85 | writeStream.on('error', err => console.warn(`ncms-plugin-images: failed to write image file: ${err}`)) 86 | writeStream.write(webPBuffer) 87 | } 88 | } 89 | return filename 90 | } 91 | 92 | function detectExisting(path: string, imageUrl: string): ImageCacheEntry { 93 | const entries = imageCache[path] 94 | return entries.filter((entry) => { 95 | return multiStringMatch(entry.url, imageUrl) || multiStringMatch(entry.location, imageUrl) 96 | })[0] 97 | } 98 | 99 | async function processAsset( 100 | path: string, 101 | imageUrl: string, 102 | updator: { update: Content | string }, 103 | isPdf: boolean, 104 | debug?: boolean, 105 | ): Promise { 106 | if (imageUrl && path) { 107 | let filename = '' 108 | try { 109 | filename = await writeOutAsset(imageUrl, detectExisting(path, imageUrl), isPdf) 110 | } 111 | catch (e) { 112 | if (debug) 113 | console.warn('ncms-plugin-images: File type could not be reliably determined! The binary data may be malformed! No file saved!') 114 | return 115 | } 116 | if (filename) { 117 | imageCache[path].push({ 118 | filename, 119 | location: `/remote/${filename}`, 120 | url: imageUrl, 121 | }) 122 | // if we don't do this, the replaceall cant find the proper url below 123 | if (typeof updator.update !== 'string') { 124 | if (updator.update?.html.includes('amazonaws')) 125 | updator.update.html = updator.update.html.replaceAll('&', '&') 126 | updator.update.html = updator.update.html.replace(imageUrl, `/remote/${filename}`) 127 | } 128 | else { 129 | // This replaces the coverImage 130 | updator.update = updator.update.replace(imageUrl, `/remote/${filename}`) 131 | } 132 | if (debug) 133 | console.log('ncms-plugin-images: rewriting', path, 'at', filename) 134 | } 135 | } 136 | } 137 | 138 | return { 139 | name: 'ncms-plugin-images', 140 | hook: 'during-tree', 141 | core: true, 142 | exec: async (context: PageContent, options: PluginExecOptions) => { 143 | const copyOfContext = structuredClone(context) 144 | if (copyOfContext._updateNeeded) { 145 | if (!copyOfContext.path) 146 | return 147 | 148 | const matchables = [ 149 | GENERIC_MATCH, 150 | IMAGE_SOURCE_MATCH, 151 | OBJECT_PDF_MATCH, 152 | ...customMatchers, 153 | ] 154 | if (!imageCache[copyOfContext.path]) 155 | imageCache[copyOfContext.path] = [] as ImageCacheEntry[] 156 | const contents = { 157 | update: copyOfContext.content as Content, 158 | } 159 | const coverImage = { 160 | update: copyOfContext.coverImage as string, 161 | } 162 | // Must run all async in series so that we don't end up with duplicates 163 | for (const match of matchables) { 164 | if (!copyOfContext.path) 165 | return 166 | const path = copyOfContext.path 167 | const matched = (contents.update && Array.from(contents.update.html.matchAll(match), m => m[1])) || [] 168 | const matchedCoverImages = (coverImage.update && [coverImage.update]) || [] 169 | for (const imageUrl of matched) { 170 | const isPdf = imageUrl.includes('.pdf') 171 | await processAsset(path, imageUrl, contents, isPdf, options.debug) 172 | } 173 | 174 | for (const imageUrl of matchedCoverImages) 175 | await processAsset(path, imageUrl, coverImage, false, options.debug) 176 | } 177 | copyOfContext.content = contents.update 178 | copyOfContext.coverImage = coverImage.update 179 | try { 180 | if (!fs.existsSync(`${imageCacheDirectory}/remote`)) 181 | fs.mkdirSync(`${imageCacheDirectory}/remote`) 182 | fs.writeFileSync(`${imageCacheDirectory}/remote/${IMAGE_CACHE_FILENAME}`, JSON.stringify(imageCache)) 183 | if (options.debug) 184 | fs.writeFileSync('debug/images.json', JSON.stringify(imageCache)) 185 | } 186 | catch (e) { 187 | if (options.debug) 188 | console.warn(e, 'ncms-plugin-images: error writing to image cache.') 189 | } 190 | } 191 | return copyOfContext 192 | }, 193 | } 194 | } 195 | -------------------------------------------------------------------------------- /src/plugins/linker.ts: -------------------------------------------------------------------------------- 1 | import { idToUuid } from 'notion-utils' 2 | import type { CMS, PageContent } from '../types' 3 | import NotionCMS from '../notion-cms' 4 | 5 | const links: Map = new Map() 6 | 7 | const ESCAPED_HREF_REGEX = /]*?\s+)?href=(["'])\/([0-9a-f]{8}[0-9a-f]{4}[0-9a-f]{4}[0-9a-f]{4}[0-9a-f]{12})\1/g 8 | 9 | export default function () { 10 | return [{ 11 | name: 'ncms-plugin-linker', 12 | hook: 'during-tree', 13 | core: true, 14 | exec: (context: PageContent) => { 15 | const copyOfContext = structuredClone(context) 16 | if (copyOfContext._notion?.id && copyOfContext.path) 17 | links.set(copyOfContext._notion?.id, copyOfContext.path) 18 | return copyOfContext 19 | }, 20 | }, 21 | { 22 | name: 'ncms-plugin-linker', 23 | hook: 'post-tree', 24 | core: true, 25 | exec: (context: CMS) => { 26 | const copyOfContext = structuredClone(context) 27 | 28 | const cmsWalker = NotionCMS._createCMSWalker((node: PageContent) => { 29 | let html: string | undefined = node.content?.html 30 | let md: string | undefined = node.content?.markdown 31 | let plaintext: string | undefined = node.content?.plaintext 32 | 33 | if (html) { 34 | const pathsToReplace = [...html.matchAll(ESCAPED_HREF_REGEX)] 35 | pathsToReplace.forEach((path) => { 36 | const id = path[2] 37 | const uuid = idToUuid(id) 38 | 39 | const replacerPath = links.get(uuid) 40 | if (uuid && !replacerPath) { 41 | console.warn( 42 | `ncms-plugin-linker: found a uuid without a corresponding path. This page: ${uuid} is not included in NotionCMS.`) 43 | } 44 | if (replacerPath && uuid && html) { 45 | html = html.replaceAll(`/${id}`, replacerPath) 46 | md = md?.replaceAll(`/${id}`, replacerPath) 47 | plaintext = plaintext?.replaceAll(`/${id}`, replacerPath) 48 | } 49 | }) 50 | if (node.content) { 51 | node.content.html = html 52 | if (md) 53 | node.content.markdown = md 54 | if (plaintext) 55 | node.content.plaintext = plaintext 56 | } 57 | } 58 | }) 59 | 60 | cmsWalker.walk(copyOfContext) 61 | 62 | return copyOfContext 63 | }, 64 | }] 65 | } 66 | -------------------------------------------------------------------------------- /src/plugins/render.ts: -------------------------------------------------------------------------------- 1 | import type { Blocks } from '@notion-stuff/v4-types' 2 | import _ from 'lodash' 3 | import type { BlockRenderers } from '../notion-blocks-parser' 4 | import NotionBlocksParser from '../notion-blocks-parser' 5 | import type { PluginPassthrough, UnsafePlugin } from '../types' 6 | 7 | export default function ({ 8 | blockRenderers, 9 | debug, 10 | }: { blockRenderers: BlockRenderers; debug?: boolean }) { 11 | const parser = new NotionBlocksParser({ blockRenderers, debug }) 12 | return { 13 | parser, 14 | name: 'ncms-plugin-blocks-render', 15 | core: true, 16 | hook: 'parse', 17 | exec: (context: PluginPassthrough): string => { 18 | const copyOfContext = _.cloneDeep(context) as Blocks 19 | return parser.blocksToHtml(copyOfContext) 20 | }, 21 | } satisfies UnsafePlugin 22 | } 23 | -------------------------------------------------------------------------------- /src/tests/custom-render.spec.ts: -------------------------------------------------------------------------------- 1 | // eslint-disable-next-line @typescript-eslint/ban-ts-comment 2 | // @ts-nocheck 3 | import { suite } from 'uvu' 4 | import * as assert from 'uvu/assert' 5 | import dotenv from 'dotenv' 6 | import type { Block } from '@notion-stuff/v4-types' 7 | import NotionCMS, { NotionBlocksParser, blocksRenderPlugin } from '../index' 8 | import type { Content, PageContent } from '../types' 9 | 10 | import { 11 | expectedKitchenSinkSiteData, 12 | expectedRoutes, 13 | expectedSiteData, 14 | expectedTags, 15 | } from './notion-api-mock.spec' 16 | 17 | dotenv.config() 18 | 19 | // Kitchen sink database 20 | const databaseId = '21608fc7-c1c5-40a1-908f-9ade89585111' 21 | const notionAPIKey = process.env.NOTION 22 | 23 | let PluginsDefaultCMS: NotionCMS, 24 | PluginsDefaultOtherCMS: NotionCMS, 25 | PluginsCustomCMS: NotionCMS, 26 | PluginsCustomFallbackCMS: NotionCMS 27 | 28 | // Helper Function for parsing internal block rich text 29 | // eslint-disable-next-line jest/unbound-method 30 | const parseRichText = NotionBlocksParser.parseRichText 31 | 32 | // temporarily ignore md and plaintext versions of content 33 | function filterContent(content: Content) { 34 | delete content.plaintext 35 | delete content.markdown 36 | return content 37 | } 38 | 39 | export const PluginsDefault = suite('PluginsDefault') 40 | 41 | PluginsDefault.before(async () => { 42 | PluginsDefaultCMS = new NotionCMS({ 43 | databaseId: '610627a9-28b1-4477-b660-c00c5364435b', 44 | notionAPIKey, 45 | draftMode: true, 46 | // No Plugins - use default renderer plugin behind the scenes 47 | }) 48 | console.log('custom render test: purging cache') 49 | PluginsDefaultCMS.purgeCache() 50 | await PluginsDefaultCMS.fetch() 51 | PluginsDefaultCMS.walk((node: PageContent) => filterContent(node.content)) 52 | }) 53 | 54 | // routes 55 | // walk is used by routes so this is tested here implicitly 56 | PluginsDefault('routes', () => { 57 | assert.equal(PluginsDefaultCMS.routes.sort(), expectedRoutes.sort()) 58 | }) 59 | 60 | // tags 61 | PluginsDefault('tags', () => { 62 | assert.equal(PluginsDefaultCMS.cms.tags.sort(), expectedTags.sort()) 63 | }) 64 | 65 | // Tree structures 66 | PluginsDefault('siteData', () => { 67 | assert.equal(PluginsDefaultCMS.cms.siteData, expectedSiteData) 68 | }) 69 | 70 | export const PluginsDefaultOther = suite('PluginsDefaultOther') 71 | 72 | PluginsDefaultOther.before(async () => { 73 | PluginsDefaultOtherCMS = new NotionCMS({ 74 | databaseId: '610627a9-28b1-4477-b660-c00c5364435b', 75 | notionAPIKey, 76 | draftMode: true, 77 | // Standin Plugin - use default renderer plugin behind the scenes 78 | plugins: [() => ({ 79 | name: 'ncms-placeholder-plugin', 80 | hook: 'post-parse', 81 | exec: (block: Block) => block, 82 | })], 83 | }) 84 | console.log('custom render test: purging cache') 85 | PluginsDefaultOtherCMS.purgeCache() 86 | await PluginsDefaultOtherCMS.fetch() 87 | PluginsDefaultOtherCMS.walk((node: PageContent) => filterContent(node.content)) 88 | }) 89 | 90 | // routes 91 | // walk is used by routes so this is tested here implicitly 92 | PluginsDefaultOther('routes', () => { 93 | assert.equal(PluginsDefaultOtherCMS.routes.sort(), expectedRoutes.sort()) 94 | }) 95 | 96 | // tags 97 | PluginsDefaultOther('tags', () => { 98 | assert.equal(PluginsDefaultOtherCMS.cms.tags.sort(), expectedTags.sort()) 99 | }) 100 | 101 | // Tree structures 102 | PluginsDefaultOther('siteData', () => { 103 | assert.equal(PluginsDefaultOtherCMS.cms.siteData, expectedSiteData) 104 | }) 105 | 106 | export const PluginsCustom = suite('PluginsCustom') 107 | 108 | PluginsCustom.before(async () => { 109 | PluginsCustomCMS = new NotionCMS({ 110 | // Kitchen sink DB in community/tests 111 | databaseId, 112 | notionAPIKey, 113 | // Should work with other plugin too 114 | plugins: [() => ({ 115 | name: 'ncms-placeholder-plugin', 116 | hook: 'post-parse', 117 | exec: (block: Block) => block, 118 | }), 119 | // use custom renderer plugin behind the scenes 120 | blocksRenderPlugin({ 121 | blockRenderers: { 122 | // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access 123 | CalloutBlock: block => `
${parseRichText(block.callout.rich_text as string)}
\n\n`, 124 | // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access 125 | QuoteBlock: block => `
${parseRichText(block.quote.rich_text as string)}
\n\n`, 126 | }, 127 | })], 128 | }) 129 | console.log('custom render test: purging cache') 130 | PluginsCustomCMS.purgeCache() 131 | await PluginsCustomCMS.fetch() 132 | PluginsCustomCMS.walk((node: PageContent) => filterContent(node.content)) 133 | }) 134 | 135 | PluginsCustom('Custom render correctly alters blocks', () => { 136 | assert.ok((PluginsCustomCMS.data['/kitchen-sink'] as PageContent) 137 | .content.html.match(/
([.|\s\S]*)<\/div ncms-test callout>/g)) 138 | assert.ok((PluginsCustomCMS.data['/kitchen-sink'] as PageContent) 139 | .content.html.match(/
([.|\s\S]*)<\/div ncms-test quote>/g)) 140 | }) 141 | 142 | export const PluginsCustomFallback = suite('PluginsCustomFallback') 143 | 144 | PluginsCustomFallback.before(async () => { 145 | PluginsCustomFallbackCMS = new NotionCMS({ 146 | databaseId, 147 | notionAPIKey, 148 | draftMode: true, 149 | // Should work with other plugin too 150 | plugins: [ 151 | // use custom renderer plugin behind the scenes 152 | blocksRenderPlugin({ 153 | blockRenderers: { 154 | CalloutBlock: (block: Block) => null, // Nulls should invoke default renderer 155 | QuoteBlock: (block: Block) => null, // Nulls should invoke default renderer 156 | }, 157 | })], 158 | }) 159 | console.log('custom render test: purging cache') 160 | PluginsCustomFallbackCMS.purgeCache() 161 | await PluginsCustomFallbackCMS.fetch() 162 | PluginsCustomFallbackCMS.walk((node: PageContent) => filterContent(node.content)) 163 | }) 164 | 165 | PluginsCustomFallback('Custom render correctly uses fallback block renderer ', () => { 166 | assert.equal((PluginsCustomFallbackCMS.cms.siteData['/kitchen-sink'] as PageContent) 167 | .content.html, (expectedKitchenSinkSiteData['/kitchen-sink'] as PageContent).content.html) 168 | }) 169 | -------------------------------------------------------------------------------- /src/tests/limiter.spec.ts: -------------------------------------------------------------------------------- 1 | // eslint-disable-next-line @typescript-eslint/ban-ts-comment 2 | // @ts-nocheck 3 | import { setTimeout } from 'node:timers/promises' 4 | import Bottleneck from 'bottleneck' 5 | import dotenv from 'dotenv' 6 | import { suite } from 'uvu' 7 | import * as assert from 'uvu/assert' 8 | import NotionCMS from '../index' 9 | 10 | import type { Content, PageContent } from '../types' 11 | import { 12 | expectedRejectedPageData, 13 | expectedRoutes, 14 | expectedSiteData, 15 | expectedTagGroups, 16 | expectedTaggedCollection, 17 | expectedTags, 18 | } from './notion-api-mock.spec' 19 | 20 | dotenv.config() 21 | 22 | export const TestLimiter = suite('TestLimiter') 23 | 24 | const databaseId = '610627a9-28b1-4477-b660-c00c5364435b' 25 | 26 | const limiter = new Bottleneck({ 27 | maxConcurrent: 1, 28 | minTime: 333, 29 | }) 30 | 31 | const testCMS = new NotionCMS({ 32 | databaseId, 33 | notionAPIKey: process.env.NOTION, 34 | draftMode: true, 35 | limiter, 36 | }) 37 | 38 | console.log('limiter test: purging cache') 39 | testCMS.purgeCache() 40 | 41 | await testCMS.fetch() 42 | 43 | // temporarily ignore md and plaintext versions of content 44 | function filterContent(content: Content) { 45 | delete content.plaintext 46 | delete content.markdown 47 | return content 48 | } 49 | 50 | testCMS.walk((node: PageContent) => filterContent(node.content)) 51 | 52 | // routes 53 | // walk is used by routes so this is tested here implicitly 54 | TestLimiter('routes', () => { 55 | assert.equal(testCMS.routes.sort(), expectedRoutes.sort()) 56 | }) 57 | 58 | // tags 59 | TestLimiter('tags', () => { 60 | assert.equal(testCMS.cms.tags.sort(), expectedTags.sort()) 61 | }) 62 | 63 | // Tree structures 64 | TestLimiter('siteData', () => { 65 | assert.equal(testCMS.cms.siteData, expectedSiteData) 66 | }) 67 | 68 | // data getter 69 | TestLimiter('data (getter)', () => { 70 | assert.equal(testCMS.cms.siteData, testCMS.data) 71 | }) 72 | 73 | // taggedCollection 74 | TestLimiter('taggedCollection', () => { 75 | const results = testCMS.getTaggedCollection(['notion', 'programming']) 76 | assert.equal(results, expectedTaggedCollection) 77 | }) 78 | 79 | // query by path 80 | TestLimiter('query by path', () => { 81 | const productB = testCMS.queryByPath('/products/category/product-b') 82 | assert.equal(productB.name, 'Product B') 83 | const category = testCMS.queryByPath('/products/category') 84 | assert.equal(category.name, 'Category') 85 | }) 86 | 87 | // filter sub pages 88 | TestLimiter('filter sub pages', () => { 89 | const category = testCMS.queryByPath('/products/category') 90 | const productsInCategory = testCMS.filterSubPages(category) 91 | const names = productsInCategory.map((product: PageContent) => product.name) 92 | assert.equal(names, ['Product B', 'Product A']) 93 | 94 | // Uncomment when fuzzy search is built 95 | // const categoryB = testCMS.queryByPath('/category') 96 | // const productsInCategoryB = testCMS.filterSubPages(categoryB) 97 | // const namesB = productsInCategoryB.map(product => product.name) 98 | // assert.equal(namesB, ['Product B', 'Product A']) 99 | }) 100 | 101 | // reject sub pages 102 | TestLimiter('reject sub pages', () => { 103 | const category = testCMS.queryByPath('/products/category') 104 | const categoryProps = testCMS.rejectSubPages(category) 105 | assert.equal(categoryProps, expectedRejectedPageData) 106 | 107 | // Uncomment when fuzzy search is built 108 | // const categoryB = testCMS.queryByPath('/category') 109 | // const categoryPropsB = testCMS.rejectSubPages(categoryB) 110 | // assert.equal(categoryPropsB, expectedRejectedPageData) 111 | }) 112 | 113 | TestLimiter('walk from partial path', () => { 114 | const test = [] 115 | testCMS.walk((node: PageContent) => test.push(node.name), '/products/category') 116 | assert.equal(test, ['Product B', 'Product A']) 117 | }) 118 | 119 | TestLimiter('async walk from partial path', async () => { 120 | const test = [] 121 | await testCMS.asyncWalk(async (node: PageContent) => await setTimeout(test.push(node.name), 300), '/products/category') 122 | assert.equal(test, ['Product B', 'Product A']) 123 | }) 124 | 125 | TestLimiter('import', async () => { 126 | assert.equal( 127 | await testCMS.import(JSON.stringify({ 128 | metadata: { 129 | databaseId, 130 | rootUrl: '', 131 | }, 132 | stages: [ 133 | 'db', 134 | 'content', 135 | 'complete', 136 | ], 137 | routes: [], 138 | tags: expectedTags, 139 | tagGroups: expectedTagGroups, 140 | siteData: expectedSiteData, 141 | })), 142 | testCMS.cms, 143 | ) 144 | }) 145 | 146 | TestLimiter('import fails', async () => { 147 | try { 148 | await testCMS.import() 149 | assert.unreachable('should have thrown') 150 | } 151 | catch (err) { 152 | assert.instance(err, Error) 153 | } 154 | }) 155 | -------------------------------------------------------------------------------- /src/tests/notion-cms-caching.spec.ts: -------------------------------------------------------------------------------- 1 | import dotenv from 'dotenv' 2 | import { suite } from 'uvu' 3 | import * as assert from 'uvu/assert' 4 | import nock from 'nock' 5 | import NotionCMS from '../index' 6 | import type { CMS, PluginPassthrough } from '../types' 7 | 8 | dotenv.config() 9 | 10 | export const TestNotionCMSCache = suite('TestNotionCMSCache') 11 | 12 | const cachingDatabaseId = '1234' // this does not exist in notion 13 | 14 | const baseUrl = 'https://api.notion.com/v1' 15 | 16 | const cachingTestCMS: NotionCMS = new NotionCMS({ 17 | databaseId: cachingDatabaseId, 18 | notionAPIKey: process.env.NOTION as string, 19 | draftMode: true, 20 | }) 21 | 22 | cachingTestCMS.purgeCache() 23 | 24 | const topLevelPageId = '456' 25 | const topLevelPageId2 = '789' 26 | 27 | TestNotionCMSCache('Unchanged since last edit time uses cache', async () => { 28 | nock(baseUrl) 29 | .post(`/databases/${cachingDatabaseId}/query`) 30 | .query(true) 31 | .reply(200, { 32 | object: 'list', 33 | results: [ 34 | { 35 | object: 'page', 36 | id: topLevelPageId, 37 | created_time: '2023-04-09T06:03:00.000Z', 38 | last_edited_time: '2023-04-09T06:03:00.000Z', 39 | created_by: { 40 | object: 'user', 41 | id: '4e38fa57-609c-4beb-8e28-271b11cc81a3', 42 | }, 43 | last_edited_by: { 44 | object: 'user', 45 | id: '4e38fa57-609c-4beb-8e28-271b11cc81a3', 46 | }, 47 | properties: { 48 | name: { 49 | id: 'title', 50 | type: 'title', 51 | title: [ 52 | { 53 | type: 'text', 54 | text: { 55 | content: 'Page 1', 56 | link: null, 57 | }, 58 | annotations: {}, 59 | plain_text: 'Page 1', 60 | href: null, 61 | }, 62 | ], 63 | }, 64 | Author: { 65 | id: 'SQeZ', 66 | type: 'people', 67 | people: [ 68 | { 69 | object: 'user', 70 | id: '4e38fa57-609c-4beb-8e28-271b11cc81a3', 71 | name: 'Jacob', 72 | avatar_url: null, 73 | type: 'person', 74 | person: { 75 | email: 'jacob@agencykit.so', 76 | }, 77 | }, 78 | ], 79 | }, 80 | Tags: { 81 | id: 'NNmP', 82 | type: 'multi_select', 83 | multi_select: [ 84 | { 85 | id: '098acfda-2fb1-4ecf-8737-c03b80b5cb18', 86 | name: 'programming', 87 | color: 'default', 88 | }, 89 | ], 90 | }, 91 | }, 92 | cover: null, 93 | icon: null, 94 | archived: false, 95 | url: 'https://www.notion.so/Product-B-7fc90a1dca4d49ad91b5136c3d5a304d', 96 | }, 97 | { 98 | object: 'page', 99 | id: topLevelPageId2, 100 | created_time: '2023-04-09T06:03:00.000Z', 101 | last_edited_time: '2023-04-09T06:03:00.000Z', 102 | created_by: { 103 | object: 'user', 104 | id: '4e38fa57-609c-4beb-8e28-271b11cc81a3', 105 | }, 106 | last_edited_by: { 107 | object: 'user', 108 | id: '4e38fa57-609c-4beb-8e28-271b11cc81a3', 109 | }, 110 | properties: { 111 | name: { 112 | id: 'title', 113 | type: 'title', 114 | title: [ 115 | { 116 | type: 'text', 117 | text: { 118 | content: 'Page 2', 119 | link: null, 120 | }, 121 | annotations: {}, 122 | plain_text: 'Page 2', 123 | href: null, 124 | }, 125 | ], 126 | }, 127 | Author: { 128 | id: 'SQeZ', 129 | type: 'people', 130 | people: [ 131 | { 132 | object: 'user', 133 | id: '4e38fa57-609c-4beb-8e28-271b11cc81a3', 134 | name: 'Jacob', 135 | avatar_url: null, 136 | type: 'person', 137 | person: { 138 | email: 'jacob@agencykit.so', 139 | }, 140 | }, 141 | ], 142 | }, 143 | Tags: { 144 | id: 'NNmP', 145 | type: 'multi_select', 146 | multi_select: [ 147 | { 148 | id: '098acfda-2fb1-4ecf-8737-c03b80b5cb18', 149 | name: 'programming', 150 | color: 'default', 151 | }, 152 | ], 153 | }, 154 | }, 155 | cover: null, 156 | icon: null, 157 | archived: false, 158 | url: 'https://www.notion.so/Product-B-7fc90a1dca4d49ad91b5136c3d5a304d', 159 | }, 160 | ], 161 | next_cursor: null, 162 | has_more: false, 163 | type: 'page', 164 | page: {}, 165 | }) 166 | 167 | nock(baseUrl) 168 | .get(`/blocks/${topLevelPageId}/children`) 169 | .query(true) 170 | .reply(200, { 171 | object: 'list', 172 | results: [ 173 | { 174 | object: 'block', 175 | id: '1c92a5ea-dfeb-4c8f-b662-cde078bb02ad', 176 | parent: { 177 | type: 'page_id', 178 | page_id: topLevelPageId, 179 | }, 180 | created_time: '2023-04-22T04:34:00.000Z', 181 | last_edited_time: '2023-04-22T04:34:00.000Z', 182 | created_by: { 183 | object: 'user', 184 | id: '4e38fa57-609c-4beb-8e28-271b11cc81a3', 185 | }, 186 | last_edited_by: { 187 | object: 'user', 188 | id: '4e38fa57-609c-4beb-8e28-271b11cc81a3', 189 | }, 190 | has_children: false, 191 | archived: false, 192 | type: 'heading_1', 193 | heading_1: { 194 | rich_text: [ 195 | { 196 | type: 'text', 197 | text: { 198 | content: 'Block 1: Has not changed.', 199 | link: null, 200 | }, 201 | annotations: {}, 202 | plain_text: 'Block 1: Has not changed.', 203 | href: null, 204 | }, 205 | ], 206 | is_toggleable: false, 207 | color: 'default', 208 | }, 209 | }, 210 | ], 211 | }) 212 | 213 | nock(baseUrl) 214 | .get(`/blocks/${topLevelPageId2}/children`) 215 | .query(true) 216 | .reply(200, { 217 | object: 'list', 218 | results: [ 219 | { 220 | object: 'block', 221 | id: '1c92a5ea-dfeb-4c8f-b662-cde078bb02ad', 222 | parent: { 223 | type: 'page_id', 224 | page_id: topLevelPageId2, 225 | }, 226 | created_time: '2023-04-22T04:34:00.000Z', 227 | last_edited_time: '2023-04-22T04:34:00.000Z', 228 | created_by: { 229 | object: 'user', 230 | id: '4e38fa57-609c-4beb-8e28-271b11cc81a3', 231 | }, 232 | last_edited_by: { 233 | object: 'user', 234 | id: '4e38fa57-609c-4beb-8e28-271b11cc81a3', 235 | }, 236 | has_children: false, 237 | archived: false, 238 | type: 'heading_1', 239 | heading_1: { 240 | rich_text: [ 241 | { 242 | type: 'text', 243 | text: { 244 | content: 'Block 2: Has not changed.', 245 | link: null, 246 | }, 247 | annotations: {}, 248 | plain_text: 'Block 2: Has not changed.', 249 | href: null, 250 | }, 251 | ], 252 | is_toggleable: false, 253 | color: 'default', 254 | }, 255 | }, 256 | ], 257 | }) 258 | // Build the cache 259 | const cms: CMS = await cachingTestCMS.pull() 260 | 261 | assert.ok(cms.siteData['/page-1'].content?.plaintext === 'Block 1: Has not changed.') 262 | assert.ok(cms.siteData['/page-2'].content?.plaintext === 'Block 2: Has not changed.') 263 | 264 | // Change some data in only page 2 265 | nock(baseUrl) 266 | .post(`/databases/${cachingDatabaseId}/query`) 267 | .query(true) 268 | .reply(200, { 269 | object: 'list', 270 | results: [ 271 | { 272 | object: 'page', 273 | id: topLevelPageId, 274 | created_time: '2023-04-09T06:03:00.000Z', 275 | last_edited_time: '2023-04-09T06:03:00.000Z', 276 | created_by: { 277 | object: 'user', 278 | id: '4e38fa57-609c-4beb-8e28-271b11cc81a3', 279 | }, 280 | last_edited_by: { 281 | object: 'user', 282 | id: '4e38fa57-609c-4beb-8e28-271b11cc81a3', 283 | }, 284 | properties: { 285 | name: { 286 | id: 'title', 287 | type: 'title', 288 | title: [ 289 | { 290 | type: 'text', 291 | text: { 292 | content: 'Page 1', 293 | link: null, 294 | }, 295 | annotations: {}, 296 | plain_text: 'Page 1', 297 | href: null, 298 | }, 299 | ], 300 | }, 301 | Author: { 302 | id: 'SQeZ', 303 | type: 'people', 304 | people: [ 305 | { 306 | object: 'user', 307 | id: '4e38fa57-609c-4beb-8e28-271b11cc81a3', 308 | name: 'Jacob', 309 | avatar_url: null, 310 | type: 'person', 311 | person: { 312 | email: 'jacob@agencykit.so', 313 | }, 314 | }, 315 | ], 316 | }, 317 | Tags: { 318 | id: 'NNmP', 319 | type: 'multi_select', 320 | multi_select: [ 321 | { 322 | id: '098acfda-2fb1-4ecf-8737-c03b80b5cb18', 323 | name: 'programming', 324 | color: 'default', 325 | }, 326 | ], 327 | }, 328 | }, 329 | cover: null, 330 | icon: null, 331 | archived: false, 332 | url: 'https://www.notion.so/Product-B-7fc90a1dca4d49ad91b5136c3d5a304d', 333 | }, 334 | { 335 | object: 'page', 336 | id: topLevelPageId2, 337 | created_time: '2023-04-09T06:03:00.000Z', 338 | last_edited_time: '2023-04-09T06:16:00.000Z', 339 | created_by: { 340 | object: 'user', 341 | id: '4e38fa57-609c-4beb-8e28-271b11cc81a3', 342 | }, 343 | last_edited_by: { 344 | object: 'user', 345 | id: '4e38fa57-609c-4beb-8e28-271b11cc81a3', 346 | }, 347 | properties: { 348 | name: { 349 | id: 'title', 350 | type: 'title', 351 | title: [ 352 | { 353 | type: 'text', 354 | text: { 355 | content: 'Page 2', 356 | link: null, 357 | }, 358 | annotations: {}, 359 | plain_text: 'Page 2', 360 | href: null, 361 | }, 362 | ], 363 | }, 364 | Author: { 365 | id: 'SQeZ', 366 | type: 'people', 367 | people: [ 368 | { 369 | object: 'user', 370 | id: '4e38fa57-609c-4beb-8e28-271b11cc81a3', 371 | name: 'Jacob', 372 | avatar_url: null, 373 | type: 'person', 374 | person: { 375 | email: 'jacob@agencykit.so', 376 | }, 377 | }, 378 | ], 379 | }, 380 | Tags: { 381 | id: 'NNmP', 382 | type: 'multi_select', 383 | multi_select: [ 384 | { 385 | id: '098acfda-2fb1-4ecf-8737-c03b80b5cb18', 386 | name: 'programming', 387 | color: 'default', 388 | }, 389 | ], 390 | }, 391 | }, 392 | cover: null, 393 | icon: null, 394 | archived: false, 395 | url: 'https://www.notion.so/Product-B-7fc90a1dca4d49ad91b5136c3d5a304d', 396 | }, 397 | ], 398 | next_cursor: null, 399 | has_more: false, 400 | type: 'page', 401 | page: {}, 402 | }) 403 | nock(baseUrl) 404 | .get(`/blocks/${topLevelPageId2}/children`) 405 | .query(true) 406 | .reply(200, { 407 | object: 'list', 408 | results: [ 409 | { 410 | object: 'block', 411 | id: '1c92a5ea-dfeb-4c8f-b662-cde078bb02ad', 412 | parent: { 413 | type: 'page_id', 414 | page_id: topLevelPageId2, 415 | }, 416 | created_time: '2023-04-22T04:34:00.000Z', 417 | last_edited_time: '2023-04-22T04:34:00.000Z', 418 | created_by: { 419 | object: 'user', 420 | id: '4e38fa57-609c-4beb-8e28-271b11cc81a3', 421 | }, 422 | last_edited_by: { 423 | object: 'user', 424 | id: '4e38fa57-609c-4beb-8e28-271b11cc81a3', 425 | }, 426 | has_children: false, 427 | archived: false, 428 | type: 'heading_1', 429 | heading_1: { 430 | rich_text: [ 431 | { 432 | type: 'text', 433 | text: { 434 | content: 'Block 2: Has changed.', 435 | link: null, 436 | }, 437 | annotations: {}, 438 | plain_text: 'Block 2: Has changed.', 439 | href: null, 440 | }, 441 | ], 442 | is_toggleable: false, 443 | color: 'default', 444 | }, 445 | }, 446 | ], 447 | }) 448 | 449 | const cms2: CMS = await cachingTestCMS.pull() 450 | 451 | assert.ok(cms2.siteData['/page-1'].content?.plaintext === 'Block 1: Has not changed.') 452 | assert.ok(cms2.siteData['/page-2'].content?.plaintext === 'Block 2: Has changed.') 453 | }) 454 | 455 | TestNotionCMSCache('Plugins run even when using cached state.', async () => { 456 | const databaseId = '610627a9-28b1-4477-b660-c00c5364435b' 457 | 458 | let counter = 0 459 | 460 | const testCMS: NotionCMS = new NotionCMS({ 461 | databaseId, 462 | notionAPIKey: process.env.NOTION as string, 463 | draftMode: true, 464 | refreshTimeout: '1 minute', 465 | plugins: [ 466 | { 467 | name: 'counter-plugin', 468 | hook: '*', // this will run every hook 469 | exec: (ctx: PluginPassthrough) => { 470 | counter++ 471 | return ctx 472 | }, 473 | }, 474 | ], 475 | }) 476 | console.log('caching test: purging cache') 477 | testCMS.purgeCache() 478 | await testCMS.pull() 479 | assert.equal(counter, 22) 480 | }) 481 | -------------------------------------------------------------------------------- /src/tests/notion-cms.spec.ts: -------------------------------------------------------------------------------- 1 | // eslint-disable-next-line @typescript-eslint/ban-ts-comment 2 | // @ts-nocheck 3 | import { setTimeout } from 'node:timers/promises' 4 | import dotenv from 'dotenv' 5 | import { suite } from 'uvu' 6 | import * as assert from 'uvu/assert' 7 | import NotionCMS from '../index' 8 | 9 | import type { Content, PageContent } from '../types' 10 | 11 | import { 12 | expectedRejectedPageData, 13 | expectedRoutes, 14 | expectedSiteData, 15 | expectedTagGroups, 16 | expectedTaggedCollection, 17 | expectedTags, 18 | } from './notion-api-mock.spec' 19 | 20 | dotenv.config() 21 | 22 | export const TestNotionCMS = suite('TestNotionCMS') 23 | 24 | const databaseId = '610627a9-28b1-4477-b660-c00c5364435b' 25 | 26 | const testCMS: NotionCMS = new NotionCMS({ 27 | databaseId, 28 | notionAPIKey: process.env.NOTION, 29 | draftMode: true, 30 | }) 31 | 32 | await testCMS.pull() 33 | 34 | // temporarily ignore md and plaintext versions of content 35 | function filterContent(content: Content) { 36 | delete content.plaintext 37 | delete content.markdown 38 | return content 39 | } 40 | 41 | testCMS.walk((node: PageContent) => filterContent(node.content)) 42 | 43 | // routes 44 | // walk is used by routes so this is tested here implicitly 45 | TestNotionCMS('routes', () => { 46 | assert.equal(testCMS.routes.sort(), expectedRoutes.sort()) 47 | }) 48 | 49 | // tags 50 | TestNotionCMS('tags', () => { 51 | assert.equal(testCMS.cms.tags.sort(), expectedTags.sort()) 52 | }) 53 | 54 | // Tree structures 55 | TestNotionCMS('siteData', () => { 56 | assert.equal(testCMS.cms.siteData, expectedSiteData) 57 | }) 58 | 59 | // data getter 60 | TestNotionCMS('data (getter)', () => { 61 | assert.equal(testCMS.cms.siteData, testCMS.data) 62 | }) 63 | 64 | // taggedCollection 65 | TestNotionCMS('taggedCollection', () => { 66 | const results = testCMS.getTaggedCollection(['notion', 'programming']) 67 | assert.equal(results, expectedTaggedCollection) 68 | }) 69 | 70 | // query by path 71 | TestNotionCMS('query by path', () => { 72 | const productB: PageContent = testCMS.queryByPath('/products/category/product-b') 73 | assert.equal(productB.name, 'Product B') 74 | const category: PageContent = testCMS.queryByPath('/products/category') 75 | assert.equal(category.name, 'Category') 76 | }) 77 | 78 | // filter sub pages 79 | TestNotionCMS('filter sub pages', () => { 80 | const category = testCMS.queryByPath('/products/category') 81 | const productsInCategory = testCMS.filterSubPages(category) 82 | const names = productsInCategory.map((product: PageContent) => product.name) 83 | assert.equal(names, ['Product B', 'Product A']) 84 | 85 | // Uncomment when fuzzy search is built 86 | // const categoryB = testCMS.queryByPath('/category') 87 | // const productsInCategoryB = testCMS.filterSubPages(categoryB) 88 | // const namesB = productsInCategoryB.map(product => product.name) 89 | // assert.equal(namesB, ['Product B', 'Product A']) 90 | }) 91 | 92 | // reject sub pages 93 | TestNotionCMS('reject sub pages', () => { 94 | const category = testCMS.queryByPath('/products/category') 95 | const categoryProps = testCMS.rejectSubPages(category) 96 | assert.equal(categoryProps, expectedRejectedPageData) 97 | 98 | // Uncomment when fuzzy search is built 99 | // const categoryB = testCMS.queryByPath('/category') 100 | // const categoryPropsB = testCMS.rejectSubPages(categoryB) 101 | // assert.equal(categoryPropsB, expectedRejectedPageData) 102 | }) 103 | 104 | TestNotionCMS('walk from partial path', () => { 105 | const test = [] 106 | testCMS.walk((node: PageContent) => test.push(node.name), '/products/category') 107 | assert.equal(test, ['Product B', 'Product A']) 108 | }) 109 | 110 | TestNotionCMS('async walk from partial path', async () => { 111 | const test = [] 112 | await testCMS.asyncWalk(async (node: PageContent) => await setTimeout(test.push(node.name), 300), '/products/category') 113 | assert.equal(test, ['Product B', 'Product A']) 114 | }) 115 | 116 | TestNotionCMS('import', async () => { 117 | assert.equal( 118 | await testCMS.import(JSON.stringify({ 119 | metadata: { 120 | databaseId, 121 | rootUrl: '', 122 | }, 123 | stages: [ 124 | 'db', 125 | 'content', 126 | 'complete', 127 | ], 128 | routes: [], 129 | tags: expectedTags, 130 | tagGroups: expectedTagGroups, 131 | siteData: expectedSiteData, 132 | })), 133 | testCMS.cms, 134 | ) 135 | }) 136 | 137 | TestNotionCMS('import fails', async () => { 138 | try { 139 | await testCMS.import() 140 | assert.unreachable('should have thrown') 141 | } 142 | catch (err) { 143 | assert.instance(err, Error) 144 | } 145 | }) 146 | 147 | TestNotionCMS('Options have changed', () => { 148 | const options = { 149 | databaseId, 150 | notionAPIKey: process.env.NOTION, 151 | draftMode: true, 152 | plugins: [() => 'test plugin'], 153 | } 154 | 155 | const otherOptions = { 156 | databaseId, 157 | notionAPIKey: process.env.NOTION, 158 | draftMode: true, 159 | plugins: [() => 'test plugin'], 160 | } 161 | 162 | const newOptions = { 163 | databaseId, 164 | notionAPIKey: process.env.NOTION, 165 | draftMode: false, 166 | } 167 | 168 | const testOptionsCMS: NotionCMS = new NotionCMS(options) 169 | 170 | assert.ok( 171 | testOptionsCMS._documentOptions(options) === testOptionsCMS._documentOptions(otherOptions)) 172 | 173 | assert.not( 174 | testOptionsCMS._documentOptions(options) === testOptionsCMS._documentOptions(newOptions)) 175 | }) 176 | 177 | TestNotionCMS('Human readable refresh timeout', () => { 178 | const testOptionsCMS: NotionCMS = new NotionCMS({ 179 | databaseId, 180 | notionAPIKey: process.env.NOTION, 181 | draftMode: true, 182 | refreshTimeout: 'one hour', 183 | }) 184 | assert.ok( 185 | testOptionsCMS.refreshTimeout === 3_600_000) 186 | }) 187 | -------------------------------------------------------------------------------- /src/tests/test.ts: -------------------------------------------------------------------------------- 1 | import { TestNotionCMS } from './notion-cms.spec' 2 | import { 3 | PluginsCustom, 4 | PluginsCustomFallback, 5 | PluginsDefault, 6 | PluginsDefaultOther, 7 | } from './custom-render.spec' 8 | import { TestLimiter } from './limiter.spec' 9 | import { TestNotionCMSCache } from './notion-cms-caching.spec' 10 | 11 | TestNotionCMS.run() 12 | 13 | TestLimiter.run() 14 | 15 | PluginsDefault.run() 16 | PluginsDefaultOther.run() 17 | PluginsCustom.run() 18 | PluginsCustomFallback.run() 19 | 20 | TestNotionCMSCache.run() 21 | -------------------------------------------------------------------------------- /src/types.ts: -------------------------------------------------------------------------------- 1 | import type { 2 | PageObjectResponse, 3 | PersonUserObjectResponse, 4 | RichTextItemResponse, 5 | } from '@notionhq/client/build/src/api-endpoints' 6 | 7 | import type { Blocks } from '@notion-stuff/v4-types' 8 | import type { Client } from '@notionhq/client' 9 | import type NotionBlocksParser from './notion-blocks-parser' 10 | 11 | declare global { 12 | interface String { 13 | route: string 14 | slug: string 15 | } 16 | } 17 | 18 | export type Properties = Partial 19 | 20 | export interface Options { 21 | databaseId: string 22 | notionAPIKey: string 23 | debug?: boolean 24 | draftMode?: boolean 25 | autoUpdate?: boolean 26 | refreshTimeout?: number | string // in ms or converted from human readable string 27 | localCacheDirectory?: string 28 | rootAlias?: string 29 | quiet?: boolean 30 | rootUrl?: string | URL | undefined // Used to generate full path links, 31 | limiter?: { schedule: Function } 32 | plugins?: Array 33 | } 34 | 35 | export interface Stats { 36 | durationSeconds: number 37 | totalAPICalls: number 38 | succeededCalls: number 39 | failedCalls: number 40 | totalPages: number 41 | } 42 | 43 | export interface Content { 44 | plaintext: string 45 | markdown: string 46 | html: string 47 | } 48 | 49 | export interface CMS { 50 | stages: Array 51 | lastUpdateTimestamp?: number 52 | metadata: { 53 | options?: string 54 | rootUrl?: string | URL | undefined 55 | databaseId: string 56 | stats: Partial 57 | } 58 | routes: Array> 59 | tags: Array 60 | tagGroups: Record> 61 | siteData: Record 62 | } 63 | 64 | export interface PageContent { 65 | _ancestors?: PageContent[] 66 | _updateNeeded?: boolean 67 | name?: string 68 | path?: string 69 | url?: string 70 | otherProps?: PageObjectResponse['properties'] 71 | _notion?: { 72 | parent?: PageContent 73 | id: string 74 | last_edited_time: string 75 | } 76 | authors?: Array 77 | slug?: string 78 | tags?: Array 79 | coverImage?: string 80 | content?: Content 81 | } 82 | 83 | export interface Route { 84 | [key: string]: PageContent 85 | } 86 | 87 | export type ExtendedPageContent = PageContent & { 88 | [key: string]: unknown 89 | } 90 | 91 | export type Page = PageContent | (PageContent & Route) | ExtendedPageContent 92 | 93 | // Directly stolen from PageObjectResponse Record Type in notionClient/API-endpoints.ts 94 | export interface PageObjectTitle { type: 'title'; title: Array; id: string } 95 | 96 | export interface PageObjectRelation { type: 'relation'; relation: Array<{ id: string }>; id: string } 97 | 98 | export interface PageObjectUser { 99 | type: 'people' 100 | people: Array 101 | id: string 102 | } 103 | 104 | export interface PageMultiSelect { 105 | type: 'multi_select' 106 | multi_select: Array<{ 107 | id: string 108 | name: string 109 | color: string 110 | }> 111 | id: string 112 | } 113 | 114 | export interface PageSelect { 115 | type: 'select' 116 | select: { 117 | id: string 118 | name: string 119 | color: string 120 | } | null 121 | id: string 122 | } 123 | 124 | export interface PageRichText { 125 | type: 'rich_text' 126 | rich_text: Array 127 | id: string 128 | } 129 | 130 | export type Cover = { type: 'external'; external: { url: string } } | { type: 'file'; file: { url: string; expiry_time: string } } | null 131 | 132 | export type RouteObject = [string, object] 133 | 134 | export type PluginPassthrough = Blocks | CMS | Page | string 135 | 136 | export interface PluginExecOptions { 137 | debug?: boolean 138 | localCacheDirectory?: string 139 | notion?: Client 140 | } 141 | 142 | export interface Plugin { 143 | name: string 144 | hook: 'import' | 'pre-tree' | 'pre-parse' | 'post-parse' | 'during-tree' | 'post-tree' | '*' 145 | exec: (context: PluginPassthrough, instanceOptions?: PluginExecOptions) => PluginPassthrough 146 | } 147 | 148 | export interface UnsafePlugin { 149 | parser: NotionBlocksParser 150 | name: string 151 | core: boolean 152 | hook: 'parse' | 'import' | 'pre-tree' | 'pre-parse' | 'post-parse' | 'during-tree' | 'post-tree' | '*' 153 | exec: (context: PluginPassthrough, instanceOptions?: PluginExecOptions) => PluginPassthrough 154 | } 155 | 156 | export interface FlatListItem { 157 | _key: string 158 | [pid: string]: string | Partial | unknown 159 | } 160 | 161 | export type FlatList = FlatListItem[] 162 | -------------------------------------------------------------------------------- /src/utilities.ts: -------------------------------------------------------------------------------- 1 | import { dirname } from 'node:path' 2 | import fs from 'node:fs' 3 | import serializeJS from 'serialize-javascript' 4 | import { parse, stringify } from 'flatted' 5 | import _ from 'lodash' 6 | 7 | export function deserialize(serializedJavascript: string): Function { 8 | // eslint-disable-next-line no-eval 9 | return eval?.(`(${serializedJavascript})`) as Function 10 | } 11 | 12 | export function replaceFuncs(key: string, value: unknown) { 13 | return typeof value === 'function' 14 | ? `__func__${serializeJS(value)}` 15 | : value 16 | } 17 | 18 | export function reviveFuncs(key: string, value: unknown) { 19 | return (typeof value === 'string' && value.startsWith('__func__')) 20 | ? deserialize(value.replace('__func__', '')) 21 | : value 22 | } 23 | 24 | export function filterAncestors(key: string, value: unknown) { 25 | if (key === '_ancestors') 26 | return '[ancestors ref]' 27 | return value 28 | } 29 | 30 | export function JSONStringifyWithFunctions(obj: Object): string { 31 | return stringify(obj, replaceFuncs) 32 | } 33 | 34 | export function JSONParseWithFunctions(string: string): Object { 35 | return parse(string, reviveFuncs) as Object 36 | } 37 | 38 | export function writeFile(path: string, contents: string): void { 39 | fs.mkdirSync(dirname(path), { recursive: true }) 40 | fs.writeFileSync(path, contents) 41 | } 42 | 43 | export function slugify(name: string): string { 44 | return _.kebabCase(name) 45 | } 46 | 47 | export function routify(name: string): string { 48 | const slug = slugify(name) 49 | return slug.padStart(slug.length + 1, '/') 50 | } 51 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ESNEXT", 4 | "module": "ESNext", 5 | "emitDeclarationOnly": true, 6 | "declaration": true, 7 | "outDir": ".tmp", 8 | "strict": true, 9 | "moduleResolution": "node", 10 | "esModuleInterop": true, 11 | "skipLibCheck": true, 12 | "forceConsistentCasingInFileNames": true 13 | }, 14 | "include": ["src/**/*"], 15 | "exclude": [ 16 | "dist", 17 | "node_modules" 18 | ] 19 | } 20 | --------------------------------------------------------------------------------