├── .eslintrc.js ├── .gitignore ├── .nvmrc ├── .vscode └── settings.json ├── LICENSE ├── README.md ├── doc0.png ├── doc1.png ├── logo.png ├── notion.code-workspace ├── package-lock.json ├── package.json ├── packages ├── notion-graph-backend │ ├── README.md │ ├── env.ts │ ├── environment.d.ts │ ├── index.ts │ ├── package.json │ └── tsconfig.json ├── notion-graph-frontend │ ├── .storybook │ │ ├── main.js │ │ └── preview.js │ ├── jest │ │ ├── jest.config.ts │ │ └── setupTest.js │ ├── package.json │ ├── public │ │ ├── index.html │ │ └── robots.txt │ ├── src │ │ ├── app.tsx │ │ ├── assets │ │ │ ├── guide0.png │ │ │ ├── guide1.png │ │ │ ├── guide2.png │ │ │ ├── guide3.png │ │ │ └── guide4.png │ │ ├── components │ │ │ ├── Example │ │ │ │ ├── fallback.tsx │ │ │ │ ├── index.stories.tsx │ │ │ │ └── index.tsx │ │ │ └── Util │ │ │ │ └── WithErrorBoundary │ │ │ │ └── index.tsx │ │ ├── constants │ │ │ └── index.ts │ │ ├── globalStyle.tsx │ │ ├── index.tsx │ │ ├── pages │ │ │ └── Main │ │ │ │ ├── fallback.tsx │ │ │ │ ├── index.tsx │ │ │ │ └── localFragments │ │ │ │ ├── HowToAndTroubleShooting │ │ │ │ └── index.tsx │ │ │ │ └── NotionPublicPageLinkInput │ │ │ │ └── index.tsx │ │ ├── theme.ts │ │ ├── utilities │ │ │ ├── essentials.ts │ │ │ ├── notion.test.ts │ │ │ └── notion.ts │ │ └── webpack.d.ts │ ├── tsconfig.esbuild.json │ ├── tsconfig.json │ └── webpack │ │ ├── webpack.config.common.ts │ │ ├── webpack.config.dev.ts │ │ └── webpack.config.prod.ts └── notion-graph-scraper │ ├── .gitignore │ ├── README.md │ ├── doc0.png │ ├── doc1.png │ ├── environment.d.ts │ ├── errors.ts │ ├── index.ts │ ├── legacy │ └── official │ │ └── get-graph-from-root-block.ts │ ├── lib │ ├── global-util.ts │ ├── isomorphic-notion-util.ts │ ├── lib.ts │ ├── logger.ts │ ├── notion-graph.ts │ ├── request-queue.ts │ ├── undirected-nodes-graph.ts │ └── unofficial-notion-api-util.ts │ ├── logo.png │ ├── package.json │ ├── tsconfig.json │ ├── tsup.config.ts │ └── types │ ├── block-map.ts │ ├── notion-content-node.ts │ └── util-types.ts └── renovate.json /.eslintrc.js: -------------------------------------------------------------------------------- 1 | // eslint-disable-next-line no-undef 2 | module.exports = { 3 | root: true, 4 | parser: `@typescript-eslint/parser`, 5 | env: { 6 | browser: true, 7 | }, 8 | extends: [ 9 | `eslint:recommended`, 10 | `plugin:@typescript-eslint/eslint-recommended`, 11 | `plugin:@typescript-eslint/recommended`, 12 | `plugin:react/recommended`, 13 | `prettier`, 14 | ], 15 | // https://github.com/yannickcr/eslint-plugin-react#configuration 16 | settings: { 17 | react: { 18 | version: `detect`, 19 | }, 20 | }, 21 | parserOptions: { 22 | ecmaFeatures: { 23 | jsx: true, 24 | }, 25 | ecmaVersion: 12, 26 | sourceType: `module`, 27 | }, 28 | plugins: [`@typescript-eslint`, `react`, `prettier`], 29 | rules: { 30 | "prettier/prettier": `error`, 31 | "react/prop-types": 0, 32 | "linebreak-style": [`error`, `unix`], 33 | "arrow-body-style": `off`, 34 | "prefer-arrow-callback": `off`, 35 | "@typescript-eslint/ban-ts-comment": `off`, 36 | quotes: [`error`, `backtick`], 37 | }, 38 | }; 39 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | node_modules 3 | dist 4 | .env -------------------------------------------------------------------------------- /.nvmrc: -------------------------------------------------------------------------------- 1 | v16.15.1 -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "editor.codeActionsOnSave": { 3 | "source.fixAll.eslint": true 4 | }, 5 | "eslint.validate": [ 6 | "javascript", 7 | "javascriptreact", 8 | "typescript", 9 | "typescriptreact" 10 | ], 11 | "files.associations": { 12 | "*.jsx": "javascriptreact", 13 | "*.tsx": "typescriptreact" 14 | } 15 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Joel M 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # notion 2 | 3 | ![logo.png](./packages/notion-graph-scraper/logo.png) 4 | 5 | ```bash 6 | npm i --save @graphcentral/notion-graph-scraper 7 | ``` 8 | 9 | You will be able to visualize the output from @graphcentral/notion-graph-scraper with [@graphcentral/graph](https://github.com/graphcentral/graph) like below: 10 | ![doc1.png](./packages/notion-graph-scraper/doc1.png) 11 | 12 | ## Example 13 | 14 | First, get a public Notion page (if you just want to test it out), preferably with some sub-pages inside it, so that you can get a graph with some nodes. You can also request private pages if you create your own instance of `NotionAPI` from `notion-client`. The example below is only for public pages. 15 | 16 | Then, get the id of the page like below: 17 | 18 | ![doc0.png](./packages/notion-graph-scraper/doc0.png) 19 | 20 | Then, input the id of the page as a parameter to `notionGraph.buildGraphFromRootNode` 21 | 22 | ```ts 23 | import { NotionGraph } from "@graphcentral/notion-graph-scraper" 24 | import fs from "fs" 25 | /** 26 | * example of how to use `@graphcentral/notion-graph-scraper` 27 | */ 28 | ;(async () => { 29 | const notionGraph = new NotionGraph({ 30 | maxDiscoverableNodes: 2000, 31 | maxDiscoverableNodesInOtherSpaces: 2000, 32 | verbose: true, 33 | }) 34 | const graph = await notionGraph.buildGraphFromRootNode( 35 | // Some random Japanese blog 36 | `95fcfe03257541c5aaa21dd43bdbc381` 37 | ) 38 | console.log(graph.nodes.length) 39 | console.log(graph.links.length) 40 | await new Promise((resolve, reject) => { 41 | fs.writeFile(`test0.json`, JSON.stringify(graph), (err) => { 42 | if (err) reject(err) 43 | else resolve(``) 44 | }) 45 | }); 46 | 47 | process.exit(0) 48 | })() 49 | ``` 50 | 51 | The graph will be an object of this type: 52 | 53 | ```ts 54 | { 55 | nodes: Node[] 56 | links: Link[] 57 | errors: Error[] 58 | } 59 | ``` 60 | 61 | Then, you can directly use this as an input to [@graphcentral/graph](https://github.com/graphcentral/graph) to visualize it on the web as a force layout graph. 62 | 63 | ## Example setup 64 | 65 | An example setup is at [@graphcentral/notion-scrape-example](https://github.com/graphcentral/notion-scrape-example). 66 | 67 | ## Use ES6 module instead of Commonjs 68 | 69 | This project uses es6 module (.mjs). Therefore, the following setup is needed: 70 | 71 | In your `package.json`, specify "type" as "module" 72 | ```json 73 | { 74 | ... 75 | "type": "module", 76 | ... 77 | } 78 | ``` 79 | 80 | Then, using the latest version of node (this project uses whatever is specified in `.nvmrc` which is as of now `v16.15.1`), just run `node index.js` where `index.js` contains the previous example code. 81 | 82 | -------------------------------------------------------------------------------- /doc0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graphcentral/notion/70cd31090c324f36537c0f60160a4c22452f54e3/doc0.png -------------------------------------------------------------------------------- /doc1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graphcentral/notion/70cd31090c324f36537c0f60160a4c22452f54e3/doc1.png -------------------------------------------------------------------------------- /logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graphcentral/notion/70cd31090c324f36537c0f60160a4c22452f54e3/logo.png -------------------------------------------------------------------------------- /notion.code-workspace: -------------------------------------------------------------------------------- 1 | { 2 | "folders": [ 3 | { 4 | "path": "." 5 | } 6 | ], 7 | "settings": { 8 | "eslint.workingDirectories": [ 9 | {"mode": "auto"} 10 | ] 11 | } 12 | } -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@graphcentral/root", 3 | "version": "1.0.0", 4 | "repository": "https://github.com/graphcentral/notion.git", 5 | "author": "9oelM ", 6 | "license": "MIT", 7 | "description": "Notion utilities for graph", 8 | "private": false, 9 | "workspaces": { 10 | "packages": [ 11 | "packages/*" 12 | ] 13 | }, 14 | "scripts": { 15 | "lint": "eslint .", 16 | "lint:debug": "eslint . --debug", 17 | "lint:fix": "eslint . --fix", 18 | "scraper:dev": "npm run dev -w packages/notion-graph-scraper", 19 | "scraper:compile": "npm run compile -w packages/notion-graph-scraper", 20 | "scraper:package": "npm run package -w packages/notion-graph-scraper", 21 | "fe:dev": "npm run dev -w packages/notion-graph-frontend", 22 | "fe:test": "npm run test -w packages/notion-graph-frontend", 23 | "be:dev": "npm run dev -w packages/notion-graph-backend" 24 | }, 25 | "devDependencies": { 26 | "@typescript-eslint/eslint-plugin": "5.9.1", 27 | "@typescript-eslint/parser": "5.9.1", 28 | "eslint": "8.6.0", 29 | "eslint-config-prettier": "8.3.0", 30 | "eslint-plugin-prettier": "4.0.0", 31 | "eslint-plugin-react": "7.28.0", 32 | "prettier": "2.5.1" 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /packages/notion-graph-backend/README.md: -------------------------------------------------------------------------------- 1 | # backend 2 | 3 | This is the backend of user-facing tryout page for Notion knowledge graph. -------------------------------------------------------------------------------- /packages/notion-graph-backend/env.ts: -------------------------------------------------------------------------------- 1 | import * as dotenv from "dotenv"; 2 | import path from "path"; 3 | import { dirname } from "path"; 4 | import { fileURLToPath } from "url"; 5 | 6 | const __dirname = dirname(fileURLToPath(import.meta.url)); 7 | 8 | dotenv.config({ path: path.resolve(__dirname, `..`, `..`, `.env`) }); 9 | 10 | export const ENV = { 11 | PUSHER_APP_ID: process.env.pusher_app_id as string, 12 | PUSHER_KEY: process.env.pusher_key as string, 13 | PUSHER_SECRET: process.env.pusher_secret as string, 14 | PUSHER_CLUSTER: process.env.pusher_cluster as string, 15 | }; 16 | 17 | const maybeUndefinedEnvVals = Object.entries(ENV).filter( 18 | ([, val]) => val === undefined 19 | ); 20 | if (maybeUndefinedEnvVals.length > 0) { 21 | throw new Error( 22 | `There is an undefined environment variable: ${JSON.stringify( 23 | maybeUndefinedEnvVals 24 | )}` 25 | ); 26 | } 27 | -------------------------------------------------------------------------------- /packages/notion-graph-backend/environment.d.ts: -------------------------------------------------------------------------------- 1 | declare global { 2 | namespace NodeJS { 3 | interface ProcessEnv { 4 | pusher_app_id: string; 5 | pusher_key: string; 6 | pusher_secret: string; 7 | pusher_cluster: string; 8 | GITHUB_AUTH_TOKEN: string; 9 | NODE_ENV: `development` | `production`; 10 | PORT?: string; 11 | PWD: string; 12 | } 13 | } 14 | } 15 | 16 | // If this file has no import/export statements (i.e. is a script) 17 | // convert it into a module by adding an empty export statement. 18 | export {}; 19 | -------------------------------------------------------------------------------- /packages/notion-graph-backend/index.ts: -------------------------------------------------------------------------------- 1 | import { NotionGraph } from "@graphcentral/notion-graph-scraper"; 2 | import fs from "fs"; 3 | import Pusher from "pusher"; 4 | import { ENV } from "./env"; 5 | 6 | const pusher = new Pusher({ 7 | appId: ENV.PUSHER_APP_ID, 8 | key: ENV.PUSHER_KEY, 9 | secret: ENV.PUSHER_SECRET, 10 | cluster: ENV.PUSHER_CLUSTER, 11 | useTLS: true, 12 | }); 13 | 14 | pusher.trigger(`my-channel`, `my-event`, { 15 | message: `hello world`, 16 | }); 17 | 18 | const run = async () => { 19 | const notionGraph = new NotionGraph({ 20 | maxDiscoverableNodes: 2000, 21 | maxDiscoverableNodesInOtherSpaces: 2000, 22 | verbose: false, 23 | maxConcurrentRequest: 80, 24 | }); 25 | const graph = await notionGraph.buildGraphFromRootNode( 26 | // notion help page 27 | `e040febf70a94950b8620e6f00005004` 28 | ); 29 | console.log(graph.nodes.length); 30 | console.log(graph.links.length); 31 | await new Promise((resolve, reject) => { 32 | fs.writeFile(`test0.json`, JSON.stringify(graph), (err) => { 33 | if (err) reject(err); 34 | else resolve(``); 35 | }); 36 | }); 37 | 38 | process.exit(0); 39 | }; 40 | 41 | run(); 42 | -------------------------------------------------------------------------------- /packages/notion-graph-backend/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@graphcentral/notion-graph-backend", 3 | "version": "1.0.0", 4 | "description": "", 5 | "type": "module", 6 | "main": "index.mjs", 7 | "scripts": { 8 | "dev": "node --experimental-specifier-resolution=node --loader=ts-node/esm ./index.ts" 9 | }, 10 | "keywords": [], 11 | "author": "", 12 | "license": "MIT", 13 | "dependencies": { 14 | "@graphcentral/notion-graph-scraper": "*", 15 | "@netlify/functions": "^1.0.0", 16 | "dotenv": "^16.0.1", 17 | "pusher": "^5.1.1-beta" 18 | }, 19 | "devDependencies": { 20 | "ts-node": "^10.9.1", 21 | "tsconfig-paths": "^4.1.0" 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /packages/notion-graph-backend/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "ts-node": { 3 | "esm": true 4 | }, 5 | "compilerOptions": { 6 | /* Basic Options */ 7 | // "incremental": true, /* Enable incremental compilation */ 8 | "target": "ESNext", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019' or 'ESNEXT'. */ 9 | "module": "es2022", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ 10 | "lib": ["ESNext"], /* Specify library files to be included in the compilation. */ 11 | // "allowJs": true, /* Allow javascript files to be compiled. */ 12 | // "checkJs": true, /* Report errors in .js files. */ 13 | // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */ 14 | // "declaration": true, /* Generates corresponding '.d.ts' file. */ 15 | // "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */ 16 | // "sourceMap": true, /* Generates corresponding '.map' file. */ 17 | // "outFile": "./", /* Concatenate and emit output to single file. */ 18 | // "outDir": "./", /* Redirect output structure to the directory. */ 19 | // "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ 20 | // "composite": true, /* Enable project compilation */ 21 | // "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */ 22 | // "removeComments": true, /* Do not emit comments to output. */ 23 | // "noEmit": true, /* Do not emit outputs. */ 24 | // "importHelpers": true, /* Import emit helpers from 'tslib'. */ 25 | // "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */ 26 | "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */ 27 | 28 | /* Strict Type-Checking Options */ 29 | "strict": true, /* Enable all strict type-checking options. */ 30 | "noImplicitAny": true, /* Raise error on expressions and declarations with an implied 'any' type. */ 31 | "strictNullChecks": true, /* Enable strict null checks. */ 32 | "strictFunctionTypes": true, /* Enable strict checking of function types. */ 33 | "strictBindCallApply": true, /* Enable strict 'bind', 'call', and 'apply' methods on functions. */ 34 | "strictPropertyInitialization": true, /* Enable strict checking of property initialization in classes. */ 35 | "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */ 36 | "alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */ 37 | 38 | /* Additional Checks */ 39 | // "noUnusedLocals": true, /* Report errors on unused locals. */ 40 | // "noUnusedParameters": true, /* Report errors on unused parameters. */ 41 | // "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */ 42 | "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */ 43 | 44 | /* Module Resolution Options */ 45 | "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */ 46 | // "baseUrl": "./", /* Base directory to resolve non-absolute module names. */ 47 | // "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */ 48 | // "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */ 49 | // "typeRoots": [], /* List of folders to include type definitions from. */ 50 | // "types": [], /* Type declaration files to be included in compilation. */ 51 | // "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */ 52 | "esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */ 53 | // "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */ 54 | // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ 55 | 56 | /* Source Map Options */ 57 | // "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */ 58 | // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ 59 | // "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */ 60 | // "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */ 61 | 62 | /* Experimental Options */ 63 | // "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */ 64 | // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /packages/notion-graph-frontend/.storybook/main.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | const ForkTsCheckerWebpackPlugin = require(`fork-ts-checker-webpack-plugin`); 3 | const CircularDependencyPlugin = require('circular-dependency-plugin') 4 | 5 | module.exports = { 6 | "stories": [ 7 | "../src/**/*.stories.mdx", 8 | "../src/**/*.stories.@(js|jsx|ts|tsx)" 9 | ], 10 | "addons": [ 11 | "@storybook/addon-links", 12 | "@storybook/addon-essentials", 13 | '@storybook/addon-storysource', 14 | '@storybook/addon-a11y', 15 | ], 16 | webpackFinal: (config) => { 17 | config.plugins = [ 18 | ...config.plugins, 19 | new ForkTsCheckerWebpackPlugin(), 20 | new CircularDependencyPlugin({ 21 | // exclude detection of files based on a RegExp 22 | exclude: /a\.js|node_modules/, 23 | // include specific files based on a RegExp 24 | // include: /src/, 25 | // add errors to webpack instead of warnings 26 | failOnError: false, 27 | // allow import cycles that include an asyncronous import, 28 | // e.g. via import(/* webpackMode: "weak" */ './file.js') 29 | allowAsyncCycles: false, 30 | // set the current working directory for displaying module paths 31 | cwd: process.cwd(), 32 | }) 33 | ] 34 | // https://stackoverflow.com/questions/67070802/webpack-5-and-storybook-6-integration-throws-an-error-in-defineplugin-js 35 | config.resolve.fallback = { 36 | http: false, 37 | path: false, 38 | crypto: false, 39 | } 40 | 41 | return config; 42 | }, 43 | core: { 44 | builder: "webpack5", 45 | } 46 | } -------------------------------------------------------------------------------- /packages/notion-graph-frontend/.storybook/preview.js: -------------------------------------------------------------------------------- 1 | 2 | export const parameters = { 3 | actions: { argTypesRegex: "^on[A-Z].*" }, 4 | controls: { 5 | matchers: { 6 | color: /(background|color)$/i, 7 | date: /Date$/, 8 | }, 9 | }, 10 | } -------------------------------------------------------------------------------- /packages/notion-graph-frontend/jest/jest.config.ts: -------------------------------------------------------------------------------- 1 | import type { Config } from "@jest/types"; 2 | 3 | const config: Config.InitialOptions = { 4 | verbose: true, 5 | rootDir: `..`, 6 | setupFiles: [`/jest/setupTest.js`], 7 | preset: `ts-jest`, 8 | testEnvironment: `jsdom`, 9 | moduleNameMapper: { 10 | "src/(.*)": `/src/$1`, 11 | }, 12 | }; 13 | export default config; 14 | -------------------------------------------------------------------------------- /packages/notion-graph-frontend/jest/setupTest.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-undef */ 2 | /* eslint-disable @typescript-eslint/no-var-requires */ 3 | const Enzyme = require(`enzyme`); 4 | const Adapter = require(`@wojtekmaj/enzyme-adapter-react-17`); 5 | 6 | Enzyme.configure({ adapter: new Adapter() }); 7 | -------------------------------------------------------------------------------- /packages/notion-graph-frontend/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@graphcentral/notion-graph-frontend", 3 | "version": "1.0.0", 4 | "main": "index.js", 5 | "repository": "https://github.com/graphcentral/notion.git", 6 | "author": "9oelM ", 7 | "license": "MIT", 8 | "private": false, 9 | "scripts": { 10 | "test": "jest --config ./jest/jest.config.ts", 11 | "dev": "../../node_modules/.bin/webpack-dev-server --config ./webpack/webpack.config.dev.ts --mode development", 12 | "prod": "webpack --config ./webpack/webpack.config.prod.ts --mode production", 13 | "storybook": "start-storybook -p 6006", 14 | "build-storybook": "build-storybook" 15 | }, 16 | "devDependencies": { 17 | "@babel/core": "7.16.7", 18 | "@jest/types": "27.4.2", 19 | "@types/enzyme": "3.10.11", 20 | "@types/html-webpack-plugin": "^3.2.6", 21 | "@types/jest": "^27.4.0", 22 | "@types/lodash.debounce": "^4.0.7", 23 | "@types/lodash.flow": "3.5.6", 24 | "@types/react": "^18.0.17", 25 | "@types/react-dom": "^18.0.6", 26 | "@types/webpack-dev-server": "4.7.1", 27 | "@wojtekmaj/enzyme-adapter-react-17": "0.6.6", 28 | "babel-loader": "8.2.3", 29 | "circular-dependency-plugin": "5.2.2", 30 | "css-loader": "^6.7.1", 31 | "dotenv-webpack": "7.0.3", 32 | "enzyme": "3.11.0", 33 | "esbuild-loader": "^2.19.0", 34 | "fork-ts-checker-webpack-plugin": "6.5.0", 35 | "html-webpack-plugin": "5.5.0", 36 | "jest": "27.4.7", 37 | "style-loader": "^3.3.1", 38 | "ts-jest": "27.1.2", 39 | "ts-loader": "9.2.6", 40 | "ts-node": "^10.9.1", 41 | "typescript": "4.5.4", 42 | "wait-for-expect": "3.0.2", 43 | "webpack": "5.65.0", 44 | "webpack-cli": "4.10.0", 45 | "webpack-dev-server": "4.7.2" 46 | }, 47 | "dependencies": { 48 | "@emotion/react": "^11.10.0", 49 | "axios": "^0.24.0", 50 | "lodash.debounce": "^4.0.8", 51 | "lodash.flow": "^3.5.0", 52 | "normalize.css": "^8.0.1", 53 | "pusher-js": "^7.4.0", 54 | "react": "^18.2.0", 55 | "react-dom": "^18.2.0", 56 | "zustand": "^4.1.0" 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /packages/notion-graph-frontend/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | My project name 12 | 14 | 15 | 16 | 17 |
18 |
19 | 20 | 21 | -------------------------------------------------------------------------------- /packages/notion-graph-frontend/public/robots.txt: -------------------------------------------------------------------------------- 1 | User-agent: * 2 | Disallow: -------------------------------------------------------------------------------- /packages/notion-graph-frontend/src/app.tsx: -------------------------------------------------------------------------------- 1 | import { css, Global, ThemeProvider } from "@emotion/react"; 2 | import React from "react"; 3 | import { GlobalStyle } from "src/globalStyle"; 4 | import { MainPage } from "src/pages/Main"; 5 | import { theme } from "src/theme"; 6 | import { enhance } from "src/utilities/essentials"; 7 | 8 | export const App = enhance(() => { 9 | return ( 10 | 11 | 12 | 13 | 14 | ); 15 | })(); 16 | -------------------------------------------------------------------------------- /packages/notion-graph-frontend/src/assets/guide0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graphcentral/notion/70cd31090c324f36537c0f60160a4c22452f54e3/packages/notion-graph-frontend/src/assets/guide0.png -------------------------------------------------------------------------------- /packages/notion-graph-frontend/src/assets/guide1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graphcentral/notion/70cd31090c324f36537c0f60160a4c22452f54e3/packages/notion-graph-frontend/src/assets/guide1.png -------------------------------------------------------------------------------- /packages/notion-graph-frontend/src/assets/guide2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graphcentral/notion/70cd31090c324f36537c0f60160a4c22452f54e3/packages/notion-graph-frontend/src/assets/guide2.png -------------------------------------------------------------------------------- /packages/notion-graph-frontend/src/assets/guide3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graphcentral/notion/70cd31090c324f36537c0f60160a4c22452f54e3/packages/notion-graph-frontend/src/assets/guide3.png -------------------------------------------------------------------------------- /packages/notion-graph-frontend/src/assets/guide4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graphcentral/notion/70cd31090c324f36537c0f60160a4c22452f54e3/packages/notion-graph-frontend/src/assets/guide4.png -------------------------------------------------------------------------------- /packages/notion-graph-frontend/src/components/Example/fallback.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import { FC } from "react"; 3 | 4 | export const ExampleFallback: FC = () => ( 5 |
6 |

11 | Oops. Something went wrong. Please try again. 12 |

13 |
14 | ); 15 | -------------------------------------------------------------------------------- /packages/notion-graph-frontend/src/components/Example/index.stories.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | 3 | import { Meta, Story } from "@storybook/react"; 4 | import { ExampleImpure, ExampleImpureProps } from "."; 5 | 6 | const Template: Story = (args: ExampleImpureProps) => ( 7 | 8 | ); 9 | 10 | export const ExampleImpure1: Story = Template.bind({}); 11 | ExampleImpure1.args = { 12 | color: `blue`, 13 | }; 14 | 15 | export default { 16 | title: `Example`, 17 | component: ExampleImpure, 18 | parameters: { 19 | layout: `centered`, 20 | actions: { 21 | handles: [`click`], 22 | }, 23 | }, 24 | argTypes: { 25 | color: { control: `color` }, 26 | }, 27 | } as Meta; 28 | -------------------------------------------------------------------------------- /packages/notion-graph-frontend/src/components/Example/index.tsx: -------------------------------------------------------------------------------- 1 | import axios, { AxiosError, AxiosResponse } from "axios"; 2 | import React, { useEffect, useState } from "react"; 3 | import { FC } from "react"; 4 | import { enhance, tcAsync } from "src/utilities/essentials"; 5 | import { ExampleFallback } from "./fallback"; 6 | 7 | enum NetworkRequestStatus { 8 | LOADING = `LOADING`, 9 | ERROR = `ERROR`, 10 | SUCCESS = `SUCCESS`, 11 | } 12 | 13 | // eslint-disable-next-line @typescript-eslint/ban-types 14 | export type ExampleImpureProps = { 15 | color: string; 16 | }; 17 | 18 | export const ExampleImpure: FC = 19 | enhance(({ color }) => { 20 | const [exampleText, setExampleText] = 21 | useState>(null); 22 | const [networkRequestStatus, setnetworkRequestStatus] = 23 | useState(NetworkRequestStatus.LOADING); 24 | 25 | useEffect(() => { 26 | async function getIpsumText() { 27 | const [getIpsumTextRequestError, getIpsumTextRequest] = await tcAsync< 28 | AxiosResponse, 29 | AxiosError 30 | >( 31 | axios.get( 32 | `https://baconipsum.com/api/?type=all-meat¶s=2&start-with-lorem=1` 33 | ) 34 | ); 35 | 36 | if (getIpsumTextRequestError || !getIpsumTextRequest) { 37 | setnetworkRequestStatus(NetworkRequestStatus.ERROR); 38 | return; 39 | } 40 | 41 | setnetworkRequestStatus(NetworkRequestStatus.SUCCESS); 42 | setExampleText(getIpsumTextRequest.data); 43 | } 44 | 45 | getIpsumText(); 46 | }, []); 47 | 48 | if (networkRequestStatus === NetworkRequestStatus.ERROR) { 49 | return ( 50 |

55 | error occurred while getting the text from server. Please try again. 56 |

57 | ); 58 | } 59 | 60 | if (networkRequestStatus === NetworkRequestStatus.LOADING) { 61 | return

loading...

; 62 | } 63 | 64 | return {exampleText}; 65 | })(ExampleFallback); 66 | 67 | // eslint-disable-next-line @typescript-eslint/ban-types 68 | export type ExamplePureProps = { 69 | color: string; 70 | }; 71 | 72 | export const ExamplePure: FC = enhance( 73 | ({ color, children }) => ( 74 |
75 |

80 | {children} 81 |

82 |
83 | ) 84 | )(ExampleFallback); 85 | -------------------------------------------------------------------------------- /packages/notion-graph-frontend/src/components/Util/WithErrorBoundary/index.tsx: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/explicit-module-boundary-types */ 2 | /* eslint-disable @typescript-eslint/ban-types */ 3 | import React, { ComponentType, FC, memo } from "react"; 4 | import { ErrorInfo, PureComponent, ReactNode } from "react"; 5 | 6 | export const NullFallback: FC = () => null; 7 | 8 | export type ErrorBoundaryProps = { 9 | Fallback: ReactNode; 10 | }; 11 | 12 | export type ErrorBoundaryState = { 13 | error?: Error; 14 | errorInfo?: ErrorInfo; 15 | }; 16 | 17 | export class ErrorBoundary extends PureComponent< 18 | ErrorBoundaryProps, 19 | ErrorBoundaryState 20 | > { 21 | constructor(props: ErrorBoundaryProps) { 22 | super(props); 23 | this.state = { error: undefined, errorInfo: undefined }; 24 | } 25 | 26 | public componentDidCatch(error: Error, errorInfo: ErrorInfo): void { 27 | this.setState({ 28 | error: error, 29 | errorInfo: errorInfo, 30 | }); 31 | /** 32 | * @todo log Sentry here 33 | */ 34 | } 35 | 36 | public render(): ReactNode { 37 | if (this.state.error) return this.props.Fallback; 38 | return this.props.children; 39 | } 40 | } 41 | 42 | export function withErrorBoundary(Component: ComponentType) { 43 | return (Fallback = NullFallback) => { 44 | // eslint-disable-next-line react/display-name 45 | return memo(({ ...props }: Props) => { 46 | return ( 47 | }> 48 | 49 | 50 | ); 51 | }); 52 | }; 53 | } 54 | -------------------------------------------------------------------------------- /packages/notion-graph-frontend/src/constants/index.ts: -------------------------------------------------------------------------------- 1 | const NOTION_HELP_PAGE = `https://www.notion.so/Help-Support-Documentation-e040febf70a94950b8620e6f00005004`; 2 | 3 | export const CONSTANTS = { 4 | NOTION_HELP_PAGE, 5 | }; 6 | -------------------------------------------------------------------------------- /packages/notion-graph-frontend/src/globalStyle.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import { css, Global } from "@emotion/react"; 3 | /** @jsx jsx */ 4 | import { jsx } from "@emotion/react"; 5 | import { useTypedTheme } from "src/theme"; 6 | 7 | export const GlobalStyle = () => { 8 | const theme = useTypedTheme(); 9 | return ( 10 | 32 | ); 33 | }; 34 | -------------------------------------------------------------------------------- /packages/notion-graph-frontend/src/index.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | // @ts-ignore 3 | import { createRoot } from "react-dom/client"; 4 | import { App } from "src/app"; 5 | import "normalize.css"; 6 | 7 | const container = document.getElementById(`root`); 8 | const root = createRoot(container!); 9 | root.render(); 10 | -------------------------------------------------------------------------------- /packages/notion-graph-frontend/src/pages/Main/fallback.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import { FC } from "react"; 3 | 4 | export const ExampleFallback: FC = () => ( 5 |
6 |

11 | Oops. Something went wrong. Please try again. 12 |

13 |
14 | ); 15 | -------------------------------------------------------------------------------- /packages/notion-graph-frontend/src/pages/Main/index.tsx: -------------------------------------------------------------------------------- 1 | import React, { useCallback, useEffect, useRef, useState } from "react"; 2 | import { enhance } from "src/utilities/essentials"; 3 | import { useTypedTheme } from "src/theme"; 4 | /** @jsx jsx */ 5 | import { jsx } from "@emotion/react"; 6 | import { HowToAndTroubleShooting } from "src/pages/Main/localFragments/HowToAndTroubleShooting"; 7 | import { NotionPublicPageLinkInput } from "src/pages/Main/localFragments/NotionPublicPageLinkInput"; 8 | import { CONSTANTS } from "src/constants"; 9 | import { isValidNotionURL } from "src/utilities/notion"; 10 | 11 | export function useNotionLink() { 12 | const [notionPublicPageLink, setNotionPublicPageLink] = useState( 13 | CONSTANTS.NOTION_HELP_PAGE 14 | ); 15 | const [isNotionPublicPageLinkValid, setNotionPagePublicLinkValid] = useState( 16 | isValidNotionURL(CONSTANTS.NOTION_HELP_PAGE) 17 | ); 18 | const onNotionPublicPageLinkChange: NonNullable< 19 | React.DetailedHTMLProps< 20 | React.InputHTMLAttributes, 21 | HTMLInputElement 22 | >[`onChange`] 23 | > = useCallback((event) => { 24 | console.log( 25 | event, 26 | event.target.value, 27 | isValidNotionURL(event.target.value) 28 | ); 29 | setNotionPublicPageLink(event.target.value); 30 | }, []); 31 | 32 | useEffect(() => { 33 | setNotionPagePublicLinkValid(isValidNotionURL(notionPublicPageLink)); 34 | }, [notionPublicPageLink]); 35 | 36 | return { 37 | isNotionPublicPageLinkValid, 38 | onNotionPublicPageLinkChange, 39 | notionPublicPageLink, 40 | }; 41 | } 42 | 43 | export const MainPage = enhance(() => { 44 | const theme = useTypedTheme(); 45 | 46 | const troubleShootingSectionRef = useRef(null); 47 | const onClickProblems = useCallback(() => { 48 | if (!troubleShootingSectionRef.current) return; 49 | 50 | troubleShootingSectionRef.current.scrollIntoView({ 51 | behavior: `smooth`, 52 | }); 53 | }, []); 54 | const { 55 | isNotionPublicPageLinkValid, 56 | onNotionPublicPageLinkChange, 57 | notionPublicPageLink, 58 | } = useNotionLink(); 59 | 60 | return ( 61 |
72 |
84 |

93 | Enter your Notion page's public link here! 👇🏼 94 |

95 | 103 | 122 | 139 |
140 |
148 | 149 |
150 |
151 | ); 152 | })(); 153 | -------------------------------------------------------------------------------- /packages/notion-graph-frontend/src/pages/Main/localFragments/HowToAndTroubleShooting/index.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import { useTypedTheme } from "src/theme"; 3 | import { enhance } from "src/utilities/essentials"; 4 | import Guide0Image from "src/assets/guide0.png"; 5 | import Guide1Image from "src/assets/guide1.png"; 6 | import Guide2Image from "src/assets/guide2.png"; 7 | import Guide3Image from "src/assets/guide3.png"; 8 | import Guide4Image from "src/assets/guide4.png"; 9 | /** @jsx jsx */ 10 | import { jsx } from "@emotion/react"; 11 | 12 | export const HowToAndTroubleShooting = enhance(() => { 13 | const theme = useTypedTheme(); 14 | 15 | return ( 16 | <> 17 |

23 | How to & Troubleshooting 24 |

25 |
33 |

39 | 1. Locate your Notion page 40 |

41 |

47 | Preferably choose the page that has many children, because you want to 48 | see a knowledge graph of some extent. 49 |

50 | Locate your Notion page 51 |
52 |
60 |

66 | 2. Make your page public 67 |

68 |

74 | Click on 'Share' at the top right corner, and turn on the toggle for 75 | 'Share to web'. 76 |

77 | Click on 'Share' at the top right corner 85 | Turn on the toggle for 'Share to web' step #1 93 | Turn on the toggle for 'Share to web' step #2 101 |
102 |
110 |

116 | 3. Bring the link from Notion to this website 117 |

118 |

124 | Click on 'Copy web link', and paste the link into the input above on 125 | this website 126 |

127 | Bring the link from Notion to this website 132 |
133 | 134 | ); 135 | })(); 136 | -------------------------------------------------------------------------------- /packages/notion-graph-frontend/src/pages/Main/localFragments/NotionPublicPageLinkInput/index.tsx: -------------------------------------------------------------------------------- 1 | import React, { FC, useCallback, useEffect, useState } from "react"; 2 | import { useTypedTheme } from "src/theme"; 3 | import { enhance } from "src/utilities/essentials"; 4 | /** @jsx jsx */ 5 | import { jsx, css } from "@emotion/react"; 6 | import { CONSTANTS } from "src/constants"; 7 | import { isValidNotionURL } from "src/utilities/notion"; 8 | import { useNotionLink } from "src/pages/Main"; 9 | 10 | export type NotionPublicPageLinkInputProps = { 11 | onClickTroubleShooting: VoidFunction; 12 | } & ReturnType; 13 | 14 | export const NotionPublicPageLinkInput: FC = 15 | enhance( 16 | ({ 17 | onClickTroubleShooting, 18 | isNotionPublicPageLinkValid, 19 | notionPublicPageLink, 20 | onNotionPublicPageLinkChange, 21 | }) => { 22 | const theme = useTypedTheme(); 23 | 24 | return ( 25 |
30 | 54 |

62 | Not a valid Notion URL.{` `} 63 | 72 | Check troubleshooting below? 73 | 74 |

75 |
76 | ); 77 | } 78 | )(); 79 | -------------------------------------------------------------------------------- /packages/notion-graph-frontend/src/theme.ts: -------------------------------------------------------------------------------- 1 | import { useTheme } from "@emotion/react"; 2 | 3 | export const theme = { 4 | colors: { 5 | darkPrimary: `#181818`, 6 | darkSecondary: `#131313`, 7 | whitePrimary: `#C9CACA`, 8 | whiteSecondary: `#898989`, 9 | whiteThird: `#2A2A2A`, 10 | whiteFourth: `#1D1D1D`, 11 | interactivePrimary: `#269AD3`, 12 | interactiveSecondary: `#138AC1`, 13 | interactiveText: `#FFFFFF`, 14 | warningText: `#E33F45`, 15 | warningPrimary: `#281D1C`, 16 | warningSeconary: `#5A2824`, 17 | }, 18 | lines: { 19 | link: `#363636`, 20 | }, 21 | border: { 22 | smallRadius: `0.2rem`, 23 | }, 24 | } as const; 25 | 26 | export function useTypedTheme(): typeof theme { 27 | return useTheme() as typeof theme; 28 | } 29 | -------------------------------------------------------------------------------- /packages/notion-graph-frontend/src/utilities/essentials.ts: -------------------------------------------------------------------------------- 1 | import { FC, memo } from "react"; 2 | import { withErrorBoundary } from "../components/Util/WithErrorBoundary"; 3 | import flow from "lodash.flow"; 4 | 5 | export const enhance: ( 6 | Component: FC 7 | ) => ( 8 | Fallback?: FC 9 | ) => React.MemoExoticComponent<({ ...props }: Props) => JSX.Element> = flow( 10 | memo, 11 | withErrorBoundary 12 | ); 13 | 14 | export type TcResult = [null, Data] | [Throws]; 15 | 16 | export async function tcAsync( 17 | promise: Promise 18 | ): Promise> { 19 | try { 20 | const response: T = await promise; 21 | 22 | return [null, response]; 23 | } catch (error) { 24 | return [error] as [Throws]; 25 | } 26 | } 27 | 28 | export function tcSync< 29 | ArrType, 30 | Params extends Array, 31 | Returns, 32 | Throws = Error 33 | >( 34 | fn: (...params: Params) => Returns, 35 | ...deps: Params 36 | ): TcResult { 37 | try { 38 | const data: Returns = fn(...deps); 39 | 40 | return [null, data]; 41 | } catch (e) { 42 | return [e] as [Throws]; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /packages/notion-graph-frontend/src/utilities/notion.test.ts: -------------------------------------------------------------------------------- 1 | import { 2 | NORMAL_USERS_NOTION_URL_REGEX, 3 | OFFICIAL_NOTION_URL_REGEX, 4 | } from "src/utilities/notion"; 5 | 6 | describe(`NORMAL_USERS_NOTION_URL_REGEX`, () => { 7 | it.concurrent.each([ 8 | [ 9 | `https://aaa.notion.site/blah-blah-blah-12345678901234567890123456789012`, 10 | true, 11 | ], 12 | [ 13 | `https://.notion.site/blah-blah-blah-12345678901234567890123456789012`, 14 | false, 15 | ], 16 | [ 17 | `https:/aaa.notion.site/blah-blah-blah-12345678901234567890123456789012`, 18 | false, 19 | ], 20 | [ 21 | `httpsaaa.notion.site/blah-blah-blah-12345678901234567890123456789012`, 22 | true, 23 | ], 24 | [`.notion.site/blah-blah-blah-12345678901234567890123456789012`, false], 25 | [ 26 | `blah.notion.site/blah-blah-blah-1234567890123456789012345678901234567`, 27 | false, 28 | ], 29 | [ 30 | `blah.notion.site/asdfasfasdfasfasdfasdfasdf-asdfasdfadf-1234567890123456789012345678901234567`, 31 | false, 32 | ], 33 | [ 34 | `blah.notion.so/asdfasfasdfasfasdfasdfasdf-asdfasdfadf-1234567890123456789012345678901234567`, 35 | false, 36 | ], 37 | [ 38 | `blah.notion.site/asdfasfasdfasfasdfasdfasdf-asdfasdfadf-12345678901234567890123456789012`, 39 | true, 40 | ], 41 | ])(`%p should be a %p normal user's notion url`, (url, expected) => { 42 | expect(NORMAL_USERS_NOTION_URL_REGEX.test(url)).toBe(expected); 43 | }); 44 | }); 45 | describe(`OFFICIAL_NOTION_URL_REGEX`, () => { 46 | it.concurrent.each([ 47 | [`https://notion.so/blah-blah-blah-12345678901234567890123456789012`, true], 48 | [`notion.so/blah-blah-blah-12345678901234567890123456789012`, true], 49 | [`www.notion.so/blah-blah-blah-12345678901234567890123456789012`, true], 50 | [ 51 | `www.notion.so/blah-blah-blahaaaaa-12345678901234567890123456789012`, 52 | true, 53 | ], 54 | [`.notion.so/blah-blah-blahaaaaa-12345678901234567890123456789012`, false], 55 | [`..notion.so/blah-blah-blahaaaaa-12345678901234567890123456789012`, false], 56 | [ 57 | `//..notion.so/blah-blah-blahaaaaa-12345678901234567890123456789012`, 58 | false, 59 | ], 60 | [ 61 | `https://..notion.so/blah-blah-blahaaaaa-12345678901234567890123456789012`, 62 | false, 63 | ], 64 | [ 65 | `https://.notion.so/blah-blah-blahaaaaa-12345678901234567890123456789012`, 66 | false, 67 | ], 68 | [ 69 | `https://ww.notion.so/blah-blah-blahaaaaa-12345678901234567890123456789012`, 70 | false, 71 | ], 72 | [ 73 | `https://notion.so/blah-blah-blahaaaaa-1234567890123456789012345678901a`, 74 | true, 75 | ], 76 | ])(`%p should be a %p official notion url`, (url, expected) => { 77 | expect(OFFICIAL_NOTION_URL_REGEX.test(url)).toBe(expected); 78 | }); 79 | }); 80 | -------------------------------------------------------------------------------- /packages/notion-graph-frontend/src/utilities/notion.ts: -------------------------------------------------------------------------------- 1 | export const NORMAL_USERS_NOTION_URL_REGEX = 2 | /^(https:\/\/)?([-a-zA-Z0-9]+\.)?notion\.site\/[-a-zA-Z0-9]*-[a-zA-Z0-9]{32}$/; 3 | export const OFFICIAL_NOTION_URL_REGEX = 4 | /^(https:\/\/)?(www\.)?notion\.so\/[-a-zA-Z0-9]*-[a-zA-Z0-9]{32}$/; 5 | 6 | /** 7 | * 8 | * @param url should be in the form of https://blahblah.notion.site/Some-long-title-like-this-3c718fc5c0c84a92855df8e6edca2cb5 or 9 | * https://notion.so/Some-long-title-like-this-3c718fc5c0c84a92855df8e6edca2cb5. 10 | * These are only two types of valid URLs (but with the exclusion/inclusion of https://, www, and so on) 11 | * @returns whether the url is in the specified form 12 | */ 13 | export function isValidNotionURL(url: string): boolean { 14 | return ( 15 | NORMAL_USERS_NOTION_URL_REGEX.test(url) || 16 | OFFICIAL_NOTION_URL_REGEX.test(url) 17 | ); 18 | } 19 | -------------------------------------------------------------------------------- /packages/notion-graph-frontend/src/webpack.d.ts: -------------------------------------------------------------------------------- 1 | // eslint-disable-next-line quotes 2 | declare module "*.png" { 3 | const content: any; 4 | export default content; 5 | } 6 | -------------------------------------------------------------------------------- /packages/notion-graph-frontend/tsconfig.esbuild.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es5", 4 | "module": "commonjs", 5 | "lib": ["DOM", "ESNext"], 6 | "jsx": "react", 7 | "sourceMap": true, 8 | "downlevelIteration": true, 9 | 10 | "strict": true, 11 | "noImplicitAny": true, 12 | "strictNullChecks": true, 13 | "strictFunctionTypes": true, 14 | "strictBindCallApply": true, 15 | "strictPropertyInitialization": true, 16 | "noImplicitThis": true, 17 | "alwaysStrict": true, 18 | 19 | 20 | "noUnusedLocals": false, 21 | "noUnusedParameters": false, 22 | "noImplicitReturns": true, 23 | "noFallthroughCasesInSwitch": true, 24 | "noUncheckedIndexedAccess": true, 25 | "noPropertyAccessFromIndexSignature": true, 26 | 27 | "moduleResolution": "node", 28 | "baseUrl": "./src", 29 | "paths": { 30 | "src/*": [ 31 | "./src/*" 32 | ] 33 | }, 34 | "types": ["@emotion/core"], 35 | "allowSyntheticDefaultImports": true, 36 | "esModuleInterop": true, 37 | "skipLibCheck": true, 38 | "forceConsistentCasingInFileNames": true 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /packages/notion-graph-frontend/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | /* Visit https://aka.ms/tsconfig.json to read more about this file */ 4 | 5 | /* Basic Options */ 6 | // "incremental": true, /* Enable incremental compilation */ 7 | "target": "es5", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */ 8 | "module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */ 9 | "lib": ["DOM", "ESNext"], /* Specify library files to be included in the compilation. */ 10 | // "allowJs": true, /* Allow javascript files to be compiled. */ 11 | // "checkJs": true, /* Report errors in .js files. */ 12 | "jsx": "react", /* Specify JSX code generation: 'preserve', 'react-native', 'react', 'react-jsx' or 'react-jsxdev'. */ 13 | // "declaration": true, /* Generates corresponding '.d.ts' file. */ 14 | // "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */ 15 | "sourceMap": true, /* Generates corresponding '.map' file. */ 16 | // "outFile": "./", /* Concatenate and emit output to single file. */ 17 | // "outDir": "./", /* Redirect output structure to the directory. */ 18 | // "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ 19 | // "composite": true, /* Enable project compilation */ 20 | // "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */ 21 | // "removeComments": true, /* Do not emit comments to output. */ 22 | // "noEmit": true, /* Do not emit outputs. */ 23 | // "importHelpers": true, /* Import emit helpers from 'tslib'. */ 24 | "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */ 25 | // "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */ 26 | 27 | /* Strict Type-Checking Options */ 28 | "strict": true, /* Enable all strict type-checking options. */ 29 | "noImplicitAny": true, /* Raise error on expressions and declarations with an implied 'any' type. */ 30 | "strictNullChecks": true, /* Enable strict null checks. */ 31 | "strictFunctionTypes": true, /* Enable strict checking of function types. */ 32 | "strictBindCallApply": true, /* Enable strict 'bind', 'call', and 'apply' methods on functions. */ 33 | "strictPropertyInitialization": true, /* Enable strict checking of property initialization in classes. */ 34 | "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */ 35 | "alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */ 36 | 37 | /* Additional Checks */ 38 | "noUnusedLocals": true, /* Report errors on unused locals. */ 39 | "noUnusedParameters": true, /* Report errors on unused parameters. */ 40 | "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */ 41 | "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */ 42 | "noUncheckedIndexedAccess": true, /* Include 'undefined' in index signature results */ 43 | "noPropertyAccessFromIndexSignature": true, /* Require undeclared properties from index signatures to use element accesses. */ 44 | 45 | /* Module Resolution Options */ 46 | "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */ 47 | "baseUrl": "./", /* Base directory to resolve non-absolute module names. */ 48 | "paths": { 49 | "src/*": [ 50 | "./src/*" 51 | ] 52 | }, 53 | "jsxImportSource": "@emotion/react", 54 | "types": ["@emotion/core", "jest"], 55 | // "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */ 56 | // "typeRoots": [], /* List of folders to include type definitions from. */ 57 | // "types": [], /* Type declaration files to be included in compilation. */ 58 | "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */ 59 | "esModuleInterop": true, /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */ 60 | // "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */ 61 | // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ 62 | 63 | /* Source Map Options */ 64 | // "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */ 65 | // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ 66 | // "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */ 67 | // "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */ 68 | 69 | /* Experimental Options */ 70 | // "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */ 71 | // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ 72 | 73 | /* Advanced Options */ 74 | "skipLibCheck": true, /* Skip type checking of declaration files. */ 75 | "forceConsistentCasingInFileNames": true, /* Disallow inconsistently-cased references to the same file. */ 76 | "resolveJsonModule": true 77 | }, 78 | } 79 | -------------------------------------------------------------------------------- /packages/notion-graph-frontend/webpack/webpack.config.common.ts: -------------------------------------------------------------------------------- 1 | import path from "path"; 2 | import webpack from "webpack"; 3 | import HtmlWebpackPlugin from "html-webpack-plugin"; 4 | import tsconfigRaw from "../tsconfig.esbuild.json"; 5 | 6 | export const commonConfig: webpack.Configuration = { 7 | entry: `./src/index.tsx`, 8 | // https://webpack.js.org/plugins/split-chunks-plugin/ 9 | optimization: { 10 | splitChunks: { 11 | chunks: `all`, 12 | minSize: 500, 13 | // minRemainingSize: 0, 14 | minChunks: 1, 15 | maxAsyncRequests: 30, 16 | maxInitialRequests: 30, 17 | // enforceSizeThreshold: 50000, 18 | cacheGroups: { 19 | defaultVendors: { 20 | test: /[\\/]node_modules[\\/]/, 21 | priority: -10, 22 | reuseExistingChunk: true, 23 | }, 24 | default: { 25 | minChunks: 2, 26 | priority: -20, 27 | reuseExistingChunk: true, 28 | }, 29 | }, 30 | }, 31 | }, 32 | module: { 33 | rules: [ 34 | { 35 | test: /\.tsx?$/, 36 | loader: `esbuild-loader`, 37 | options: { 38 | loader: `tsx`, 39 | target: `es2015`, 40 | tsconfigRaw, 41 | }, 42 | }, 43 | { 44 | test: /\.css?$/, 45 | use: [`style-loader`, `css-loader`], 46 | }, 47 | { 48 | test: /\.(png|svg|jpg|jpeg|gif)$/i, 49 | type: `asset/resource`, 50 | }, 51 | ], 52 | }, 53 | resolve: { 54 | extensions: [`.tsx`, `.ts`, `.js`], 55 | alias: { 56 | src: path.resolve(__dirname, `..`, `src/`), 57 | }, 58 | }, 59 | output: { 60 | filename: `[chunkhash].[name].js`, 61 | path: path.resolve(__dirname, `dist`), 62 | }, 63 | plugins: [ 64 | // eslint-disable-next-line @typescript-eslint/ban-ts-comment 65 | // @ts-ignore 66 | new HtmlWebpackPlugin({ 67 | template: path.join(__dirname, `..`, `public`, `index.html`), 68 | }), 69 | ], 70 | }; 71 | -------------------------------------------------------------------------------- /packages/notion-graph-frontend/webpack/webpack.config.dev.ts: -------------------------------------------------------------------------------- 1 | import path from "path"; 2 | import webpack from "webpack"; 3 | // https://github.com/DefinitelyTyped/DefinitelyTyped/issues/27570#issuecomment-437115227 4 | // eslint-disable-next-line @typescript-eslint/ban-ts-comment 5 | // @ts-ignore 6 | // eslint-disable-next-line @typescript-eslint/no-unused-vars 7 | import * as webpackDevServer from "webpack-dev-server"; 8 | import { commonConfig } from "./webpack.config.common"; 9 | 10 | const config: webpack.Configuration = { 11 | mode: `development`, 12 | devtool: `inline-source-map`, 13 | devServer: { 14 | static: path.join(__dirname, `dist`), 15 | compress: true, 16 | port: 8080, 17 | open: true, 18 | }, 19 | ...commonConfig, 20 | }; 21 | 22 | export default config; 23 | -------------------------------------------------------------------------------- /packages/notion-graph-frontend/webpack/webpack.config.prod.ts: -------------------------------------------------------------------------------- 1 | import webpack from "webpack"; 2 | import { commonConfig } from "./webpack.config.common"; 3 | 4 | const config: webpack.Configuration = { 5 | mode: `production`, 6 | ...commonConfig, 7 | }; 8 | 9 | export default config; 10 | -------------------------------------------------------------------------------- /packages/notion-graph-scraper/.gitignore: -------------------------------------------------------------------------------- 1 | dist -------------------------------------------------------------------------------- /packages/notion-graph-scraper/README.md: -------------------------------------------------------------------------------- 1 | # notion 2 | 3 | ![logo.png](./logo.png) 4 | 5 | ```bash 6 | npm i --save @graphcentral/notion-graph-scraper 7 | ``` 8 | 9 | # `@graphcentral/notion-graph-scraper` 10 | 11 | Scrapes all Notion pages given a root page. For the graphing library itself, visit [graphcentral/graph](https://github.com/graphcentral/graph). 12 | 13 | ## Example 14 | 15 | First, get a public Notion page (if you just want to test it out), preferably with some sub-pages inside it, so that you can get a graph with some nodes. You can also request private pages if you create your own instance of `NotionAPI` from `notion-client`. The example below is only for public pages. 16 | 17 | Then, get the id of the page like below: 18 | 19 | ![doc0.png](./doc0.png) 20 | 21 | Then, input the id of the page as a parameter to `notionGraph.buildGraphFromRootNode` 22 | 23 | ```ts 24 | import { NotionGraph } from "@graphcentral/notion-graph-scraper" 25 | import fs from "fs" 26 | /** 27 | * example of how to use `@graphcentral/notion-graph-scraper` 28 | */ 29 | ;(async () => { 30 | const notionGraph = new NotionGraph({ 31 | maxDiscoverableNodes: 2000, 32 | maxDiscoverableNodesInOtherSpaces: 2000, 33 | verbose: true, 34 | }) 35 | const graph = await notionGraph.buildGraphFromRootNode( 36 | // notion help page 37 | `e040febf70a94950b8620e6f00005004` 38 | ) 39 | console.log(graph.nodes.length) 40 | console.log(graph.links.length) 41 | await new Promise((resolve, reject) => { 42 | fs.writeFile(`test0.json`, JSON.stringify(graph), (err) => { 43 | if (err) reject(err) 44 | else resolve(``) 45 | }) 46 | }); 47 | 48 | process.exit(0) 49 | })() 50 | ``` 51 | 52 | The graph will be an object of this type: 53 | 54 | ```ts 55 | { 56 | nodes: Node[] 57 | links: Link[] 58 | errors: Error[] 59 | } 60 | ``` 61 | 62 | Then, you can directly use this as an input to [@graphcentral/graph](https://github.com/graphcentral/graph) to visualize it on the web as a force layout graph, like below: 63 | 64 | ![doc1.png](./doc1.png) -------------------------------------------------------------------------------- /packages/notion-graph-scraper/doc0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graphcentral/notion/70cd31090c324f36537c0f60160a4c22452f54e3/packages/notion-graph-scraper/doc0.png -------------------------------------------------------------------------------- /packages/notion-graph-scraper/doc1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graphcentral/notion/70cd31090c324f36537c0f60160a4c22452f54e3/packages/notion-graph-scraper/doc1.png -------------------------------------------------------------------------------- /packages/notion-graph-scraper/environment.d.ts: -------------------------------------------------------------------------------- 1 | declare global { 2 | namespace NodeJS { 3 | interface ProcessEnv { 4 | NODE_ENV: `development` | `production`; 5 | NOTION_TOKEN: string; 6 | } 7 | } 8 | } 9 | 10 | // If this file has no import/export statements (i.e. is a script) 11 | // convert it into a module by adding an empty export statement. 12 | export {}; 13 | -------------------------------------------------------------------------------- /packages/notion-graph-scraper/errors.ts: -------------------------------------------------------------------------------- 1 | import { Block, BlockMap } from "./types/block-map"; 2 | 3 | export class Errors { 4 | public static NKG_0000(blockId: string): string { 5 | return `Could not find topmost block from blockId ${blockId}`; 6 | } 7 | public static NKG_0001(block: BlockMap[keyof BlockMap]): string { 8 | return `Root block ${block.value.id} is not an acceptable type: ${ 9 | block.value.type 10 | }. Full block: 11 | ${JSON.stringify(block, null, 2)}`; 12 | } 13 | public static NKG_0002(block: BlockMap[keyof BlockMap]): string { 14 | return `Failed before requesting collectionData because either collection_id or collection_view_id is not available (or both). Full block: 15 | ${JSON.stringify(block, null, 2)}`; 16 | } 17 | public static NKG_0003(propertyMustBeDefined: string): string { 18 | return `Failed to process collection_view. ${propertyMustBeDefined} is not defined.`; 19 | } 20 | public static NKG_0004(collectionId: string): string { 21 | return `Failed to process collection_view. collectionId: ${collectionId} is not a key in collection.`; 22 | } 23 | public static NKG_0005(block: Block): string { 24 | return `Block does not have value?.format?.alias_pointer?.id. 25 | ${JSON.stringify(block, null, 2)}`; 26 | } 27 | public static NKG_0006( 28 | maxDiscoverableNodes: number, 29 | maxDiscoverableNodesInOtherSpaces: number 30 | ): string { 31 | return `Expected maxDiscoverableNodes (${maxDiscoverableNodes}) to be bigger than or equal to maxDiscoverableNodesInOtherSpaces (${maxDiscoverableNodesInOtherSpaces}). This combination of numbers is impossible because at least one node must be from your workspace.`; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /packages/notion-graph-scraper/index.ts: -------------------------------------------------------------------------------- 1 | import { NotionGraph } from "./lib/lib"; 2 | import fs from "fs"; 3 | /** 4 | * example of how to use `@graphcentral/notion-graph-scraper` 5 | */ 6 | (async () => { 7 | const notionGraph = new NotionGraph({ 8 | maxDiscoverableNodes: 300, 9 | maxDiscoverableNodesInOtherSpaces: 300, 10 | verbose: true, 11 | }); 12 | const graph = await notionGraph.buildGraphFromRootNode( 13 | `e040febf70a94950b8620e6f00005004` 14 | ); 15 | console.log(graph.nodes.length); 16 | console.log(graph.links.length); 17 | await new Promise((resolve, reject) => { 18 | fs.writeFile(`test0.json`, JSON.stringify(graph), (err) => { 19 | if (err) reject(err); 20 | else resolve(``); 21 | }); 22 | }); 23 | 24 | process.exit(0); 25 | })(); 26 | -------------------------------------------------------------------------------- /packages/notion-graph-scraper/legacy/official/get-graph-from-root-block.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/ban-ts-comment */ 2 | import { Client } from "@notionhq/client"; 3 | import { 4 | ListBlockChildrenResponse, 5 | QueryDatabaseResponse, 6 | } from "@notionhq/client/build/src/api-endpoints"; 7 | import to from "await-to-js"; 8 | import { RequestQueue } from "../../lib/request-queue"; 9 | import { UndirectedNodesGraph } from "../../lib/undirected-nodes-graph"; 10 | import { NotionContentNode } from "../../types/notion-content-node"; 11 | import { 12 | separateIdWithDashSafe, 13 | identifyObjectTitle, 14 | } from "../../lib/isomorphic-notion-util"; 15 | 16 | function blockTypeToNotionContentNodeType( 17 | blockType: `child_page` | `child_database` 18 | ) { 19 | switch (blockType) { 20 | case `child_database`: 21 | return `database`; 22 | case `child_page`: 23 | return `page`; 24 | default: 25 | return `error`; 26 | } 27 | } 28 | 29 | /** 30 | * 31 | * @param notion notion client 32 | * @param rootBlockId the id of a root page or database 33 | * @returns `null` on error. Otherwise `database` or `page` 34 | */ 35 | async function retrieveRootNode( 36 | notion: Client, 37 | rootBlockId: NotionContentNode[`id`] 38 | ): Promise { 39 | const [err, blockInfo] = await to( 40 | notion.blocks.retrieve({ 41 | block_id: separateIdWithDashSafe(rootBlockId), 42 | }) 43 | ); 44 | 45 | if (err || !blockInfo) { 46 | return null; 47 | } 48 | 49 | return { 50 | type: blockTypeToNotionContentNodeType( 51 | // @ts-ignore: sdk bad typing 52 | blockInfo.type 53 | ), 54 | id: separateIdWithDashSafe(rootBlockId), 55 | title: identifyObjectTitle(blockInfo), 56 | }; 57 | } 58 | 59 | /** 60 | * 61 | * @param notion 62 | * @param parentNode 63 | * @returns `null` on error, otherwise databaseChildren OR pageChildren 64 | */ 65 | async function retrieveDatabaseOrPageChildren( 66 | notion: Client, 67 | parentNode: NotionContentNode 68 | ): Promise<{ 69 | databaseChildren: QueryDatabaseResponse | null; 70 | pageChildren: ListBlockChildrenResponse | null; 71 | } | null> { 72 | let pageChildren: Awaited< 73 | ReturnType 74 | > | null = null; 75 | let databaseChildren: Awaited< 76 | ReturnType 77 | > | null = null; 78 | switch (parentNode.type) { 79 | case `database`: { 80 | const [err, databaseChildrenQueryResult] = await to( 81 | notion.databases.query({ 82 | database_id: separateIdWithDashSafe(parentNode.id), 83 | page_size: 50, 84 | }) 85 | ); 86 | if (databaseChildrenQueryResult) { 87 | databaseChildren = databaseChildrenQueryResult; 88 | } 89 | // if (err) console.log(err) 90 | break; 91 | } 92 | case `page`: { 93 | const [err, pageChildrenListResult] = await to( 94 | notion.blocks.children.list({ 95 | block_id: separateIdWithDashSafe(parentNode.id), 96 | page_size: 50, 97 | }) 98 | ); 99 | if (pageChildrenListResult) { 100 | pageChildren = pageChildrenListResult; 101 | } 102 | // if (err) console.log(err) 103 | } 104 | } 105 | 106 | // something went wrong 107 | if (!databaseChildren && !pageChildren) { 108 | return null; 109 | } 110 | 111 | return { 112 | databaseChildren, 113 | pageChildren, 114 | }; 115 | } 116 | 117 | function createNotionContentNodeFromPageChild( 118 | pageChild: ListBlockChildrenResponse[`results`][0] 119 | ): NotionContentNode { 120 | return { 121 | title: identifyObjectTitle(pageChild), 122 | id: pageChild.id, 123 | // @ts-ignore: sdk doesn't support good typing 124 | type: blockTypeToNotionContentNodeType( 125 | // @ts-ignore: sdk doesn't support good typing 126 | pageChild.type 127 | ), 128 | }; 129 | } 130 | 131 | function createNotionContentNodeFromDatabaseChild( 132 | databaseChild: QueryDatabaseResponse[`results`][0] 133 | ): NotionContentNode { 134 | return { 135 | title: identifyObjectTitle(databaseChild), 136 | id: databaseChild.id, 137 | type: databaseChild.object, 138 | }; 139 | } 140 | 141 | /** 142 | * Notion API currently does not support getting all children of a page at once 143 | * so the only way is to recursively extract all pages and databases from the root block (page or database) 144 | * @param notion Notion client 145 | * @param rootBlockId the id of the root page or database. 146 | * Either format of 1429989fe8ac4effbc8f57f56486db54 or 147 | * 1429989f-e8ac-4eff-bc8f-57f56486db54 are all fine. 148 | * @returns all recurisvely discovered children of the root page 149 | */ 150 | export async function buildGraphFromRootNode( 151 | notion: Client, 152 | rootBlockId: string 153 | ): Promise<{ 154 | nodes: NotionContentNode[]; 155 | links: ReturnType< 156 | UndirectedNodesGraph[`getD3JsEdgeFormat`] 157 | >; 158 | }> { 159 | const rootNode = await retrieveRootNode(notion, rootBlockId); 160 | 161 | if (!rootNode) { 162 | throw new Error(`Error while retrieving rootNode`); 163 | } 164 | const nodes: NotionContentNode[] = [rootNode]; 165 | const nodesGraph = new UndirectedNodesGraph(); 166 | const requestQueue = new RequestQueue({ maxConcurrentRequest: 50 }); 167 | 168 | async function retrieveNodesRecursively(parentNode: NotionContentNode) { 169 | const queryChild = (child: NotionContentNode) => { 170 | requestQueue.enqueue(() => retrieveNodesRecursively(child)); 171 | }; 172 | const processNewBlock = (newBlock: NotionContentNode) => { 173 | nodesGraph.addEdge(parentNode, newBlock); 174 | nodes.push(newBlock); 175 | queryChild(newBlock); 176 | }; 177 | 178 | const databaseOrPageChildren = await retrieveDatabaseOrPageChildren( 179 | notion, 180 | parentNode 181 | ); 182 | 183 | if (!databaseOrPageChildren) { 184 | return; 185 | } 186 | 187 | const { pageChildren, databaseChildren } = databaseOrPageChildren; 188 | 189 | if (pageChildren) { 190 | for (const child of pageChildren.results) { 191 | try { 192 | // @ts-ignore: sdk doesn't support good typing 193 | if (child.type === `child_database` || child.type === `child_page`) { 194 | const newBlock = createNotionContentNodeFromPageChild(child); 195 | processNewBlock(newBlock); 196 | } 197 | } catch (e) { 198 | // console.log(e) 199 | console.log(`e`); 200 | } 201 | } 202 | } else if (databaseChildren) { 203 | for (const child of databaseChildren.results) { 204 | try { 205 | const newBlock = createNotionContentNodeFromDatabaseChild(child); 206 | processNewBlock(newBlock); 207 | } catch (e) { 208 | // console.log(e) 209 | console.log(`e`); 210 | } 211 | } 212 | } 213 | } 214 | 215 | const [err] = await to( 216 | Promise.allSettled([ 217 | retrieveNodesRecursively(rootNode), 218 | new Promise((resolve) => { 219 | requestQueue.onComplete(resolve); 220 | }), 221 | ]) 222 | ); 223 | 224 | if (err) { 225 | throw err; 226 | } 227 | 228 | return { 229 | nodes, 230 | links: nodesGraph.getD3JsEdgeFormat(), 231 | }; 232 | } 233 | -------------------------------------------------------------------------------- /packages/notion-graph-scraper/lib/global-util.ts: -------------------------------------------------------------------------------- 1 | import { to } from "await-to-js"; 2 | import { serializeError } from "serialize-error"; 3 | 4 | /** 5 | * await-to-js wrapper to enable serializing the error to 6 | * a normal javascript object 7 | */ 8 | export async function toEnhanced( 9 | p: Promise 10 | ): Promise< 11 | [ReturnType | Err | null, Result | undefined] 12 | > { 13 | const [err, result] = await to(p); 14 | 15 | return [serializeError(err), result]; 16 | } 17 | 18 | export function debugObject(obj: T) { 19 | console.log(JSON.stringify(obj, null, 2)); 20 | } 21 | -------------------------------------------------------------------------------- /packages/notion-graph-scraper/lib/isomorphic-notion-util.ts: -------------------------------------------------------------------------------- 1 | export function identifyObjectTitle< 2 | Obj extends { object: `database` | `page` | `block` } 3 | >(obj: Obj): string { 4 | const identify = () => { 5 | if (obj.object === `database`) { 6 | // eslint-disable-next-line @typescript-eslint/ban-ts-comment 7 | // @ts-ignore: sdk bad typing 8 | return obj.title?.[0].plain_text; 9 | } else if (obj.object === `page`) { 10 | return ( 11 | // eslint-disable-next-line @typescript-eslint/ban-ts-comment 12 | // @ts-ignore: sdk bad typing 13 | obj.properties?.Name?.title?.[0].plain_text ?? 14 | // eslint-disable-next-line @typescript-eslint/ban-ts-comment 15 | // @ts-ignore: sdk bad typing 16 | obj.properties?.title?.title?.[0].plain_text 17 | ); 18 | } else if (obj.object === `block`) { 19 | // eslint-disable-next-line @typescript-eslint/ban-ts-comment 20 | // @ts-ignore: sdk bad typing 21 | return obj.child_page?.title ?? obj.child_database.title; 22 | } 23 | 24 | throw new Error(`should never get here`); 25 | }; 26 | 27 | return nameUntitledIfEmpty(identify()); 28 | } 29 | 30 | /** 31 | * 32 | * @param maybe_without_dash 1429989fe8ac4effbc8f57f56486db54 33 | * @returns 1429989f-e8ac-4eff-bc8f-57f56486db54 34 | */ 35 | export function separateIdWithDashSafe(maybe_without_dash: string): string { 36 | if (isIdAlreadySeparateByDash(maybe_without_dash)) { 37 | return maybe_without_dash; 38 | } 39 | 40 | if (maybe_without_dash.length != 32) { 41 | throw new Error(`Incorrect length of id: ${maybe_without_dash.length}`); 42 | } 43 | 44 | if (!/^[a-zA-Z0-9]{32}$/.test(maybe_without_dash)) { 45 | throw new Error( 46 | `Incorrect format of id: ${maybe_without_dash}. It must be /^[a-zA-Z0-9]{32}$/` 47 | ); 48 | } 49 | 50 | return `${maybe_without_dash.substring(0, 8)}-${maybe_without_dash.substring( 51 | 8, 52 | 12 53 | )}-${maybe_without_dash.substring(12, 16)}-${maybe_without_dash.substring( 54 | 16, 55 | 20 56 | )}-${maybe_without_dash.substring(20, 32)}`; 57 | } 58 | 59 | export function isIdAlreadySeparateByDash( 60 | maybe_separate_with_dash: string 61 | ): boolean { 62 | if (maybe_separate_with_dash.length !== 36) { 63 | return false; 64 | } 65 | return /^[a-zA-Z0-9]{8}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{12}$/.test( 66 | maybe_separate_with_dash 67 | ); 68 | } 69 | 70 | export function nameUntitledIfEmpty(title: string): string { 71 | if (title === ``) { 72 | return `Untitled`; 73 | } 74 | 75 | return title; 76 | } 77 | -------------------------------------------------------------------------------- /packages/notion-graph-scraper/lib/lib.ts: -------------------------------------------------------------------------------- 1 | export * from "./notion-graph"; 2 | export * from "./isomorphic-notion-util"; 3 | export * from "./undirected-nodes-graph"; 4 | export * from "./request-queue"; 5 | export * from "./unofficial-notion-api-util"; 6 | -------------------------------------------------------------------------------- /packages/notion-graph-scraper/lib/logger.ts: -------------------------------------------------------------------------------- 1 | export function createLogger(isVerbose: boolean) { 2 | if (!isVerbose) return null; 3 | return (msg: string) => { 4 | const date = new Date(); 5 | console.log(`[${date.toLocaleString()}] ${msg}`); 6 | }; 7 | } 8 | -------------------------------------------------------------------------------- /packages/notion-graph-scraper/lib/notion-graph.ts: -------------------------------------------------------------------------------- 1 | import { NotionAPI } from "notion-client"; 2 | import { ErrorObject } from "serialize-error"; 3 | import { Errors } from "../errors"; 4 | import { toEnhanced } from "./global-util"; 5 | import { RequestQueue } from "./request-queue"; 6 | import { separateIdWithDashSafe } from "./isomorphic-notion-util"; 7 | import { Block, BlockMap } from "../types/block-map"; 8 | import { UndirectedNodesGraph } from "./undirected-nodes-graph"; 9 | import { 10 | isNotionContentNodeType, 11 | NotionContentNodeUnofficialAPI, 12 | } from "../types/notion-content-node"; 13 | import { UnofficialNotionAPIUtil } from "./unofficial-notion-api-util"; 14 | import { createLogger } from "./logger"; 15 | 16 | /** 17 | * Graph of notion blocks. 18 | * 19 | * Does a lot of async calls, 20 | * so it's possible that async error occurs anywhere. 21 | * 22 | * The key to handling the error is how we approach the UX in the frontend 23 | * regarding the error. What will we do for the user when we encounter an error? 24 | * 25 | * - Case 1: Some of the blocks are missing, but it's still viewable by the user. Then still include the errors and error messages from the server, but show the contents 26 | * - Case 2: All blocks are missing (ex. internet not working for some reason from the server). Then send a complete error, possibly with a helpful if message if any 27 | */ 28 | export class NotionGraph { 29 | private unofficialNotionAPI: NotionAPI; 30 | private errors: (ErrorObject | Error)[] = []; 31 | private nodes: Record< 32 | NotionContentNodeUnofficialAPI[`id`], 33 | NotionContentNodeUnofficialAPI 34 | > = {}; 35 | /** 36 | * total number of discovered, unique nodes 37 | */ 38 | private nodesLength = 0; 39 | /** 40 | * total number of discovered, unique nodes in other spaces (= Notion workspace). 41 | */ 42 | private otherSpacesNodesLength = 0; 43 | /** 44 | * represents a graph of nodes. 45 | * contains info about how nodes are connected by edges 46 | */ 47 | private nodesGraph = 48 | new UndirectedNodesGraph(); 49 | /** 50 | * @see NotionGraph['constructor'] 51 | */ 52 | private maxDiscoverableNodes: number; 53 | /** 54 | * @see NotionGraph['constructor'] 55 | */ 56 | private maxDiscoverableNodesInOtherSpaces: number; 57 | /** 58 | * @see NotionGraph['constructor'] 59 | */ 60 | private maxConcurrentRequest: number; 61 | private lastRequestTimeoutMs: number; 62 | private verbose = true; 63 | private logger: ReturnType; 64 | 65 | constructor({ 66 | unofficialNotionAPI = new NotionAPI(), 67 | maxDiscoverableNodes = 500, 68 | maxDiscoverableNodesInOtherSpaces = 250, 69 | maxConcurrentRequest = 35, 70 | lastRequestTimeoutMs = 15_000, 71 | verbose = true, 72 | }: { 73 | /** 74 | * If you want to use your `NotionAPI` instance, you can do it yourself 75 | * ```ts 76 | * import { NotionAPI } from "notion-client" 77 | * 78 | * const notionUnofficialClient = new NotionAPI({ ...customConfig }) 79 | * const ng = new NotionGraph({ unofficialNotionAPI: notionUnofficialClient, ... }) 80 | * ``` 81 | * 82 | * Otherwise, leave this field as undefined. This means that 83 | * you are only going to be able to request public pages on Notion. 84 | */ 85 | unofficialNotionAPI?: NotionAPI; 86 | /** 87 | * user-defined value of maximum discoverable number of unique nodes. 88 | * must stop discovery once the program finds nodes over 89 | * the discoverable number of unique nodes set by the user. 90 | * 91 | * this is useful when your notion workspace 92 | * (or 'space' as it is from the actual notion API) 93 | * has lots of pages and you want to stop before the amount of pages you accumulate 94 | * becomes too many. 95 | * 96 | * setting it to null means infinity, but it's never 97 | * recommended because you can't guarantee how long it will take to discover 98 | * all nodes. Make sure you know what you are doing if you set it to `null`. 99 | * 100 | * @throws when this is smaller than maxDiscoverableNodes, which is impossible to happen 101 | * @default 500 nodes. 102 | */ 103 | maxDiscoverableNodes?: number | null; 104 | /** 105 | * This parameter is only needed due to the existence of backlinks 106 | * (= 'link to page' function on Notion). 107 | * 108 | * If your page happens to have lots of backlinks to OTHER `space`s (= workspaces) out of your 109 | * current space, then you would need to decide if you will try to discover 110 | * the nodes from those spaces as well. 111 | * 112 | * Otherwise, in the worst case, the program may not halt because 113 | * a chain of Notion nodes that include many backlinks to other workspaces 114 | * will probably take hours or days to crawl all of the nodes. 115 | * 116 | * if you don't need pages or databases outside your workspace, 117 | * simply set this to 0. 118 | * 119 | * @throws when this is bigger than maxDiscoverableNodes, which is impossible to happen 120 | * @default 250 nodes 121 | */ 122 | maxDiscoverableNodesInOtherSpaces?: number; 123 | /** 124 | * # network requests to be sent the same time. 125 | * if too big it might end up causing some delay 126 | * @default 35 127 | */ 128 | maxConcurrentRequest?: number; 129 | /** 130 | * 131 | * @deprecated 132 | * will be removed later. 133 | * there is a better way to handle the last request without this. 134 | * 135 | * If `maxDiscoverableNodes` is bigger than the total pages discovered 136 | * and there are no more request in the duration of `lastRequestTimeoutMs`, 137 | * the program exits by then 138 | */ 139 | lastRequestTimeoutMs?: number; 140 | /** 141 | * If set as true, will output progress as it scrapes pages 142 | */ 143 | verbose?: boolean; 144 | }) { 145 | if ( 146 | maxDiscoverableNodes !== null && 147 | maxDiscoverableNodes < maxDiscoverableNodesInOtherSpaces 148 | ) { 149 | throw new Error( 150 | Errors.NKG_0006(maxDiscoverableNodes, maxDiscoverableNodesInOtherSpaces) 151 | ); 152 | } 153 | this.unofficialNotionAPI = unofficialNotionAPI; 154 | this.maxDiscoverableNodes = maxDiscoverableNodes ?? Infinity; 155 | this.maxConcurrentRequest = maxConcurrentRequest; 156 | this.maxDiscoverableNodesInOtherSpaces = maxDiscoverableNodesInOtherSpaces; 157 | this.lastRequestTimeoutMs = lastRequestTimeoutMs; 158 | this.verbose = verbose; 159 | this.logger = createLogger(verbose); 160 | } 161 | 162 | private accumulateError(err: ErrorObject | Error) { 163 | this.errors.push(err); 164 | } 165 | 166 | /** 167 | * Finds the topmost block from any block id. 168 | * Notion API is structured in a way that any call to a getPage 169 | * would return its recursive parents in its response. 170 | * The last recursive parent will be the topmost block. 171 | * @param blockIdWithoutDash 172 | * @returns `null` if an error happens or nothing is found 173 | * @throws nothing 174 | */ 175 | private async findTopmostBlock( 176 | blockIdWithoutDash: string 177 | ): Promise { 178 | const [err, page] = await toEnhanced( 179 | this.unofficialNotionAPI.getPage(blockIdWithoutDash) 180 | ); 181 | 182 | if (err || !page) { 183 | if (err) { 184 | this.errors.push(err); 185 | this.errors.push(new Error(Errors.NKG_0000(blockIdWithoutDash))); 186 | } 187 | return null; 188 | } 189 | 190 | const topmostBlock = Object.values(page.block).find( 191 | (b) => 192 | // the block itself or the block that has parent as a 'space' 193 | b.value.id === separateIdWithDashSafe(blockIdWithoutDash) || 194 | UnofficialNotionAPIUtil.isBlockToplevelPageOrCollectionViewPage(b) 195 | ); 196 | 197 | if (!topmostBlock) { 198 | this.errors.push(new Error(Errors.NKG_0000(blockIdWithoutDash))); 199 | return null; 200 | } 201 | 202 | return topmostBlock; 203 | } 204 | 205 | private addDiscoveredNode({ 206 | childNode, 207 | parentNode, 208 | requestQueue, 209 | rootBlockSpaceId, 210 | }: { 211 | childNode: NotionContentNodeUnofficialAPI; 212 | parentNode: NotionContentNodeUnofficialAPI; 213 | requestQueue: RequestQueue; 214 | rootBlockSpaceId: string | undefined; 215 | }) { 216 | if (!(childNode.id in this.nodes)) { 217 | this.nodesLength += 1; 218 | } 219 | 220 | this.nodes[childNode.id] = childNode; 221 | this.nodesGraph.addEdge(childNode, parentNode); 222 | if (parentNode.cc) parentNode.cc += 1; 223 | else parentNode.cc = 1; 224 | requestQueue.incrementExternalRequestMatchCount(); 225 | requestQueue.enqueue(() => 226 | this.recursivelyDiscoverBlocks({ 227 | rootBlockSpaceId, 228 | requestQueue, 229 | // now childnode will become a parent of other nodes 230 | parentNode: childNode, 231 | }) 232 | ); 233 | } 234 | 235 | /** 236 | * Notion API has a weird structure 237 | * where you can't get the database(=collection)'s title at once if it is a child of a page in the response. 238 | * You need to request the database as a parent directly again. 239 | * This function just uses the second response to update the database's title 240 | * @param page 241 | * @param parentNode 242 | * @returns nothing 243 | */ 244 | private addCollectionViewTitleInNextRecursiveCall( 245 | page: Awaited>, 246 | parentNode: NotionContentNodeUnofficialAPI 247 | ): void { 248 | if ( 249 | parentNode.type !== `collection_view` && 250 | parentNode.type !== `collection_view_page` 251 | ) 252 | return; 253 | 254 | const blocks = page.block; 255 | const collection = page.collection; 256 | // this contains the id of the 'collection' (not 'collection_view') 257 | // 'collection' contains the name of the database, which is what we want 258 | const collectionViewBlock = blocks[parentNode.id]; 259 | // use this to get the collection (database) title 260 | const collectionId: string | undefined = 261 | // eslint-disable-next-line @typescript-eslint/ban-ts-comment 262 | // @ts-ignore 263 | collectionViewBlock.value.collection_id; 264 | // extra careful output typing 265 | if (!collection) 266 | this.accumulateError(new Error(Errors.NKG_0003(`collection`))); 267 | if (!collectionId) 268 | this.accumulateError(new Error(Errors.NKG_0003(`collectionId`))); 269 | if (!collectionViewBlock) 270 | this.accumulateError(new Error(Errors.NKG_0003(`collectionViewBlock`))); 271 | if (collectionId && collection && !(collectionId in collection)) { 272 | this.accumulateError(new Error(Errors.NKG_0004(collectionId))); 273 | } 274 | 275 | if ( 276 | collection && 277 | collectionId && 278 | collectionViewBlock && 279 | collectionId in collection 280 | ) { 281 | // { 282 | // ... 283 | // "collection": { 284 | // "58e7440f-fad4-4a30-9de3-2dc5f5673b62": { 285 | // "role": "reader", 286 | // "value": { 287 | // "id": "58e7440f-fad4-4a30-9de3-2dc5f5673b62", 288 | // "version": 14, 289 | // "name": [ 290 | // [ 291 | // "Database-test" 292 | // ] 293 | // ], 294 | // @ts-ignore: wrong library typing 295 | const collectionBlock: Block = collection[collectionId]; 296 | const title = 297 | UnofficialNotionAPIUtil.getTitleFromCollectionBlock(collectionBlock); 298 | // just being extra careful 299 | if (parentNode.id in this.nodes) { 300 | this.nodes[parentNode.id].title = title; 301 | } 302 | } 303 | } 304 | 305 | /** 306 | * Recursively discovers notion blocks (= nodes). 307 | * 308 | * All block types we care about is `NotionContentNodeUnofficialAPI['type']`. 309 | * Check it out. 310 | */ 311 | public async recursivelyDiscoverBlocks({ 312 | rootBlockSpaceId, 313 | parentNode, 314 | requestQueue, 315 | }: { 316 | rootBlockSpaceId: string | undefined; 317 | parentNode: NotionContentNodeUnofficialAPI; 318 | requestQueue: RequestQueue; 319 | }): Promise { 320 | const [err, page] = await toEnhanced( 321 | // getPageRaw must NOT be used 322 | // as it returns insufficient information 323 | this.unofficialNotionAPI.getPage(parentNode.id) 324 | ); 325 | if (err) this.accumulateError(err); 326 | if (!page) return; 327 | // if the parent node was collection_view, 328 | // the response must contain `collection` and `collection_view` keys 329 | if ( 330 | parentNode.type === `collection_view` || 331 | parentNode.type === `collection_view_page` 332 | ) { 333 | this.addCollectionViewTitleInNextRecursiveCall(page, parentNode); 334 | } 335 | 336 | for (const selfOrChildBlockId of Object.keys(page.block)) { 337 | // if the number of discovered nodes is more 338 | // than we want 339 | if ( 340 | this.maxDiscoverableNodes && 341 | this.nodesLength > this.maxDiscoverableNodes 342 | ) { 343 | requestQueue.setNoMoreRequestEnqueued(); 344 | return; 345 | } 346 | const childBlock = page.block[selfOrChildBlockId]; 347 | 348 | // somtimes .value is undefined for some reason 349 | if (!childBlock || !childBlock.value) { 350 | continue; 351 | } 352 | 353 | const childBlockType = page.block[selfOrChildBlockId].value.type; 354 | const spaceId = page.block[selfOrChildBlockId].value.space_id; 355 | /** 356 | * Ignore unwanted content type 357 | */ 358 | if (!isNotionContentNodeType(childBlockType)) continue; 359 | /** 360 | * Ignore the block itself returned inside the response 361 | * or the block that has already been discovered 362 | */ 363 | if ( 364 | selfOrChildBlockId === separateIdWithDashSafe(parentNode.id) || 365 | selfOrChildBlockId in this.nodes 366 | ) { 367 | continue; 368 | } 369 | /** 370 | * If spaceId is undefined, we can't proceed anyway 371 | * not sure if this typing from the sdk is correct 372 | * it seems that spaceId is always defined for 373 | * the node types we use though (`NotionContentNodeUnofficialAPI['type']`) 374 | */ 375 | if (!spaceId) { 376 | continue; 377 | } 378 | 379 | if (childBlockType !== `alias` && spaceId !== rootBlockSpaceId) { 380 | /** 381 | * If there are too many nodes (except 'alias' because it's not technically a page) discovered from 382 | * other spaces, ignore them 383 | */ 384 | if ( 385 | this.otherSpacesNodesLength > this.maxDiscoverableNodesInOtherSpaces 386 | ) { 387 | continue; 388 | } 389 | this.otherSpacesNodesLength += 1; 390 | } 391 | 392 | const childBlockId = selfOrChildBlockId; 393 | switch (childBlockType) { 394 | // for alias, don't add another block 395 | // just add edges between the pages 396 | case `alias`: { 397 | const aliasedBlockId = childBlock.value?.format?.alias_pointer?.id; 398 | const aliasedBlockSpaceId = 399 | childBlock.value?.format?.alias_pointer?.spaceId; 400 | if (aliasedBlockId) { 401 | this.nodesGraph.addEdgeByIds(parentNode.id, aliasedBlockId); 402 | // if aliased block id is in another space, 403 | // need to request that block separately 404 | // because it is not going to be discovered 405 | if (aliasedBlockSpaceId !== rootBlockSpaceId) { 406 | // @todo 407 | } 408 | } else { 409 | this.errors.push(new Error(Errors.NKG_0005(childBlock))); 410 | } 411 | break; 412 | } 413 | case `collection_view`: { 414 | const childNode: NotionContentNodeUnofficialAPI = { 415 | // title will be known in the next request 416 | title: `Unknown database title`, 417 | id: childBlockId, 418 | spaceId, 419 | parentId: parentNode.id, 420 | type: childBlockType, 421 | }; 422 | this.addDiscoveredNode({ 423 | childNode, 424 | parentNode, 425 | requestQueue, 426 | rootBlockSpaceId, 427 | }); 428 | break; 429 | } 430 | case `collection_view_page`: { 431 | const childNode: NotionContentNodeUnofficialAPI = { 432 | // title will be known in the next request 433 | title: `Unknown database page title`, 434 | id: childBlockId, 435 | collection_id: 436 | // @ts-ignore 437 | childBlock.value.collection_id, 438 | parentId: parentNode.id, 439 | spaceId, 440 | type: childBlockType, 441 | }; 442 | this.addDiscoveredNode({ 443 | childNode, 444 | parentNode, 445 | requestQueue, 446 | rootBlockSpaceId, 447 | }); 448 | break; 449 | } 450 | case `page`: { 451 | const title = 452 | UnofficialNotionAPIUtil.getTitleFromPageBlock(childBlock); 453 | const spaceId = childBlock.value.space_id ?? `Unknown space id`; 454 | const typeSafeChildNode = { 455 | id: childBlockId, 456 | parentId: parentNode.id, 457 | spaceId, 458 | type: childBlockType, 459 | title, 460 | }; 461 | this.addDiscoveredNode({ 462 | childNode: typeSafeChildNode, 463 | parentNode, 464 | requestQueue, 465 | rootBlockSpaceId, 466 | }); 467 | break; 468 | } 469 | } 470 | } 471 | } 472 | 473 | /** 474 | * Builds a graph from a node (also called a page or block in Notion) 475 | * 476 | * You can easily find the page's block id from the URL. 477 | * The sequence of last 32 characters in the URL is the block id. 478 | * For example: 479 | * 480 | * https://my.notion.site/My-Page-Title-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx 481 | * 482 | * `xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx` is the block id. 483 | * @param rootBlockId the block id of the page, favorably the root page in your workspace 484 | * so that as many pages as possible will be discovered. 485 | * @returns graph information relevant to frontend's graph visualization 486 | */ 487 | public async buildGraphFromRootNode(rootBlockId: string): Promise<{ 488 | nodes: NotionContentNodeUnofficialAPI[]; 489 | links: ReturnType< 490 | UndirectedNodesGraph[`getD3JsEdgeFormat`] 491 | >; 492 | errors: (ErrorObject | Error)[]; 493 | }> { 494 | const defaultReturn = { 495 | links: [], 496 | nodes: Object.values(this.nodes), 497 | errors: this.errors, 498 | }; 499 | const requestQueue = new RequestQueue({ 500 | maxRequestCount: this.maxDiscoverableNodes, 501 | maxConcurrentRequest: this.maxConcurrentRequest, 502 | lastRequestTimeoutMs: this.lastRequestTimeoutMs, 503 | logger: this.logger, 504 | }); 505 | 506 | const topMostBlock = await this.findTopmostBlock(rootBlockId); 507 | 508 | if (!topMostBlock) { 509 | return defaultReturn; 510 | } 511 | 512 | const typeSafeRootBlockNode = 513 | UnofficialNotionAPIUtil.extractTypeUnsafeNotionContentNodeFromBlock( 514 | topMostBlock 515 | ); 516 | 517 | const rootBlockSpaceId = topMostBlock.value.space_id; 518 | if (!typeSafeRootBlockNode) { 519 | this.errors.push(new Error(Errors.NKG_0001(topMostBlock))); 520 | return defaultReturn; 521 | } 522 | 523 | this.nodes[typeSafeRootBlockNode.id] = typeSafeRootBlockNode; 524 | await toEnhanced( 525 | Promise.allSettled([ 526 | this.recursivelyDiscoverBlocks({ 527 | // @ts-ignore: todo fix this (the topmost block can be a collection_view_page) 528 | parentNode: typeSafeRootBlockNode, 529 | rootBlockSpaceId, 530 | requestQueue, 531 | }), 532 | new Promise((resolve) => requestQueue.onComplete(resolve)), 533 | ]) 534 | ); 535 | 536 | // edges may contain undiscovered nodes 537 | // so remove them 538 | const links = this.nodesGraph 539 | .getD3JsEdgeFormat() 540 | .filter( 541 | ({ source, target }) => source in this.nodes && target in this.nodes 542 | ); 543 | 544 | return { 545 | nodes: Object.values(this.nodes), 546 | links, 547 | errors: this.errors, 548 | }; 549 | } 550 | } 551 | -------------------------------------------------------------------------------- /packages/notion-graph-scraper/lib/request-queue.ts: -------------------------------------------------------------------------------- 1 | import to from "await-to-js"; 2 | import EventEmitter from "events"; 3 | import { createLogger } from "./logger"; 4 | 5 | /** 6 | * https://developers.notion.com/reference/request-limits 7 | * Notion API has a request limit 8 | * also, although we are using unofficial API, 9 | * it's a good idea not to overload the server too much anyway 10 | * So use a simple request queue to control number of concurrent requests 11 | */ 12 | export class RequestQueue { 13 | private queue: (() => Promise)[] = []; 14 | private responses: (Res | Err)[] = []; 15 | private currentRequestCount = 0; 16 | /** 17 | * Once you reach max request count, 18 | * it will send no more requests 19 | */ 20 | private maxRequestCount = Infinity; 21 | private maxConcurrentRequest = -1; 22 | private eventEmitter = new EventEmitter(); 23 | private lastRequestTimeoutMs: number; 24 | private intervalId: NodeJS.Timer | null = null; 25 | private hasNoMoreRequestEnqueued = false; 26 | private externalSuccessfulRequestCount = 0; 27 | private totalSuccessfulRequestCount = 0; 28 | private logger: ReturnType; 29 | constructor({ 30 | maxConcurrentRequest, 31 | maxRequestCount = Infinity, 32 | lastRequestTimeoutMs = 15_000, 33 | logger, 34 | }: { 35 | maxConcurrentRequest: number; 36 | /** 37 | * default: 15000 ms 38 | */ 39 | lastRequestTimeoutMs?: number; 40 | maxRequestCount?: number; 41 | logger: ReturnType; 42 | }) { 43 | if (maxConcurrentRequest <= 0) { 44 | throw new Error(`maxConcurrentRequest must be bigger than 0`); 45 | } 46 | this.maxConcurrentRequest = maxConcurrentRequest; 47 | this.lastRequestTimeoutMs = lastRequestTimeoutMs; 48 | this.maxRequestCount = maxRequestCount; 49 | this.logger = logger; 50 | this.checkAndSendRequest(); 51 | } 52 | 53 | private terminate() { 54 | this.eventEmitter.emit(`complete`, this.responses); 55 | if (this.intervalId) clearInterval(this.intervalId); 56 | } 57 | 58 | private terminateIfPossible() { 59 | if (this.currentRequestCount === 0 && this.queue.length === 0) { 60 | this.terminate(); 61 | } 62 | } 63 | 64 | /** 65 | * This function is used to periodically check the number of concurrent 66 | * requests at a time and send the request if the number of concurrent requests 67 | * is less than `maxConcurrentRequest`. 68 | * 69 | * If there are no more requests to send, it will emit `complete` event and terminate. 70 | */ 71 | private checkAndSendRequest() { 72 | let timeoutIds: NodeJS.Timeout[] = []; 73 | let totalRequestCount = 0; 74 | const run = () => { 75 | // wait until external requests finish 76 | if ( 77 | this.externalSuccessfulRequestCount !== 0 && 78 | this.totalSuccessfulRequestCount !== 0 && 79 | this.externalSuccessfulRequestCount < this.totalSuccessfulRequestCount 80 | ) { 81 | return; 82 | } 83 | this.logger?.( 84 | `# current requests: ${this.currentRequestCount} / # items in the queue: ${this.queue.length}` 85 | ); 86 | this.logger?.(`# total requests sent: ${totalRequestCount}`); 87 | this.logger?.( 88 | `# total successful requests: ${this.externalSuccessfulRequestCount}` 89 | ); 90 | if ( 91 | // only care out external, since 92 | // `this.totalSuccessfulRequestCount` is the items in the current queue. 93 | // it may be the case that the external requests have already achieved `maxRequestCount` items, 94 | // which means there is no point of continuing already 95 | this.externalSuccessfulRequestCount >= this.maxRequestCount 96 | ) { 97 | this.terminate(); 98 | } 99 | if ( 100 | !(this.currentRequestCount === 0 && this.queue.length === 0) && 101 | this.currentRequestCount < this.maxConcurrentRequest 102 | ) { 103 | while (this.currentRequestCount < this.maxConcurrentRequest) { 104 | if (this.externalSuccessfulRequestCount >= this.maxRequestCount) { 105 | this.queue = []; 106 | this.currentRequestCount = 0; 107 | this.hasNoMoreRequestEnqueued = true; 108 | break; 109 | } 110 | ++totalRequestCount; 111 | 112 | timeoutIds.forEach((id) => clearTimeout(id)); 113 | timeoutIds = []; 114 | this.sendRequest() 115 | .catch((err: Err) => { 116 | this.responses.push(err); 117 | }) 118 | .then((res) => { 119 | if (res) this.responses.push(res); 120 | }) 121 | .finally(() => { 122 | ++this.totalSuccessfulRequestCount; 123 | --this.currentRequestCount; 124 | // if it is clear that no more requests will be enqueued, 125 | // check if the function can end right away 126 | if (this.hasNoMoreRequestEnqueued) { 127 | this.terminateIfPossible(); 128 | } 129 | timeoutIds.push( 130 | setTimeout(() => { 131 | // if things seem to be completed, check again after 1 second, 132 | // and if it is empty, that means new request has not been sent anymore 133 | // which means every request has been sent and there's no more work to do 134 | this.terminateIfPossible(); 135 | }, this.lastRequestTimeoutMs) 136 | ); 137 | }); 138 | ++this.currentRequestCount; 139 | } 140 | } 141 | }; 142 | run(); 143 | this.intervalId = setInterval(run, 10); 144 | } 145 | 146 | private async sendRequest(): Promise { 147 | const req = this.queue.shift(); 148 | 149 | if (req === undefined) { 150 | return null; 151 | } 152 | const [err, res] = await to(req()); 153 | 154 | if (res === undefined || err !== null) { 155 | return err; 156 | } 157 | 158 | return res; 159 | } 160 | 161 | /** 162 | * Let RequestQueue know that there is going to be no more 163 | * request input from the user. 164 | * 165 | * This is important because RequestQueue will be able to quit 166 | * immediately after the last request completes knowing that 167 | * no more requests will be enqueued. 168 | */ 169 | public setNoMoreRequestEnqueued() { 170 | this.hasNoMoreRequestEnqueued = true; 171 | } 172 | 173 | /** 174 | * User only has to enqueue his request here and RequestQueue will take 175 | * care of the rest. 176 | * @param retriveBlockRequestFn 177 | * any function that returns a promise (i.e. sends an async request) 178 | */ 179 | public enqueue(retriveBlockRequestFn: () => Promise) { 180 | if (this.hasNoMoreRequestEnqueued) return; 181 | this.queue.push(retriveBlockRequestFn); 182 | } 183 | 184 | /** 185 | * @param listener any callback to be called when RequestQueue finishes its work 186 | * and meaning that the queue is empty 187 | */ 188 | public onComplete void>(listener: Fn) { 189 | this.eventEmitter.on(`complete`, listener); 190 | 191 | this.queue = []; 192 | this.responses = []; 193 | } 194 | 195 | public incrementExternalRequestMatchCount(increaseBy = 1) { 196 | if (increaseBy <= 0) return; 197 | this.externalSuccessfulRequestCount += increaseBy; 198 | } 199 | } 200 | -------------------------------------------------------------------------------- /packages/notion-graph-scraper/lib/undirected-nodes-graph.ts: -------------------------------------------------------------------------------- 1 | import { NotionContentNode } from "../types/notion-content-node"; 2 | import { DeepReadonly } from "../types/util-types"; 3 | 4 | /** 5 | * represents the graph of nodes. 6 | * 7 | * example: 8 | * ``` 9 | * { 10 | * [node_id0]: { 11 | * [node_id1]: true, 12 | * [node_id2]: true, 13 | * }, 14 | * [node_id1]: { 15 | * [node_id3]: true 16 | * } 17 | * } 18 | * ``` 19 | * this means node_id0 is connected with node_id1 and node_id2, and 20 | * node_id1 with node_id3. This is an undirected graph; Therfore, there 21 | * must not be any duplicate edge in this data structure. 22 | * 23 | * For example, 24 | * ``` 25 | * { 26 | * [node_id0]: { 27 | * [node_id1]: true, 28 | * [node_id2]: true, 29 | * }, 30 | * [node_id1]: { 31 | * [node_id0]: true 32 | * } 33 | * } 34 | * ``` 35 | * Such a graph must not be made because edge(node_id1, node_id0) exists twice. 36 | * 37 | * This kind of data structure is used in an effort to efficiently create the graph 38 | */ 39 | export type RawUndirectedNodesGraph = Record< 40 | NotionContentNode[`id`], 41 | Record | undefined 42 | >; 43 | 44 | /** 45 | * d3.js uses `source: ... target: ...` format to abstract 46 | * the concept of an edge. 47 | */ 48 | interface D3JsEdge { 49 | source: NotionContentNode[`id`]; 50 | target: NotionContentNode[`id`]; 51 | } 52 | 53 | /** 54 | * Represents an undirected graph of nodes. 55 | */ 56 | export class UndirectedNodesGraph< 57 | Node extends { id: NotionContentNode[`id`] } 58 | > { 59 | private graph: RawUndirectedNodesGraph = {}; 60 | private nodesLength = 0; 61 | 62 | /** 63 | * Adds an edge between two nodes, but avoids making duplicates 64 | * if the edge already exists 65 | * 66 | * Existence of an edge at a time cannot guarantee the existence 67 | * of a vertex stored in another data structure (it depends on your implementation) 68 | */ 69 | public addEdgeByIds(node0id: string, node1id: string) { 70 | // node0 may already have node1, but we just update it anyway 71 | let node0Edges = this.graph[node0id]; 72 | const node1Edges = this.graph[node1id]; 73 | // check if edge already exists in node1Edges 74 | if (node1Edges && node1Edges[node0id]) { 75 | // if edge already exists, return 76 | return; 77 | } 78 | // otherwise, add a new edge to node0 79 | if (!node0Edges) node0Edges = {}; 80 | node0Edges[node1id] = true; 81 | 82 | this.graph[node0id] = node0Edges; 83 | this.nodesLength += 1; 84 | } 85 | /** 86 | * Adds an edge between two nodes, but avoids making duplicates 87 | * if the edge already exists 88 | * 89 | * usually, `target` will be stored as parent of a node 90 | * `source` will be the child node 91 | */ 92 | public addEdge(node0: Node, node1: Node): void { 93 | this.addEdgeByIds(node0.id, node1.id); 94 | } 95 | 96 | public getGraph(): DeepReadonly { 97 | return this.graph; 98 | } 99 | 100 | /** 101 | * transform `this.graph` to d3.js edge (link) shape. 102 | * The output will be used directly in frontend. 103 | * 104 | */ 105 | public getD3JsEdgeFormat(): DeepReadonly { 106 | const d3JsEdges: D3JsEdge[] = []; 107 | Object.keys(this.graph).map((nodeId) => { 108 | const edges = this.graph[nodeId]; 109 | if (!edges) return; 110 | for (const edge of Object.keys(edges)) { 111 | d3JsEdges.push({ 112 | source: nodeId, 113 | target: edge, 114 | }); 115 | } 116 | }); 117 | 118 | return d3JsEdges; 119 | } 120 | 121 | public get length(): number { 122 | return this.length; 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /packages/notion-graph-scraper/lib/unofficial-notion-api-util.ts: -------------------------------------------------------------------------------- 1 | import { nameUntitledIfEmpty } from "./isomorphic-notion-util"; 2 | import { Block, BlockMap } from "../types/block-map"; 3 | import { 4 | isNotionContentNodeType, 5 | NotionContentNodeUnofficialAPI, 6 | } from "../types/notion-content-node"; 7 | 8 | /** 9 | * Utils specific to unofficial notion api 10 | */ 11 | export class UnofficialNotionAPIUtil { 12 | /** 13 | * Be as conservative as possible because 14 | * Notion API may change any time 15 | * @param page 16 | * @returns 17 | */ 18 | public static getTitleFromPageBlock(page: BlockMap[keyof BlockMap]): string { 19 | const { properties } = page.value; 20 | 21 | // if a page is untitled, properties does not exist at all 22 | if (!properties) { 23 | return `Untitled`; 24 | } 25 | 26 | const title = properties?.title?.[0]?.[0]; 27 | 28 | if (title === undefined || title === null) { 29 | return `Untitled`; 30 | } 31 | 32 | return nameUntitledIfEmpty(title); 33 | } 34 | 35 | public static getTitleFromCollectionBlock(collectionBlock: Block): string { 36 | const name: string | undefined = 37 | // eslint-disable-next-line @typescript-eslint/ban-ts-comment 38 | // @ts-ignore 39 | collectionBlock.value?.name?.[0]?.[0]; 40 | 41 | if (name === undefined || name === null) { 42 | return `Unknown database title`; 43 | } 44 | 45 | return name; 46 | } 47 | /** 48 | * `parent_table` is space if the block is at the top level of all pages (= the block is 49 | * one of the pages we can click on from the left navigation panel on Notion app) 50 | * @param block any block (page, collection view page, ...) 51 | * @returns whether the block is a top level page or collection view page. 52 | */ 53 | public static isBlockToplevelPageOrCollectionViewPage( 54 | block: BlockMap[keyof BlockMap] 55 | ): boolean { 56 | // parent_table can be undefined 57 | return block.value?.parent_table === `space`; 58 | } 59 | 60 | /** 61 | * Gets a notion content node from a block. 62 | * But the type could be anything. It's not one of the four block types we want yet. 63 | * @deprecated don't use this except for finding the root block 64 | */ 65 | public static extractTypeUnsafeNotionContentNodeFromBlock( 66 | block: BlockMap[keyof BlockMap] 67 | ): null | NotionContentNodeUnofficialAPI { 68 | const title = UnofficialNotionAPIUtil.getTitleFromPageBlock(block); 69 | const type = block.value.type; 70 | const spaceId = block.value.space_id ?? `Unknown space id`; 71 | 72 | if (!isNotionContentNodeType(type)) return null; 73 | 74 | if (type === `collection_view_page`) { 75 | if (!block.value.collection_id) return null; 76 | 77 | return { 78 | id: block.value.id, 79 | title, 80 | type, 81 | spaceId, 82 | parentId: `none`, 83 | collection_id: block.value.collection_id, 84 | }; 85 | } 86 | 87 | return { 88 | id: block.value.id, 89 | title, 90 | type, 91 | spaceId, 92 | parentId: `none`, 93 | }; 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /packages/notion-graph-scraper/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graphcentral/notion/70cd31090c324f36537c0f60160a4c22452f54e3/packages/notion-graph-scraper/logo.png -------------------------------------------------------------------------------- /packages/notion-graph-scraper/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@graphcentral/notion-graph-scraper", 3 | "version": "0.1.0-rc.6", 4 | "description": "scrapes pages from Notion given a root page", 5 | "main": "dist/lib.mjs", 6 | "types": "dist/lib.d.ts", 7 | "files": ["dist"], 8 | "bugs": { 9 | "url": "https://github.com/graphcentral/notion/issues" 10 | }, 11 | "repository": { 12 | "type": "git", 13 | "url": "git+https://github.com/graphcentral/notion.git" 14 | }, 15 | "scripts": { 16 | "dev": "tsup index.ts --onSuccess \"node --loader=ts-node/esm dist/index.mjs\"", 17 | "compile": "tsup index.ts", 18 | "run-compiled": "node --loader=ts-node/esm dist/index.mjs", 19 | "package": "rm -rf dist && tsup ./lib/lib.ts" 20 | }, 21 | "keywords": ["notion", "graph", "notion-graph", "scraper", "obsidian", "knowledge-graph"], 22 | "author": "9oelM", 23 | "license": "MIT", 24 | "dependencies": { 25 | "@notionhq/client": "^1.0.4", 26 | "await-to-js": "^3.0.0", 27 | "dotenv": "^16.0.1", 28 | "notion-client": "^6.12.9", 29 | "serialize-error": "^11.0.0" 30 | }, 31 | "devDependencies": { 32 | "@babel/core": "^7.18.2", 33 | "@babel/preset-env": "^7.18.2", 34 | "esbuild-plugin-babel": "^0.2.3", 35 | "tsup": "^6.1.2", 36 | "typescript": "^4.7.3" 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /packages/notion-graph-scraper/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | /* Basic Options */ 4 | // "incremental": true, /* Enable incremental compilation */ 5 | "target": "es6", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019' or 'ESNEXT'. */ 6 | "module": "esnext", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ 7 | // "lib": [], /* Specify library files to be included in the compilation. */ 8 | // "allowJs": true, /* Allow javascript files to be compiled. */ 9 | // "checkJs": true, /* Report errors in .js files. */ 10 | // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */ 11 | // "declaration": true, /* Generates corresponding '.d.ts' file. */ 12 | // "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */ 13 | // "sourceMap": true, /* Generates corresponding '.map' file. */ 14 | // "outFile": "./", /* Concatenate and emit output to single file. */ 15 | // "outDir": "./", /* Redirect output structure to the directory. */ 16 | // "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ 17 | // "composite": true, /* Enable project compilation */ 18 | // "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */ 19 | // "removeComments": true, /* Do not emit comments to output. */ 20 | // "noEmit": true, /* Do not emit outputs. */ 21 | // "importHelpers": true, /* Import emit helpers from 'tslib'. */ 22 | // "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */ 23 | // "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */ 24 | 25 | /* Strict Type-Checking Options */ 26 | "strict": true, /* Enable all strict type-checking options. */ 27 | "noImplicitAny": true, /* Raise error on expressions and declarations with an implied 'any' type. */ 28 | "strictNullChecks": true, /* Enable strict null checks. */ 29 | "strictFunctionTypes": true, /* Enable strict checking of function types. */ 30 | "strictBindCallApply": true, /* Enable strict 'bind', 'call', and 'apply' methods on functions. */ 31 | "strictPropertyInitialization": true, /* Enable strict checking of property initialization in classes. */ 32 | "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */ 33 | "alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */ 34 | 35 | /* Additional Checks */ 36 | // "noUnusedLocals": true, /* Report errors on unused locals. */ 37 | // "noUnusedParameters": true, /* Report errors on unused parameters. */ 38 | // "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */ 39 | "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */ 40 | 41 | /* Module Resolution Options */ 42 | "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */ 43 | // "baseUrl": "./", /* Base directory to resolve non-absolute module names. */ 44 | // "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */ 45 | // "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */ 46 | // "typeRoots": [], /* List of folders to include type definitions from. */ 47 | // "types": [], /* Type declaration files to be included in compilation. */ 48 | "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */ 49 | "esModuleInterop": true, /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */ 50 | 51 | // "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */ 52 | // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ 53 | 54 | /* Source Map Options */ 55 | // "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */ 56 | // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ 57 | // "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */ 58 | // "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */ 59 | 60 | /* Experimental Options */ 61 | // "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */ 62 | // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ 63 | "skipLibCheck": true 64 | }, 65 | "include": ["./**/*"], 66 | "exclude": ["**/node_modules"] 67 | } 68 | -------------------------------------------------------------------------------- /packages/notion-graph-scraper/tsup.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from "tsup"; 2 | import path from "path"; 3 | import * as dotenv from "dotenv"; 4 | 5 | dotenv.config({ path: path.resolve(__dirname, `..`, `..`, `.env`) }); 6 | 7 | export default defineConfig({ 8 | outDir: path.resolve(`.`, `dist`), 9 | minify: true, 10 | dts: true, 11 | target: `node16`, 12 | platform: `node`, 13 | format: [`esm`], 14 | }); 15 | -------------------------------------------------------------------------------- /packages/notion-graph-scraper/types/block-map.ts: -------------------------------------------------------------------------------- 1 | import { NotionAPI } from "notion-client"; 2 | 3 | export type BlockMap = Awaited>[`block`]; 4 | 5 | export type Block = BlockMap[keyof BlockMap]; 6 | -------------------------------------------------------------------------------- /packages/notion-graph-scraper/types/notion-content-node.ts: -------------------------------------------------------------------------------- 1 | export interface NotionContentNode { 2 | title: string; 3 | id: string; 4 | type: `database` | `page` | `error`; 5 | } 6 | 7 | /** 8 | * A type used to represent a single Notion 'block' 9 | * or 'node' as we'd like to call it in this graph-related project 10 | */ 11 | export type NotionContentNodeUnofficialAPI = 12 | | { 13 | title: string; 14 | id: string; 15 | /** 16 | * Notion workspace id 17 | */ 18 | spaceId: string; 19 | /** 20 | * parent node's id 21 | */ 22 | parentId: NotionContentNodeUnofficialAPI[`id`]; 23 | /** 24 | * children count 25 | */ 26 | cc?: number; 27 | type: `page` | `collection_view` | `alias`; 28 | } 29 | | { 30 | title: string; 31 | /** 32 | * collection view page id 33 | * */ 34 | id: string; 35 | /** 36 | * Notion workspace id 37 | */ 38 | spaceId: string; 39 | /** 40 | * children count 41 | */ 42 | cc?: number; 43 | parentId: NotionContentNodeUnofficialAPI[`id`]; 44 | type: `collection_view_page`; 45 | /** 46 | * collection id 47 | * */ 48 | collection_id: string; 49 | }; 50 | 51 | export function isNotionContentNodeType( 52 | s: string 53 | ): s is NotionContentNodeUnofficialAPI[`type`] { 54 | return ( 55 | s === `page` || 56 | s === `collection_view` || 57 | s === `collection_view_page` || 58 | s === `alias` 59 | ); 60 | } 61 | -------------------------------------------------------------------------------- /packages/notion-graph-scraper/types/util-types.ts: -------------------------------------------------------------------------------- 1 | export type DeepReadonly = T extends (infer R)[] 2 | ? DeepReadonlyArray 3 | : // eslint-disable-next-line @typescript-eslint/ban-types 4 | T extends Function 5 | ? T 6 | : T extends object 7 | ? DeepReadonlyObject 8 | : T; 9 | 10 | type DeepReadonlyArray = ReadonlyArray>; 11 | 12 | type DeepReadonlyObject = { 13 | readonly [P in keyof T]: DeepReadonly; 14 | }; 15 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": [ 3 | "config:base" 4 | ], 5 | "packageRules": [ 6 | { 7 | "matchPackagePatterns": [ 8 | "*" 9 | ], 10 | "matchUpdateTypes": [ 11 | "minor", 12 | "patch" 13 | ], 14 | "groupName": "all non-major dependencies", 15 | "groupSlug": "all-minor-patch" 16 | } 17 | ] 18 | } --------------------------------------------------------------------------------