├── .eslintrc.json ├── .github ├── dependabot.yml └── workflows │ ├── deploy-demo-to-pages.yml │ ├── development.yml │ ├── published-simulation.yml │ ├── test-analyze.yml │ └── test.yml ├── .gitignore ├── .husky └── pre-commit ├── .prettierignore ├── LICENSE ├── README.md ├── builtin-plugins ├── commonheadings.mjs ├── depth.mjs ├── pres.mjs ├── size.mjs └── wordcount.mjs ├── cli └── index.mjs ├── example-directories ├── github-docs-sample.tgz └── my-jamstack-site │ └── index.md ├── example-plugins └── my-docsql-plugins │ └── chocolate-icecream-mentions.mjs ├── justfile ├── lib └── analyze.mjs ├── next-env.d.ts ├── next.config.js ├── package-lock.json ├── package.json ├── public ├── .nojekyll ├── favicon-16x16.png ├── favicon-32x32.png └── favicon.ico ├── screenshots ├── click-to-open.png ├── dark-mode.png ├── download.png ├── downloaded-csv.png ├── downloaded-json.png ├── example-queries.png ├── less-trivial-query.png ├── open-help.png ├── opened-in-vscode.png ├── post-pretty-format.png ├── pre-pretty-format.png ├── sample-plugin.png ├── saved-queries.png ├── simple-query.png └── urls.png ├── src ├── components │ ├── about-metadata.tsx │ ├── code-input.tsx │ ├── demo-alert.tsx │ ├── download-found-records.tsx │ ├── example-queries.tsx │ ├── footer.tsx │ ├── found-records.tsx │ ├── help.tsx │ ├── home.tsx │ ├── saved-queries.tsx │ ├── searchable-data.tsx │ ├── theme-switcher.tsx │ ├── toolbar-menu.tsx │ └── toolbar.tsx ├── contexts │ └── possible-keys.ts ├── hooks │ └── use-router-replace.ts ├── lib │ └── sources.ts ├── pages │ ├── _app.tsx │ ├── _document.tsx │ ├── api │ │ └── open.ts │ └── index.tsx ├── styles │ ├── about-metadata.module.css │ ├── code-input.module.css │ ├── example-queries.module.css │ ├── footer.module.css │ ├── found-records.module.css │ ├── globals.css │ ├── home.module.css │ └── saved-queries.module.css ├── types.ts └── utils │ └── syntax-highlighter.tsx └── tsconfig.json /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "next/core-web-vitals", 3 | "ignorePatterns": [ 4 | "out/", 5 | "/.next/" 6 | ] 7 | } 8 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: npm 4 | directory: "/" 5 | schedule: 6 | interval: monthly 7 | open-pull-requests-limit: 20 8 | - package-ecosystem: "github-actions" 9 | directory: "/" 10 | schedule: 11 | interval: monthly 12 | -------------------------------------------------------------------------------- /.github/workflows/deploy-demo-to-pages.yml: -------------------------------------------------------------------------------- 1 | name: Build demo to GitHub Pages 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | branches: 7 | - main 8 | 9 | env: 10 | GH_PAGES_PREFIX: "/docsql" 11 | NEXT_PUBLIC_DEMO_ALERT_TITLE: "This is just a demo!" 12 | NEXT_PUBLIC_DEMO_ALERT_BODY: 'With docsQL, you're supposed to run it with your ownprojects Markdown files. What you're seeing here is just a sample instance that uses a sample of content from the GitHub Docs project.' 13 | 14 | jobs: 15 | build-and-deploy: 16 | runs-on: ubuntu-latest 17 | steps: 18 | - name: Check out repo 19 | uses: actions/checkout@v4 20 | 21 | - name: Setup node 22 | uses: actions/setup-node@v4 23 | with: 24 | cache: npm 25 | 26 | - name: Install dependencies 27 | run: npm ci 28 | 29 | - name: Cache nextjs build 30 | uses: actions/cache@v4 31 | with: 32 | path: .next/cache 33 | key: ${{ runner.os }}-nextjs-${{ hashFiles('package*.json') }} 34 | restore-keys: | 35 | ${{ runner.os }}-nextjs- 36 | 37 | - name: Build 38 | run: npm run build 39 | 40 | - name: Analyze and generate JSON file 41 | run: | 42 | 43 | set -ex 44 | 45 | tar -zxf example-directories/github-docs-sample.tgz 46 | npm run analyze -- github-docs-sample/ 47 | 48 | - name: Debug what was built 49 | run: | 50 | ls -ltr out 51 | cat out/index.html 52 | 53 | - name: Deploy 🚀 54 | uses: JamesIves/github-pages-deploy-action@v4 55 | with: 56 | folder: out 57 | -------------------------------------------------------------------------------- /.github/workflows/development.yml: -------------------------------------------------------------------------------- 1 | name: Test development 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | 9 | jobs: 10 | development: 11 | runs-on: ubuntu-latest 12 | strategy: 13 | fail-fast: false 14 | matrix: 15 | node: 16 | - 18 17 | - 20 18 | 19 | steps: 20 | - name: Check out repo 21 | uses: actions/checkout@v4 22 | 23 | - name: Setup node ${{ matrix.node }} 24 | uses: actions/setup-node@v4 25 | with: 26 | cache: npm 27 | node-version: ${{ matrix.node }} 28 | 29 | - name: Install dependencies 30 | run: npm ci 31 | 32 | - name: Run development 33 | env: 34 | PORT: 3000 35 | run: | 36 | echo CONTENT_SOURCES=example-directories/my-jamstack-site >> .env 37 | npm run dev > /tmp/stdout.log 2> /tmp/stderr.log & 38 | 39 | - name: Check that the server started 40 | run: curl --retry-connrefused --retry 3 -I http://localhost:3000/ 41 | 42 | - name: Show server outputs 43 | run: | 44 | echo "____STDOUT____" 45 | cat /tmp/stdout.log 46 | echo "____STDERR____" 47 | cat /tmp/stderr.log 48 | -------------------------------------------------------------------------------- /.github/workflows/published-simulation.yml: -------------------------------------------------------------------------------- 1 | name: Published simulation 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | 9 | jobs: 10 | simulate-publishing: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Check out repo 14 | uses: actions/checkout@v4 15 | 16 | - name: Setup node 17 | uses: actions/setup-node@v4 18 | with: 19 | cache: npm 20 | 21 | - name: Install dependencies 22 | run: npm ci 23 | 24 | - name: Cache nextjs build 25 | uses: actions/cache@v4 26 | with: 27 | path: .next/cache 28 | key: ${{ runner.os }}-nextjs-${{ hashFiles('package*.json') }} 29 | restore-keys: | 30 | ${{ runner.os }}-nextjs- 31 | 32 | - name: Build 33 | run: npm run build 34 | 35 | - name: Package 36 | run: npm pack 37 | 38 | - name: Extract and run from tarball 39 | env: 40 | # It's usually the default for `serve` but good to be explicit 41 | PORT: 3000 42 | run: | 43 | set -ex 44 | 45 | TARBALL=`ls docsql-*.tgz` 46 | echo $TARBALL 47 | TEMP_DIR=`mktemp -d` 48 | tar -xf $TARBALL --directory $TEMP_DIR 49 | cd $TEMP_DIR/package 50 | 51 | npm install 52 | npx . $GITHUB_WORKSPACE/example-directories > /tmp/stdout.log 2> /tmp/stderr.log & 53 | 54 | - name: Check that the server started 55 | run: curl --retry-connrefused --retry 4 -I http://localhost:3000/ 56 | 57 | - name: Debug any server outputs 58 | if: failure() 59 | run: | 60 | echo "____STDOUT____" 61 | cat /tmp/stdout.log 62 | echo "____STDERR____" 63 | cat /tmp/stderr.log 64 | -------------------------------------------------------------------------------- /.github/workflows/test-analyze.yml: -------------------------------------------------------------------------------- 1 | name: Test analyze 2 | 3 | on: 4 | pull_request: 5 | 6 | jobs: 7 | analyze-sample-stuff: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - name: Check out repo 11 | uses: actions/checkout@v4 12 | 13 | - name: Set up node 14 | uses: actions/setup-node@v4 15 | with: 16 | cache: npm 17 | 18 | - name: Install dependencies 19 | run: npm ci 20 | 21 | - name: Analyze 22 | run: | 23 | node cli/index.mjs \ 24 | --plugins example-plugins/my-docsql-plugins \ 25 | --analyze-only \ 26 | example-directories/my-jamstack-site 27 | 28 | cat out/docs.json 29 | 30 | - name: Check what was generated 31 | uses: actions/github-script@v7 32 | with: 33 | script: | 34 | const fs = require('fs') 35 | const assert = require('assert') 36 | 37 | const all = JSON.parse(fs.readFileSync('out/docs.json')) 38 | 39 | assert(all.meta) 40 | assert(all.meta.took) 41 | assert(all.meta.rows) 42 | assert(all.pages) 43 | assert(Array.isArray(all.pages)) 44 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: 7 | - main 8 | 9 | jobs: 10 | lint: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v4 14 | - uses: actions/setup-node@v4 15 | with: 16 | cache: npm 17 | - run: npm ci 18 | - run: npm run lint 19 | 20 | test: 21 | name: Test with Node ${{ matrix.node }} 22 | runs-on: ubuntu-latest 23 | strategy: 24 | fail-fast: false 25 | matrix: 26 | node: 27 | - 20 28 | - 22 29 | 30 | steps: 31 | - name: Check out repo 32 | uses: actions/checkout@v4 33 | 34 | - name: Set up Node ${{ matrix.node }} 35 | uses: actions/setup-node@v4 36 | with: 37 | node-version: ${{ matrix.node }} 38 | cache: npm 39 | 40 | - name: Install dependencies 41 | run: npm ci 42 | 43 | - name: Cache nextjs build 44 | uses: actions/cache@v4 45 | with: 46 | path: .next/cache 47 | key: ${{ runner.os }}-nextjs-${{ hashFiles('package*.json') }} 48 | restore-keys: | 49 | ${{ runner.os }}-nextjs- 50 | 51 | - name: Lint 52 | run: npm run lint 53 | 54 | - name: Build 55 | run: npm run build 56 | 57 | - name: Start server 58 | env: 59 | # It's usually the default for `serve` but good to be explicit 60 | PORT: 3000 61 | run: | 62 | echo CONTENT_SOURCES=example-directories/my-jamstack-site >> .env 63 | npm run run > /tmp/stdout.log 2> /tmp/stderr.log & 64 | 65 | - name: Check that the server started 66 | run: curl --retry-connrefused --retry 3 -I http://localhost:3000/ 67 | 68 | - name: View the home page 69 | run: | 70 | curl -v --fail http://localhost:3000/ 71 | 72 | - name: Show server outputs 73 | run: | 74 | echo "____STDOUT____" 75 | cat /tmp/stdout.log 76 | echo "____STDERR____" 77 | cat /tmp/stderr.log 78 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | 8 | # testing 9 | /coverage 10 | 11 | # next.js 12 | /.next/ 13 | /out/ 14 | 15 | # production 16 | /build 17 | 18 | # misc 19 | .DS_Store 20 | *.pem 21 | 22 | # debug 23 | npm-debug.log* 24 | yarn-debug.log* 25 | yarn-error.log* 26 | 27 | # local env files 28 | .env.local 29 | .env.development.local 30 | .env.test.local 31 | .env.production.local 32 | 33 | # vercel 34 | .vercel 35 | 36 | # typescript 37 | *.tsbuildinfo 38 | .docsqldb/ 39 | docsql-*.tgz 40 | public/docs.json 41 | -------------------------------------------------------------------------------- /.husky/pre-commit: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | 4 | npm run eslint:fix 5 | npm run prettier-writer 6 | npm run lint 7 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | .next/ 2 | out/ 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2022 Peter Bengtsson (mail@peterbe.com) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # docsQL 2 | 3 | Getting an overview of your Jamstack Markdown files. 4 | 5 | ## Demo 6 | 7 | Play with: [https://peterbe.github.io/docsql/](https://peterbe.github.io/docsql/) 8 | 9 | You're supposed to run `docsQL` with your own projects Markdown files. This 10 | demo uses a subset of the content behind [GitHub Docs](https://github.com/github/docs). 11 | 12 | ## How it works 13 | 14 | You give the CLI program one or more directories that contain Markdown files. 15 | Most of the time it's just one directory; where your jamstack website files 16 | are. 17 | 18 | Each file is opened and the front-matter becomes the key-value pairs that you 19 | can later query. I.e. if you have a front-matter key called `title` 20 | you'll be able to query `SELECT title FROM ? WHERE title ILIKE "%peterbe%"`. 21 | The content is not included in the database. That would make the searchable 22 | database too big. 23 | 24 | Additionally, plugins are executed for each file. There are built-in plugins 25 | and there are plugins you write and point to yourself. 26 | One of the built-in plugins is called `commonheadings.mjs` and it counts 27 | the number of `## ` and `### ` rows there are in the content so you can 28 | query `SELECT h2s, h3s, h2s+h3s AS combined FROM ? ORDER BY 3 DESC`. 29 | 30 | To specify your own plugins for your particular project, see the 31 | section on ["Plugins"](#plugins). 32 | 33 | ## Getting started 34 | 35 | ```sh 36 | npx docsql /path/to/my/project/with/lots/of/markdown/files 37 | ``` 38 | 39 | ## Getting start (after `git` clone) 40 | 41 | ```sh 42 | export CONTENT_SOURCES=/path/to/my/project/with/lots/of/markdown/files 43 | npm run run 44 | ``` 45 | 46 | ## Getting Started (for local development) 47 | 48 | ```sh 49 | echo CONTENT_SOURCES=/path/to/my/project/with/lots/of/markdown/files >> .env 50 | npm run dev 51 | ``` 52 | 53 | ## Plugins 54 | 55 | The built-in plugins are can be found in the source code (TODO: add link). 56 | These are hopefully generic enough and useful enough for most people. 57 | 58 | To write your own plugin, you create a `.mjs` file. Your `.mjs` files 59 | just need to export a default function that returns an object. Best 60 | demonstrated with an example: 61 | 62 | 1. Create a folder called `my-docsql-plugins`. 63 | 2. Create the file `my-docsql-plugins/chocolate-icecream-mentions.mjs` 64 | 3. Enter something like this: 65 | 66 | ```js 67 | const regex = /💩/g; 68 | export default function countCocoIceMentions({ data, content }) { 69 | const inTitle = (data.title.match(regex) || []).length; 70 | const inBody = (content.match(regex) || []).length; 71 | return { 72 | chocolateIcecreamMentions: inTitle + inBody, 73 | }; 74 | } 75 | ``` 76 | 77 | The name of the function isn't important. You could have used 78 | `export default function whatever(`. What is important is that you 79 | get a context object that contains the keys `data` and `content`. 80 | And it's important you return an object with keys and values that 81 | make sense to search on. You can even return a namespace which 82 | you can search on as if it was JSON. 83 | 84 | Now start the CLI with `--plugins my-docsql-plugins` and your new plugin 85 | will be included. Once the server starts, you can click "Open help" 86 | in the web interface and expect to see it mentioned there. With this, 87 | you can now run: 88 | 89 | ```sql 90 | SELECT _file, chocolateIcecreamMentions FROM ? WHERE chocolateIcecreamMentions > 0 91 | ``` 92 | 93 | Instead of passing `--plugins my-plugins --plugins /my/other/custom-plugins` 94 | you can equally set the environment variable: 95 | 96 | ```sh 97 | # Example of setting plugins directories 98 | DOCSQL_PLUGINS="myplugins, /my/other/custom-plugins" 99 | ``` 100 | 101 | ### Important custom plugin key ending in `_url` 102 | 103 | Here's an example plugin that speaks for itself: 104 | 105 | ```js 106 | // In /path/to/my/custom/plugins 107 | 108 | export default function getURL({ _file }) { 109 | const pathname = _file.replace(/\.md$/, '') 110 | return { 111 | _url: `https://example.com/${pathname}`, 112 | local_url: `http://localhost:4000/${pathname}`, 113 | } 114 | } 115 | ``` 116 | 117 | Because the keys end with `_url` these are treated as external 118 | hyperlinks in the UI when queried. For example: 119 | 120 | ```sql 121 | SELECT _url, local_url FROM ? ORDER BY RANDOM() LIMIT 10 122 | ``` 123 | 124 | Suppose that your URLs depend on something from the front-matter of 125 | each document, here's an example: 126 | 127 | ```js 128 | // In /path/to/my/custom/plugins 129 | 130 | export default function getURL({ data: {slug} }) { 131 | return { 132 | _url: `https://example.com/en/${slug}`, 133 | } 134 | } 135 | ``` 136 | 137 | ### Share plugins with your team 138 | 139 | At the moment, the best way is that one of you writes some plugins that 140 | suites your content. Once that works well, you can either zip up that 141 | directory and share with your team. Or, you can simply create a 142 | `git` repo and put them in there. 143 | 144 | ### Caveats on plugins 145 | 146 | - They don't self-document. Yet. It would be nice if you could include 147 | a string of text from within your plugin code that shows up in the 148 | "Help" section. 149 | - The order matters for overriding. There's a built-in plugin called 150 | `wordcount.mjs` which is really basic. If you don't like it, write 151 | your own plugin that returns a key called `wordCount` and it will 152 | override the built-in computation. 153 | - Debugging bad plugins is a bit rough but an error thrown is stopping 154 | the CLI and the stacktrace should be sufficiently clear. 155 | 156 | ## Open found files in your editor 157 | 158 | If you have an environment variable called `EDITOR` set, and you make a 159 | query that includes the key `_file` it will make that a clickable link, 160 | which when running on `localhost` will open that file on your computer. 161 | 162 | A lot of systems have a default `$EDITOR` which might be something 163 | terminal based, like `nano`. If you, for example, what your files to 164 | open in VS Code you can set: 165 | 166 | ```sh 167 | echo EDITOR=code >> .env 168 | ``` 169 | 170 | ## Screenshots (as of Mar 2022) 171 | 172 | *Simple query* 173 | ![Basic query](screenshots/simple-query.png) 174 | 175 | *Saved queries* 176 | ![Saved queries](screenshots/saved-queries.png) 177 | 178 | *Open help* 179 | ![Open help](screenshots/open-help.png) 180 | 181 | *Example queries help you learn* 182 | ![Example queries](screenshots/example-queries.png) 183 | 184 | *Pre- pretty format* 185 | ![Pre- pretty format](screenshots/pre-pretty-format.png) 186 | 187 | *Post- pretty format* 188 | ![Post- pretty format](screenshots/post-pretty-format.png) 189 | 190 | *Less trivial query* 191 | ![Less trivial query](screenshots/less-trivial-query.png) 192 | 193 | *URLs become clickable links* 194 | ![URLs become clickable links](screenshots/urls.png) 195 | 196 | *Dark mode* 197 | ![Dark mode](screenshots/dark-mode.png) 198 | 199 | *Export by downloading* 200 | ![Download](screenshots/download.png) 201 | 202 | *Downloaded JSON file* 203 | ![Downloaded JSON file](screenshots/downloaded-json.png) 204 | 205 | *Downloaded CSV file* 206 | ![Downloaded CSV file](screenshots/downloaded-csv.png) 207 | 208 | *Click to open in your local editor (only when running on localhost)* 209 | ![Click to open](screenshots/click-to-open.png) 210 | 211 | *Automatically opened in VS Code (only when running on localhost)* 212 | ![Opened in VS Code](screenshots/opened-in-vscode.png) 213 | 214 | *Write your own plugins (to generate columns)* 215 | ![Sample plugin code](screenshots/sample-plugin.png) 216 | 217 | ## Icon 218 | 219 | Icon by [Yannick Lung](https://www.iconfinder.com/icons/315196/documents_icon) 220 | 221 | ## How to release 222 | 223 | Run: 224 | 225 | ```sh 226 | npm run release 227 | ``` 228 | -------------------------------------------------------------------------------- /builtin-plugins/commonheadings.mjs: -------------------------------------------------------------------------------- 1 | export default function commonheadings({ content }) { 2 | const h2s = (content.match(/^##\s/gm) || []).length; 3 | const h3s = (content.match(/^###\s/gm) || []).length; 4 | return { 5 | h2s, 6 | h3s, 7 | }; 8 | } 9 | -------------------------------------------------------------------------------- /builtin-plugins/depth.mjs: -------------------------------------------------------------------------------- 1 | import { sep } from "path"; 2 | export default function getDepth({ _file }) { 3 | return { depth: _file.split(sep).length }; 4 | } 5 | -------------------------------------------------------------------------------- /builtin-plugins/pres.mjs: -------------------------------------------------------------------------------- 1 | export default function get({ content }) { 2 | return { pres: (content.match(/```/g) || []).length / 2 }; 3 | } 4 | -------------------------------------------------------------------------------- /builtin-plugins/size.mjs: -------------------------------------------------------------------------------- 1 | export default function sizePlugin({ content, rawContent }) { 2 | return { textLength: content.length, fileSize: rawContent.length }; 3 | } 4 | -------------------------------------------------------------------------------- /builtin-plugins/wordcount.mjs: -------------------------------------------------------------------------------- 1 | // import path from "path"; 2 | 3 | export default function plugin({ content }) { 4 | const words = content.replace(/#+/g, "").split(/\s+/g).filter(Boolean); 5 | const wordCount = words.filter((word) => /[a-z]/i.test(word)).length; 6 | return { wordCount }; 7 | } 8 | -------------------------------------------------------------------------------- /cli/index.mjs: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | import fs from "fs"; 4 | import path from "path"; 5 | import { fileURLToPath } from "url"; 6 | 7 | import polka from "polka"; 8 | import sirv from "sirv"; 9 | import dotenv from "dotenv"; 10 | import { Command } from "commander"; 11 | import cliProgress from "cli-progress"; 12 | 13 | import { 14 | analyzeFiles, 15 | findFiles, 16 | // findAllFiles, 17 | getAllPlugins, 18 | getAllPluginFiles, 19 | getHashFiles, 20 | } from "../lib/analyze.mjs"; 21 | 22 | // Until --experimental-json-modules is the norm... 23 | const packageInfo = JSON.parse(fs.readFileSync("./package.json", "utf-8")); 24 | const VERSION = packageInfo.version || "unversioned"; 25 | 26 | const program = new Command(); 27 | 28 | dotenv.config(); 29 | 30 | const { PORT = 3000, NODE_ENV } = process.env; 31 | 32 | const __dirname = path.dirname(fileURLToPath(import.meta.url)); 33 | 34 | // const dev = NODE_ENV !== "production"; 35 | // const app = next({ dev }); 36 | // const handle = app.getRequestHandler(); 37 | 38 | program 39 | .version(packageInfo.version, "--version", "output the current version") 40 | .description("Analyze all the Markdown files with SQL") 41 | .option("-v, --verbose", "Verbose outputs") 42 | .option("--analyze-only", "Generate the out.json without starting a server") 43 | .option("--plugin [files...]", "Filepath to .mjs plugin files") 44 | .option("--plugins [directories...]", "Directory with your own .mjs plugins") 45 | .arguments("[directories...]", "Specific directories to analyze") 46 | .parse(process.argv); 47 | 48 | main(program.opts(), program.args); 49 | 50 | async function main(opts, sources) { 51 | const { verbose } = opts; 52 | if (!sources.length) { 53 | if (!process.env.CONTENT_SOURCES) { 54 | throw new Error( 55 | "If you don't pass at least one directory, a value must be set in environment variable 'CONTENT_SOURCES'", 56 | ); 57 | } 58 | sources = process.env.CONTENT_SOURCES.split(",") 59 | .map((x) => x.trim()) 60 | .filter(Boolean); 61 | if (!sources.length) { 62 | throw new Error("CONTENT_SOURCES set but empty"); 63 | } 64 | if (verbose) { 65 | console.log(`Using ${sources} from $CONTENT_SOURCES`); 66 | } 67 | } else { 68 | if (verbose) { 69 | console.log(`Sources are: ${sources}`); 70 | } 71 | } 72 | 73 | const pluginsDirectories = [path.join(__dirname, "..", "builtin-plugins")]; 74 | 75 | let customPluginsDirectories = opts.plugins; 76 | if (process.env.DOCSQL_PLUGINS && !customPluginsDirectories) { 77 | customPluginsDirectories = process.env.DOCSQL_PLUGINS.split(",") 78 | .map((x) => x.trim()) 79 | .filter(Boolean); 80 | } 81 | 82 | // const hashCurry = (lastBit: string) => `v${VERSION}.${pluginsHash}.${lastBit}`; 83 | if (customPluginsDirectories) { 84 | pluginsDirectories.push(...customPluginsDirectories); 85 | } 86 | 87 | const t0 = new Date(); 88 | // const { plugins, pluginsHash } = await getAllPlugins(["plugins"]); 89 | 90 | const pluginFiles = getAllPluginFiles(pluginsDirectories); 91 | 92 | for (const pluginFile of opts.plugin || []) { 93 | if (!pluginFile.endsWith(".mjs")) { 94 | throw new Error("own plugin must end with .mjs"); 95 | } 96 | pluginFiles.push(path.resolve(pluginFile)); 97 | } 98 | 99 | const pluginsHash = getHashFiles(pluginFiles); 100 | console.log(`${pluginFiles.length.toLocaleString()} plugins found.`); 101 | 102 | const plugins = await getAllPlugins(pluginFiles); 103 | 104 | const hashCurry = (lastBit) => `v${VERSION}.${pluginsHash}.${lastBit}`; 105 | const allDocs = []; 106 | 107 | const allSources = []; 108 | for (const source of sources) { 109 | const files = findFiles(source); 110 | allSources.push({ source, files: files.length }); 111 | console.log(`${files.length.toLocaleString()} files found in ${source}.`); 112 | 113 | const progressBar = new cliProgress.SingleBar( 114 | {}, 115 | { 116 | format: 117 | "Analyzing \u001b[90m{bar}\u001b[0m {percentage}% | {value}/{total}", 118 | barCompleteChar: "\u2588", 119 | barIncompleteChar: "\u2591", 120 | }, 121 | ); 122 | progressBar.start(files.length, 0); 123 | const docs = await analyzeFiles(source, files, plugins, hashCurry, (i) => { 124 | progressBar.increment(); 125 | }); 126 | progressBar.stop(); 127 | allDocs.push(...docs); 128 | } 129 | 130 | const t1 = new Date(); 131 | 132 | const meta = { 133 | took: t1.getTime() - t0.getTime(), 134 | rows: allDocs.length, 135 | sources: allSources, 136 | version: VERSION, 137 | }; 138 | const combined = { 139 | pages: allDocs, 140 | meta, 141 | }; 142 | const outRoot = "out"; 143 | 144 | // When doing local dev, using `npm run dev`, the `out` directory 145 | // might not have been created. 146 | if (!fs.existsSync(outRoot)) { 147 | fs.mkdirSync(outRoot); 148 | } 149 | 150 | const outFile = path.join(outRoot, "docs.json"); 151 | fs.writeFileSync(outFile, JSON.stringify(combined, null, 2)); 152 | 153 | if (opts.analyzeOnly) { 154 | console.log(`Created ${path.resolve(outFile)}`); 155 | return; 156 | } 157 | 158 | const serve = sirv(outRoot); 159 | 160 | // app.prepare().then(() => { 161 | polka() 162 | .use(serve) 163 | // .get("*", handle) 164 | .listen(PORT, (err) => { 165 | if (err) throw err; 166 | console.log(`> Ready on http://localhost:${PORT}`); 167 | }); 168 | // }); 169 | } 170 | -------------------------------------------------------------------------------- /example-directories/github-docs-sample.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/peterbe/docsql/0d8486e4dd299ce41882e719b54df5a7572ea6cd/example-directories/github-docs-sample.tgz -------------------------------------------------------------------------------- /example-directories/my-jamstack-site/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Root page 3 | intro: This is a chocolate ice cream 💩 4 | --- 5 | 6 | The first paragraph starts here. 7 | -------------------------------------------------------------------------------- /example-plugins/my-docsql-plugins/chocolate-icecream-mentions.mjs: -------------------------------------------------------------------------------- 1 | const regex = /💩/g; 2 | export default function countCocoIceMentions({ data, content }) { 3 | const inTitle = (data.title.match(regex) || []).length; 4 | const inBody = (content.match(regex) || []).length; 5 | return { 6 | chocolateIcecreamMentions: inTitle + inBody, 7 | }; 8 | } 9 | -------------------------------------------------------------------------------- /justfile: -------------------------------------------------------------------------------- 1 | # https://github.com/casey/just 2 | # https://just.systems/ 3 | 4 | dev: 5 | npm run dev 6 | 7 | format: 8 | npm run prettier-writer 9 | 10 | build: 11 | npm run build 12 | 13 | start: build 14 | npm run run 15 | 16 | install: 17 | npm install 18 | 19 | lint: 20 | npm run lint 21 | 22 | release: 23 | npm run release 24 | 25 | upgrade: 26 | npx npm-check-updates --interactive 27 | -------------------------------------------------------------------------------- /lib/analyze.mjs: -------------------------------------------------------------------------------- 1 | import fs from "fs"; 2 | import path from "path"; 3 | import crypto from "crypto"; 4 | 5 | import { fdir } from "fdir"; 6 | import matter from "gray-matter"; 7 | 8 | const _excluding = ( 9 | process.env.DOCSQL_EXCLUDING_FILES || 10 | ` 11 | README.md 12 | LICENSE.md 13 | CODE_OF_CONDUCT.md 14 | REVIEWING.md 15 | `.replace(/\n/g, ",") 16 | ) 17 | .split(",") 18 | .map((x) => x.trim()) 19 | .filter(Boolean); 20 | const EXCEPTION_FILES = new Set(_excluding); 21 | 22 | export function findFiles(source) { 23 | const files = new fdir({ 24 | includeBasePath: true, 25 | suppressErrors: false, 26 | filters: [ 27 | (pathName) => 28 | pathName.endsWith(".md") && 29 | !EXCEPTION_FILES.has(path.basename(pathName)), 30 | ], 31 | exclude: (dirName) => { 32 | return dirName === "node_modules" || dirName.startsWith("."); 33 | }, 34 | }) 35 | .crawl(source) 36 | .sync(); 37 | return files; 38 | } 39 | 40 | export function getAllPluginFiles(roots) { 41 | return roots.map((root) => getPluginFiles(root)).flat(); 42 | } 43 | 44 | function getPluginFiles(root) { 45 | const staticRoot = path.resolve(root); 46 | const pluginsRoot = fs.readdirSync(staticRoot); 47 | 48 | return pluginsRoot 49 | .filter((name) => { 50 | if (name.endsWith(".js")) { 51 | console.warn( 52 | `Only .mjs file extensions allowed. Going to ignore ${path.join( 53 | root, 54 | name, 55 | )}`, 56 | ); 57 | return false; 58 | } 59 | if (name.startsWith(".")) { 60 | console.warn( 61 | `Will ignore private looking files. Going to ignore ${path.join( 62 | root, 63 | name, 64 | )}`, 65 | ); 66 | return false; 67 | } 68 | return name.endsWith(".mjs"); 69 | }) 70 | .sort() 71 | .map((name) => path.join(staticRoot, name)); 72 | } 73 | 74 | // export async function getAllPlugins(roots) { 75 | // return roots.map((root) => getPlugins(root)).flat(); 76 | // } 77 | 78 | export async function getAllPlugins(absoluteFilePaths) { 79 | return Promise.all( 80 | absoluteFilePaths.map(async (filePath) => { 81 | const mod = await import(filePath); 82 | return mod.default; 83 | }), 84 | ); 85 | } 86 | 87 | // const DB = new Map(); 88 | const DB = new Map(); // XXX one day this could be the cache 89 | 90 | export async function analyzeFiles( 91 | source, 92 | files, 93 | plugins, 94 | hashCurry, 95 | cb = (count) => {}, 96 | ) { 97 | let count = 0; 98 | for (const sourceFilePath of files) { 99 | const rawContent = fs.readFileSync(sourceFilePath, "utf-8"); 100 | const _hash = hashCurry(getHash(rawContent, sourceFilePath)); 101 | const _file = path.relative(source, sourceFilePath); 102 | const cached = DB.get(_file); 103 | if (cached) { 104 | if (cached._hash === _hash) { 105 | continue; 106 | } 107 | } 108 | 109 | const { content, data } = matter(rawContent); 110 | 111 | // const page: Page = { 112 | const page = { 113 | _file, 114 | // _hash, 115 | // _source: source 116 | }; 117 | // Put in the front-matter 118 | Object.assign(page, data); 119 | 120 | for (const plugin of plugins) { 121 | const pluginResult = plugin({ data, _file, content, rawContent }); 122 | if (pluginResult) { 123 | if (typeof pluginResult !== "object") { 124 | throw new Error( 125 | `Plugin ${plugin.filePath} returned a non-object on ${page}`, 126 | ); 127 | } 128 | Object.entries(pluginResult).forEach(([key, value]) => { 129 | page[key] = value; 130 | // if ( 131 | // typeof value === "string" || 132 | // typeof value === "number" || 133 | // Array.isArray(value) 134 | // ) { 135 | // page[key] = value; 136 | // } else { 137 | // console.warn(`${key}:${value} (${typeof value}) plugin`); 138 | // } 139 | }); 140 | } 141 | } 142 | 143 | DB.set(_file, page); 144 | cb(count++); 145 | } 146 | // await dumpDB(DB); 147 | // return { docs: Array.from(DB.values()), keys: allPossibleKeys }; 148 | return Array.from(DB.values()); 149 | } 150 | 151 | const DB_FOLDER_NAME = ".docsqldb"; 152 | 153 | async function dumpDB() { 154 | if (!fs.existsSync(DB_FOLDER_NAME)) fs.mkdirSync(DB_FOLDER_NAME); 155 | const dbName = `${getHash(...CONTENT_SOURCES)}.json`; 156 | const dbPath = path.join(DB_FOLDER_NAME, dbName); 157 | await writeFile( 158 | dbPath, 159 | JSON.stringify(Object.fromEntries(DB.entries()), undefined, 2), 160 | ); 161 | const { size } = fs.statSync(dbPath); 162 | console.log(`Wrote DB cache to ${path.resolve(dbPath)} (${fileSize(size)})`); 163 | } 164 | 165 | function fileSize(bytes) { 166 | if (bytes > 1024 * 1024) return `${(bytes / 1024 / 1024).toFixed(1)}MB`; 167 | if (bytes > 1024) return `${(bytes / 1024).toFixed(1)}KB`; 168 | return `${bytes} bytes`; 169 | } 170 | 171 | // function getHash(...args: string[]) { 172 | function getHash(...args) { 173 | const md5sum = crypto.createHash("md5"); 174 | for (const arg of args) { 175 | md5sum.update(arg); 176 | } 177 | // md5sum.update(content); 178 | // md5sum.update(filename); 179 | return md5sum.digest("hex").slice(0, 7); 180 | } 181 | 182 | // function getHashFiles(filePaths: string[]) { 183 | export function getHashFiles(filePaths) { 184 | const md5sum = crypto.createHash("md5"); 185 | for (const filePath of filePaths) { 186 | md5sum.update(filePath); 187 | md5sum.update(fs.readFileSync(filePath)); 188 | } 189 | return md5sum.digest("hex").slice(0, 7); 190 | } 191 | -------------------------------------------------------------------------------- /next-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | /// 3 | 4 | // NOTE: This file should not be edited 5 | // see https://nextjs.org/docs/pages/api-reference/config/typescript for more information. 6 | -------------------------------------------------------------------------------- /next.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('next').NextConfig} */ 2 | 3 | let assetPrefix = undefined; 4 | let trailingSlash = false; 5 | if (process.env.GH_PAGES_PREFIX) { 6 | console.warn("Building for GitHub Pages specifically"); 7 | assetPrefix = process.env.GH_PAGES_PREFIX + "/"; 8 | trailingSlash = true; 9 | } 10 | 11 | const nextConfig = { 12 | output: "export", 13 | reactStrictMode: true, 14 | 15 | trailingSlash, 16 | assetPrefix, 17 | }; 18 | 19 | module.exports = nextConfig; 20 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "docsql", 3 | "version": "0.12.4", 4 | "homepage": "https://github.com/peterbe/docsql", 5 | "license": "MIT", 6 | "scripts": { 7 | "analyze": "npm run run -- --analyze-only", 8 | "dev": "npm run analyze && cp out/docs.json public/ && next dev", 9 | "build": "next build", 10 | "clear": "rm -rf .next out", 11 | "start": "next start", 12 | "lint": "next lint && npm run prettier-check && npm run eslint", 13 | "prettier-check": "prettier -c \"**/*.{ts,tsx,js,mjs,scss,yml,yaml}\"", 14 | "prettier-writer": "prettier -w \"**/*.{ts,tsx,js,mjs,scss,yml,yaml}\"", 15 | "run": "node cli/index.mjs", 16 | "build:run": "npm run build && npm run run", 17 | "test": "echo 'Tested in Actions'", 18 | "prepare": "husky", 19 | "tsc": "tsc --noEmit", 20 | "release": "npm run build && np", 21 | "eslint": "eslint '**/*.{js,mjs,ts,tsx}'", 22 | "eslint:fix": "npm run eslint -- --fix" 23 | }, 24 | "bin": "cli/index.mjs", 25 | "files": [ 26 | "/out", 27 | "/lib", 28 | "/cli", 29 | "/builtin-plugins" 30 | ], 31 | "dependencies": { 32 | "@tabler/icons-react": "3.30.0", 33 | "cli-progress": "3.12.0", 34 | "commander": "13.1.0", 35 | "dotenv": "16.4.7", 36 | "fdir": "6.4.3", 37 | "gray-matter": "4.0.3", 38 | "open-editor": "5.1.0", 39 | "polka": "0.5.2", 40 | "sirv": "3.0.1" 41 | }, 42 | "devDependencies": { 43 | "@mantine/core": "7.17.1", 44 | "@mantine/hooks": "7.17.1", 45 | "@mantine/next": "6.0.22", 46 | "@types/node": "22.13.8", 47 | "@types/react": "18.3.12", 48 | "@types/refractor": "3.4.1", 49 | "alasql": "4.6.4", 50 | "eslint": "8.57.0", 51 | "eslint-config-next": "15.2.3", 52 | "husky": "9.1.7", 53 | "next": "15.2.3", 54 | "np": "10.2.0", 55 | "prettier": "3.5.3", 56 | "react": "18.3.1", 57 | "react-dom": "18.3.1", 58 | "react-refractor": "2.1.7", 59 | "swr": "2.3.3", 60 | "typescript": "5.8.2" 61 | }, 62 | "engines": { 63 | "node": "^16 || ^18 || ^19 || ^20 || ^22" 64 | }, 65 | "repository": { 66 | "type": "git", 67 | "url": "git://github.com/peterbe/docsql.git" 68 | }, 69 | "bugs": { 70 | "url": "https://github.com/peterbe/docsql/issues" 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /public/.nojekyll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/peterbe/docsql/0d8486e4dd299ce41882e719b54df5a7572ea6cd/public/.nojekyll -------------------------------------------------------------------------------- /public/favicon-16x16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/peterbe/docsql/0d8486e4dd299ce41882e719b54df5a7572ea6cd/public/favicon-16x16.png -------------------------------------------------------------------------------- /public/favicon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/peterbe/docsql/0d8486e4dd299ce41882e719b54df5a7572ea6cd/public/favicon-32x32.png -------------------------------------------------------------------------------- /public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/peterbe/docsql/0d8486e4dd299ce41882e719b54df5a7572ea6cd/public/favicon.ico -------------------------------------------------------------------------------- /screenshots/click-to-open.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/peterbe/docsql/0d8486e4dd299ce41882e719b54df5a7572ea6cd/screenshots/click-to-open.png -------------------------------------------------------------------------------- /screenshots/dark-mode.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/peterbe/docsql/0d8486e4dd299ce41882e719b54df5a7572ea6cd/screenshots/dark-mode.png -------------------------------------------------------------------------------- /screenshots/download.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/peterbe/docsql/0d8486e4dd299ce41882e719b54df5a7572ea6cd/screenshots/download.png -------------------------------------------------------------------------------- /screenshots/downloaded-csv.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/peterbe/docsql/0d8486e4dd299ce41882e719b54df5a7572ea6cd/screenshots/downloaded-csv.png -------------------------------------------------------------------------------- /screenshots/downloaded-json.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/peterbe/docsql/0d8486e4dd299ce41882e719b54df5a7572ea6cd/screenshots/downloaded-json.png -------------------------------------------------------------------------------- /screenshots/example-queries.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/peterbe/docsql/0d8486e4dd299ce41882e719b54df5a7572ea6cd/screenshots/example-queries.png -------------------------------------------------------------------------------- /screenshots/less-trivial-query.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/peterbe/docsql/0d8486e4dd299ce41882e719b54df5a7572ea6cd/screenshots/less-trivial-query.png -------------------------------------------------------------------------------- /screenshots/open-help.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/peterbe/docsql/0d8486e4dd299ce41882e719b54df5a7572ea6cd/screenshots/open-help.png -------------------------------------------------------------------------------- /screenshots/opened-in-vscode.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/peterbe/docsql/0d8486e4dd299ce41882e719b54df5a7572ea6cd/screenshots/opened-in-vscode.png -------------------------------------------------------------------------------- /screenshots/post-pretty-format.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/peterbe/docsql/0d8486e4dd299ce41882e719b54df5a7572ea6cd/screenshots/post-pretty-format.png -------------------------------------------------------------------------------- /screenshots/pre-pretty-format.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/peterbe/docsql/0d8486e4dd299ce41882e719b54df5a7572ea6cd/screenshots/pre-pretty-format.png -------------------------------------------------------------------------------- /screenshots/sample-plugin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/peterbe/docsql/0d8486e4dd299ce41882e719b54df5a7572ea6cd/screenshots/sample-plugin.png -------------------------------------------------------------------------------- /screenshots/saved-queries.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/peterbe/docsql/0d8486e4dd299ce41882e719b54df5a7572ea6cd/screenshots/saved-queries.png -------------------------------------------------------------------------------- /screenshots/simple-query.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/peterbe/docsql/0d8486e4dd299ce41882e719b54df5a7572ea6cd/screenshots/simple-query.png -------------------------------------------------------------------------------- /screenshots/urls.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/peterbe/docsql/0d8486e4dd299ce41882e719b54df5a7572ea6cd/screenshots/urls.png -------------------------------------------------------------------------------- /src/components/about-metadata.tsx: -------------------------------------------------------------------------------- 1 | import { useState } from "react"; 2 | import { Alert } from "@mantine/core"; 3 | import type { Meta } from "../types"; 4 | import styles from "../styles/about-metadata.module.css"; 5 | 6 | export function AboutMetadata({ meta }: { meta: Meta }) { 7 | const [showMore, setShowMore] = useState(false); 8 | return ( 9 |
10 | {meta.rows === 0 ? ( 11 | 12 | No rows loaded! Something is wrong with the sources or the crawler. 13 | 14 | ) : ( 15 | <> 16 |

17 | It took {formatMilliSeconds(meta.took)} to load{" "} 18 | {meta.rows.toLocaleString()} records.{" "} 19 | { 23 | event.preventDefault(); 24 | setShowMore((prevState) => !prevState); 25 | }} 26 | > 27 | [{showMore ? "less" : "more"}] 28 | 29 |

30 | {showMore && ( 31 |
32 |

33 | Sources 34 |

35 |
    36 | {meta.sources.map((source) => ( 37 |
  • 38 | {source.files.toLocaleString()} files from{" "} 39 | {source.source} 40 |
    • 41 | ))} 42 |
43 | {meta.version && ( 44 |

45 | docsQL version {meta.version} 46 |

47 | )} 48 |
49 | )} 50 | 51 | )} 52 |
53 | ); 54 | } 55 | 56 | function formatMilliSeconds(ms: number) { 57 | if (ms > 1000) { 58 | return `${(ms / 1000).toFixed(2)}s`; 59 | } 60 | return `${ms.toFixed(0)}ms`; 61 | } 62 | -------------------------------------------------------------------------------- /src/components/code-input.tsx: -------------------------------------------------------------------------------- 1 | import { useCallback, useEffect, useRef } from "react"; 2 | import { Textarea, Button, Grid, Text } from "@mantine/core"; 3 | import { Kbd } from "@mantine/core"; 4 | import styles from "../styles/code-input.module.css"; 5 | 6 | import { ToolbarMenu } from "./toolbar-menu"; 7 | import { SavedQuery, ToolbarMenuOption } from "../types"; 8 | 9 | export function CodeInput({ 10 | onChange, 11 | query, 12 | prettyQuery, 13 | typedQuery, 14 | setTypedQuery, 15 | hasError, 16 | savedQueries, 17 | currentMenu, 18 | toggleMenu, 19 | }: { 20 | onChange: (query: string) => void; 21 | query: string; 22 | prettyQuery: string; 23 | typedQuery: string; 24 | setTypedQuery: (query: string) => void; 25 | hasError: boolean; 26 | savedQueries: SavedQuery[]; 27 | currentMenu: ToolbarMenuOption; 28 | toggleMenu: (menu: ToolbarMenuOption) => void; 29 | }) { 30 | const formSubmit = useCallback(() => { 31 | onChange(typedQuery.trim()); 32 | }, [typedQuery]); 33 | 34 | const textareaRef = useRef(null); 35 | const textareaElement = textareaRef.current; 36 | useEffect(() => { 37 | const listener = (event: KeyboardEvent) => { 38 | if (event.key === "Enter" && event.metaKey) { 39 | formSubmit(); 40 | } 41 | }; 42 | if (textareaElement) textareaElement.addEventListener("keydown", listener); 43 | 44 | return () => { 45 | if (textareaElement) 46 | textareaElement.removeEventListener("keydown", listener); 47 | }; 48 | }, [textareaElement, formSubmit]); 49 | 50 | return ( 51 |
{ 53 | event.preventDefault(); 54 | formSubmit(); 55 | }} 56 | > 57 | 58 | Tip! Use ⌘-Enter to run the query when 59 | focus is inside textarea 60 | 61 | 78 | 79 | 80 | 81 | {" "} 94 | 105 | 106 | 107 | 112 | 113 | 114 |
115 | ); 116 | } 117 | -------------------------------------------------------------------------------- /src/components/demo-alert.tsx: -------------------------------------------------------------------------------- 1 | import { useState } from "react"; 2 | import { Alert } from "@mantine/core"; 3 | 4 | const DEMO_ALERT_TITLE = process.env.NEXT_PUBLIC_DEMO_ALERT_TITLE; 5 | const DEMO_ALERT_BODY = process.env.NEXT_PUBLIC_DEMO_ALERT_BODY; 6 | 7 | export function DemoAlert() { 8 | const [show, setShow] = useState(Boolean(DEMO_ALERT_TITLE)); 9 | 10 | const bodyHTML = DEMO_ALERT_BODY || "This is a demo implementation only."; 11 | 12 | if (show) { 13 | return ( 14 | { 22 | setShow(false); 23 | }} 24 | > 25 |
26 | 27 | ); 28 | } 29 | return null; 30 | } 31 | -------------------------------------------------------------------------------- /src/components/download-found-records.tsx: -------------------------------------------------------------------------------- 1 | import { Button, Group } from "@mantine/core"; 2 | 3 | import type { Records } from "../types"; 4 | 5 | export function DownloadFoundRecords({ records }: { records: Records }) { 6 | function triggerJSONDownload() { 7 | const blob = new Blob([JSON.stringify(records, undefined, 2)], { 8 | type: "application/json", 9 | }); 10 | // create hidden link, just force a click on it and then remove it from the DOM. 11 | const element = document.createElement("a"); 12 | document.body.appendChild(element); 13 | element.setAttribute("href", window.URL.createObjectURL(blob)); 14 | element.setAttribute("download", "results.json"); 15 | element.style.display = "none"; 16 | element.click(); 17 | document.body.removeChild(element); 18 | } 19 | 20 | function triggerCSVDownload() { 21 | let blobstring = ""; 22 | let keys: string[] | null = null; 23 | records.forEach((record, i) => { 24 | if (!i) { 25 | keys = Object.keys(record); 26 | blobstring += keys.map((key) => csvEncodeString(key)); 27 | blobstring += "\n"; 28 | } 29 | blobstring += keys?.map((key) => csvEncodeString(record[key])); 30 | blobstring += "\n"; 31 | }); 32 | 33 | const blob = new Blob([blobstring], { 34 | type: "text/csv", 35 | }); 36 | // create hidden link, just force a click on it and then remove it from the DOM. 37 | const element = document.createElement("a"); 38 | document.body.appendChild(element); 39 | element.setAttribute("href", window.URL.createObjectURL(blob)); 40 | element.setAttribute("download", "results.csv"); 41 | element.style.display = "none"; 42 | element.click(); 43 | document.body.removeChild(element); 44 | } 45 | 46 | return ( 47 | 48 | 58 | 59 | 69 | 70 | ); 71 | } 72 | 73 | function csvEncodeString(value: any) { 74 | if (value === null) { 75 | value = "null"; 76 | } else if (value === undefined) { 77 | value = ""; 78 | } else if (Array.isArray(value) || typeof value === "object") { 79 | value = JSON.stringify(value); 80 | } else { 81 | value = value.toString(); 82 | } 83 | value = value.replace(/"/g, '""'); 84 | 85 | if (value.search(/("|,|\n)/g) >= 0) { 86 | return `"${value}"`; 87 | } 88 | return value; 89 | } 90 | -------------------------------------------------------------------------------- /src/components/example-queries.tsx: -------------------------------------------------------------------------------- 1 | import { useState } from "react"; 2 | import { Button, Highlight, TextInput, Text, Title } from "@mantine/core"; 3 | import { IconSearch } from "@tabler/icons-react"; 4 | 5 | import { SQL } from "../utils/syntax-highlighter"; 6 | import styles from "../styles/example-queries.module.css"; 7 | 8 | const EXAMPLES = [ 9 | { sql: "SELECT title FROM ?", description: "Select just 'title'" }, 10 | { 11 | sql: "SELECT title FROM ? LIMIT 10 OFFSET 10", 12 | description: "Select first rows 10 to 20 ", 13 | }, 14 | { 15 | sql: "SELECT title, length(title) FROM ? ORDER BY 2 DESC LIMIT 5", 16 | description: "Top 5 longest titles", 17 | }, 18 | { 19 | sql: "SELECT title, title->length as length FROM ? ORDER BY 2 ASC LIMIT 5", 20 | description: "Top 5 shortest titles", 21 | }, 22 | { 23 | sql: 'SELECT topics, topics->length AS length FROM ? WHERE "Accounts" IN topics', 24 | description: 25 | "Filter by presence inside an array and count entries in array", 26 | }, 27 | { 28 | sql: 'SELECT title FROM ? WHERE title ILIKE "%github%"', 29 | description: "Case insensitive filter by wildcard operator on string", 30 | }, 31 | { 32 | sql: 'SELECT title FROM ? WHERE REGEXP_LIKE(title, "\\bactions?\\b", "i")', 33 | description: 34 | "Match whole word 'action' or 'actions' but not 'transaction' or 'actionable'", 35 | }, 36 | { 37 | sql: "SELECT changelog, changelog->label FROM ? WHERE changelog AND changelog->label", 38 | description: 39 | "Select from JSON object and filter by those that have a truthy value on that key", 40 | }, 41 | { 42 | sql: "SELECT topics, topics->length FROM ? WHERE topics->label", 43 | description: "Select arrays and filter out those that are null", 44 | }, 45 | { 46 | sql: "SELECT children->(0) FROM ? WHERE children->length", 47 | description: "Select first element in array where the array is something", 48 | }, 49 | { 50 | sql: "SELECT changelog FROM ? WHERE NOT changelog->label", 51 | description: 52 | "Select from JSON objects those that do not have a certain key", 53 | }, 54 | { 55 | sql: "SELECT title FROM ? ORDER BY RANDOM() LIMIT 10", 56 | description: "10 random titles", 57 | }, 58 | { 59 | sql: 60 | "SELECT _id, wordCount, textLength, ROUND(textLength / wordCount, 2) FROM ? WHERE wordCount > 10 " + 61 | "ORDER BY 4 DESC LIMIT 25", 62 | description: "Order average longest words rounded to 2 significant figures", 63 | }, 64 | ]; 65 | 66 | export function ExampleQueries({ 67 | loadQuery, 68 | }: { 69 | loadQuery: (s: string) => void; 70 | }) { 71 | const [search, setSearch] = useState(""); 72 | 73 | const examples = EXAMPLES.filter(({ sql, description }) => { 74 | if (search.trim()) { 75 | return ( 76 | sql.toLowerCase().includes(search.toLowerCase()) || 77 | description.toLowerCase().includes(search.toLowerCase()) 78 | ); 79 | } 80 | return true; 81 | }); 82 | return ( 83 |
84 | Saved queries 85 | 86 |
87 | setSearch(event.target.value)} 91 | placeholder="Search filter..." 92 | leftSection={} 93 | /> 94 | 95 | 96 | {examples.map(({ sql, description }, i) => { 97 | return ( 98 |
99 | {description} 100 | 101 | 110 |
111 | ); 112 | })} 113 | {examples.length === 0 && ( 114 |

115 | No examples found 116 |

117 | )} 118 | 119 | These are static examples they might not work with your data. 120 | 121 |
122 | ); 123 | } 124 | -------------------------------------------------------------------------------- /src/components/footer.tsx: -------------------------------------------------------------------------------- 1 | import { Anchor, Text } from "@mantine/core"; 2 | 3 | import styles from "../styles/footer.module.css"; 4 | 5 | export function Footer() { 6 | return ( 7 |
8 | 9 | 14 | github.com/peterbe/docsql 15 | 16 | 17 |
18 | ); 19 | } 20 | -------------------------------------------------------------------------------- /src/components/found-records.tsx: -------------------------------------------------------------------------------- 1 | import type { MouseEvent } from "react"; 2 | import { useEffect, useState } from "react"; 3 | import { Anchor, Table } from "@mantine/core"; 4 | 5 | import type { Records, OpenFile } from "../types"; 6 | import styles from "../styles/found-records.module.css"; 7 | import useSWR from "swr"; 8 | import { DownloadFoundRecords } from "./download-found-records"; 9 | 10 | export function ShowFoundRecords({ records }: { records: Records }) { 11 | // Event if $EDITOR is set current server, it might not be 12 | // an actual local server. Opening a file in your local 13 | // editor only makes sense when the server is running on your laptop. 14 | const [allowLocalLinks, setAllowLocalLinks] = useState(false); 15 | useEffect(() => { 16 | const { hostname } = document.location; 17 | if ( 18 | hostname === "localhost" || 19 | hostname === "127.0.0.1" || 20 | hostname === "0.0.0.0" 21 | ) { 22 | setAllowLocalLinks(true); 23 | } 24 | }, []); 25 | 26 | const [opening, setOpening] = useState(null); 27 | useEffect(() => { 28 | const timeout = opening 29 | ? setTimeout(() => { 30 | setOpening(null); 31 | }, 3000) 32 | : null; 33 | return () => { 34 | if (timeout) clearTimeout(timeout); 35 | }; 36 | }, [opening]); 37 | 38 | const fetchURL = opening 39 | ? "/api/open?" + new URLSearchParams({ filePath: opening }) 40 | : null; 41 | const { data: openFileResult, error: openFileError } = useSWR( 42 | fetchURL, 43 | async (url) => { 44 | const res = await fetch(url); 45 | if (res.status === 404 || res.status === 400) { 46 | const error = (await res.json()).error as string; 47 | throw new Error(error); 48 | } else if (res.status === 200) { 49 | return (await res.json()) as OpenFile; 50 | } 51 | throw new Error(`${res.status} on ${url}`); 52 | }, 53 | { 54 | revalidateOnFocus: false, 55 | revalidateOnReconnect: false, 56 | }, 57 | ); 58 | 59 | // console.log({ openFileResult, openFileError }); 60 | 61 | if (records.length === 0) { 62 | return

Absolutely diddly squat found. Sorry not sorry 🙃

; 63 | } 64 | const keys = Object.keys(records[0]); 65 | const keyTemplate = keys.join(""); 66 | 67 | const MAX_ROWS = 1000; 68 | 69 | return ( 70 |
71 | {opening && ( 72 |

73 | opening {opening} 74 |

75 | )} 76 | {openFileResult && ( 77 |

78 | tried to open {openFileResult.filePath} 79 | using {openFileResult.binary} 80 | {openFileResult.isTerminalEditor && `(is terminal editor)`} 81 |

82 | )} 83 | {openFileError && ( 84 |

85 | an error occurred trying to open the file:{" "} 86 | {openFileError.toString()} 87 |

88 | )} 89 | 90 |

91 | Found {records.length.toLocaleString()}{" "} 92 | {records.length > MAX_ROWS 93 | ? `(only showing first ${MAX_ROWS.toLocaleString()})` 94 | : null} 95 |

96 | 97 | 98 | 99 | 100 | {keys.map((key) => { 101 | return ; 102 | })} 103 | 104 | 105 | 106 | {records.slice(0, MAX_ROWS).map((record, i) => { 107 | return ( 108 | 113 | 114 | {i + 1} 115 | 116 | {keys.map((key) => { 117 | const value = record[key]; 118 | return ( 119 | 120 | { 125 | setOpening(value); 126 | }} 127 | /> 128 | 129 | ); 130 | })} 131 | 132 | ); 133 | })} 134 | 135 |
 {key}
136 | {records.length > 0 && } 137 | 138 | {records.length > MAX_ROWS && ( 139 |

140 | 141 | Capped to the first {MAX_ROWS.toLocaleString()} rows 142 | 143 |

144 | )} 145 |
146 | ); 147 | } 148 | 149 | export function ShowValue({ 150 | key_, 151 | value, 152 | allowLocalLinks, 153 | setOpening, 154 | }: { 155 | key_: string; 156 | value: any; 157 | allowLocalLinks: boolean; 158 | setOpening: (value: string) => void; 159 | }) { 160 | if (key_.endsWith("_url") && typeof value === "string") { 161 | return ( 162 | 163 | {value} 164 | 165 | ); 166 | } 167 | if (key_ === "_file") { 168 | if (allowLocalLinks) { 169 | return ( 170 | ) => { 173 | event.preventDefault(); 174 | setOpening(value); 175 | }} 176 | > 177 | 178 | 179 | ); 180 | } else { 181 | return ; 182 | } 183 | } 184 | 185 | return <>{formatValue(value)}; 186 | } 187 | 188 | function FilePath({ 189 | value, 190 | maxLength = 100, 191 | }: { 192 | value: string; 193 | maxLength?: number; 194 | }) { 195 | // Possibly truncate, nicely, because it can be really long 196 | if (value.length > maxLength) { 197 | const middle = Math.floor(value.length / 2); 198 | let left = value.slice(0, middle); 199 | let right = value.slice(middle); 200 | let padding = 1; 201 | while (left.length + right.length > maxLength) { 202 | left = value.slice(0, middle - padding); 203 | right = value.slice(middle + padding); 204 | padding++; 205 | } 206 | 207 | return ( 208 | 209 | {left}[…]{right} 210 | 211 | ); 212 | } 213 | return {value}; 214 | } 215 | 216 | function formatValue(input: any) { 217 | if (typeof input === "string") { 218 | return input; 219 | } 220 | if (input === 0) { 221 | return "0"; 222 | } 223 | if (typeof input === "number") { 224 | if (input > 1000) { 225 | return input.toLocaleString(); 226 | } 227 | } 228 | if (input === null) { 229 | return "null"; 230 | } 231 | if (Array.isArray(input)) { 232 | // return input.join(", "); 233 | return JSON.stringify(input); 234 | } 235 | if (typeof input === "object") { 236 | return JSON.stringify(input); 237 | } 238 | if (input === undefined) { 239 | return undefined; 240 | } 241 | return input.toString(); 242 | } 243 | -------------------------------------------------------------------------------- /src/components/help.tsx: -------------------------------------------------------------------------------- 1 | import { Anchor, Container, Paper, Table, Title } from "@mantine/core"; 2 | 3 | import type { PossibleKeys } from "../contexts/possible-keys"; 4 | 5 | export function ShowHelp({ possibleKeys }: { possibleKeys: PossibleKeys }) { 6 | const allKeys: { 7 | name: string; 8 | type: string; 9 | }[] = []; 10 | 11 | for (const [name, type] of Array.from(possibleKeys)) { 12 | allKeys.push({ name, type }); 13 | } 14 | allKeys.sort((a, b) => a.name.localeCompare(b.name)); 15 | 16 | return ( 17 |
18 | 19 | 20 | Possible keys 21 | 22 | 23 | 24 | Name 25 | Type 26 | 27 | 28 | 29 | {allKeys.map(({ name, type }) => ( 30 | 31 | 32 | {name} 33 | 34 | {type} 35 | 36 | ))} 37 | 38 |
39 | 40 | 41 | 42 | 43 | 44 | SQL is case-insensitive, but keys aren't 45 |

46 | SELECT title FROM ? ORDER BY textLength LIMIT 10 47 |
48 | == 49 |
50 | Select title from ? order bY textLength limiT 10 51 |

52 |

53 | But... 54 |
55 | SELECT title FROM ? ORDER BY textLength LIMIT 10 56 |
57 | != 58 |
59 | SELECT TiTLe FROM ? ORDER BY TextlengTH LIMIT 10 60 |

61 | 62 | 63 | 64 | 65 | 66 | Read the docs 67 |

68 | The most important documentation is{" "} 69 | 74 | AlaSQL 75 | {" "} 76 | which is what the SQL engine is based on. 77 |

78 |

79 | Tip! Since AlaSQL is based on JavaScript you can substitute 80 | the . 81 | for ->. For example, in JavaScript you would do:{" "} 82 | myString.length.toLocaleString(), and in AlaSQL that 83 | becomes SELECT mystring->length->toLocaleString() 84 | . 85 |

86 | 87 | 88 |
89 | ); 90 | } 91 | -------------------------------------------------------------------------------- /src/components/home.tsx: -------------------------------------------------------------------------------- 1 | import type { NextPage } from "next"; 2 | import useSWR from "swr"; 3 | import { Alert, LoadingOverlay } from "@mantine/core"; 4 | 5 | import type { PagesAndMeta } from "../types"; 6 | import { SearchableData } from "./searchable-data"; 7 | import { Footer } from "./footer"; 8 | import { DemoAlert } from "./demo-alert"; 9 | import { ThemeSwitcher } from "./theme-switcher"; 10 | import styles from "../styles/home.module.css"; 11 | 12 | const API_URL = "docs.json"; 13 | 14 | export const Home: NextPage = () => { 15 | const { data, error } = useSWR( 16 | API_URL, 17 | async (url) => { 18 | const res = await fetch(url); 19 | if (res.ok) { 20 | return await res.json(); 21 | } 22 | let message = `${res.status} on ${url}`; 23 | try { 24 | message = (await res.json()).error; 25 | } catch {} 26 | throw new Error(message); 27 | }, 28 | { 29 | revalidateOnFocus: false, 30 | }, 31 | ); 32 | 33 | return ( 34 | <> 35 | 36 |
37 |

docsQL

38 | 39 | 40 |
41 | {error && ( 42 | 43 |
46 | {error.toString()} 47 | {data &&

Showing "old" data.

} 48 |
49 | 50 | )} 51 | 55 | {data && } 56 |
57 | 58 |
59 |
60 | 61 | ); 62 | }; 63 | -------------------------------------------------------------------------------- /src/components/saved-queries.tsx: -------------------------------------------------------------------------------- 1 | import { useState } from "react"; 2 | import { Title, Button, Group, TextInput, Text } from "@mantine/core"; 3 | import { IconSearch, IconStar } from "@tabler/icons-react"; 4 | 5 | import { SQL } from "../utils/syntax-highlighter"; 6 | import type { SavedQuery } from "../types"; 7 | 8 | import styles from "../styles/saved-queries.module.css"; 9 | 10 | export function ShowSavedQueries({ 11 | savedQueries, 12 | loadQuery, 13 | deleteSavedQuery, 14 | starQuery, 15 | deleteAllSavedQueries, 16 | }: { 17 | savedQueries: SavedQuery[]; 18 | loadQuery: (s: string) => void; 19 | deleteSavedQuery: (s: string) => void; 20 | starQuery: (s: string) => void; 21 | deleteAllSavedQueries: (includingStarred?: boolean) => void; 22 | }) { 23 | const [searchFilter, setSearchFilter] = useState(""); 24 | 25 | const filteredSavedQueries = savedQueries.filter((savedQuery) => { 26 | return ( 27 | !searchFilter.trim() || 28 | savedQuery.query.toLowerCase().includes(searchFilter.toLowerCase().trim()) 29 | ); 30 | }); 31 | 32 | return ( 33 |
34 | Saved queries 35 | 36 | {savedQueries.length > 1 && ( 37 |
38 | setSearchFilter(e.target.value)} 43 | leftSection={} 44 | /> 45 | {searchFilter.trim() && ( 46 |
47 | 48 | Found {filteredSavedQueries.length}. Filtered out{" "} 49 | {filteredSavedQueries.length > 0 50 | ? savedQueries.length - filteredSavedQueries.length 51 | : "everything"} 52 | 53 |
54 | )} 55 | 56 | )} 57 | 58 | {filteredSavedQueries.map((savedQuery) => { 59 | const star = Boolean(savedQuery.star); 60 | return ( 61 |
65 |

66 | Found {savedQuery.count.toLocaleString()} records{" "} 67 | {new Date(savedQuery.ts).toLocaleTimeString()} 68 |

69 | 70 | 71 | 80 | 91 | 101 | 102 |
103 | ); 104 | })} 105 | {savedQueries.length > 1 && ( 106 | 107 | 118 | 119 | 130 | 131 | )} 132 |
133 | ); 134 | } 135 | -------------------------------------------------------------------------------- /src/components/searchable-data.tsx: -------------------------------------------------------------------------------- 1 | import { useEffect, useState } from "react"; 2 | import { useRouter } from "next/router"; 3 | import alasql from "alasql"; 4 | import { Alert } from "@mantine/core"; 5 | 6 | import type { 7 | PagesAndMeta, 8 | SavedQuery, 9 | Records, 10 | ToolbarMenuOption, 11 | } from "../types"; 12 | 13 | import { Toolbar } from "./toolbar"; 14 | import { ShowFoundRecords } from "./found-records"; 15 | import { AboutMetadata } from "./about-metadata"; 16 | import useRouterReplace from "../hooks/use-router-replace"; 17 | import { CodeInput } from "./code-input"; 18 | import type { PossibleKeys } from "../contexts/possible-keys"; 19 | import { pagesToPossibleKeys } from "../contexts/possible-keys"; 20 | 21 | function firstString(thing: string[] | string) { 22 | if (Array.isArray(thing)) return thing[0]; 23 | return thing; 24 | } 25 | 26 | function fixDoubleAliases(sql: string): string { 27 | // Necessary because of https://github.com/agershun/alasql/issues/1426 28 | const dupes = new Set(); 29 | for (const match of sql.match(/\sAS \w+/g) || []) { 30 | if (dupes.has(match)) { 31 | sql = sql.replace(match, ""); 32 | } 33 | dupes.add(match); 34 | } 35 | return sql; 36 | } 37 | 38 | const DEFAULT_QUERY = 39 | "SELECT title, length(title) AS length FROM ? ORDER BY 2 DESC LIMIT 10"; 40 | 41 | export function SearchableData({ data }: { data: PagesAndMeta }) { 42 | const router = useRouter(); 43 | 44 | const defaultQuery = firstString(router.query.query || ""); 45 | 46 | const [foundRecords, setFoundRecords] = useState(null); 47 | const [queryError, setQueryError] = useState(null); 48 | 49 | const [query, setQuery] = useState(defaultQuery); 50 | 51 | const { pages } = data; 52 | 53 | const routerReplace = useRouterReplace(); 54 | useEffect(() => { 55 | if (query) { 56 | try { 57 | setFoundRecords(alasql(query, [pages])); 58 | setQueryError(null); 59 | } catch (err) { 60 | if (err instanceof Error) { 61 | setQueryError(err); 62 | } else { 63 | throw err; 64 | } 65 | } 66 | } 67 | }, [query, pages]); 68 | 69 | const [possiblePrettySQL, setPossiblePrettySQL] = useState(""); 70 | useEffect(() => { 71 | if (query) { 72 | try { 73 | const ast = alasql.parse(query); 74 | const pretty = fixDoubleAliases( 75 | ast.toString().replace("FROM $0 AS default", "FROM ?"), 76 | ); 77 | if (pretty !== query) { 78 | setPossiblePrettySQL(pretty); 79 | } 80 | } catch { 81 | // Deliberately do nothing because nothing good can come out 82 | // of prettying a broken query. 83 | setPossiblePrettySQL(""); 84 | } 85 | } 86 | }, [query]); 87 | 88 | const asPath = router.asPath; 89 | useEffect(() => { 90 | const [asPathRoot, asPathQuery = ""] = asPath.split("?"); 91 | const params = new URLSearchParams(asPathQuery); 92 | if (query) { 93 | params.set("query", query); 94 | } else { 95 | params.delete("query"); 96 | } 97 | let asPathNew = asPathRoot; 98 | if (params.toString()) { 99 | asPathNew += `?${params.toString()}`; 100 | } 101 | 102 | if (asPathNew !== asPath) { 103 | routerReplace(asPathNew); 104 | } 105 | }, [query, asPath, routerReplace]); 106 | 107 | const [savedQueries, setSavedQueries] = useState([]); 108 | 109 | useEffect(() => { 110 | const storage = 111 | process.env.NODE_ENV === "development" ? sessionStorage : localStorage; 112 | try { 113 | let previous = JSON.parse( 114 | storage.getItem("saved_queries") || "[]", 115 | ) as SavedQuery[]; 116 | if (!Array.isArray(previous)) { 117 | previous = []; 118 | } 119 | setSavedQueries(previous); 120 | } catch (err) { 121 | if (process.env.NODE_ENV === "development") { 122 | throw err; 123 | } else { 124 | console.warn("Unable to save to local storage", err); 125 | } 126 | } 127 | }, []); 128 | 129 | useEffect(() => { 130 | if (!defaultQuery) { 131 | if (savedQueries.length) { 132 | // Use the most recently used one 133 | setQuery(savedQueries[0].query); 134 | } else { 135 | setQuery(DEFAULT_QUERY); 136 | } 137 | } 138 | }, [defaultQuery, savedQueries]); 139 | 140 | useEffect(() => { 141 | const storage = 142 | process.env.NODE_ENV === "development" ? sessionStorage : localStorage; 143 | try { 144 | storage.setItem("saved_queries", JSON.stringify(savedQueries)); 145 | } catch (err) { 146 | if (process.env.NODE_ENV === "development") { 147 | throw err; 148 | } else { 149 | console.warn("Unable to save to local storage", err); 150 | } 151 | } 152 | }, [savedQueries]); 153 | 154 | useEffect(() => { 155 | if (queryError) { 156 | setSavedQueries((prevState) => { 157 | return prevState.filter((entry) => entry.query != query); 158 | }); 159 | } else if (foundRecords) { 160 | setSavedQueries((prevState) => { 161 | if ( 162 | prevState.length > 0 && 163 | prevState[0].query === query && 164 | prevState[0].count === foundRecords.length 165 | ) { 166 | return prevState; 167 | } 168 | const keepQueries: SavedQuery[] = []; 169 | // Can't just .slice() we because we want to make sure we don't delete 170 | // those that are starred 171 | const max = 50; 172 | prevState.forEach((old, i) => { 173 | if (old.query === query) return; 174 | if (i > max && !old.star) return; 175 | keepQueries.push(old); 176 | }); 177 | 178 | return [ 179 | { query, count: foundRecords.length, ts: new Date().getTime() }, 180 | ...keepQueries, 181 | ]; 182 | }); 183 | } 184 | }, [query, foundRecords, queryError]); 185 | 186 | const [currentMenu, setCurrentMenu] = useState(""); 187 | 188 | function toggleMenu(key: ToolbarMenuOption) { 189 | setCurrentMenu((prevState) => { 190 | return prevState === key ? "" : key; 191 | }); 192 | } 193 | 194 | const possibleKeys = pagesToPossibleKeys(data.pages); 195 | 196 | return ( 197 |
198 | {queryError && ( 199 | 200 | {queryError.toString()} 201 | 202 | )} 203 | { 210 | setSavedQueries((prevState) => [ 211 | ...prevState.filter((p) => p.query !== query), 212 | ]); 213 | }} 214 | starQuery={(query: string) => { 215 | setSavedQueries((prevState) => [ 216 | ...prevState.map((p) => { 217 | if (p.query === query) { 218 | return Object.assign({}, p, { star: !Boolean(p.star) }); 219 | } else { 220 | return Object.assign({}, p); 221 | } 222 | }), 223 | ]); 224 | }} 225 | currentMenu={currentMenu} 226 | toggleMenu={toggleMenu} 227 | deleteAllSavedQueries={(includeStarred = false) => { 228 | setSavedQueries((prevState) => 229 | prevState.filter((entry) => { 230 | return !includeStarred && entry.star; 231 | }), 232 | ); 233 | }} 234 | possibleKeys={possibleKeys} 235 | /> 236 | {foundRecords !== null && currentMenu === "" && ( 237 | 238 | )} 239 | 240 |
241 | ); 242 | } 243 | 244 | function CodeInputAndToolbar({ 245 | query, 246 | setQuery, 247 | prettyQuery, 248 | queryError, 249 | savedQueries, 250 | deleteSavedQuery, 251 | starQuery, 252 | currentMenu, 253 | toggleMenu, 254 | deleteAllSavedQueries, 255 | possibleKeys, 256 | }: { 257 | query: string; 258 | setQuery: (x: string) => void; 259 | prettyQuery: string; 260 | queryError: Error | null; 261 | savedQueries: SavedQuery[]; 262 | deleteSavedQuery: (query: string) => void; 263 | starQuery: (query: string) => void; 264 | currentMenu: ToolbarMenuOption; 265 | toggleMenu: (menu: ToolbarMenuOption) => void; 266 | deleteAllSavedQueries: (includingStarred?: boolean) => void; 267 | possibleKeys: PossibleKeys; 268 | }) { 269 | const [typedQuery, setTypedQuery] = useState(""); 270 | 271 | useEffect(() => { 272 | if (query) { 273 | setTypedQuery(query); 274 | } 275 | }, [query]); 276 | 277 | return ( 278 |
279 | { 285 | setQuery(value.trim()); 286 | toggleMenu(""); 287 | }} 288 | hasError={Boolean(queryError)} 289 | savedQueries={savedQueries} 290 | currentMenu={currentMenu} 291 | toggleMenu={toggleMenu} 292 | /> 293 | { 298 | setTypedQuery(query); 299 | setQuery(query.trim()); 300 | }} 301 | deleteSavedQuery={(query: string) => { 302 | deleteSavedQuery(query); 303 | 304 | if (typedQuery === query) { 305 | setTypedQuery(""); 306 | } 307 | }} 308 | starQuery={starQuery} 309 | deleteAllSavedQueries={deleteAllSavedQueries} 310 | possibleKeys={possibleKeys} 311 | /> 312 |
313 | ); 314 | } 315 | -------------------------------------------------------------------------------- /src/components/theme-switcher.tsx: -------------------------------------------------------------------------------- 1 | import { useMantineColorScheme, Button } from "@mantine/core"; 2 | import { IconSun, IconMoonStars } from "@tabler/icons-react"; 3 | 4 | export function ThemeSwitcher() { 5 | const { colorScheme, toggleColorScheme } = useMantineColorScheme(); 6 | const dark = colorScheme === "dark"; 7 | return ( 8 | 20 | ); 21 | } 22 | -------------------------------------------------------------------------------- /src/components/toolbar-menu.tsx: -------------------------------------------------------------------------------- 1 | import { Group, Button } from "@mantine/core"; 2 | 3 | import type { ToolbarMenuOption, SavedQuery } from "../types"; 4 | 5 | export function ToolbarMenu({ 6 | current, 7 | toggle, 8 | savedQueries, 9 | }: { 10 | current: ToolbarMenuOption; 11 | toggle: (name: ToolbarMenuOption) => void; 12 | savedQueries: SavedQuery[]; 13 | }) { 14 | return ( 15 | 16 | 23 | 24 | 33 | 34 | 43 | 44 | ); 45 | } 46 | -------------------------------------------------------------------------------- /src/components/toolbar.tsx: -------------------------------------------------------------------------------- 1 | import { Box } from "@mantine/core"; 2 | 3 | import type { SavedQuery, ToolbarMenuOption } from "../types"; 4 | import { ShowHelp } from "./help"; 5 | import { ShowSavedQueries } from "./saved-queries"; 6 | import { ExampleQueries } from "./example-queries"; 7 | import type { PossibleKeys } from "../contexts/possible-keys"; 8 | 9 | export function Toolbar({ 10 | currentMenu, 11 | toggleMenu, 12 | savedQueries, 13 | loadQuery, 14 | deleteSavedQuery, 15 | starQuery, 16 | deleteAllSavedQueries, 17 | possibleKeys, 18 | }: { 19 | currentMenu: ToolbarMenuOption; 20 | toggleMenu: (menu: ToolbarMenuOption) => void; 21 | savedQueries: SavedQuery[]; 22 | loadQuery: (query: string) => void; 23 | deleteSavedQuery: (query: string) => void; 24 | starQuery: (query: string) => void; 25 | deleteAllSavedQueries: (includingStarred?: boolean) => void; 26 | possibleKeys: PossibleKeys; 27 | }) { 28 | return ( 29 | 30 | {currentMenu === "help" && } 31 | {currentMenu === "saved" && ( 32 | { 35 | loadQuery(query); 36 | toggleMenu(""); 37 | }} 38 | deleteSavedQuery={(query: string) => { 39 | deleteSavedQuery(query); 40 | }} 41 | starQuery={starQuery} 42 | deleteAllSavedQueries={deleteAllSavedQueries} 43 | /> 44 | )} 45 | {currentMenu === "examples" && ( 46 | { 48 | loadQuery(query); 49 | toggleMenu(""); 50 | }} 51 | /> 52 | )} 53 | 54 | ); 55 | } 56 | -------------------------------------------------------------------------------- /src/contexts/possible-keys.ts: -------------------------------------------------------------------------------- 1 | import type { Page } from "../types"; 2 | 3 | type ArrayTypes = "array" | "array(numbers)" | "array(strings)"; 4 | type Type = 5 | | "boolean" 6 | | "number" 7 | | "string" 8 | | "array(numbers)" 9 | | "array(strings)" 10 | | "array" 11 | | "json" 12 | | "unknown"; 13 | export type PossibleKeys = Map; 14 | 15 | function sluethArrayType(value: any[]): ArrayTypes { 16 | if (value.every((entry) => typeof entry === "number")) { 17 | return "array(numbers)"; 18 | } 19 | if (value.every((entry) => typeof entry === "string")) { 20 | return "array(strings)"; 21 | } 22 | return "array"; 23 | } 24 | 25 | export function pagesToPossibleKeys(pages: Page[]) { 26 | const found: PossibleKeys = new Map(); 27 | 28 | let rowsTested = 0; 29 | for (const page of pages) { 30 | for (const [name, value] of Object.entries(page)) { 31 | if (found.has(name)) continue; 32 | 33 | let type: Type = "unknown"; 34 | if (typeof value === "boolean") { 35 | type = "boolean"; 36 | } else if (typeof value === "number") { 37 | type = "number"; 38 | } else if (typeof value === "string") { 39 | type = "string"; 40 | } else if (Array.isArray(value)) { 41 | type = sluethArrayType(value); 42 | } else if (typeof value === "object") { 43 | type = "json"; 44 | } else { 45 | console.warn("Unknown type of value:", { name, value }, typeof value); 46 | } 47 | found.set(name, type); 48 | } 49 | rowsTested++; 50 | if (rowsTested >= 100) break; 51 | } 52 | return found; 53 | } 54 | -------------------------------------------------------------------------------- /src/hooks/use-router-replace.ts: -------------------------------------------------------------------------------- 1 | import { useRouter } from "next/router"; 2 | import type { NextRouter } from "next/router"; 3 | import { useRef, useState } from "react"; 4 | 5 | export default function useRouterReplace(): NextRouter["replace"] { 6 | const router = useRouter(); 7 | const routerRef = useRef(router); 8 | 9 | routerRef.current = router; 10 | 11 | const [{ replace }] = useState>({ 12 | replace: (path) => routerRef.current.replace(path), 13 | }); 14 | 15 | return replace; 16 | } 17 | -------------------------------------------------------------------------------- /src/lib/sources.ts: -------------------------------------------------------------------------------- 1 | const _source = process.env.CONTENT_SOURCES || ""; 2 | export const CONTENT_SOURCES = _source 3 | .split(",") 4 | .map((x) => x.trim()) 5 | .filter(Boolean); 6 | 7 | if (CONTENT_SOURCES.length === 0) { 8 | throw new Error( 9 | "Configuration error. You have to specify a $CONTENT_SOURCES environment variable.", 10 | ); 11 | } 12 | 13 | const _plugins = process.env.PLUGINS_SOURCES || ""; 14 | export const PLUGINS_SOURCES = _plugins 15 | .split(",") 16 | .map((x) => x.trim()) 17 | .filter(Boolean); 18 | 19 | export const EDITOR = process.env.EDITOR || ""; 20 | -------------------------------------------------------------------------------- /src/pages/_app.tsx: -------------------------------------------------------------------------------- 1 | import Head from "next/head"; 2 | import { MantineProvider } from "@mantine/core"; 3 | import "@mantine/core/styles.css"; 4 | import "../styles/globals.css"; 5 | import type { AppProps } from "next/app"; 6 | 7 | function MyApp({ Component, pageProps }: AppProps) { 8 | return ( 9 | <> 10 | 11 | docsQL 12 | 16 | 17 | 18 | 19 | 25 | 26 | 27 | 28 | ); 29 | } 30 | 31 | export default MyApp; 32 | -------------------------------------------------------------------------------- /src/pages/_document.tsx: -------------------------------------------------------------------------------- 1 | import Document from "next/document"; 2 | import { createGetInitialProps } from "@mantine/next"; 3 | 4 | const getInitialProps = createGetInitialProps(); 5 | 6 | export default class _Document extends Document { 7 | static getInitialProps = getInitialProps; 8 | } 9 | -------------------------------------------------------------------------------- /src/pages/api/open.ts: -------------------------------------------------------------------------------- 1 | import fs from "fs"; 2 | import path from "path"; 3 | 4 | import openEditor, { getEditorInfo } from "open-editor"; 5 | 6 | import { CONTENT_SOURCES, EDITOR } from "../../lib/sources"; 7 | 8 | // Next.js API route support: https://nextjs.org/docs/api-routes/introduction 9 | import type { NextApiRequest, NextApiResponse } from "next"; 10 | import type { OpenFile } from "../../types"; 11 | 12 | type BadRequestError = { 13 | editor: string; 14 | error: string; 15 | }; 16 | 17 | export default async function handler( 18 | req: NextApiRequest, 19 | res: NextApiResponse, 20 | ) { 21 | const filePath = Array.isArray(req.query.filePath) 22 | ? req.query.filePath[0] 23 | : req.query.filePath; 24 | 25 | if (!filePath) { 26 | res 27 | .status(400) 28 | .json({ editor: EDITOR, error: "No 'filePath' query string" }); 29 | return; 30 | } 31 | 32 | let absolutePath = ""; 33 | for (const source of CONTENT_SOURCES) { 34 | absolutePath = path.join(source, filePath); 35 | if (fs.existsSync(absolutePath)) { 36 | break; 37 | } 38 | } 39 | 40 | if (!absolutePath) { 41 | res.status(404).json({ 42 | filePath, 43 | editor: EDITOR, 44 | error: `Never able to find ${filePath} in ${CONTENT_SOURCES}`, 45 | }); 46 | return; 47 | } 48 | 49 | const params = [ 50 | { 51 | file: absolutePath, 52 | // line: 1, 53 | // column: 5, 54 | }, 55 | ]; 56 | 57 | const editorInfo = getEditorInfo(params); 58 | // Note! Can't know if it actually worked. 59 | openEditor(params); 60 | 61 | const outcome = Object.assign({}, editorInfo, { 62 | filePath, 63 | absolutePath, 64 | editor: EDITOR, 65 | }); 66 | res.status(200).json(outcome); 67 | } 68 | -------------------------------------------------------------------------------- /src/pages/index.tsx: -------------------------------------------------------------------------------- 1 | import { Home } from "../components/home"; 2 | 3 | const Page = () => { 4 | return ; 5 | }; 6 | export default Page; 7 | -------------------------------------------------------------------------------- /src/styles/about-metadata.module.css: -------------------------------------------------------------------------------- 1 | .footer { 2 | margin-top: 30px; 3 | 4 | } 5 | .footer p, .footer li { 6 | font-size: 80%; 7 | opacity: 0.8; 8 | } 9 | .more_meta { 10 | color: #666; 11 | } 12 | -------------------------------------------------------------------------------- /src/styles/code-input.module.css: -------------------------------------------------------------------------------- 1 | .textarea { 2 | width: 100%; 3 | margin-bottom: 5px; 4 | } 5 | -------------------------------------------------------------------------------- /src/styles/example-queries.module.css: -------------------------------------------------------------------------------- 1 | .example { 2 | margin: 40px 5px; 3 | } 4 | -------------------------------------------------------------------------------- /src/styles/footer.module.css: -------------------------------------------------------------------------------- 1 | .footer { 2 | display: flex; 3 | flex: 1; 4 | padding: 2rem 0; 5 | border-top: 1px solid #eaeaea; 6 | justify-content: center; 7 | align-items: center; 8 | } 9 | 10 | .footer a { 11 | display: flex; 12 | justify-content: center; 13 | align-items: center; 14 | flex-grow: 1; 15 | } 16 | -------------------------------------------------------------------------------- /src/styles/found-records.module.css: -------------------------------------------------------------------------------- 1 | .row:hover { 2 | background-color: #efefef; 3 | } 4 | 5 | .row_number { 6 | opacity: 0.5; 7 | font-size: 75%; 8 | text-decoration: none; 9 | color: #888; 10 | } 11 | 12 | .undefined_value { 13 | font-style: italic; 14 | font-size: 80%; 15 | opacity: .9; 16 | } 17 | 18 | .opening, .opened { 19 | opacity: 0.8; 20 | font-size: 80%; 21 | } 22 | 23 | .found_records td, 24 | .found_records th { 25 | font-family: Consolas, Monaco, 'Andale Mono', 'Ubuntu Mono', monospace; 26 | font-size: .9em; 27 | } 28 | -------------------------------------------------------------------------------- /src/styles/globals.css: -------------------------------------------------------------------------------- 1 | html, 2 | body { 3 | padding: 0; 4 | margin: 0; 5 | font-family: -apple-system, BlinkMacSystemFont, Segoe UI, Roboto, Oxygen, 6 | Ubuntu, Cantarell, Fira Sans, Droid Sans, Helvetica Neue, sans-serif; 7 | } 8 | 9 | /* a { 10 | color: inherit; 11 | text-decoration: none; 12 | } */ 13 | 14 | * { 15 | box-sizing: border-box; 16 | } 17 | 18 | textarea { 19 | font-family: Consolas, Monaco, 'Andale Mono', 'Ubuntu Mono', monospace !important; 20 | } 21 | /* code { 22 | background-color: #f5f2f0; 23 | font-family: Consolas, Monaco, 'Andale Mono', 'Ubuntu Mono', monospace; 24 | padding: 0.2em; 25 | border-radius: 3px; 26 | } */ 27 | 28 | /* th { 29 | background-color: #efefef; 30 | } */ 31 | td, th { 32 | font-family: Consolas, Monaco, 'Andale Mono', 'Ubuntu Mono', monospace; 33 | font-size: .9em; 34 | } 35 | -------------------------------------------------------------------------------- /src/styles/home.module.css: -------------------------------------------------------------------------------- 1 | .container { 2 | padding: 0 2rem; 3 | } 4 | 5 | .loading_error { 6 | padding: 6rem; 7 | } 8 | .reloading_error { 9 | padding: 1rem; 10 | } 11 | 12 | .heading { 13 | margin: 0; 14 | opacity: 0.7; 15 | } 16 | -------------------------------------------------------------------------------- /src/styles/saved-queries.module.css: -------------------------------------------------------------------------------- 1 | .saved_query { 2 | margin: 40px 5px; 3 | } 4 | -------------------------------------------------------------------------------- /src/types.ts: -------------------------------------------------------------------------------- 1 | export interface Page { 2 | _file: string; 3 | [key: string]: string | number; 4 | } 5 | 6 | export interface Source { 7 | source: string; 8 | files: number; 9 | } 10 | 11 | export interface Meta { 12 | took: number; 13 | rows: number; 14 | sources: Source[]; 15 | version: string; 16 | } 17 | export interface PagesAndMeta { 18 | pages: Page[]; 19 | meta: Meta; 20 | } 21 | 22 | export interface SavedQuery { 23 | query: string; 24 | count: number; 25 | ts: number; 26 | star?: boolean; 27 | } 28 | 29 | export type Records = any[]; 30 | 31 | export interface OpenFile { 32 | filePath: string; 33 | editor?: string; 34 | absolutePath?: string; 35 | binary?: string; 36 | isTerminalEditor?: boolean; 37 | error?: string; 38 | } 39 | 40 | export type ToolbarMenuOption = "" | "help" | "saved" | "examples"; 41 | -------------------------------------------------------------------------------- /src/utils/syntax-highlighter.tsx: -------------------------------------------------------------------------------- 1 | import Refractor from "react-refractor"; 2 | import sql from "refractor/lang/sql"; 3 | import "prismjs/themes/prism.css"; 4 | 5 | Refractor.registerLanguage(sql); 6 | 7 | export function SQL({ code }: { code: string }) { 8 | return ; 9 | } 10 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es5", 4 | "lib": ["dom", "dom.iterable", "esnext"], 5 | "allowJs": true, 6 | "skipLibCheck": true, 7 | "strict": true, 8 | "forceConsistentCasingInFileNames": true, 9 | "noEmit": true, 10 | "esModuleInterop": true, 11 | "module": "esnext", 12 | "moduleResolution": "node", 13 | "resolveJsonModule": true, 14 | "isolatedModules": true, 15 | "jsx": "preserve", 16 | "incremental": true 17 | }, 18 | "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"], 19 | "exclude": ["node_modules"] 20 | } 21 | --------------------------------------------------------------------------------