├── .env.template ├── .eslintrc ├── .github ├── FUNDING.yml └── workflows │ └── main.yml ├── .gitignore ├── .husky └── pre-commit ├── .npmrc ├── .prettierignore ├── .prettierrc.json ├── LICENSE.md ├── README.md ├── __tests__ ├── GistDatabase.test.ts └── data.ts ├── commit.config.js ├── jest.config.json ├── package.json ├── renovate.json ├── src ├── GistDatabase.ts ├── cli.ts ├── gistApi.ts └── index.ts ├── tsconfig.json └── yarn.lock /.env.template: -------------------------------------------------------------------------------- 1 | # https://github.com/settings/tokens?type=beta with the gist read/write permissions 2 | GIST_TOKEN="" 3 | # Some random string to use for encryption 4 | GIST_ENCRYPTION_KEY="" 5 | -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "es2021": true, 4 | "node": true, 5 | "jest": true 6 | }, 7 | "extends": ["standard", "plugin:typescript-sort-keys/recommended"], 8 | "parser": "@typescript-eslint/parser", 9 | "parserOptions": { 10 | "ecmaVersion": 13, 11 | "sourceType": "module" 12 | }, 13 | "plugins": ["@typescript-eslint"], 14 | "rules": { 15 | "indent": "off", 16 | "space-before-function-paren": "off", 17 | "no-unused-vars": "warn" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | ko_fi: linesofcodedev 2 | custom: ['https://www.paypal.me/TimMikeladze'] 3 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: Main CI workflow 2 | 3 | on: [push] 4 | 5 | jobs: 6 | run-ci: 7 | name: Run Type Check & Linters 8 | runs-on: ubuntu-latest 9 | timeout-minutes: 10 10 | env: 11 | GIST_TOKEN: ${{ secrets.GIST_TOKEN }} 12 | 13 | steps: 14 | - name: Checkout repository 15 | uses: actions/checkout@v3 16 | with: 17 | fetch-depth: 0 18 | 19 | - name: Set up Node 20 | uses: actions/setup-node@v3 21 | with: 22 | node-version: 18 23 | 24 | - name: Install dependencies (with cache) 25 | uses: bahmutov/npm-install@v1 26 | 27 | - name: Check types 28 | run: yarn type-check 29 | 30 | - name: Check linting 31 | run: yarn lint 32 | 33 | - name: Test 34 | run: yarn test:ci 35 | 36 | - name: Build 37 | run: yarn build 38 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist 3 | .idea 4 | yarn-error.log 5 | coverage 6 | .env 7 | -------------------------------------------------------------------------------- /.husky/pre-commit: -------------------------------------------------------------------------------- 1 | yarn lint-staged 2 | -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | save-exact = true 2 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | dist 2 | coverage 3 | *.lock 4 | *.snap 5 | .prettierignore 6 | .env.template 7 | .gitignore 8 | .husky/** 9 | .npmrc 10 | -------------------------------------------------------------------------------- /.prettierrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "semi": false, 3 | "singleQuote": true, 4 | "trailingComma": "none" 5 | } 6 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Tim Mikeladze 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 🗄️ Gist Database 2 | 3 | ✨ Transform [gist](https://gist.github.com/) into your personal key/value data store. 4 | 5 | ```console 6 | npm install gist-database 7 | 8 | yarn add gist-database 9 | 10 | pnpm add gist-database 11 | ``` 12 | 13 | ## 🚪 Introduction 14 | 15 | Sometimes all a project needs is the ability to read/write small amounts of JSON data and have it saved in some persistent storage. Imagine a simple data-model which receives infrequent updates and could be represented as JSON object. It doesn't demand a full-blown database, but it would be neat to have a way to interact with this data and have it persist across sessions. 16 | 17 | This is where `gist-database` comes in handy, by leveraging the power of the [gist api](https://gist.github.com/) you can easily create a key/value data-store for your project. 18 | 19 | This is a perfect solution for low write / high read scenarios when serving static site content with [Next.js](https://nextjs.org/) and using [Incremental Static Regeneration](https://nextjs.org/docs/basic-features/data-fetching/incremental-static-regeneration) to keep your cached content fresh. 20 | 21 | > 👋 Hello there! Follow me [@linesofcode](https://twitter.com/linesofcode) or visit [linesofcode.dev](https://linesofcode.dev) for more cool projects like this one. 22 | 23 | ## ⚖️‍ Acceptable use policy 24 | 25 | When using this library you **must comply** with Github's [acceptable use policy](https://docs.github.com/en/github/site-policy/github-acceptable-use-policies). Do not use this library to store data that violates Github's guidelines, violates laws, is malicious, unethical, or harmful to others. 26 | 27 | ## 🏃 Getting started 28 | 29 | In order to communicate with the Gist API you need to create a [personal access token](https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token) with the `gist` scope or use the [beta tokens](https://github.com/settings/tokens?type=beta) with the `gist read/write scope`. 30 | 31 | Save this token somewhere safe, you will need it to authenticate with the Gist API. 32 | 33 | Now let's create a new database. The empty database will be created as single gist containing a single file called `database.json` with an empty JSON object: `{}`. 34 | 35 | This package comes with a cli tool to help you perform common database operations. 36 | 37 | ```console 38 | Usage: gist-database [options] 39 | 40 | Transform gist into a key/value datastore. 41 | 42 | Options: 43 | -c --create Create a new gist database. 44 | -p --public Make the gist public. (default: false) 45 | -de --description Description of the gist. (default: "") 46 | -des --destroy Destroy a gist database. Provide the gist id of the database. 47 | -t --token Gist token. Required for all operations. 48 | -h, --help display help for command 49 | ``` 50 | 51 | To create a new database run the following command in your terminal: 52 | 53 | ```console 54 | npx gist-database -c -t 55 | ``` 56 | 57 | If successful, you should see output similar to: 58 | 59 | ```json 60 | { 61 | "id": "xxxxxxxxxxxx", 62 | "rawUrl": "https://api.github.com/gists/xxxxxxxxxxxx", 63 | "url": "https://gist.github.com/xxxxxxxxxxxx", 64 | "public": false, 65 | "description": "" 66 | } 67 | ``` 68 | 69 | This is the gist containing your main database file. Save the `id` somewhere safe. You will need it to initialize your database instance. 70 | 71 | ## 📖 API 72 | 73 | ```ts 74 | import { GistDatabase, CompressionType } from 'gist-database' 75 | 76 | // Initialize the database 77 | 78 | const db = new GistDatabase({ 79 | token: process.env.GIST_TOKEN, 80 | id: process.env.GIST_ID, 81 | encryptionKey: process.env.GIST_ENCRYPTION_KEY, // Optional - Encrypt your data 82 | compression: CompressionType.pretty // Optional - Compress your data 83 | }) 84 | 85 | // Before we begin let's define an optional Tyescript interface to add some type-safety to the shape of our data. Tip: combine this with Zod for even more safety around your data and business logic. 86 | 87 | interface ExampleData { 88 | hello: string 89 | foo?: string 90 | } 91 | 92 | const original = await db.set('key', { 93 | value: { 94 | hello: 'world' 95 | } 96 | }) 97 | 98 | const found = await db.get('key') 99 | 100 | /** 101 | { 102 | value : { 103 | hello: "world" 104 | }, 105 | id: "xxxxxxxxxxxxxxxxxxx", 106 | url: "https://api.github.com/gists/xxxxxxxxxxx", 107 | rev: "xxxxx" 108 | } 109 | **/ 110 | 111 | const updated = await db.set('key', { 112 | value: { 113 | hello: 'world', 114 | foo: 'bar' 115 | } 116 | }) 117 | 118 | /** 119 | { 120 | value : { 121 | hello: "world" 122 | foo: "bar" 123 | }, 124 | id: "xxxxxxxxxxxxxxxxxxx", 125 | url: "https://api.github.com/gists/xxxxxxxxxxx" 126 | rev: "yyyyy", 127 | } 128 | **/ 129 | 130 | // A rev can be used to ensure that the data is not overwritten by another process. If the rev does not match the current rev, the update will fail. 131 | try { 132 | await updated.set('key', { 133 | value: { 134 | hello: 'world', 135 | foo: 'bar' 136 | }, 137 | rev: original.rev // this will throw an error 138 | // rev: Database.rev() // leave field blank or manually generate a new rev 139 | }) 140 | } catch (err) { 141 | // An error will be thrown due to the rev mismatch 142 | console.log(err) 143 | } 144 | 145 | // Trying to fetch an outdated rev will also throw an error 146 | try { 147 | await updated.get('key', { 148 | rev: original.rev // this will throw an error 149 | // rev: updated.rev // this will succeed 150 | }) 151 | } catch (err) { 152 | // An error will be thrown due to the rev mismatch 153 | console.log(err) 154 | } 155 | 156 | // It's possible to pass arrays as key names. This is especially useful if you want to scope your data to a specific user or group. Internally all array keys will be joined with a `.` character. 157 | 158 | await db.set(['user', 'testUserId'], { 159 | value: { 160 | displayName: 'Test User' 161 | } 162 | }) 163 | 164 | await db.get(['user', 'testUserId']) 165 | 166 | await db.has('key') // true 167 | 168 | await db.keys() // ['key'] 169 | 170 | await db.delete('key') // void 171 | 172 | await db.set('key_with_ttl', { 173 | ttl: 1000, // 1 second 174 | description: "I'll expire soon and be deleted upon retrieval" 175 | }) 176 | 177 | // Get or delete many keys at once. `undefined` will be returned for keys that don't exist. 178 | await db.getMany(['key1', 'key2', 'key3']) 179 | 180 | await db.deleteMany(['key1', 'key2', 'key3']) 181 | 182 | // Remove all gist files and delete the database 183 | await db.destroy() 184 | ``` 185 | 186 | ## 🏗️ How it works 187 | 188 | The gist of it: each database is stored as multiple `.json` files with one or more of these files maintaining additional metadata about the database. 189 | 190 | The main file is called `database.json` (this is the file corresponding to the id you provided during initialization). It serves multiple purposes, but is primarily used as a lookup table for gistIds with a specific key. It also contains additional metadata such as associating TTL values with keys. Take care when editing or removing this file as it is the source of truth for your database. 191 | 192 | When a value is created or updated a new `.json` gist is created for the document. It contains the provided value plus additional metadata such as TTL. The id of this newly created gist is then added to the lookup table in `database.json`. 193 | 194 | Each gist can contain up to 10 files, with each file having a maximum size of 1mb. 195 | 196 | When data is written or read for a specific key, this library will chunk the data and pack it into multiple files within the gist to optimize storage. 197 | 198 | ## 📄 Storing markdown files 199 | 200 | Gists lend themselves perfectly to storing markdown files which you can then revise over time. This is a great way to keep track of your notes, ideas, or even use Gist as a headless CMS to manage your blog posts. 201 | 202 | > **❗Important note:** These files will be stored as is without any **compression** or **encryption** and there is no additional guarding around inconsistent writes using revision ids when writing to the gist containing these files. 203 | 204 | Out of the box this library supports storing additional files beyond the `value` argument passed to `set`. They will be stored as a separate gist file and be part of the response when calling `get` or `set`. 205 | 206 | This is useful for storing `.md` files or other assets like `.yml` files alongside some data while circumventing the packing, compression and encryption that is typically applied to the `value` argument. 207 | 208 | ```ts 209 | const blogId = 'xxxxxx' 210 | 211 | await db.set(`blog_${blogId}`, { 212 | value: { 213 | tags: ['javascript', 'typescript', 'gist-database'], 214 | title: 'My blog post' 215 | }, 216 | files: [ 217 | { 218 | name: `blog_${id}.md`, 219 | content: `# My blog post 220 | Gist Database is pretty cool.` 221 | } 222 | ] 223 | }) 224 | 225 | await db.get(`blog_${blogId}`) 226 | 227 | /** 228 | { 229 | value : { 230 | tags: ['javascript', 'typescript', 'gist-database'], 231 | title: 'My blog post', 232 | }, 233 | id: "xxxxxxxxxxxxxxxxxxx", 234 | url: "https://api.github.com/gists/xxxxxxxxxxx" 235 | rev: "xxxxx", 236 | files: [ 237 | { 238 | name: 'blog_${id}.md', 239 | content: `# My blog post 240 | Gist Database is pretty cool.` 241 | } 242 | ] 243 | } 244 | **/ 245 | ``` 246 | 247 | ## 🗜️ Compression 248 | 249 | When initializing `GistDatabase` you can pass an optional parameter `compression` to control how data is serialized and deserialized. By default, the data is not compressed at all and is stored as plain JSON. 250 | 251 | **Available compression options:** 252 | 253 | - `none` - no compression 254 | - `msgpck` - [msgpack](https://msgpack.org/) compression using [msgpackr](https://www.npmjs.com/package/msgpackr) 255 | - `pretty` - Store data as well-formatted JSON, this is useful for debugging purposes or databases where the content needs to be easily human-readable. 256 | 257 | ## 🔐 Encryption 258 | 259 | When initializing `GistDatabase` you can pass an optional parameter called `encryptionKey` to enable `aes-256-gcm` encryption and decryption using the [cryptr](https://github.com/MauriceButler/cryptr) package. 260 | 261 | ```ts 262 | const db = new GistDatabase({ 263 | token: process.env.GIST_TOKEN, 264 | id: process.env.GIST_ID, 265 | encryptionKey: process.env.GIST_ENCRYPTION_KEY 266 | }) 267 | ``` 268 | 269 | ## 🧮 Revisions 270 | 271 | Each time a value is set, a new `rev` id is generated using the [nanoid](https://github.com/ai/nanoid) package. This revision is used to ensure that the data is not overwritten by another process. Before data is written the document for the corresponding key will be fetched its revision id checked with one provided. If they do not match the update will fail and an error will be thrown. 272 | 273 | By default, revisions are not checked when getting or setting data. To enable revision checking, pass the `rev` parameter to `get` or `set`. Typically, this would be the `rev` value returned from the previous `get` or `set` call for the same key. 274 | 275 | This is a dirty implementation of optimistic locking. It is not a perfect solution, but it is a simple way of **trying** to keep data consistent during concurrent writes. If you're looking for consistency guarantees then you should use a proper database solution, not this library. 276 | 277 | ## ⚠️ Limitations 278 | 279 | 1. This is **not** a replacement for a **production database!** Do not store data that you cannot afford to lose or that needs to remain consistent. If it's important, use the proper database solution for your problem. 280 | 1. This is not intended for **high write** scenarios. You will be rate limited by the GitHub API. This is package is intended for **low write**, **single session** scenarios. 281 | 1. The maximum size that a value can be is approximately 10mb. However, I suspect a request that large would simply be rejected by the API. It's not a scenario I'm building for as sophisticated storage is beyond the scope of this library. Once again this is not a real database, it should not be used for storing large documents. 282 | -------------------------------------------------------------------------------- /__tests__/GistDatabase.test.ts: -------------------------------------------------------------------------------- 1 | import { CompressionType, GistDatabase, GistResponse } from '../src' 2 | import { pendingAlbums } from './data' 3 | 4 | const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) 5 | 6 | let index = 0 7 | 8 | // for (const compressionType of Object.values(CompressionType)) { 9 | for (const compressionType of [CompressionType.msgpack]) { 10 | it('GistDatabase - initialize with existing gist id', async () => { 11 | const db = new GistDatabase({ 12 | token: process.env.GIST_TOKEN 13 | }) 14 | 15 | await db.set('test', { 16 | value: { 17 | name: 'one' 18 | } 19 | }) 20 | 21 | expect(db.getDatabaseId()).toBeDefined() 22 | 23 | const db2 = new GistDatabase({ 24 | token: process.env.GIST_TOKEN, 25 | id: db.getDatabaseId() 26 | }) 27 | 28 | await db2.set('test', { 29 | value: { 30 | name: 'two' 31 | } 32 | }) 33 | }) 34 | 35 | describe(`GistDatabase - compression: ${compressionType}`, () => { 36 | let db: GistDatabase 37 | beforeAll(async () => { 38 | db = new GistDatabase({ 39 | token: process.env.GIST_TOKEN, 40 | compression: compressionType, 41 | encryptionKey: 42 | index % 2 === 0 ? process.env.GIST_ENCRYPTION_KEY : undefined 43 | }) 44 | }) 45 | afterAll(async () => { 46 | await db.destroy() 47 | index = index + 1 48 | }) 49 | it('sets and gets', async () => { 50 | const res = await db.set('test_one', { 51 | value: { 52 | name: 'test_one' 53 | } 54 | }) 55 | expect(res).toMatchObject({ 56 | value: { 57 | name: 'test_one' 58 | }, 59 | id: expect.any(String), 60 | gist: expect.any(Object) 61 | }) 62 | expect(await db.get('test_one')).toMatchObject({ 63 | value: { 64 | name: 'test_one' 65 | }, 66 | id: expect.any(String), 67 | gist: expect.any(Object) 68 | }) 69 | }) 70 | it('gets all keys', async () => { 71 | await db.set('test_two', { 72 | value: { 73 | name: 'test_two' 74 | } 75 | }) 76 | expect(await db.keys()).toEqual(['test_one', 'test_two']) 77 | }) 78 | it('deletes', async () => { 79 | await db.set('test_two', { 80 | value: { 81 | name: 'test_two' 82 | } 83 | }) 84 | expect(await db.get('test_two')).toMatchObject({ 85 | value: { 86 | name: 'test_two' 87 | }, 88 | id: expect.any(String), 89 | gist: expect.any(Object), 90 | rev: expect.any(String) 91 | }) 92 | await db.delete('test_two') 93 | expect(await db.get('test_two')).toBeUndefined() 94 | }) 95 | it('key with a ttl gets deleted', async () => { 96 | const res = await db.set('test_ttl', { 97 | value: {}, 98 | ttl: 250 99 | }) 100 | 101 | await sleep(500) 102 | 103 | expect(await db.get('test_ttl')).toBeUndefined() 104 | 105 | const found = (await db.gistApi( 106 | `/gists/${res.id}`, 107 | 'GET' 108 | )) as GistResponse 109 | 110 | expect(found).toEqual({}) 111 | }) 112 | it('gets and deletes many', async () => { 113 | await db.set('test_many_one', {}) 114 | await db.set('test_many_two', {}) 115 | 116 | expect(await db.getMany(['test_many_one', 'test_many_two'])).toHaveLength( 117 | 2 118 | ) 119 | 120 | await db.deleteMany(['test_many_one', 'test_many_two']) 121 | 122 | expect(await db.getMany(['test_many_one', 'test_many_two'])).toEqual([ 123 | undefined, 124 | undefined 125 | ]) 126 | }) 127 | 128 | it('sets and gets with revs', async () => { 129 | const initialRevision = GistDatabase.rev() 130 | 131 | const key = 'revs_tests' 132 | 133 | const res = await db.set(key, { 134 | value: { 135 | name: key 136 | }, 137 | rev: initialRevision 138 | }) 139 | 140 | expect(res).toMatchObject({ 141 | value: { 142 | name: key 143 | }, 144 | id: expect.any(String), 145 | gist: expect.any(Object), 146 | rev: initialRevision 147 | }) 148 | 149 | const updated = await db.set(key, { 150 | value: { 151 | name: key 152 | } 153 | }) 154 | 155 | await expect( 156 | db.set(key, { 157 | value: { 158 | name: key 159 | }, 160 | rev: initialRevision 161 | }) 162 | ).rejects.toThrowError() 163 | 164 | await db.set(key, { 165 | value: { 166 | name: key 167 | }, 168 | rev: updated.rev 169 | }) 170 | 171 | const found = await db.get(key) 172 | 173 | expect(found.rev).toBeDefined() 174 | expect(found.rev).not.toEqual(initialRevision) 175 | expect(found.rev).not.toEqual(updated.rev) 176 | 177 | await expect( 178 | db.get(key, { 179 | rev: initialRevision 180 | }) 181 | ).rejects.toThrowError() 182 | }) 183 | }) 184 | } 185 | 186 | it('get and set and del static util functions', () => { 187 | const obj = { 188 | a: 1, 189 | b: { 190 | c: {} 191 | } 192 | } 193 | expect(GistDatabase.get(obj, ['a'])).toBe(1) 194 | expect(GistDatabase.get(obj, ['b', 'c'])).toBeUndefined() 195 | 196 | let res = GistDatabase.set(obj, ['a'], 2) 197 | expect(GistDatabase.get(res, ['a'])).toBe(2) 198 | res = GistDatabase.set(res, ['b', 'c'], { d: 3 }) 199 | expect(GistDatabase.get(res, ['b', 'c'])).toEqual({ d: 3 }) 200 | 201 | res = GistDatabase.del(res, ['b', 'c']) 202 | 203 | expect(GistDatabase.get(res, ['b', 'c'])).toBeUndefined() 204 | expect(GistDatabase.get(res, ['a'])).toBe(2) 205 | }) 206 | 207 | describe('GistDatabase - works with nested keys', () => { 208 | let db: GistDatabase 209 | beforeAll(async () => { 210 | db = new GistDatabase({ 211 | token: process.env.GIST_TOKEN, 212 | compression: CompressionType.pretty 213 | }) 214 | }) 215 | afterAll(async () => { 216 | await db.destroy() 217 | }) 218 | it('writes and reads a nested key', async () => { 219 | await db.set(['parent'], { 220 | value: { 221 | name: 'parent' 222 | } 223 | }) 224 | 225 | await db.set(['parent', 'child'], { 226 | value: { 227 | name: 'child' 228 | } 229 | }) 230 | 231 | expect(await db.get(['parent'])).toMatchObject({ 232 | value: { 233 | name: 'parent' 234 | } 235 | }) 236 | 237 | expect(await db.get(['parent', 'child'])).toMatchObject({ 238 | value: { 239 | name: 'child' 240 | } 241 | }) 242 | }) 243 | }) 244 | 245 | describe('GistDatabase - validates key names', () => { 246 | let db: GistDatabase 247 | beforeAll(async () => { 248 | db = new GistDatabase({ 249 | token: process.env.GIST_TOKEN, 250 | compression: CompressionType.pretty 251 | }) 252 | }) 253 | afterAll(async () => { 254 | await db.destroy() 255 | }) 256 | it('checks key name', async () => { 257 | await db.set('test-test', { 258 | value: { 259 | name: 'test' 260 | } 261 | }) 262 | 263 | await db.set('test_test', { 264 | value: { 265 | name: 'test' 266 | } 267 | }) 268 | 269 | expect(await db.get('test-test')).toMatchObject({ 270 | value: { 271 | name: 'test' 272 | } 273 | }) 274 | 275 | expect(await db.get('test_test')).toMatchObject({ 276 | value: { 277 | name: 'test' 278 | } 279 | }) 280 | }) 281 | }) 282 | 283 | describe('GistDatabase - advanced scenario 1', () => { 284 | let db: GistDatabase 285 | beforeAll(async () => { 286 | db = new GistDatabase({ 287 | token: process.env.GIST_TOKEN, 288 | compression: CompressionType.pretty 289 | }) 290 | }) 291 | afterAll(async () => { 292 | await db.destroy() 293 | }) 294 | it('stores markdown files', async () => { 295 | const res = await db.set('test_markdown', { 296 | value: { 297 | name: 'test_markdown' 298 | }, 299 | files: { 300 | 'test.md': { 301 | content: '# Hello world' 302 | } 303 | } 304 | }) 305 | 306 | expect(res).toMatchObject({ 307 | files: { 308 | 'test.md': { 309 | content: '# Hello world', 310 | url: expect.any(String) 311 | } 312 | } 313 | }) 314 | 315 | const found = await db.get('test_markdown') 316 | 317 | expect(found).toMatchObject({ 318 | files: { 319 | 'test.md': { 320 | content: '# Hello world', 321 | url: expect.any(String) 322 | } 323 | } 324 | }) 325 | 326 | await db.set('test_markdown', { 327 | value: { 328 | name: 'test_markdown' 329 | }, 330 | files: { 331 | 'test.md': { 332 | content: '# Hello world updated' 333 | }, 334 | 'test2.md': { 335 | content: '# Hello world 2' 336 | } 337 | } 338 | }) 339 | 340 | const updated = await db.get('test_markdown') 341 | 342 | expect(updated).toMatchObject({ 343 | value: { 344 | name: 'test_markdown' 345 | }, 346 | files: { 347 | 'test.md': { 348 | content: '# Hello world updated', 349 | url: expect.any(String) 350 | }, 351 | 'test2.md': { 352 | content: '# Hello world 2', 353 | url: expect.any(String) 354 | } 355 | } 356 | }) 357 | 358 | const gist = await db.gistApi( 359 | `/gists/${updated.files['test.md'].id}`, 360 | 'GET' 361 | ) 362 | 363 | expect(gist).toMatchObject({ 364 | files: { 365 | 'test.md': { 366 | content: '# Hello world updated' 367 | } 368 | } 369 | }) 370 | 371 | await db.delete('test_markdown') 372 | 373 | expect( 374 | await db.gistApi(`/gists/${updated.files['test.md'].id}`, 'GET') 375 | ).toEqual({}) 376 | }) 377 | }) 378 | 379 | describe('GistDatabase - advanced scenario 2', () => { 380 | let db: GistDatabase 381 | beforeAll(async () => { 382 | db = new GistDatabase({ 383 | token: process.env.GIST_TOKEN, 384 | compression: CompressionType.pretty 385 | }) 386 | }) 387 | afterAll(async () => { 388 | await db.destroy() 389 | }) 390 | 391 | it('sets and gets value', async () => { 392 | await db.set('pendingAlbums', { 393 | value: pendingAlbums 394 | }) 395 | 396 | let found = await db.get('pendingAlbums') 397 | 398 | expect(found).toMatchObject({ 399 | value: pendingAlbums 400 | }) 401 | 402 | expect(found.value.albums).toHaveLength(3) 403 | 404 | pendingAlbums.albums.pop() 405 | 406 | expect(found).not.toMatchObject({ 407 | value: pendingAlbums 408 | }) 409 | 410 | expect(found.value.albums).toHaveLength(3) 411 | 412 | expect(pendingAlbums.albums).toHaveLength(2) 413 | 414 | await db.set('pendingAlbums', { 415 | value: pendingAlbums 416 | }) 417 | 418 | found = await db.get('pendingAlbums') 419 | 420 | expect(found.value.albums).toHaveLength(2) 421 | 422 | expect(found).toMatchObject({ 423 | value: pendingAlbums 424 | }) 425 | }) 426 | }) 427 | -------------------------------------------------------------------------------- /__tests__/data.ts: -------------------------------------------------------------------------------- 1 | export const pendingAlbums = { 2 | lastSyncAt: 1672692167887, 3 | albums: [ 4 | { 5 | detail: { 6 | artists: [ 7 | { 8 | external_urls: { 9 | spotify: 'https://open.spotify.com/artist/4snI0qikpQST1U1VWAxEY6' 10 | }, 11 | id: '4snI0qikpQST1U1VWAxEY6', 12 | name: 'Alai Oli' 13 | } 14 | ], 15 | external_urls: { 16 | spotify: 'https://open.spotify.com/album/4VpptIyFNY5qigfujqM3bs' 17 | }, 18 | id: '4VpptIyFNY5qigfujqM3bs', 19 | images: [ 20 | { 21 | height: 640, 22 | url: 'https://i.scdn.co/image/ab67616d0000b273c4dac9da97cbf7651701ab3d', 23 | width: 640 24 | }, 25 | { 26 | height: 300, 27 | url: 'https://i.scdn.co/image/ab67616d00001e02c4dac9da97cbf7651701ab3d', 28 | width: 300 29 | }, 30 | { 31 | height: 64, 32 | url: 'https://i.scdn.co/image/ab67616d00004851c4dac9da97cbf7651701ab3d', 33 | width: 64 34 | } 35 | ], 36 | name: 'Снег и пепел, Volume 1: синглы и раритеты', 37 | release_date: '2022-07-01' 38 | } 39 | }, 40 | { 41 | detail: { 42 | artists: [ 43 | { 44 | external_urls: { 45 | spotify: 'https://open.spotify.com/artist/5LfIyLdBqyQ6dubTemDmr9' 46 | }, 47 | id: '5LfIyLdBqyQ6dubTemDmr9', 48 | name: 'Стереополина' 49 | } 50 | ], 51 | external_urls: { 52 | spotify: 'https://open.spotify.com/album/0EM9DoKLZX5Rm62TvPxuoy' 53 | }, 54 | id: '0EM9DoKLZX5Rm62TvPxuoy', 55 | images: [ 56 | { 57 | height: 640, 58 | url: 'https://i.scdn.co/image/ab67616d0000b273dc6099f7988415e3595bcec8', 59 | width: 640 60 | }, 61 | { 62 | height: 300, 63 | url: 'https://i.scdn.co/image/ab67616d00001e02dc6099f7988415e3595bcec8', 64 | width: 300 65 | }, 66 | { 67 | height: 64, 68 | url: 'https://i.scdn.co/image/ab67616d00004851dc6099f7988415e3595bcec8', 69 | width: 64 70 | } 71 | ], 72 | name: 'Гости без будущего', 73 | release_date: '2022-11-11' 74 | } 75 | }, 76 | { 77 | detail: { 78 | artists: [ 79 | { 80 | external_urls: { 81 | spotify: 'https://open.spotify.com/artist/5tlNJfV9UIpgnbWmvUEFu7' 82 | }, 83 | id: '5tlNJfV9UIpgnbWmvUEFu7', 84 | name: 'Gentleman' 85 | } 86 | ], 87 | external_urls: { 88 | spotify: 'https://open.spotify.com/album/7timh5uVzLFbEt8bDPaIzq' 89 | }, 90 | id: '7timh5uVzLFbEt8bDPaIzq', 91 | images: [ 92 | { 93 | height: 640, 94 | url: 'https://i.scdn.co/image/ab67616d0000b2730994588d076f8470f6b9c8aa', 95 | width: 640 96 | }, 97 | { 98 | height: 300, 99 | url: 'https://i.scdn.co/image/ab67616d00001e020994588d076f8470f6b9c8aa', 100 | width: 300 101 | }, 102 | { 103 | height: 64, 104 | url: 'https://i.scdn.co/image/ab67616d000048510994588d076f8470f6b9c8aa', 105 | width: 64 106 | } 107 | ], 108 | name: 'Blaue Stunde (Deluxe Version)', 109 | release_date: '2021-05-14' 110 | } 111 | } 112 | ] 113 | } 114 | -------------------------------------------------------------------------------- /commit.config.js: -------------------------------------------------------------------------------- 1 | import { GitEmoji } from 'commit-it' 2 | 3 | export default { 4 | plugins: [ 5 | new GitEmoji({ 6 | askForShortDescription: false, 7 | commitBodyRequired: false 8 | }) 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /jest.config.json: -------------------------------------------------------------------------------- 1 | { 2 | "preset": "ts-jest/presets/default-esm", 3 | "testEnvironment": "node", 4 | "testPathIgnorePatterns": ["/node_modules/"], 5 | "setupFiles": ["dotenv/config"], 6 | "testMatch": ["**/?(*.)+(spec|test).[jt]s?(x)"], 7 | "extensionsToTreatAsEsm": [".ts"], 8 | "transform": { 9 | "^.+\\.tsx?$": [ 10 | "ts-jest", 11 | { 12 | "isolatedModules": true, 13 | "useESM": true 14 | } 15 | ] 16 | }, 17 | "testTimeout": 60000 18 | } 19 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "gist-database", 3 | "version": "1.8.0", 4 | "description": "Transform gist into your personal key/value data store. Pair this with Next.js and incremental static regeneration to add dynamic content to your static site. Built with TypeScript.", 5 | "author": "Tim Mikeladze ", 6 | "keywords": [ 7 | "gist", 8 | "gist-database", 9 | "gistdb", 10 | "gist-db", 11 | "gist key value store", 12 | "simple key value", 13 | "key value", 14 | "github-database" 15 | ], 16 | "repository": { 17 | "type": "git", 18 | "url": "https://github.com/TimMikeladze/gist-database.git" 19 | }, 20 | "license": "MIT", 21 | "files": [ 22 | "./dist" 23 | ], 24 | "source": "src/index.ts", 25 | "types": "dist/index.d.ts", 26 | "type": "module", 27 | "exports": { 28 | "require": "./dist/index.cjs", 29 | "default": "./dist/index.modern.js" 30 | }, 31 | "main": "./dist/index.cjs", 32 | "module": "./dist/index.module.js", 33 | "unpkg": "./dist/index.umd.js", 34 | "bin": "./dist/cli.module.js", 35 | "scripts": { 36 | "dev": "microbundle watch src/{index,cli}.ts --target node -f modern", 37 | "build": "rm -rf dist && microbundle src/{index,cli}.ts", 38 | "lint": "eslint --fix \"{src,__tests__}/**/*.+(ts|tsx|js|jsx)\" && prettier --write .", 39 | "test": "yarn node --experimental-vm-modules $(yarn bin jest) --passWithNoTests", 40 | "test:ci": "yarn test --ci --coverage", 41 | "prepublishOnly": "yarn type-check && yarn lint && yarn test && yarn build", 42 | "type-check": "tsc", 43 | "release": "release-it", 44 | "commit": "commit-it", 45 | "cli": "yarn build && node dist/cli.modern.js" 46 | }, 47 | "release-it": { 48 | "git": { 49 | "commitMessage": "🔖 | v${version}" 50 | }, 51 | "github": { 52 | "release": true 53 | }, 54 | "npm": { 55 | "publish": false 56 | } 57 | }, 58 | "lint-staged": { 59 | "**/*.{ts,js,jsx,tsx}": "eslint --fix", 60 | "*": "prettier --write" 61 | }, 62 | "husky": { 63 | "hooks": { 64 | "pre-commit": "lint-staged" 65 | } 66 | }, 67 | "devDependencies": { 68 | "@types/cryptr": "4.0.1", 69 | "@types/jest": "29.5.1", 70 | "@types/node": "18.16.16", 71 | "@typescript-eslint/eslint-plugin": "5.59.8", 72 | "@typescript-eslint/parser": "5.59.8", 73 | "commit-it": "0.0.11", 74 | "dotenv": "16.1.1", 75 | "eslint": "8.41.0", 76 | "eslint-config-standard": "17.1.0", 77 | "eslint-plugin-import": "2.27.5", 78 | "eslint-plugin-n": "15.7.0", 79 | "eslint-plugin-node": "11.1.0", 80 | "eslint-plugin-promise": "6.1.1", 81 | "eslint-plugin-typescript-sort-keys": "2.3.0", 82 | "husky": "8.0.3", 83 | "jest": "29.5.0", 84 | "lint-staged": "13.2.2", 85 | "microbundle": "0.15.1", 86 | "prettier": "2.8.8", 87 | "release-it": "15.10.3", 88 | "ts-jest": "29.1.0", 89 | "typescript": "5.0.4" 90 | }, 91 | "dependencies": { 92 | "buffer": "6.0.3", 93 | "commander": "10.0.1", 94 | "cross-fetch": "3.1.6", 95 | "cryptr": "6.2.0", 96 | "is-plain-obj": "4.1.0", 97 | "msgpackr": "1.9.2", 98 | "nanoid": "4.0.2" 99 | }, 100 | "resolutions": { 101 | "json5": ">=2.2.2" 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": ["config:base"], 3 | "stabilityDays": 3, 4 | "timezone": "America/Los_Angeles", 5 | "schedule": ["on the first day of the month"], 6 | "packageRules": [ 7 | { 8 | "matchPackagePatterns": ["*"], 9 | "matchUpdateTypes": ["minor", "patch"], 10 | "groupName": "all non-major dependencies", 11 | "groupSlug": "all-minor-patch" 12 | } 13 | ], 14 | "ignoreDeps": [] 15 | } 16 | -------------------------------------------------------------------------------- /src/GistDatabase.ts: -------------------------------------------------------------------------------- 1 | import isPlainObject from 'is-plain-obj' 2 | import { getGistApi } from './gistApi' 3 | import { Blob } from 'buffer' 4 | import { pack, unpack } from 'msgpackr' 5 | import Cryptr from 'cryptr' 6 | import { nanoid } from 'nanoid' 7 | 8 | export enum CompressionType { 9 | // eslint-disable-next-line no-unused-vars 10 | msgpack = 'msgpack', 11 | // eslint-disable-next-line no-unused-vars 12 | none = 'none', 13 | // eslint-disable-next-line no-unused-vars 14 | pretty = 'pretty' 15 | } 16 | 17 | export interface GistDatabaseOptions { 18 | compression?: CompressionType 19 | description?: string 20 | encryptionKey?: string 21 | id?: string 22 | public?: boolean 23 | token: string 24 | } 25 | 26 | export type GistResponse = { 27 | files: Record< 28 | string, 29 | { 30 | content: string 31 | size: number 32 | } 33 | > 34 | id: string 35 | url: string 36 | } 37 | 38 | export interface ExtraFile { 39 | content?: string 40 | gist?: GistResponse 41 | id: string 42 | url: string 43 | } 44 | 45 | export type ExtraFiles = Record 46 | 47 | export type Doc = { 48 | extraFile: ExtraFile 49 | files: ExtraFiles 50 | gist: { 51 | [key: string]: any 52 | id: string 53 | } 54 | id: string 55 | rev: string 56 | value: T 57 | } 58 | 59 | export type DocRef = { 60 | id: string 61 | ttl: { 62 | createdAt: number 63 | ttl: number 64 | } 65 | } 66 | 67 | export const defaultOptions: Partial = { 68 | public: false 69 | } 70 | 71 | export class GistDatabase { 72 | private readonly options: GistDatabaseOptions 73 | public readonly gistApi: ReturnType 74 | public static MAX_FILE_SIZE_BYTES = 1000000 // 1mb 75 | public static MAX_FILES_PER_GIST = 10 76 | public isNewDatabase: boolean 77 | public initialized: boolean = false 78 | private readonly cryptr: Cryptr 79 | public static ROOT_GIST_NAME = 'database.json' 80 | 81 | constructor(options: GistDatabaseOptions) { 82 | this.options = { 83 | ...defaultOptions, 84 | ...options 85 | } 86 | this.gistApi = getGistApi({ 87 | token: this.options.token 88 | }) 89 | this.cryptr = this.options.encryptionKey 90 | ? new Cryptr(this.options.encryptionKey) 91 | : undefined 92 | } 93 | 94 | public getDatabaseId() { 95 | return this.options.id 96 | } 97 | 98 | public static createDatabaseRoot( 99 | options: GistDatabaseOptions 100 | ): Promise { 101 | const gistApi = getGistApi({ 102 | token: options.token 103 | }) 104 | 105 | const cryptr = options.encryptionKey 106 | ? new Cryptr(options.encryptionKey) 107 | : undefined 108 | 109 | return gistApi('/gists', 'POST', { 110 | description: options.description, 111 | public: options.public, 112 | files: { 113 | [GistDatabase.ROOT_GIST_NAME]: { 114 | content: GistDatabase.serialize({}, options.compression, cryptr) 115 | } 116 | } 117 | }) as Promise 118 | } 119 | 120 | public async init() { 121 | let gist 122 | if (!this.options.id) { 123 | gist = await GistDatabase.createDatabaseRoot(this.options) 124 | this.options.id = gist.id 125 | this.isNewDatabase = true 126 | } else { 127 | const gist = (await this.gistApi( 128 | `/gists/${this.options.id}`, 129 | 'GET' 130 | )) as GistResponse 131 | if (!gist) { 132 | throw new Error('gist not found') 133 | } 134 | this.isNewDatabase = false 135 | this.options.id = gist.id 136 | } 137 | this.initialized = true 138 | return gist 139 | } 140 | 141 | private async initIfNeeded() { 142 | if (!this.initialized) { 143 | await this.init() 144 | } 145 | } 146 | 147 | public async keys(): Promise { 148 | const root = await this.getRoot() 149 | const database = GistDatabase.deserialize( 150 | root.files[GistDatabase.ROOT_GIST_NAME].content, 151 | this.options.compression, 152 | this.cryptr 153 | ) 154 | return Object.keys(database) 155 | } 156 | 157 | public async getRoot(): Promise { 158 | await this.initIfNeeded() 159 | return (await this.gistApi( 160 | `/gists/${this.options.id}`, 161 | 'GET' 162 | )) as GistResponse 163 | } 164 | 165 | public async get( 166 | key: string | string[], 167 | { 168 | rev 169 | }: { 170 | rev?: string 171 | } = {} 172 | ): Promise> { 173 | const path = Array.isArray(key) ? key : [key] 174 | 175 | const root = await this.getRoot() 176 | 177 | const database = GistDatabase.deserialize( 178 | root.files[GistDatabase.ROOT_GIST_NAME].content, 179 | this.options.compression, 180 | this.cryptr 181 | ) 182 | 183 | const foundDocRef: DocRef = GistDatabase.get(database, path) 184 | 185 | if (!foundDocRef) { 186 | return undefined 187 | } 188 | 189 | if (foundDocRef.ttl.ttl && GistDatabase.ttlIsExpired(foundDocRef.ttl)) { 190 | await this.deleteExtraFilesForGist({ 191 | id: foundDocRef.id 192 | }) 193 | await this.gistApi(`/gists/${foundDocRef.id}`, 'DELETE') 194 | return undefined 195 | } 196 | 197 | const gist = (await this.gistApi( 198 | `/gists/${foundDocRef.id}`, 199 | 'GET' 200 | )) as GistResponse 201 | 202 | if (!gist) { 203 | return undefined 204 | } 205 | 206 | if (!gist?.files || !Object.keys(gist.files).length) { 207 | return undefined 208 | } 209 | 210 | const doc = GistDatabase.unpack( 211 | gist.files, 212 | this.options.compression, 213 | this.cryptr 214 | ) as DocRef & Doc 215 | 216 | const files: ExtraFiles = {} 217 | 218 | if (doc.extraFile) { 219 | const gist = (await this.gistApi( 220 | `/gists/${doc.extraFile.id}`, 221 | 'GET' 222 | )) as GistResponse 223 | 224 | if (gist && gist.files && Object.keys(gist.files).length) { 225 | Object.keys(gist.files).forEach((key) => { 226 | files[key] = { 227 | content: gist.files[key].content, 228 | id: doc.extraFile.id, 229 | url: doc.extraFile.url 230 | } 231 | }) 232 | } 233 | } 234 | 235 | doc.files = files 236 | 237 | const ttl = doc.ttl 238 | 239 | if (ttl.ttl && GistDatabase.ttlIsExpired(ttl)) { 240 | await this.deleteExtraFilesForGist({ 241 | doc 242 | }) 243 | await this.gistApi(`/gists/${foundDocRef.id}`, 'DELETE') 244 | return undefined 245 | } 246 | 247 | if (rev && doc.rev !== rev) { 248 | throw new Error(GistDatabase.formatRevisionError(doc.rev, rev)) 249 | } 250 | 251 | return { 252 | gist, 253 | id: foundDocRef.id, 254 | value: doc.value, 255 | rev: doc.rev, 256 | files: doc.files, 257 | extraFile: doc.extraFile 258 | } 259 | } 260 | 261 | public getMany(keys: string[]): Promise { 262 | return Promise.all(keys.map((key) => this.get(key))) 263 | } 264 | 265 | public async has(key: string | string[]): Promise { 266 | return (await this.get(key)) !== undefined 267 | } 268 | 269 | public static unpack( 270 | files: GistResponse['files'], 271 | type: CompressionType, 272 | cryptr: Cryptr 273 | ) { 274 | const keys = Object.keys(files) 275 | if (!keys.length) { 276 | return undefined 277 | } 278 | 279 | // filter all keys which match the pattern "_${index}.json" 280 | const jsonKeys = keys.filter((key) => key.match(/_\d+\.json$/)) 281 | 282 | let data = {} 283 | for (const key of jsonKeys) { 284 | data = { 285 | ...data, 286 | ...GistDatabase.deserialize(files[key].content, type, cryptr) 287 | } 288 | } 289 | 290 | return data 291 | } 292 | 293 | public static async pack( 294 | path, 295 | value, 296 | { 297 | ttl, 298 | createdAt, 299 | rev, 300 | extraFile = null 301 | }: { 302 | createdAt?: number 303 | extraFile?: ExtraFile 304 | rev?: string 305 | ttl?: number 306 | }, 307 | type: CompressionType, 308 | cryptr: Cryptr 309 | ) { 310 | const data = { 311 | value, 312 | ttl: { 313 | ttl, 314 | createdAt 315 | }, 316 | rev, 317 | extraFile 318 | } 319 | 320 | // eslint-disable-next-line no-undef 321 | const size = new Blob([ 322 | JSON.stringify(GistDatabase.serialize(value, type, cryptr)) 323 | ]).size 324 | 325 | if ( 326 | size > 327 | GistDatabase.MAX_FILE_SIZE_BYTES * GistDatabase.MAX_FILES_PER_GIST 328 | ) { 329 | throw new Error( 330 | `attempting to write a value that is too large at ${path}` 331 | ) 332 | } 333 | 334 | // cut an object in half, returning an array containing keys to the first half and the second half 335 | const bisect = (obj) => { 336 | const keys = Object.keys(obj) 337 | const half = Math.ceil(keys.length / 2) 338 | return [keys.slice(0, half), keys.slice(half)] 339 | } 340 | 341 | const keysToValues = (keys, obj) => { 342 | return keys.reduce((acc, key) => { 343 | acc[key] = obj[key] 344 | return acc 345 | }, {}) 346 | } 347 | 348 | const toFiles = ( 349 | obj, 350 | allResults = {} 351 | ): Record< 352 | string, 353 | { 354 | content: string 355 | } 356 | > => { 357 | let finished = false 358 | let index = 0 359 | let results = {} 360 | while (!finished) { 361 | const [firstHalf, secondHalf] = bisect(obj) 362 | 363 | const firstHalfSize = new Blob([ 364 | this.serialize(keysToValues(firstHalf, obj), type, cryptr) 365 | ]).size 366 | 367 | const secondHalfSize = new Blob([ 368 | this.serialize(keysToValues(secondHalf, obj), type, cryptr) 369 | ]).size 370 | 371 | if ( 372 | GistDatabase.MAX_FILE_SIZE_BYTES >= 373 | firstHalfSize + secondHalfSize 374 | ) { 375 | results[GistDatabase.formatPath(path, index)] = { 376 | content: this.serialize(obj, type, cryptr) 377 | } 378 | finished = true 379 | } else { 380 | if (firstHalfSize >= GistDatabase.MAX_FILE_SIZE_BYTES) { 381 | results = { 382 | ...allResults, 383 | ...toFiles(keysToValues(firstHalf, obj), allResults) 384 | } 385 | } 386 | if (secondHalfSize >= GistDatabase.MAX_FILE_SIZE_BYTES) { 387 | results = { 388 | ...allResults, 389 | ...toFiles(keysToValues(secondHalf, obj), allResults) 390 | } 391 | } 392 | } 393 | index++ 394 | } 395 | return { 396 | ...allResults, 397 | ...results 398 | } 399 | } 400 | 401 | const files = toFiles(data) 402 | 403 | if (Object.keys(files).length > GistDatabase.MAX_FILES_PER_GIST) { 404 | throw new Error( 405 | `attempting to write a value that has too many files at ${path}` 406 | ) 407 | } 408 | 409 | return files 410 | } 411 | 412 | public async set( 413 | key: string | string[], 414 | args: { 415 | description?: string 416 | files?: Record< 417 | string, 418 | { 419 | content: string 420 | } 421 | > 422 | rev?: string 423 | ttl?: number 424 | value?: T 425 | } 426 | ): Promise> { 427 | const { description, ttl, value = {} } = args 428 | if (!isPlainObject(value)) { 429 | throw new Error('value must be a plain javascript object') 430 | } 431 | const path = Array.isArray(key) ? key : [key] 432 | 433 | const root = await this.getRoot() 434 | 435 | const database = GistDatabase.deserialize( 436 | root.files[GistDatabase.ROOT_GIST_NAME].content, 437 | this.options.compression, 438 | this.cryptr 439 | ) 440 | 441 | const { id } = GistDatabase.get(database, path) || {} 442 | 443 | let gist: GistResponse 444 | 445 | let created = false 446 | 447 | const newRev = nanoid() 448 | 449 | let doc: Doc 450 | 451 | const extraFiles: ExtraFiles = {} 452 | let extraFile: ExtraFile 453 | 454 | // Update existing gist 455 | if (id) { 456 | if (args.rev) { 457 | doc = await this.get(key) 458 | if (doc && doc.rev !== args.rev) { 459 | throw new Error(GistDatabase.formatRevisionError(doc.rev, args.rev)) 460 | } 461 | } 462 | 463 | if (args.files && Object.keys(args.files).length) { 464 | if (!doc) { 465 | doc = await this.get(key) 466 | } 467 | const gist = (await this.gistApi( 468 | `/gists/${doc.extraFile.id}`, 469 | 'PATCH', 470 | { 471 | files: args.files 472 | } 473 | )) as GistResponse 474 | 475 | extraFile = { 476 | id: gist.id, 477 | url: gist.url 478 | } 479 | 480 | Object.keys(args.files).forEach((key) => { 481 | extraFiles[key] = { 482 | id: gist.id, 483 | url: gist.url, 484 | gist, 485 | content: gist.files[key].content 486 | } 487 | }) 488 | } 489 | 490 | const files = await GistDatabase.pack( 491 | path, 492 | value, 493 | { 494 | ttl, 495 | createdAt: Date.now(), 496 | rev: newRev, 497 | extraFile 498 | }, 499 | this.options.compression, 500 | this.cryptr 501 | ) 502 | 503 | gist = (await this.gistApi(`/gists/${id}`, 'PATCH', { 504 | description, 505 | files 506 | })) as GistResponse 507 | } else { 508 | // Create new gist 509 | 510 | if (args.files && Object.keys(args.files).length) { 511 | const gist = (await this.gistApi('/gists', 'POST', { 512 | public: this.options.public, 513 | files: args.files 514 | })) as GistResponse 515 | 516 | extraFile = { 517 | id: gist.id, 518 | url: gist.url 519 | } 520 | 521 | Object.keys(args.files).forEach((key) => { 522 | extraFiles[key] = { 523 | id: gist.id, 524 | url: gist.url, 525 | gist, 526 | content: gist.files[key].content 527 | } 528 | }) 529 | } 530 | 531 | const files = await GistDatabase.pack( 532 | path, 533 | value, 534 | { 535 | ttl, 536 | createdAt: Date.now(), 537 | rev: args.rev || GistDatabase.rev(), 538 | extraFile 539 | }, 540 | this.options.compression, 541 | this.cryptr 542 | ) 543 | 544 | gist = (await this.gistApi('/gists', 'POST', { 545 | description, 546 | public: this.options.public, 547 | files 548 | })) as GistResponse 549 | } 550 | 551 | if (!id || ttl) { 552 | database[path.join('.')] = { 553 | id: gist.id, 554 | ttl: { 555 | ...GistDatabase.get(database, [path.join('.'), 'ttl']), 556 | ttl 557 | } 558 | } 559 | 560 | await this.gistApi(`/gists/${this.options.id}`, 'PATCH', { 561 | files: { 562 | [GistDatabase.ROOT_GIST_NAME]: { 563 | content: GistDatabase.serialize( 564 | database, 565 | this.options.compression, 566 | this.cryptr 567 | ) 568 | } 569 | } 570 | }) 571 | 572 | created = true 573 | } 574 | 575 | return { 576 | value: value as T, 577 | gist, 578 | id: gist.id, 579 | rev: created && args.rev ? args.rev : newRev, 580 | files: extraFiles, 581 | extraFile 582 | } 583 | } 584 | 585 | public async delete(key: string | string[]) { 586 | const path = Array.isArray(key) ? key : [key] 587 | const root = await this.getRoot() 588 | const database = GistDatabase.deserialize( 589 | root.files[GistDatabase.ROOT_GIST_NAME].content, 590 | this.options.compression, 591 | this.cryptr 592 | ) 593 | const found: DocRef = GistDatabase.get(database, path) 594 | 595 | if (!found) { 596 | return undefined 597 | } 598 | 599 | const doc = await this.get(key) 600 | 601 | await this.deleteExtraFilesForGist({ 602 | doc 603 | }) 604 | 605 | await this.gistApi(`/gists/${found.id}`, 'DELETE') 606 | 607 | const newDatabase = GistDatabase.del(database, path) 608 | 609 | await this.gistApi(`/gists/${this.options.id}`, 'PATCH', { 610 | files: { 611 | [GistDatabase.ROOT_GIST_NAME]: { 612 | content: GistDatabase.serialize( 613 | newDatabase, 614 | this.options.compression, 615 | this.cryptr 616 | ) 617 | } 618 | } 619 | }) 620 | } 621 | 622 | public async deleteMany(keys: string[]) { 623 | return Promise.all(keys.map((key) => this.delete(key))) 624 | } 625 | 626 | public async destroy() { 627 | const root = await this.getRoot() 628 | const database = GistDatabase.deserialize( 629 | root.files[GistDatabase.ROOT_GIST_NAME].content, 630 | this.options.compression, 631 | this.cryptr 632 | ) 633 | 634 | await Promise.allSettled( 635 | Object.keys(database).map(async (key) => { 636 | await this.deleteExtraFilesForGist(database[key].id) 637 | await this.gistApi(`/gists/${database[key].id}`, 'DELETE') 638 | }) 639 | ) 640 | 641 | await this.gistApi(`/gists/${this.options.id}`, 'DELETE') 642 | } 643 | 644 | private async deleteExtraFilesForGist({ 645 | id, 646 | doc 647 | }: { 648 | doc?: Doc 649 | id?: string 650 | }) { 651 | let foundDoc: Doc 652 | if (id) { 653 | foundDoc = await this.get(id) 654 | } else { 655 | foundDoc = doc 656 | } 657 | if (Object.keys(foundDoc.files).length) { 658 | await Promise.all( 659 | Object.keys(foundDoc.files).map((key) => { 660 | const file = foundDoc.files[key] 661 | return this.gistApi(`/gists/${file.id}`, 'DELETE') 662 | }) 663 | ) 664 | } 665 | } 666 | 667 | public static get(obj: T, path: string[]): T { 668 | const key = path.join('.') 669 | return obj[key] 670 | } 671 | 672 | public static set(obj: T, path: string[], value: any): T { 673 | const key = path.join('.') 674 | return { 675 | ...obj, 676 | [key]: value 677 | } 678 | } 679 | 680 | public static del(obj: T, path: string[]): T { 681 | const key = path.join('.') 682 | 683 | delete obj[key] 684 | 685 | return obj 686 | } 687 | 688 | public static ttlIsExpired(ttl: DocRef['ttl']) { 689 | return ttl.ttl && Date.now() - ttl.createdAt > ttl.ttl 690 | } 691 | 692 | public static formatPath(path: string[], index: number = 0) { 693 | return (Array.isArray(path) ? path.join('.') : path) + '_' + index + '.json' 694 | } 695 | 696 | public static serialize(value: any, type: CompressionType, cryptr: Cryptr) { 697 | const getData = () => { 698 | if (type === CompressionType.msgpack) { 699 | const serialized = pack(value) 700 | return JSON.stringify(serialized) 701 | } else if (type === CompressionType.pretty) { 702 | return JSON.stringify(value, null, 2) 703 | } else { 704 | return JSON.stringify(value) 705 | } 706 | } 707 | if (cryptr) { 708 | return cryptr.encrypt(getData()) 709 | } 710 | return getData() 711 | } 712 | 713 | public static deserialize(value: any, type: CompressionType, cryptr: Cryptr) { 714 | if (type === CompressionType.msgpack) { 715 | const buffer = Buffer.from( 716 | JSON.parse(cryptr ? cryptr.decrypt(value) : value) 717 | ) 718 | return unpack(buffer) 719 | } else { 720 | return JSON.parse(cryptr ? cryptr.decrypt(value) : value) 721 | } 722 | } 723 | 724 | public static rev() { 725 | return nanoid() 726 | } 727 | 728 | public static formatRevisionError(expected: string, received: string) { 729 | return `rev mismatch, expected ${expected} but was received ${received}` 730 | } 731 | } 732 | -------------------------------------------------------------------------------- /src/cli.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | import { GistDatabase } from './GistDatabase' 4 | import { program } from 'commander' 5 | 6 | interface CommanderOptions { 7 | create: boolean 8 | description: string 9 | destroy: string 10 | public: boolean 11 | token: string 12 | } 13 | 14 | const main = async () => { 15 | program 16 | .name('gist-database') 17 | .description('Transform gist into a key/value datastore.') 18 | .option('-c --create', 'Create a new gist database.') 19 | .option('-p --public', 'Make the gist public.', false) 20 | .option('-de --description ', 'Description of the gist.', '') 21 | .option( 22 | '-des --destroy ', 23 | 'Destroy a gist database. Provide the gist id of the database.' 24 | ) 25 | .requiredOption( 26 | '-t --token ', 27 | 'Gist token. Required for all operations.' 28 | ) 29 | try { 30 | program.parse(process.argv) 31 | 32 | const options: CommanderOptions = program.opts() 33 | 34 | if (options.create) { 35 | console.log('Creating database...') 36 | const res = await GistDatabase.createDatabaseRoot({ 37 | token: options.token, 38 | public: options.public, 39 | description: options.description 40 | }) 41 | console.log('Database created!') 42 | console.log({ 43 | id: res.id, 44 | rawUrl: res.url, 45 | url: `https://gist.github.com/${res.id}`, 46 | public: options.public, 47 | description: options.description 48 | }) 49 | } else if (options.destroy) { 50 | console.log('Destroying database...') 51 | const db = new GistDatabase({ 52 | token: options.token, 53 | id: options.destroy 54 | }) 55 | await db.destroy() 56 | console.log('Database destroyed!') 57 | } 58 | } catch (err) { 59 | console.error(err) 60 | process.exit(1) 61 | } 62 | 63 | process.exit() 64 | } 65 | 66 | main() 67 | -------------------------------------------------------------------------------- /src/gistApi.ts: -------------------------------------------------------------------------------- 1 | import fetch from 'cross-fetch' 2 | 3 | export const getGistApi = 4 | (options: { encryptionKey?: string; token: string }) => 5 | async ( 6 | path: string, 7 | method: 'POST' | 'GET' | 'PATCH' | 'DELETE', 8 | body: Record = {} 9 | ) => { 10 | const res = await fetch(`https://api.github.com${path}`, { 11 | method, 12 | headers: { 13 | Authorization: `Bearer ${options.token}`, 14 | Accept: 'application/vnd.github+json', 15 | 'X-GitHub-Api-Version': '2022-11-28' 16 | }, 17 | body: method === 'GET' ? undefined : JSON.stringify(body) 18 | }) 19 | 20 | if (res.ok) { 21 | try { 22 | const json = (await res.json()) as { 23 | files: Record 24 | } 25 | return json 26 | } catch (err) { 27 | return {} 28 | } 29 | } else { 30 | return {} 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | export * from './GistDatabase' 2 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "esModuleInterop": true, 4 | "noEmit": true, 5 | "module": "ESNext", 6 | "target": "ESNext", 7 | "moduleResolution": "Node" 8 | } 9 | } 10 | --------------------------------------------------------------------------------