├── core ├── lib │ ├── deps.d.ts │ ├── gensource.ts │ ├── gensource.web.ts │ ├── bit.ts │ ├── stores │ │ ├── index.ts │ │ ├── readonly.ts │ │ ├── splitwrites.ts │ │ ├── poll.ts │ │ ├── singlemem.ts │ │ ├── onekey.ts │ │ ├── ot.ts │ │ ├── map.ts │ │ └── opmem.ts │ ├── iterGuard.ts │ ├── simple.ts │ ├── err.ts │ ├── index.ts │ ├── sel.ts │ ├── typeregistry.ts │ ├── subvalues.ts │ ├── version.ts │ ├── types │ │ ├── field.ts │ │ └── map.ts │ └── opcache.ts ├── test │ ├── kvmem.ts │ ├── map.ts │ └── router.ts ├── README.md ├── LICENSE └── package.json ├── demos ├── bp │ ├── output.wasm │ ├── public │ │ ├── icon.png │ │ ├── Play.woff │ │ ├── toolpanel.css │ │ ├── editor.css │ │ └── editor.html │ ├── rustpng.d.ts │ ├── rustpng_bg.js │ ├── browserclient │ │ ├── render.ts │ │ └── index.ts │ ├── rustpng.js │ └── server.ts ├── universalclient │ ├── README.md │ ├── package.json │ ├── examplestore.ts │ ├── main.ts │ └── public │ │ └── index.html ├── midi │ ├── public │ │ └── index.html │ ├── package.json │ ├── state.ts │ ├── server.ts │ └── client.ts ├── bidirectional │ ├── public │ │ └── index.html │ ├── 1.ts │ ├── server.ts │ └── client.ts ├── monitor │ ├── deps.d.ts │ ├── README.md │ ├── package.json │ ├── public │ │ └── index.html │ ├── monitor.ts │ ├── dashboard.ts │ └── server.ts ├── contentfulgraphql │ ├── README.md │ ├── post.tsx │ ├── public │ │ └── style.css │ └── server.ts ├── text │ ├── package.json │ ├── public │ │ ├── editorstyle.css │ │ └── mdstyle.css │ └── tsconfig.json └── Makefile ├── package.json ├── prozess ├── README.md ├── package.json ├── prozess.ts ├── playground.ts ├── prozess-mock.ts └── prozessops.ts ├── .gitignore ├── filestore ├── README.md └── package.json ├── contentful ├── README.md └── playground.ts ├── graphql ├── README.md ├── mapreduce.js └── lib │ └── gql.ts ├── lmdb ├── README.md ├── package.json ├── dumpdb.js ├── yarn.lock ├── playground.ts ├── deps.d.ts ├── test.ts └── tsconfig.json ├── net ├── playground.ts ├── lib │ ├── index.ts │ ├── tcpserver.ts │ ├── tcpclient.ts │ ├── util.ts │ ├── httpserver.ts │ ├── wsclient.ts │ ├── wsserver.ts │ ├── clientservermux.ts │ ├── tinystream.ts │ ├── netmessages.ts │ └── reconnectingclient.ts ├── package.json ├── README.md ├── test.ts ├── yarn.lock └── tsconfig.json └── foundationdb ├── LICENSE ├── package.json ├── yarn.lock ├── playground.ts ├── README.md └── test.ts /core/lib/deps.d.ts: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /demos/bp/output.wasm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephg/statecraft/HEAD/demos/bp/output.wasm -------------------------------------------------------------------------------- /demos/bp/public/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephg/statecraft/HEAD/demos/bp/public/icon.png -------------------------------------------------------------------------------- /demos/bp/public/Play.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephg/statecraft/HEAD/demos/bp/public/Play.woff -------------------------------------------------------------------------------- /demos/bp/rustpng.d.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | export function convert(arg0: string, arg1: number, arg2: number, arg3: number): Uint8Array; 3 | 4 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "private": true, 3 | "workspaces": ["core", "net", "lmdb", "foundationdb", "demos/text", "demos/midi", "demos/universalclient", "demos/monitor"] 4 | } 5 | -------------------------------------------------------------------------------- /core/lib/gensource.ts: -------------------------------------------------------------------------------- 1 | import {randomBytes} from 'crypto' 2 | 3 | export default function genSource() { 4 | const sourceBytes = randomBytes(12) 5 | return sourceBytes.toString('base64') 6 | } 7 | -------------------------------------------------------------------------------- /demos/universalclient/README.md: -------------------------------------------------------------------------------- 1 | # Universal Client 2 | 3 | This is a client which will happily connect to any backing store (over TCP or WebSockets) and expose a simple UI to view the contents of the given store -------------------------------------------------------------------------------- /prozess/README.md: -------------------------------------------------------------------------------- 1 | # Statecraft prozess connector 2 | 3 | This is an experimental statecraft operation log. Ignore this for now - it is not currently maintained. It will be moved out of this repo in a future commit. 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .*.swp 3 | db 4 | .DS_Store 5 | rust/target 6 | rust/Cargo.lock 7 | dist 8 | bundle.js 9 | *.tmp-browserify* 10 | oldlib 11 | oldtest 12 | yarn-error.log 13 | scratch* 14 | *.cluster 15 | testdb 16 | .vscode 17 | -------------------------------------------------------------------------------- /filestore/README.md: -------------------------------------------------------------------------------- 1 | # Statecraft file store 2 | 3 | This provides a simple store which watches a JSON file and exposes the file's contents as a store. 4 | 5 | All writes replace the entire contents of the file (so its really not efficient), but it works great for managing application configuration. 6 | -------------------------------------------------------------------------------- /filestore/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@statecraft/filestore", 3 | "version": "0.1.0", 4 | "description": "Statecraft file store", 5 | "main": "index.js", 6 | "author": "Seph Gentle ", 7 | "license": "ISC", 8 | "dependencies": { 9 | "chokidar": "^2.1.5" 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /prozess/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@statecraft/prozess", 3 | "version": "0.1.0", 4 | "description": "Statecraft prozess binding", 5 | "main": "index.js", 6 | "author": "Seph Gentle ", 7 | "license": "ISC", 8 | "dependencies": { 9 | "prozess-client": "*" 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /contentful/README.md: -------------------------------------------------------------------------------- 1 | # Statecraft contentful store 2 | 3 | This is a proof-of-concept graphql-based contentful store using the graphql map reduce store. 4 | 5 | This existed at one point as part of a contentful demo. It is experimental and will either be deleted or moved into a different repository in the future. 6 | -------------------------------------------------------------------------------- /core/lib/gensource.web.ts: -------------------------------------------------------------------------------- 1 | // require('./util') 2 | const alpha = 'abcdefghijklmnopqrstuvwxyz' 3 | const alphabet = alpha + alpha.toUpperCase() + '0123456789' 4 | 5 | export default function genSource() { 6 | let out = '' 7 | for (let i = 0; i < 12; i++) out += alphabet[(Math.random() * alphabet.length)|0] 8 | return out 9 | } -------------------------------------------------------------------------------- /demos/midi/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 |

MIDI demo

5 |

6 | The midi store is running. To see your midi controller state, connect the universal client: 7 | 8 | 

npx ts-node demos/universalclient/main.ts tcp://localhost:2002
9 |

-------------------------------------------------------------------------------- /demos/bidirectional/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | Mouse tracker 3 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /demos/monitor/deps.d.ts: -------------------------------------------------------------------------------- 1 | 2 | declare module 'cpu-stats' { 3 | type CPUInfo = { 4 | cpu: number, 5 | user: number, 6 | nice: number, 7 | sys: number, 8 | idle: number, 9 | irq: number 10 | } 11 | function stats(samplems: number, cb: (err: Error, result: CPUInfo[]) => void): void; 12 | export = stats 13 | } 14 | -------------------------------------------------------------------------------- /demos/bp/rustpng_bg.js: -------------------------------------------------------------------------------- 1 | 2 | const path = require('path').join(__dirname, 'output.wasm'); 3 | const bytes = require('fs').readFileSync(path); 4 | let imports = {}; 5 | imports['./rustpng'] = require('./rustpng'); 6 | 7 | const wasmModule = new WebAssembly.Module(bytes); 8 | const wasmInstance = new WebAssembly.Instance(wasmModule, imports); 9 | module.exports = wasmInstance.exports; 10 | -------------------------------------------------------------------------------- /graphql/README.md: -------------------------------------------------------------------------------- 1 | # Statecraft GraphQL support 2 | 3 | Eventually I want an official graphql statecraft store whereby stores can explicitly define a graphql schema and then we have a store which wraps a KV / range store, exposing that store via graphql. 4 | 5 | Currently this only really exists as a couple experiments. This code may not currently run. It will be expanded on at a later time. 6 | -------------------------------------------------------------------------------- /demos/contentfulgraphql/README.md: -------------------------------------------------------------------------------- 1 | # Contentful Graphql demo 2 | 3 | This is a blog which uses data from contentful and serves it through a graphql rendering function. 4 | 5 | The blog content is stored in contentful, and we use the contentful streaming API to pull down changes into an in-memory store as they happen. The data is run through a map-reduce to render, and the HTML is served straight out of memory. -------------------------------------------------------------------------------- /lmdb/README.md: -------------------------------------------------------------------------------- 1 | # LMDB Statecraft bindings 2 | 3 | This is a simple statecraft binding for [LMDB](https://symas.com/lmdb/technical/). LMDB is a simple, fast embedded database. 4 | 5 | The LDMB implementation currently does not act as an operation log. That is, the store depends on an externally provided operation log; which it subscribes to on startup. All changes are streamed into a local on-disk key-value store (lmdb). 6 | -------------------------------------------------------------------------------- /core/lib/bit.ts: -------------------------------------------------------------------------------- 1 | // Simple helpers for bitsets (eg supported query type fields) 2 | 3 | // TODO: Consider exporting these into a separate tiny package. 4 | export const bitSet = (...bits: number[]) => { 5 | let field = 0 6 | for (let i = 0; i < bits.length; i++) field |= (1< ( 11 | !!(field & (1 << bitNum)) 12 | ) 13 | -------------------------------------------------------------------------------- /core/test/kvmem.ts: -------------------------------------------------------------------------------- 1 | import 'mocha' 2 | import kvmem from '../lib/stores/kvmem' 3 | import runTests from './common' 4 | import assert from 'assert' 5 | 6 | // describe('maths', () => { 7 | // it.only('Everest test', () => { 8 | 9 | 10 | // // assert(3 - 2 == 1) 11 | 12 | 13 | // // assert(3 - 3 == 0) 14 | 15 | 16 | // }) 17 | // }) 18 | 19 | describe('kvmem', () => { 20 | runTests(async () => kvmem()) 21 | }) -------------------------------------------------------------------------------- /core/README.md: -------------------------------------------------------------------------------- 1 | # Statecraft core 2 | 3 | This is the core statecraft repository. 4 | 5 | It contains some simple stores and combinators - enough to get started with very few dependancies. 6 | 7 | You will probably want to use the statecraft core with other modules to provide interoperability with your data sources, provide network interoperability and so on. These modules are packaged separately to reduce the weight of dependancies in your project. 8 | 9 | -------------------------------------------------------------------------------- /demos/bp/browserclient/render.ts: -------------------------------------------------------------------------------- 1 | import html from 'nanohtml' 2 | 3 | export default function render(mimetype: string, obj: any) { 4 | switch (mimetype) { 5 | case 'image/png': 6 | case 'image/jpeg': 7 | // return html`` 8 | return html`` 9 | 10 | case 'application/json': 11 | return html`
${JSON.stringify(obj, null, 2)}
` 12 | 13 | default: 14 | throw Error('Unknown mimetype ' + mimetype) 15 | } 16 | } -------------------------------------------------------------------------------- /net/playground.ts: -------------------------------------------------------------------------------- 1 | import {I, stores} from '@statecraft/core' 2 | import server from './lib/tcpserver' 3 | import remoteStore from './lib/tcpclient' 4 | 5 | const listen = async () => { 6 | const localStore = await stores.kvmem() 7 | server(localStore).listen(3334) 8 | console.log('listening on 3334') 9 | } 10 | 11 | const testNet = async () => { 12 | await listen() 13 | 14 | const store = await remoteStore(3334, 'localhost') 15 | const results = await store.fetch({type: I.QueryType.KV, q:new Set(['x'])}) 16 | console.log(results) 17 | } 18 | -------------------------------------------------------------------------------- /core/lib/stores/index.ts: -------------------------------------------------------------------------------- 1 | import opmem from './opmem' 2 | import kvmem from './kvmem' 3 | import singlemem from './singlemem' 4 | 5 | import ot from './ot' 6 | import map from './map' 7 | import onekey from './onekey' 8 | import readonly from './readonly' 9 | import router from './router' 10 | import splitwrites from './splitwrites' 11 | 12 | import poll from './poll' 13 | 14 | export default { 15 | opmem, kvmem, singlemem, 16 | 17 | // TODO: These should probably beCapitalizedLikeThis. 18 | ot, map, onekey, readonly, router, splitwrites, 19 | 20 | poll, 21 | } -------------------------------------------------------------------------------- /contentful/playground.ts: -------------------------------------------------------------------------------- 1 | 2 | const contentful = async () => { 3 | const keys = JSON.parse(readFileSync('keys.json', 'utf8')) 4 | const ops = createContentful(await kvStore(), { 5 | space: keys.space, 6 | accessToken: keys.contentAPI, 7 | 8 | }) 9 | 10 | const store = await kvStore(undefined, {inner: ops}) 11 | const sub = store.subscribe({type: I.QueryType.StaticRange, q: [{ 12 | low: sel(''), 13 | // low: sel('post/'), 14 | high: sel('post/\xff'), 15 | }]}) 16 | // for await (const r of sub) { 17 | // console.log('results', ins(r)) 18 | // } 19 | for await (const r of subValues(I.ResultType.Range, sub)) { 20 | console.log('results', ins(r)) 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /demos/monitor/README.md: -------------------------------------------------------------------------------- 1 | # Monitor Demo 2 | 3 | This is a simple demo showing how statecraft can be used in a devops context. 4 | 5 | There's 2 processes: 6 | 7 | - The core server, which hosts a simple website showing a dashboard 8 | - A worker process which is installed on each client, that connects to the dashboard. 9 | 10 | To run this demo: 11 | 12 | - In one tab start the monitoring dashboard with `yarn start` 13 | - In another terminal, run `yarn run monitor`. You can also run this from another machine by setting the `HOST` environment variable to point to the machine running the dashboard. 14 | 15 | The server will listen for connections. Monitors can connect to the server and stream information about CPU usage. -------------------------------------------------------------------------------- /core/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2019 Joseph Gentle 2 | 3 | Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. 4 | 5 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 6 | -------------------------------------------------------------------------------- /foundationdb/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2019 Joseph Gentle 2 | 3 | Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. 4 | 5 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 6 | -------------------------------------------------------------------------------- /net/lib/index.ts: -------------------------------------------------------------------------------- 1 | import wsserver, {wrapWebSocket} from './wsserver' 2 | import wsclient, {connect as connectToWS} from './wsclient' 3 | import tcpserver, {serveToSocket} from './tcpserver' 4 | import tcpclient, {connectToSocket, createStreams as createTCPStreams} from './tcpclient' 5 | import reconnectingclient from './reconnectingclient' 6 | import connectMux, {BothMsg} from './clientservermux' 7 | import WebSocket from 'ws' 8 | 9 | import * as tinyStream from './tinystream' 10 | 11 | export { 12 | WebSocket, wsserver, wrapWebSocket, 13 | tcpserver, serveToSocket, 14 | 15 | wsclient, connectToWS, 16 | tcpclient, connectToSocket, createTCPStreams, 17 | 18 | reconnectingclient, 19 | 20 | connectMux, BothMsg, 21 | 22 | tinyStream, 23 | } -------------------------------------------------------------------------------- /lmdb/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@statecraft/lmdb", 3 | "version": "0.1.1", 4 | "main": "dist/lmdb.js", 5 | "types": "dist/lmdb.d.ts", 6 | "author": "Seph Gentle ", 7 | "license": "ISC", 8 | "files": [ 9 | "dist/*", 10 | "lmdb.ts", 11 | "deps.d.ts" 12 | ], 13 | "scripts": { 14 | "prepare": "npm run build", 15 | "build": "tsc" 16 | }, 17 | "dependencies": { 18 | "@josephg/resolvable": "^1.0.0", 19 | "@statecraft/core": "^0.1.1", 20 | "debug": "^4.1.1", 21 | "node-lmdb": "^0.7.0" 22 | }, 23 | "devDependencies": { 24 | "@types/debug": "^4.1.3", 25 | "@types/node": "^11.13.0", 26 | "mocha": "^6.1.1", 27 | "ts-node": "^8.0.3", 28 | "typescript": "^3.4.2" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /core/lib/stores/readonly.ts: -------------------------------------------------------------------------------- 1 | // Read only store wrapper 2 | import * as I from '../interfaces' 3 | 4 | export default function readOnly | I.Store>(inner: S): S { 5 | // This is pretty messy.. :/ 6 | const outer: S = { 7 | storeInfo: { 8 | ...inner.storeInfo, 9 | capabilities: { 10 | ...inner.storeInfo.capabilities, 11 | mutationTypes: new Set() 12 | } 13 | }, 14 | 15 | mutate() { 16 | throw Error('Mutation not allowed') 17 | }, 18 | } as any as S 19 | 20 | for (const k in inner) { 21 | if (k !== 'mutate' && k !== 'storeInfo') { 22 | const v = inner[k] 23 | if (typeof v === 'function') outer[k] = v.bind(inner) 24 | } 25 | } 26 | 27 | return outer 28 | } -------------------------------------------------------------------------------- /demos/universalclient/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@statecraft/demos-universalclient", 3 | "version": "1.0.0", 4 | "description": "Statecraft univeral debugger", 5 | "main": "main.js", 6 | "author": "Seph Gentle ", 7 | "license": "ISC", 8 | "scripts": { 9 | "build": "browserify -p tsify client.ts -o public/bundle.js" 10 | }, 11 | "dependencies": { 12 | "@statecraft/core": "^0.1.0", 13 | "@statecraft/net": "^0.1.0", 14 | "@types/express": "^4.16.1", 15 | "@types/node": "^11.13.0", 16 | "choo": "^6.13.3", 17 | "express": "^4.16.4", 18 | "ot-json1": "^0.2.4", 19 | "ot-text-unicode": "^3.0.0", 20 | "ts-node": "^8.0.3", 21 | "typescript": "^3.4.2" 22 | }, 23 | "devDependencies": { 24 | "browserify": "^16.2.3", 25 | "tsify": "^4.0.1" 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /demos/monitor/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@statecraft/monitor-demo", 3 | "version": "1.0.0", 4 | "description": "Statecraft server monitor demo", 5 | "main": "server.ts", 6 | "license": "ISC", 7 | "scripts": { 8 | "build": "browserify -p tsify dashboard.ts -o public/bundle.js", 9 | "start": "ts-node server.ts", 10 | "start-monitor": "ts-node monitor.ts" 11 | }, 12 | "dependencies": { 13 | "@statecraft/core": "*", 14 | "@statecraft/net": "*", 15 | "@types/express": "^4.16.1", 16 | "@types/node": "^11.13.0", 17 | "choo": "^6.13.3", 18 | "cpu-stats": "^1.0.0", 19 | "express": "^4.16.4" 20 | }, 21 | "devDependencies": { 22 | "browserify": "^16.2.3", 23 | "tinyify": "^2.5.0", 24 | "ts-node": "^8.1.0", 25 | "tsify": "^4.0.1", 26 | "typescript": "^3.4.5" 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /foundationdb/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@statecraft/foundationdb", 3 | "version": "0.1.3", 4 | "description": "Statecraft foundationdb bindings", 5 | "main": "dist/fdb.js", 6 | "author": "Seph Gentle ", 7 | "license": "ISC", 8 | "scripts": { 9 | "prepare": "npm run build", 10 | "build": "tsc", 11 | "test": "mocha -r ts-node/register test.ts" 12 | }, 13 | "files": [ 14 | "dist/*", 15 | "fdb.ts" 16 | ], 17 | "dependencies": { 18 | "@statecraft/core": "^0.1.1", 19 | "msgpack-lite": "^0.1.26" 20 | }, 21 | "peerDependencies": { 22 | "foundationdb": "^0.10.2" 23 | }, 24 | "devDependencies": { 25 | "@types/node": "^11.13.0", 26 | "foundationdb": "^0.10.2", 27 | "mocha": "^6.1.1", 28 | "ts-node": "^8.0.3", 29 | "typescript": "^3.4.2" 30 | }, 31 | "public": true 32 | } 33 | -------------------------------------------------------------------------------- /net/lib/tcpserver.ts: -------------------------------------------------------------------------------- 1 | import {I} from '@statecraft/core' 2 | import serve from './server' 3 | import {wrapReader, wrapWriter} from './tinystream' 4 | 5 | import net, { Socket } from 'net' 6 | import msgpack from 'msgpack-lite' 7 | import { write } from 'fs'; 8 | 9 | export const serveToSocket = (store: I.Store, socket: Socket) => { 10 | const writer = msgpack.createEncodeStream() 11 | writer.pipe(socket) 12 | socket.on('end', () => { 13 | // This isn't passed through for some reason. 14 | writer.end() 15 | }) 16 | 17 | const reader = msgpack.createDecodeStream() 18 | socket.pipe(reader) 19 | 20 | serve(wrapReader(reader), wrapWriter(writer), store) 21 | } 22 | 23 | export default (store: I.Store) => { 24 | return net.createServer(c => { 25 | // console.log('got connection') 26 | serveToSocket(store, c) 27 | }) 28 | } -------------------------------------------------------------------------------- /demos/midi/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@statecraft/midi-demo", 3 | "version": "1.0.0", 4 | "description": "Statecraft midi demo", 5 | "main": "server.ts", 6 | "author": "Seph Gentle ", 7 | "license": "ISC", 8 | "scripts": { 9 | "build": "browserify -p tsify client.ts -o public/bundle.js", 10 | "start": "ts-node server.ts" 11 | }, 12 | "dependencies": { 13 | "@statecraft/core": "0.1.0", 14 | "@statecraft/net": "0.1.0", 15 | "@types/node": "^11.13.0", 16 | "@types/webmidi": "^2.0.3", 17 | "express": "^4.16.4", 18 | "ot-json1": "^0.2.4", 19 | "ts-node": "^8.0.3" 20 | }, 21 | "devDependencies": { 22 | "@types/express": "^4.16.1", 23 | "@types/node": "^11.13.0", 24 | "browserify": "^16.2.3", 25 | "tinyify": "^2.5.0", 26 | "tsify": "^4.0.1", 27 | "typescript": "^3.4.2" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /net/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@statecraft/net", 3 | "version": "0.1.1", 4 | "description": "Statecraft network server & client", 5 | "main": "dist/lib/index.js", 6 | "types": "dist/lib/index.d.js", 7 | "author": "Seph Gentle ", 8 | "license": "ISC", 9 | "scripts": { 10 | "prepare": "npm run build", 11 | "build": "tsc" 12 | }, 13 | "browser": { 14 | "./dist/lib/tcpserver.js": false, 15 | "./dist/lib/wsserver.js": false, 16 | "./dist/lib/tcpclient.js": false 17 | }, 18 | "dependencies": { 19 | "@statecraft/core": "^0.1.1", 20 | "@josephg/resolvable": "^1.0.0", 21 | "isomorphic-ws": "^4.0.1", 22 | "ministreamiterator": "^1.0.0", 23 | "ws": "^6.2.1" 24 | }, 25 | "devDependencies": { 26 | "@types/node": "^11.13.0", 27 | "@types/ws": "^6.0.1", 28 | "typescript": "^3.4.2" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /core/test/map.ts: -------------------------------------------------------------------------------- 1 | import * as I from '../lib/interfaces' 2 | import 'mocha' 3 | import kvmem from '../lib/stores/kvmem' 4 | import runTests from './common' 5 | import map from '../lib/stores/map' 6 | import splitWrites from '../lib/stores/splitwrites' 7 | import assert from 'assert' 8 | import { setKV } from '../lib/simple' 9 | 10 | describe('map', () => { 11 | it('maps simple values', async () => { 12 | const root = await kvmem() 13 | const v = await setKV(root, 'x', 5) 14 | 15 | const store = map(root, x => x + 1) 16 | const result = await store.fetch({type: I.QueryType.KV, q: new Set(['x'])}, {minVersion: v}) 17 | assert.strictEqual(result.results.get('x'), 6) 18 | }) 19 | 20 | runTests(async () => { 21 | const store = await kvmem() 22 | const read = map(store, i => i) 23 | 24 | return splitWrites(read, store) 25 | }) 26 | }) 27 | -------------------------------------------------------------------------------- /lmdb/dumpdb.js: -------------------------------------------------------------------------------- 1 | const msgpack = require('msgpack-lite') 2 | const lmdb = require('node-lmdb') 3 | const {inspect} = require('util') 4 | 5 | const env = new lmdb.Env() 6 | env.open({ path: 'db', maxDbs: 2, noTls: true, }) 7 | const snapDb = env.openDbi({name:'snapshots', create:true}) 8 | 9 | const txn = env.beginTxn({readOnly:true}) 10 | 11 | const cursor = new lmdb.Cursor(txn, snapDb) 12 | 13 | for (let found = cursor.goToFirst(); found; found = cursor.goToNext()) { 14 | cursor.getCurrentBinary((k, v) => { 15 | if (k[0] === 1) { 16 | console.log('reserved', k.slice(1).toString('utf8'), inspect(msgpack.decode(v), {colors:true})) 17 | } else { 18 | const [lastMod, data] = msgpack.decode(v) 19 | console.log(`${k.toString('utf8')}: ${inspect(data, {colors:true})} (v ${lastMod})`) 20 | } 21 | }) 22 | } 23 | 24 | cursor.close() 25 | txn.commit() 26 | 27 | -------------------------------------------------------------------------------- /core/lib/iterGuard.ts: -------------------------------------------------------------------------------- 1 | // This is a helper for async iterators which calls the cleanup function when the iterator is returned. 2 | export default function iterGuard(inner: AsyncIterator, cleanupFn: () => void): AsyncIterableIterator { 3 | let isDone = false 4 | let donefn: (v: IteratorResult) => void 5 | const doneP = new Promise>(resolve => { donefn = resolve }) 6 | 7 | const iter: AsyncIterableIterator = { 8 | next() { 9 | return isDone 10 | ? Promise.resolve({done: true, value: undefined as any as T}) 11 | : Promise.race([inner.next(), doneP]) 12 | }, 13 | return(v: T) { 14 | isDone = true 15 | const result = {value: v, done: true} 16 | donefn(result) 17 | cleanupFn() 18 | return Promise.resolve(result) 19 | }, 20 | [Symbol.asyncIterator]() { return iter } 21 | } 22 | return iter 23 | } 24 | -------------------------------------------------------------------------------- /demos/contentfulgraphql/post.tsx: -------------------------------------------------------------------------------- 1 | import React, { ReactElement } from 'react' 2 | import commonmark from 'commonmark' 3 | 4 | const renderMarkdown = (content: string) => { 5 | const parser = new commonmark.Parser({smart: true}) 6 | const writer = new commonmark.HtmlRenderer({smart: true, safe: true}) 7 | 8 | const tree = parser.parse(content) 9 | return {__html: writer.render(tree)} 10 | } 11 | 12 | export default function Post({updatedAt, title, content, author: {fullName}}: any): ReactElement { 13 | console.log('updat', updatedAt) 14 | return 15 | 16 | {title} 17 | 18 | 19 |
20 |

{title}

21 |

22 |

23 |
{fullName} / {(new Date(updatedAt)).toLocaleDateString()}
24 | 25 | } 26 | -------------------------------------------------------------------------------- /core/lib/simple.ts: -------------------------------------------------------------------------------- 1 | // This contains some simple helpers to get / set values. 2 | import * as I from './interfaces' 3 | 4 | export const getKV = async (store: I.Store, k: I.Key): Promise => { 5 | const r = await store.fetch({type: I.QueryType.KV, q: new Set([k])}) 6 | return r.results.get(k) 7 | } 8 | 9 | export const setKV = (store: I.Store, k: I.Key, v: Val): Promise => ( 10 | store.mutate(I.ResultType.KV, new Map([[k, {type: 'set', data: v}]])) 11 | ) 12 | 13 | export const rmKV = (store: I.Store, k: I.Key): Promise => ( 14 | store.mutate(I.ResultType.KV, new Map([[k, {type: 'rm'}]])) 15 | ) 16 | 17 | export const getSingle = async (store: I.Store) => ( 18 | (await store.fetch({type: I.QueryType.Single, q:true})).results 19 | ) 20 | 21 | export const setSingle = (store: I.Store, value: Val) => ( 22 | store.mutate(I.ResultType.Single, {type: 'set', data: value}) 23 | ) 24 | -------------------------------------------------------------------------------- /core/lib/stores/splitwrites.ts: -------------------------------------------------------------------------------- 1 | import * as I from '../interfaces' 2 | 3 | // This store simply sends all reads to one store, and all writes to another. 4 | export default function splitWrites(readStore: I.Store, writeStore: I.Store): I.Store { 5 | return { 6 | storeInfo: { 7 | uid: readStore.storeInfo.uid, // TODO: Is this ok? 8 | sources: readStore.storeInfo.sources, 9 | sourceIsMonotonic: readStore.storeInfo.sourceIsMonotonic, 10 | capabilities: { 11 | queryTypes: readStore.storeInfo.capabilities.queryTypes, 12 | mutationTypes: writeStore.storeInfo.capabilities.mutationTypes, 13 | } 14 | }, 15 | 16 | fetch: readStore.fetch, 17 | catchup: readStore.catchup, 18 | getOps: readStore.getOps, 19 | subscribe: readStore.subscribe, 20 | 21 | mutate: writeStore.mutate, 22 | 23 | close() { 24 | readStore.close() 25 | writeStore.close() 26 | }, 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /demos/text/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@statecraft/text-demo", 3 | "version": "1.0.0", 4 | "description": "Statecraft collaborative text editor demo", 5 | "main": "server.ts", 6 | "license": "ISC", 7 | "scripts": { 8 | "build": "browserify -p tsify editor.ts -o public/bundle.js", 9 | "start": "ts-node server.ts" 10 | }, 11 | "dependencies": { 12 | "@statecraft/core": "*", 13 | "@statecraft/net": "*", 14 | "@types/commonmark": "^0.27.3", 15 | "@types/express": "^4.16.1", 16 | "@types/fresh": "^0.5.0", 17 | "@types/jsesc": "^0.4.29", 18 | "@types/node": "^11.13.0", 19 | "commonmark": "^0.28.1", 20 | "express": "^4.16.4", 21 | "fresh": "^0.5.2", 22 | "jsesc": "^2.5.2", 23 | "msgpack-lite": "^0.1.26", 24 | "nanohtml": "^1.5.0", 25 | "ot-text-unicode": "^3.0.0" 26 | }, 27 | "devDependencies": { 28 | "browserify": "^16.2.3", 29 | "tinyify": "^2.5.0", 30 | "tsify": "^4.0.1", 31 | "typescript": "^3.5.3" 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /graphql/mapreduce.js: -------------------------------------------------------------------------------- 1 | 2 | // Given a schema.. 3 | 4 | const schema = buildSchema(` 5 | type Post { 6 | title: String, 7 | content: String, 8 | author: Author, 9 | } 10 | 11 | type Author { 12 | fullName: String, 13 | } 14 | 15 | 16 | type Query { 17 | postById(id: String!): Post 18 | } 19 | `) 20 | 21 | const store = augment(kvStore(new Map([ 22 | ['auth', {fullName: 'Seph'}], 23 | ['post1', {title: 'Yo', content: 'omg I am clever', author: 'auth'}], 24 | ]))) 25 | 26 | 27 | // Might also need to pass a map-reduce function version number, so we know when to regenerate stuff. 28 | const fullposts = gqlMapReduce(store, schema, 'Post', gql`{title, author {fullName} }`, post => { 29 | // Or whatever. The fields named by the graphql query are tagged. 30 | return `Post ${post.title} author ${post.author.fullName}` 31 | }) 32 | 33 | 34 | //// 35 | 36 | // Also need a structured way to slurp up the contents of a store into another local store. LMDB would be good here 37 | 38 | -------------------------------------------------------------------------------- /foundationdb/yarn.lock: -------------------------------------------------------------------------------- 1 | # THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. 2 | # yarn lockfile v1 3 | 4 | 5 | foundationdb@^0.9.2: 6 | version "0.9.2" 7 | resolved "https://registry.yarnpkg.com/foundationdb/-/foundationdb-0.9.2.tgz#9881732ef24f32cfc20f90cad437049c19048b92" 8 | integrity sha512-3fM9Tsm82ljkGpDML8kcbbSNmCfPytjVj/+TqXqXHGCAO1dUOUkSiyvBVGrSkkJVByMbftGyeXguoApQp9cF6A== 9 | dependencies: 10 | nan "^2.10.0" 11 | node-gyp-build "^3.3.0" 12 | 13 | nan@^2.10.0: 14 | version "2.13.2" 15 | resolved "https://registry.yarnpkg.com/nan/-/nan-2.13.2.tgz#f51dc7ae66ba7d5d55e1e6d4d8092e802c9aefe7" 16 | integrity sha512-TghvYc72wlMGMVMluVo9WRJc0mB8KxxF/gZ4YYFy7V2ZQX9l7rgbPg7vjS9mt6U5HXODVFVI2bOduCzwOMv/lw== 17 | 18 | node-gyp-build@^3.3.0: 19 | version "3.8.0" 20 | resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-3.8.0.tgz#0f57efeb1971f404dfcbfab975c284de7c70f14a" 21 | integrity sha512-bYbpIHyRqZ7sVWXxGpz8QIRug5JZc/hzZH4GbdT9HTZi6WmKCZ8GLvP8OZ9TTiIBvwPFKgtGrlWQSXDAvYdsPw== 22 | -------------------------------------------------------------------------------- /core/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@statecraft/core", 3 | "version": "0.1.2", 4 | "description": "", 5 | "main": "dist/lib/index.js", 6 | "types": "dist/lib/index.d.ts", 7 | "scripts": { 8 | "prepare": "npm run build", 9 | "test": "mocha -r ts-node/register test/*.ts", 10 | "build": "tsc" 11 | }, 12 | "browser": { 13 | "./lib/gensource.ts": "./lib/gensource.web.ts", 14 | "./dist/lib/gensource.js": "./dist/lib/gensource.web.js" 15 | }, 16 | "files": [ 17 | "dist/lib/**", 18 | "lib/**" 19 | ], 20 | "author": "Joseph Gentle ", 21 | "license": "ISC", 22 | "dependencies": { 23 | "@josephg/resolvable": "^1.0.0", 24 | "binary-search": "^1.3.2", 25 | "es6-error": "^4.1.1", 26 | "ministreamiterator": "^1.0.0" 27 | }, 28 | "devDependencies": { 29 | "@types/mocha": "^5.2.5", 30 | "@types/msgpack-lite": "^0.1.6", 31 | "@types/node": "^10.12.5", 32 | "mocha": "^5.2.0", 33 | "ts-node": "^8.0.2", 34 | "typescript": "^3.4.1" 35 | }, 36 | "public": true 37 | } 38 | -------------------------------------------------------------------------------- /demos/universalclient/examplestore.ts: -------------------------------------------------------------------------------- 1 | // This is an example store that can be connected to by the universal client 2 | 3 | import net from 'net' 4 | 5 | import {I, stores, subValues, setKV, rmKV} from '@statecraft/core' 6 | import {tcpserver} from '@statecraft/net' 7 | // import serve from '../../lib/net/tcpserver' 8 | // import makeMap from '../../lib/stores/map' 9 | 10 | const {kvmem, map: makeMap} = stores 11 | 12 | process.on('unhandledRejection', err => { 13 | console.error((err as any).stack) 14 | process.exit(1) 15 | }) 16 | 17 | ;(async () => { 18 | const store = await kvmem(new Map([ 19 | ['hi', {a:3, b:4}], 20 | ['yo', {a:10, b:30}] 21 | ])) 22 | 23 | setInterval(() => { 24 | setKV(store, 'x', {a: Math.random(), b: Math.random()}) 25 | }, 1000) 26 | 27 | const server = tcpserver(store) 28 | // const mapStore = makeMap(store, ({a, b}) => { 29 | // return "a + b is " + (a+b) 30 | // }) 31 | 32 | // const server = tcpserver(mapStore) 33 | server.listen(2002) 34 | console.log('listening on TCP port 2002') 35 | })() -------------------------------------------------------------------------------- /demos/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: all clean watch-bp watch-text watch-bidirectional watch-monitor 2 | 3 | all: demos/text/public/bundle.js 4 | clean: 5 | rm demos/*/public/bundle.js 6 | 7 | 8 | demos/text/public/bundle.js: demos/text/*.ts lib/*.ts lib/*/*.ts 9 | npx browserify -p tsify -p tinyify demos/text/editor.ts -o $@ 10 | 11 | watch-bp: 12 | npx watchify -v -p tsify demos/bp/browserclient/index.ts -o demos/bp/public/bundle.js 13 | 14 | watch-text: 15 | npx watchify -v -p tsify demos/text/editor.ts -o demos/text/public/bundle.js 16 | 17 | demos/bidirectional/public/bundle.js: demos/bidirectional/*.ts lib/*.ts lib/*/*.ts 18 | npx browserify -p tsify -p tinyify demos/bidirectional/client.ts -o $@ 19 | 20 | watch-bidirectional: 21 | npx watchify -v -p tsify demos/bidirectional/client.ts -o demos/bidirectional/public/bundle.js 22 | 23 | watch-monitor: 24 | npx watchify -v -p tsify demos/monitor/dashboard.ts -o demos/monitor/public/bundle.js 25 | 26 | watch-universalclient: 27 | npx watchify -v -p tsify demos/universalclient/client.ts -o demos/universalclient/public/bundle.js 28 | -------------------------------------------------------------------------------- /core/lib/err.ts: -------------------------------------------------------------------------------- 1 | import ExtendableError from 'es6-error' 2 | 3 | export class VersionTooOldError extends ExtendableError {} 4 | export class WriteConflictError extends ExtendableError {} 5 | export class UnsupportedTypeError extends ExtendableError {} 6 | export class AccessDeniedError extends ExtendableError {} 7 | export class InvalidDataError extends ExtendableError {} 8 | export class StoreChangedError extends ExtendableError {} 9 | 10 | const constructors = {VersionTooOldError, WriteConflictError, UnsupportedTypeError, AccessDeniedError, InvalidDataError, StoreChangedError} 11 | export default constructors 12 | 13 | export interface ErrJson {msg: string, name: string} 14 | 15 | export const errToJSON = (err: Error): ErrJson => { 16 | // console.warn('Sending error to client', err.stack) 17 | return {msg: err.message, name: err.name} 18 | } 19 | export const errFromJSON = (obj: ErrJson) => { 20 | const Con = (constructors as {[k: string]: typeof ExtendableError})[obj.name] 21 | if (Con) return new Con(obj.msg) 22 | else { 23 | const err = new Error(obj.msg) 24 | err.name = obj.name 25 | return err 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /demos/monitor/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | Dashboard 3 | 62 | 63 | 64 | 65 | -------------------------------------------------------------------------------- /core/lib/stores/poll.ts: -------------------------------------------------------------------------------- 1 | import * as I from '../interfaces' 2 | // import kvmem from './kvmem' 3 | import singlemem from './singlemem' 4 | import readonly from './readonly' 5 | import {setSingle} from '../simple' 6 | 7 | export interface PollOpts { 8 | periodMS?: number, 9 | initialVersion?: number, 10 | source?: I.Source, 11 | } 12 | 13 | const wait = (timeout: number) => new Promise(resolve => setTimeout(resolve, timeout)) 14 | 15 | // Mmmmmm I wonder if this should actually also wrap a backing store to write 16 | // into... 17 | export default async function poller( 18 | poll: () => Promise, 19 | opts: PollOpts = {}): Promise> { 20 | const initial = await poll() 21 | const inner = singlemem(initial, opts.source, opts.initialVersion) 22 | 23 | const periodMS = opts.periodMS || 3000 24 | ;(async () => { 25 | while (true) { 26 | await wait(periodMS) 27 | const newVal = await poll() 28 | // console.log('..', newVal) 29 | await setSingle(inner, newVal) 30 | } 31 | })() 32 | 33 | // TODO: Read only wrapper! 34 | return readonly(inner) 35 | } 36 | -------------------------------------------------------------------------------- /demos/text/public/editorstyle.css: -------------------------------------------------------------------------------- 1 | * { 2 | box-sizing: border-box; 3 | } 4 | 5 | body { 6 | margin: 0; 7 | background-color: #9d35db; 8 | } 9 | 10 | textarea { 11 | position: fixed; 12 | height: calc(100vh - 4em); 13 | width: calc(100vw - 4em); 14 | margin: 2em; 15 | resize: none; 16 | padding: 1.2em; 17 | border: 5px solid #0f0d6b; 18 | 19 | font-family: monospace; 20 | font-size: 16px; 21 | 22 | color: #87001d; 23 | background-color: #f9fff9; 24 | } 25 | 26 | textarea:focus { 27 | outline: none; 28 | } 29 | 30 | #connstatus { 31 | position: fixed; 32 | left: 50%; 33 | transform: translateX(-50%); 34 | padding: 4px; 35 | font-family: monospace; 36 | font-size: 16px; 37 | font-weight: bold; 38 | background-color: rgba(244, 76, 255, 0.6); 39 | } 40 | 41 | #connstatus.waiting { 42 | background-color: red; 43 | } 44 | #connstatus.connecting { 45 | background-color: yellow; 46 | } 47 | 48 | #connstatus.connected:before { 49 | content: 'Connected' 50 | } 51 | #connstatus.connecting:before { 52 | content: 'Connecting' 53 | } 54 | #connstatus.waiting:before { 55 | content: 'Not connected! Waiting to reconnect' 56 | } 57 | 58 | -------------------------------------------------------------------------------- /core/lib/index.ts: -------------------------------------------------------------------------------- 1 | import * as I from './interfaces' 2 | import stores from './stores' 3 | import err, {errFromJSON, errToJSON, ErrJson} from './err' 4 | 5 | import otDoc from './otdoc' 6 | import genSource from './gensource' 7 | import {getSingle, setSingle, getKV, rmKV, setKV} from './simple' 8 | import sel from './sel' 9 | import * as version from './version' 10 | import subValues, {catchupStateMachine, subResults} from './subvalues' 11 | 12 | import transaction from './transaction' 13 | 14 | import {register as registerType, supportedTypes, typeOrThrow} from './typeregistry' 15 | import {queryTypes, resultTypes, wrapQuery} from './qrtypes' 16 | import {bitHas, bitSet} from './bit' 17 | 18 | import makeSubGroup from './subgroup' 19 | 20 | export { 21 | I, // TODO: Its weird exposing this as types. 22 | stores, 23 | catchupStateMachine, subValues, subResults, 24 | err, errFromJSON, errToJSON, ErrJson, 25 | genSource, 26 | otDoc, 27 | 28 | getSingle, setSingle, getKV, rmKV, setKV, 29 | sel, 30 | version, 31 | 32 | registerType, supportedTypes, typeOrThrow, 33 | queryTypes, resultTypes, wrapQuery, 34 | 35 | transaction, 36 | makeSubGroup, 37 | 38 | bitSet, bitHas, 39 | } -------------------------------------------------------------------------------- /core/test/router.ts: -------------------------------------------------------------------------------- 1 | import 'mocha' 2 | import kvmem from '../lib/stores/kvmem' 3 | import router, {ALL} from '../lib/stores/router' 4 | import runTests from './common' 5 | 6 | describe('router', () => { 7 | runTests(async () => { 8 | const root = await kvmem() 9 | const store = router() 10 | 11 | // This is really dumb - we're just mounting it all the way through. 12 | store.mount(root, '', ALL, '', true) 13 | 14 | // TODO: Also run all the tests with a double-mounted store 15 | // TODO: Also run all the tests via a split read / writer. 16 | return store 17 | }) 18 | 19 | it('contains sources of all mounted stores') 20 | it('maps keys in fetch both ways') 21 | it('groups subscriptions based on the sources') 22 | 23 | describe('getops', () => { 24 | // Example inputs: 25 | // [[source:x, v:1]] => [source:x, v:1] 26 | // [[source:x, v:1], [source:y, v:1]] => [{source:x, v:1}, {source:y, v:1}] 27 | // [[v:{x:1, y:2}], [v:{y:1}]] => [{v:{y:1}}, {v:{x:1, y:2}}] 28 | // [[v:{x:1, y:1}], [v:{y:2}]] => [{v:{x:1, y:1}}, {v:{y:2}}] 29 | // [[v:{x:1}], [v:{x:1}]] => [v:{x:1}] with ops merged 30 | // ... And variations. 31 | }) 32 | }) 33 | -------------------------------------------------------------------------------- /demos/midi/state.ts: -------------------------------------------------------------------------------- 1 | // Stolen from @types/webmidi 2 | type MIDIPortDeviceState = "disconnected" | "connected"; 3 | 4 | type MIDIPortConnectionState = "open" | "closed" | "pending"; 5 | 6 | export interface MIDIPort { 7 | /** 8 | * A unique ID of the port. This can be used by developers to remember ports the 9 | * user has chosen for their application. 10 | */ 11 | id: string; 12 | 13 | /** 14 | * The manufacturer of the port. 15 | */ 16 | manufacturer?: string; 17 | 18 | /** 19 | * The system name of the port. 20 | */ 21 | name?: string; 22 | 23 | /** 24 | * The version of the port. 25 | */ 26 | version?: string; 27 | 28 | /** 29 | * The state of the device. 30 | */ 31 | state: MIDIPortDeviceState; 32 | 33 | /** 34 | * The state of the connection to the device. 35 | */ 36 | // connection: MIDIPortConnectionState; 37 | } 38 | 39 | export interface MIDIInput extends MIDIPort { 40 | keys: {[k: string]: { // The keys are actually numbers, but :/ 41 | held: boolean, 42 | pressure: number, 43 | timestamp: number 44 | }}, 45 | pots: number[], 46 | sliders: number[], 47 | pitch: number, 48 | modulation: number, 49 | } 50 | 51 | export default interface State { 52 | timeOrigin: number, 53 | inputs: MIDIInput[], 54 | outputs: MIDIPort[], 55 | } -------------------------------------------------------------------------------- /net/README.md: -------------------------------------------------------------------------------- 1 | # Statecraft networking 2 | 3 | > *Status:* Beta. Working & tested, but I want to simplify the protocol slightly before 1.0. 4 | 5 | The goal of this module is to allow clients to transparently use remote statecraft stores as if they were local. 6 | 7 | There are currently two transports available: 8 | 9 | - TCP (generally for node <-> node communication) 10 | - WebSockets (for node <-> browser communication) 11 | 12 | (I'd also like to add an ICP based comms layer implementation here for multi-language support). 13 | 14 | With each transport one computer must act as the network server, and one must act as the client. But that decision is orthogonal to which of the two machines exposes a statecraft store to its remote. 15 | 16 | You can have: 17 | 18 | - Network server exposes a store which each client consumes. This is the most common architecture and currently the most tested & supported. 19 | - Each network client exposes a store. The server consumes all the client stores (TODO: Document how to do this) 20 | - The server and each client create and expose a store. This is useful for example to have clients expose some local state back to a governing SC server. This architecture works but is currently lacking an easy way to automatically reconnect. See the bidirectional example on how to implement this. 21 | 22 | -------------------------------------------------------------------------------- /lmdb/yarn.lock: -------------------------------------------------------------------------------- 1 | # THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. 2 | # yarn lockfile v1 3 | 4 | 5 | bindings@^1.2.1: 6 | version "1.5.0" 7 | resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.5.0.tgz#10353c9e945334bc0511a6d90b38fbc7c9c504df" 8 | integrity sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ== 9 | dependencies: 10 | file-uri-to-path "1.0.0" 11 | 12 | file-uri-to-path@1.0.0: 13 | version "1.0.0" 14 | resolved "https://registry.yarnpkg.com/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz#553a7b8446ff6f684359c445f1e37a05dacc33dd" 15 | integrity sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw== 16 | 17 | nan@^2.12.0: 18 | version "2.13.2" 19 | resolved "https://registry.yarnpkg.com/nan/-/nan-2.13.2.tgz#f51dc7ae66ba7d5d55e1e6d4d8092e802c9aefe7" 20 | integrity sha512-TghvYc72wlMGMVMluVo9WRJc0mB8KxxF/gZ4YYFy7V2ZQX9l7rgbPg7vjS9mt6U5HXODVFVI2bOduCzwOMv/lw== 21 | 22 | node-lmdb@^0.6.2: 23 | version "0.6.2" 24 | resolved "https://registry.yarnpkg.com/node-lmdb/-/node-lmdb-0.6.2.tgz#2adc8b3907b067b25a3cfd2ea093253aada54ebd" 25 | integrity sha512-j9K0qVBSZ1YqP2hg++zxvxKL8mDYAhaprlH2VdfCI2Y+Pfr11kFXLfZ9lXs/H5gMq+Xy39zSR2Kh/DWU8+bUUQ== 26 | dependencies: 27 | bindings "^1.2.1" 28 | nan "^2.12.0" 29 | -------------------------------------------------------------------------------- /net/test.ts: -------------------------------------------------------------------------------- 1 | // import 'mocha' 2 | // import kvStore from '../lib/stores/kvmem' 3 | // import createServer from '../lib/net/tcpserver' 4 | // import connectStore from '../lib/stores/tcpclient' 5 | // import runTests from './common' 6 | // import * as I from '../lib/interfaces' 7 | // import {AddressInfo} from 'net' 8 | 9 | // // TODO: This is currently using a simple TCP server & client and sending 10 | // // messages over msgpack. We should also test the JSON encoding path, because 11 | // // there are a bunch of errors which can show up there that we don't catch 12 | // // here. (Eg, version encoding bugs) 13 | // describe('net', () => { 14 | // const serverForStore = new WeakMap, any>() 15 | // runTests(() => new Promise((resolve, reject) => kvStore().then(store => { 16 | // const server = createServer(store) 17 | // server.listen(0, () => { 18 | // const port = (server.address() as AddressInfo).port 19 | // connectStore(port, 'localhost').then(remoteStore => { 20 | // serverForStore.set(remoteStore!, server) 21 | // resolve(remoteStore!) 22 | // }, err => { 23 | // console.error('ERROR: Could not connect to local server', err) 24 | // server.close() 25 | // return reject(err) 26 | // }) 27 | // }) 28 | // })), (store) => serverForStore.get(store).close()) 29 | 30 | // it('returns an error if connection fails before hello is received') 31 | // }) -------------------------------------------------------------------------------- /foundationdb/playground.ts: -------------------------------------------------------------------------------- 1 | import {I} from '@statecraft/core' 2 | import fdbStore from './fdb' 3 | import {Console} from 'console' 4 | import * as fdb from 'foundationdb' 5 | 6 | fdb.setAPIVersion(600) 7 | 8 | process.on('unhandledRejection', err => { 9 | console.error(err != null ? (err as any).stack : 'error') 10 | process.exit(1) 11 | }) 12 | 13 | global.console = new (Console as any)({ 14 | stdout: process.stdout, 15 | stderr: process.stderr, 16 | inspectOptions: {depth: null} 17 | }) 18 | 19 | const testFDB = async () => { 20 | const fdbConn = fdb.openSync().at('temp-playground') // TODO: Directory layer stuff. 21 | const store = await fdbStore(fdbConn) 22 | 23 | const sub = store.subscribe({type:I.QueryType.KV, q:new Set(['x', 'q', 'y'])}, {}) 24 | ;(async () => { 25 | for await (const data of sub) { 26 | console.log('subscribe data', data) 27 | } 28 | })() 29 | 30 | const txn = new Map([['a', {type:'inc', data: 10}]]) 31 | // const txn = new Map([['x', {type:'set', data: {ddd: (Math.random() * 100)|0}}]]) 32 | // const txn = new Map([['q', {type:'set', data: (Math.random() * 100)|0}]]) 33 | console.log('source', store.storeInfo.sources) 34 | const v = await store.mutate(I.ResultType.KV, txn) 35 | console.log('mutate cb', v) 36 | 37 | const results = await store.fetch({type:I.QueryType.AllKV, q: true}) 38 | console.log('fetch results', results) 39 | 40 | store.close() 41 | } 42 | 43 | testFDB() 44 | -------------------------------------------------------------------------------- /lmdb/playground.ts: -------------------------------------------------------------------------------- 1 | import {I, stores} from '@statecraft/core' 2 | import lmdbStore from './lmdb' 3 | import {Console} from 'console' 4 | 5 | const {opmem} = stores 6 | 7 | 8 | process.on('unhandledRejection', err => { 9 | console.error(err != null ? (err as any).stack : 'error') 10 | process.exit(1) 11 | }) 12 | 13 | global.console = new (Console as any)({ 14 | stdout: process.stdout, 15 | stderr: process.stderr, 16 | inspectOptions: {depth: null} 17 | }) 18 | 19 | 20 | 21 | const testLmdb = async () => { 22 | const ops = opmem() 23 | // const client = await connectProzess() 24 | 25 | const store = await lmdbStore(ops, process.argv[2] || 'testdb') 26 | 27 | const sub = store.subscribe({type:I.QueryType.KV, q:new Set(['x', 'q', 'y'])}, {}) 28 | ;(async () => { 29 | for await (const data of sub) { 30 | console.log('subscribe data', data) 31 | } 32 | })() 33 | 34 | const txn = new Map([['a', {type:'inc', data: 10}]]) 35 | // const txn = new Map([['x', {type:'set', data: {ddd: (Math.random() * 100)|0}}]]) 36 | // const txn = new Map([['q', {type:'set', data: (Math.random() * 100)|0}]]) 37 | console.log('source', store.storeInfo.sources) 38 | const v = await store.mutate(I.ResultType.KV, txn, [new Uint8Array()]) 39 | console.log('mutate cb', v) 40 | 41 | const results = await store.fetch({type:I.QueryType.AllKV, q: true}) 42 | console.log('fetch results', results) 43 | 44 | // store.close() 45 | } 46 | 47 | testLmdb() 48 | -------------------------------------------------------------------------------- /foundationdb/README.md: -------------------------------------------------------------------------------- 1 | # Statecraft Foundationdb store 2 | 3 | This is a simple implementation of a statecraft wrapper for the [foundationdb](https://www.foundationdb.org/) database maintained by Apple. 4 | 5 | This SC wrapper supports running a horizontally scaled set of statecraft stores all pointing to the same backing foundationdb cluster. 6 | 7 | > Status: Ready for use, but see caveats below 8 | 9 | 10 | ## Usage 11 | 12 | ```javascript 13 | const fdb = require('foundationdb') 14 | const fdbStore = require('@statecraft/foundationdb') 15 | 16 | // Configure foundationdb and create a database 17 | fdb.setAPIVersion(600) 18 | const db = fdb.openSync().at('my-prefix') 19 | 20 | ;(async () => { 21 | // Wrap the foundationdb database with statecraft! 22 | const store = await fdbStore(db) 23 | 24 | // ... 25 | // store.fetch(...), etc. 26 | })() 27 | ``` 28 | 29 | 30 | ## Supported queries 31 | 32 | The foundationdb store acts as a key-value store with range support. So, you can run both standard KV queries and range queries against the store. 33 | 34 | 35 | ## Caveats 36 | 37 | - There is currently no support for deleting old operations from the operation log in foundationdb. 38 | - No performance tuning has been done yet. 39 | - Large queries are handled by single FDB transactions. This means you will run into performance issues and transaction size limits (5M by default) if you're fetching a lot of data all at once. Please file an issue if this is important to you. 40 | 41 | ## License 42 | 43 | ISC -------------------------------------------------------------------------------- /net/lib/tcpclient.ts: -------------------------------------------------------------------------------- 1 | import {I} from '@statecraft/core' 2 | import * as N from './netmessages' 3 | import createStore from './client' 4 | import {wrapReader, wrapWriter, TinyReader, TinyWriter} from './tinystream' 5 | 6 | import net, {Socket} from 'net' 7 | import msgpack from 'msgpack-lite' 8 | 9 | const wrap = (socket: Socket): [TinyReader, TinyWriter] => { 10 | const writer = wrapWriter(socket, msgpack.encode) 11 | 12 | const readStream = msgpack.createDecodeStream() 13 | socket.pipe(readStream) 14 | 15 | const reader = wrapReader(readStream) 16 | return [reader, writer] 17 | } 18 | 19 | // We're exporting so many different variants here because they're useful for 20 | // reconnecting clients and stuff like that. 21 | export function createStreams(port: number, host: string): Promise<[TinyReader, TinyWriter]> { 22 | const socket = net.createConnection(port, host) 23 | // return wrap(socket) 24 | const pair = wrap(socket) 25 | 26 | return new Promise((resolve, reject) => { 27 | socket.on('connect', () => resolve(pair)) 28 | socket.on('error', reject) 29 | }) 30 | } 31 | 32 | export function connectToSocket(socket: Socket): Promise> { 33 | const [reader, writer] = wrap(socket) 34 | return createStore(reader, writer) 35 | } 36 | 37 | export default function(port: number, host: string): Promise> { 38 | const socket = net.createConnection(port, host) 39 | return connectToSocket(socket) 40 | } -------------------------------------------------------------------------------- /foundationdb/test.ts: -------------------------------------------------------------------------------- 1 | import 'mocha' 2 | import {I} from '@statecraft/core' 3 | import fdbStore from './fdb' 4 | 5 | // TODO: Its a little gross depending on the common tests like this. It would be 6 | // better to pull this suite out into yet another module. 7 | import runTests from '@statecraft/core/dist/test/common' 8 | 9 | import * as fdb from 'foundationdb' 10 | 11 | fdb.setAPIVersion(600) 12 | 13 | const TEST_PREFIX = '__test' 14 | let _dbid = 1 15 | const storeDb = new Map, fdb.Database>() 16 | 17 | const create = async () => { 18 | const prefix = TEST_PREFIX + _dbid++ 19 | const db = fdb.openSync().at(prefix) 20 | await db.clearRange('', Buffer.from([0xff])) // Hope there's no bugs in this one! 21 | const store = await fdbStore(db) 22 | // console.log('got store at prefix', prefix) 23 | storeDb.set(store, db) 24 | return store 25 | } 26 | 27 | const teardown = (store: I.Store) => { // teardown. Nuke it. 28 | const db = storeDb.get(store)! 29 | storeDb.delete(store) 30 | store.close() 31 | db.clearRange('', Buffer.from([0xff])) 32 | // TODO: And close the database. 33 | } 34 | 35 | describe('fdb', () => { 36 | try { 37 | const db = fdb.openSync() 38 | // db.close() 39 | } catch (e) { 40 | console.warn('Warning: Foundationdb instance not found. Skipping foundationdb tests. Error:', e.message) 41 | return 42 | } 43 | 44 | // it('hi', async () => { 45 | // const store = await create() 46 | // setTimeout(() => teardown(store), 10) 47 | // }) 48 | runTests(create, teardown) 49 | }) -------------------------------------------------------------------------------- /core/lib/sel.ts: -------------------------------------------------------------------------------- 1 | import {StaticKeySelector as Sel, Key} from './interfaces' 2 | 3 | const sel = (k: Key, isAfter: boolean = false): Sel => ({k, isAfter}) 4 | 5 | // A selector and a key can never be equal. 6 | const kLtSel = sel.kLt = (k: Key, s: Sel) => k < s.k || (k === s.k && s.isAfter) 7 | sel.kGt = (k: Key, s: Sel) => !kLtSel(k, s) 8 | 9 | sel.kWithin = (k: Key, s: Sel, e: Sel) => !kLtSel(k, s) && kLtSel(k, e) 10 | 11 | const selLtSel = sel.ltSel = (a: Sel, b: Sel) => a.k < b.k || (a.k === b.k && !a.isAfter && b.isAfter) 12 | const selGtSel = sel.gtSel = (a: Sel, b: Sel) => selLtSel(b, a) 13 | sel.LtESel = (a: Sel, b: Sel) => !selLtSel(b, a) 14 | sel.GtESel = (a: Sel, b: Sel) => !selLtSel(a, b) 15 | 16 | const minSel = sel.min = (a: Sel, b: Sel): Sel => ( 17 | (a.k < b.k) ? a 18 | : (a.k > b.k) ? b 19 | : {k: a.k, isAfter: a.isAfter && b.isAfter} 20 | ) 21 | const maxSel = sel.max = (a: Sel, b: Sel): Sel => ( 22 | (a.k > b.k) ? a 23 | : (a.k < b.k) ? b 24 | : {k: a.k, isAfter: a.isAfter || b.isAfter} 25 | ) 26 | 27 | sel.intersect = (as: Sel, ae: Sel, bs: Sel, be: Sel): [Sel, Sel] | null => { 28 | const start = maxSel(as, bs) 29 | const end = minSel(ae, be) 30 | return selLtSel(end, start) ? null : [start, end] 31 | } 32 | 33 | sel.union = (as: Sel, ae: Sel, bs: Sel, be: Sel): [Sel, Sel] | null => { 34 | // If the ranges aren't touching / overlapping, the returned union is invalid. 35 | return (selLtSel(ae, bs) || selLtSel(be, as)) ? null : [minSel(as, bs), maxSel(ae, be)] 36 | } 37 | 38 | sel.addPrefix = (prefix: string, s: Sel): Sel => ({k: prefix + s.k, isAfter: s.isAfter}) 39 | 40 | export default sel -------------------------------------------------------------------------------- /core/lib/typeregistry.ts: -------------------------------------------------------------------------------- 1 | // This is the master operation type documents. 2 | // 3 | // This is used for interpretting transactions sent to the server through 4 | // mutate() and transactions sent to client subscriptions. 5 | // 6 | // It implements set and remove, and forwards the actual changes to 7 | // child op types, which can be registered using the register function below. 8 | import {Type, AnyOTType} from './interfaces' 9 | 10 | export const typeRegistry: {[name: string]: AnyOTType} = {} 11 | export const supportedTypes = new Set(['rm', 'set']) 12 | 13 | export function register(type: AnyOTType) { 14 | typeRegistry[type.name] = type 15 | supportedTypes.add(type.name) 16 | } 17 | 18 | // register(require('../common/rangeops')) 19 | // register(require('../common/setops')) 20 | 21 | 22 | 23 | // The 'inc' type is a tiny dummy type. 24 | register({ 25 | name: 'inc', 26 | create(data) { return data|0 }, 27 | apply(snapshot, op) { // Op is a number 28 | console.log('inc apply', snapshot, op) 29 | if (/*typeof snapshot === 'object' ||*/ typeof op !== 'number') throw Error('Invalid data') 30 | return (snapshot|0) + op 31 | }, 32 | }) 33 | 34 | export function typeOrThrow(typeName: string): AnyOTType { 35 | const type = typeRegistry[typeName] 36 | if (!type) throw Error('Unsupported type ' + typeName) 37 | return type 38 | } 39 | 40 | 41 | // TODO: + Register string, JSON, etc. 42 | 43 | 44 | // I'm just going to export the utilities with the type. Its .. not ideal, but 45 | // there's no clear line between what should be part of the result set type and 46 | // what is just utility methods. This'll be fine for now. 47 | -------------------------------------------------------------------------------- /net/lib/util.ts: -------------------------------------------------------------------------------- 1 | import {I, queryTypes} from '@statecraft/core' 2 | import * as N from './netmessages' 3 | 4 | export const queryToNet = (q: I.Query | I.ReplaceQuery): N.NetQuery => { 5 | if (q.type === I.QueryType.Single || q.type === I.QueryType.AllKV) return q.type 6 | else return [q.type, queryTypes[q.type].toJSON(q.q)] 7 | } 8 | 9 | export const queryFromNet = (nq: N.NetQuery): I.Query | I.ReplaceQuery => { 10 | if (typeof nq === 'number') return ({type: nq} as I.Query) 11 | else { 12 | const [qtype, data] = nq 13 | return { 14 | type: qtype, 15 | q: queryTypes[qtype].fromJSON(data) 16 | } as I.Query 17 | } 18 | } 19 | 20 | // I'd really much prefer to use base64 encoding here, but the browser doesn't 21 | // have a usable base64 encoder and they aren't small. 22 | // Msgpack handles this fine, but I also want it to work over JSON. Eh.. :/ 23 | export const versionToNet = (v: I.Version): N.NetVersion => Array.from(v) 24 | export const versionFromNet = (v: N.NetVersion): I.Version => Uint8Array.from(v) 25 | 26 | export const versionRangeToNet = ({from, to}: I.VersionRange): N.NetVersionRange => [versionToNet(from), versionToNet(to)] 27 | export const versionRangeFromNet = (val: N.NetVersionRange): I.VersionRange => ({from: versionFromNet(val[0]), to: versionFromNet(val[1])}) 28 | 29 | export const fullVersionToNet = (vs: I.FullVersion): N.NetFullVersion => vs.map(v => v ? versionToNet(v) : null) 30 | export const fullVersionFromNet = (nvs: N.NetFullVersion): I.FullVersion => nvs.map(nv => nv ? versionFromNet(nv) : null) 31 | 32 | export const fullVersionRangeToNet = (vs: I.FullVersionRange): N.NetFullVersionRange => ( 33 | vs.map(v => v ? versionRangeToNet(v) : null) 34 | ) 35 | export const fullVersionRangeFromNet = (nvs: N.NetFullVersionRange): I.FullVersionRange => ( 36 | nvs.map(nv => nv ? versionRangeFromNet(nv) : null) 37 | ) 38 | -------------------------------------------------------------------------------- /net/lib/httpserver.ts: -------------------------------------------------------------------------------- 1 | import {I, bitHas} from '@statecraft/core' 2 | import http from 'http' 3 | 4 | const id = (x: T) => x 5 | 6 | export interface HttpOpts { 7 | // Defaults to id. 8 | urlToKey?: (url: string) => I.Key, 9 | 10 | // Defaults to JSON stringifying everything. 11 | valToHttp?: (val: any, key: I.Key, req: http.IncomingMessage) => { 12 | mimeType: string, 13 | content: string | Buffer, 14 | } 15 | } 16 | 17 | const defaultOpts: HttpOpts = { 18 | urlToKey: id, 19 | valToHttp: (val: any) => ({ 20 | mimeType: 'application/json', 21 | content: JSON.stringify({success: val != null, value: val}) 22 | }) 23 | } 24 | 25 | // This function should really return an express / connect Router or 26 | // something. 27 | export default function handler(store: I.Store, optsIn?: HttpOpts) { 28 | if (!bitHas(store.storeInfo.capabilities.queryTypes, I.QueryType.KV)) { 29 | throw Error('Httpserver needs kv support') 30 | } 31 | 32 | const opts = Object.assign({}, defaultOpts, optsIn) 33 | // const urlToKey = (opts && opts.urlToKey) ? opts.urlToKey : id 34 | // const valToHttp = (opts && opts.valToHttp) ? opts.valToHttp : toJson 35 | 36 | return async (req: http.IncomingMessage, res: http.ServerResponse) => { 37 | const key = opts.urlToKey!(req.url!) 38 | const result = await store.fetch({type: I.QueryType.KV, q: new Set([key])}) 39 | const value = result.results.get(key) 40 | // console.log('key', key, 'value', value, result) 41 | 42 | // TODO: Clean this up. 43 | // res.setHeader('x-sc-version', JSON.stringify(result.versions)) 44 | 45 | // TODO: Add etag. Steal code from text demo for it. 46 | 47 | const {mimeType, content} = opts.valToHttp!(value, key, req) 48 | 49 | res.setHeader('content-type', mimeType) 50 | res.writeHead(value == null ? 404 : 200) 51 | res.end(content) 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /graphql/lib/gql.ts: -------------------------------------------------------------------------------- 1 | // import {graphql, buildSchema, GraphQLFieldResolver} from 'graphql' 2 | import * as gql from 'graphql' 3 | import * as gqlTools from 'graphql-tools' 4 | 5 | import kvStore from './stores/kvmem' 6 | import doTxn, {Transaction} from './transaction' 7 | import express from 'express' 8 | import graphqlHTTP from 'express-graphql' 9 | import {inspect} from 'util' 10 | 11 | process.on('unhandledRejection', err => { throw err }) 12 | 13 | 14 | 15 | const schema = gqlTools.makeExecutableSchema({ 16 | typeDefs: ` 17 | type Post { 18 | title: String, 19 | content: String, 20 | author: Author, 21 | } 22 | 23 | type Author { 24 | fullName: String, 25 | } 26 | 27 | type Query { 28 | postById(id: String!): Post 29 | foo: String, 30 | } 31 | `, 32 | resolvers: { 33 | Post: { 34 | author(post, args, {txn}: {txn: Transaction}) { 35 | return txn.get('authors/' + post.author) 36 | } 37 | }, 38 | 39 | Query: { 40 | postById: async (self, {id}: {id: string}, {txn}: {txn: Transaction}) => { 41 | console.log('id', id) 42 | const post = await txn.get('posts/' + id) 43 | console.log('post', post) 44 | return post 45 | } 46 | } 47 | } 48 | }) 49 | 50 | 51 | 52 | 53 | ;(async () => { 54 | const store = await kvStore(new Map([ 55 | ['authors/auth', {fullName: 'Seph'}], 56 | ['posts/post1', {title: 'Yo', content: 'omg I am clever', author: 'auth'}], 57 | ])) 58 | 59 | const res = await doTxn(store, async txn => ( 60 | await gql.graphql(schema, `{postById(id: "post1") {title, author {fullName}}}`, null, {txn}) 61 | )) 62 | console.log(inspect(res, {depth: null, colors: true})) 63 | 64 | // const app = express() 65 | // app.use('/graphql', graphqlHTTP({ 66 | // graphiql: true, 67 | // schema, 68 | // rootValue: root, 69 | // })) 70 | })() 71 | -------------------------------------------------------------------------------- /lmdb/deps.d.ts: -------------------------------------------------------------------------------- 1 | 2 | declare module 'node-lmdb' { 3 | interface Database { 4 | close(): void 5 | } 6 | 7 | type Key = Buffer | string | number 8 | 9 | interface Txn { 10 | getString(dbi: Database, key: Key): string 11 | getStringUmsafe(dbi: Database, key: Key): string 12 | getBinary(dbi: Database, key: Key): Buffer 13 | getBinaryUnsafe(dbi: Database, key: Key): Buffer 14 | getNumber(dbi: Database, key: Key): number 15 | getBoolean(dbi: Database, key: Key): boolean 16 | 17 | putBinary(dbi: Database, key: Key, val: Buffer): void 18 | putString(dbi: Database, key: Key, val: string): void 19 | putString(dbi: Database, key: Key, val: number): void 20 | putString(dbi: Database, key: Key, val: boolean): void 21 | 22 | del(dbi: Database, key: Key): void 23 | abort(): void 24 | commit(): void 25 | } 26 | 27 | class Env { 28 | open(opts: { 29 | path?: string, 30 | maxDbs?: number, 31 | noTls?: boolean, 32 | }): void 33 | 34 | openDbi(opts: { 35 | name: string | null, 36 | create?: boolean, 37 | }): Database 38 | 39 | beginTxn(opts?: {readOnly?: boolean}): Txn 40 | close(): void 41 | } 42 | 43 | class Cursor { 44 | constructor(txn: Txn, dbi: Database) 45 | 46 | goToFirst(): Key | null 47 | goToLast(): Key | null 48 | goToKey(k: Key): Key | null // Go to k. Returns null if not found. 49 | goToRange(k: Key): Key | null // Go to the next key >= k. 50 | goToNext(): Key | null 51 | goToPrev(): Key | null 52 | 53 | getCurrentString(): string 54 | getCurrentNumber(): number 55 | getCurrentBoolean(): boolean 56 | getCurrentBinary(): Buffer 57 | 58 | // Valid only until next put() or txn closed. 59 | getCurrentStringUnsafe(): string 60 | getCurrentBinaryUnsafe(): Buffer 61 | 62 | // Weirdly no put methods. 63 | del(): void 64 | 65 | close(): void 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /net/lib/wsclient.ts: -------------------------------------------------------------------------------- 1 | import {I} from '@statecraft/core' 2 | import * as N from './netmessages' 3 | import createStore from './client' 4 | import {TinyReader, TinyWriter, onMsg} from './tinystream' 5 | import WebSocket from 'isomorphic-ws' 6 | 7 | // const wsurl = `ws${window.location.protocol.slice(4)}//${window.location.host}/ws` 8 | 9 | export const connect = (wsurl: string): Promise<[TinyReader, TinyWriter]> => new Promise((resolve, reject) => { 10 | const ws = new WebSocket(wsurl) 11 | 12 | const reader: TinyReader = {buf: [], isClosed: false} 13 | ws.onmessage = (msg) => { 14 | const data = JSON.parse(msg.data.toString()) 15 | // console.log('received', data) 16 | onMsg(reader, data) 17 | } 18 | 19 | ws.onclose = () => { 20 | console.warn('---- WEBSOCKET CLOSED ----') 21 | reader.isClosed = true 22 | reader.onClose && reader.onClose() 23 | } 24 | 25 | const writer: TinyWriter = { 26 | write(data) { 27 | if (ws.readyState === ws.OPEN) { 28 | // console.log('sending', data) 29 | ws.send(JSON.stringify(data)) 30 | } else { 31 | // On regular connections this won't happen because the first 32 | // message sent is server -> client. So we can't send anything until 33 | // the connection is open anyway. Its a problem with clientservermux though 34 | console.log('websocket message discarded because ws closed') 35 | } 36 | }, 37 | close() { 38 | ws.close() 39 | }, 40 | } 41 | 42 | ws.onopen = () => { 43 | console.log('ws opened') 44 | resolve([reader, writer]) 45 | } 46 | ws.onerror = (e) => { 47 | console.error('ws error', e) 48 | reject(e) 49 | } 50 | }) 51 | 52 | // TODO: Implement automatic reconnection and expose a simple server 53 | // describing the connection state 54 | export default async function(wsurl: string): Promise> { 55 | const [r, w] = await connect(wsurl) 56 | return createStore(r, w) 57 | } -------------------------------------------------------------------------------- /prozess/prozess.ts: -------------------------------------------------------------------------------- 1 | // I'm not sure the best way to structure this. This is some utility functions for interacting 2 | // with prozess stores. 3 | 4 | import * as I from './interfaces' 5 | import {Event, PClient, SubCbData, VersionConflictError} from 'prozess-client' 6 | import msgpack from 'msgpack-lite' 7 | import err from './err' 8 | import {V64, v64ToNum} from './version' 9 | 10 | export const encodeTxn = (txn: I.KVTxn, meta: I.Metadata) => msgpack.encode([Array.from(txn), meta]) 11 | export const decodeTxn = (data: Buffer): [I.KVTxn, I.Metadata] => { 12 | const [txn, meta] = msgpack.decode(data) 13 | return [new Map>(txn), meta] 14 | } 15 | 16 | // TODO: This should work with batches. 17 | export const decodeEvent = (event: Event, source: I.Source): I.TxnWithMeta => { 18 | const [txn, meta] = decodeTxn(event.data) 19 | return { versions: [V64(event.version)], txn, meta } 20 | } 21 | 22 | export function sendTxn(client: PClient, 23 | txn: I.KVTxn, 24 | meta: I.Metadata, 25 | expectedVersion: I.Version, 26 | opts: object): Promise { 27 | const data = encodeTxn(txn, meta) 28 | 29 | // TODO: This probably doesn't handle a missing version properly. 30 | // TODO: Add read conflict keys through opts. 31 | return client.send(data, { 32 | // targetVersion: expectedVersion === -1 ? -1 : expectedVersion + 1, 33 | targetVersion: v64ToNum(expectedVersion) + 1, 34 | conflictKeys: Array.from(txn.keys()), 35 | }).then(V64).catch(e => { 36 | // console.warn('WARNING: prozess detected conflict', e.stack) 37 | return Promise.reject(e instanceof VersionConflictError 38 | ? new err.WriteConflictError(e.message) 39 | : e 40 | ) 41 | }) 42 | } 43 | 44 | // Returns ops in SC style - range (from, to]. 45 | // export function getOps(client: PClient, 46 | // from: I.Version, 47 | // to: I.Version, 48 | // callback: I.Callback) { 49 | // client.getEvents(from + 1, to, {}, callback) 50 | // } 51 | -------------------------------------------------------------------------------- /demos/bidirectional/1.ts: -------------------------------------------------------------------------------- 1 | import * as I from '../../lib/interfaces' 2 | import * as N from '../../lib/net/netmessages' 3 | import singleStore from '../../lib/stores/singlemem' 4 | import {inspect} from 'util' 5 | 6 | import {TinyReader, TinyWriter, wrapReader, wrapWriter} from '../../lib/net/tinystream' 7 | import connectMux from '../../lib/net/clientservermux' 8 | // import createServer from '../../lib/net/tcpserver' 9 | // import connectStore from '../../lib/stores/tcpclient' 10 | 11 | process.on('unhandledRejection', err => { throw err }) 12 | 13 | const store1 = singleStore('store1') 14 | const store2 = singleStore('store2') 15 | 16 | const createPair = (): [TinyReader, TinyWriter] => { 17 | const reader: TinyReader = {buf: [], isClosed: false} 18 | const writer: TinyWriter = { 19 | write(msg) { 20 | // console.log('msg', msg) 21 | process.nextTick(() => reader.onMessage!(msg)) 22 | }, 23 | close() {}, 24 | } 25 | return [reader, writer] 26 | } 27 | 28 | type BothMsg = N.CSMsg | N.SCMsg 29 | const [r1, w2] = createPair() 30 | const [r2, w1] = createPair() 31 | 32 | ;(async () => { 33 | const remoteStore2P = connectMux(r1, w1, store1, true) 34 | const remoteStore1P = connectMux(r2, w2, store2, false) 35 | 36 | const [remoteStore2, remoteStore1] = await Promise.all([remoteStore2P, remoteStore1P]) 37 | 38 | console.log('store1', await remoteStore1.fetch({type: I.QueryType.Single, q:true})) 39 | console.log('store2', await remoteStore2.fetch({type: I.QueryType.Single, q:true})) 40 | 41 | const sub = remoteStore1.subscribe({type: I.QueryType.Single, q: true}, {}) 42 | const results = await sub.next() 43 | console.log('initial', results.value) 44 | 45 | ;(async () => { 46 | for await (const data of sub) { 47 | console.log('subscribe data', inspect(data, false, 10, true)) 48 | } 49 | })() 50 | 51 | const v = await remoteStore1.mutate(I.ResultType.Single, {type:'set', data: {x: 10}}) 52 | console.log('mutate run', v) 53 | 54 | })() 55 | -------------------------------------------------------------------------------- /net/lib/wsserver.ts: -------------------------------------------------------------------------------- 1 | import {I} from '@statecraft/core' 2 | import serve from './server' 3 | import {Writable} from 'stream' 4 | import {TinyReader, TinyWriter, wrapWriter, onMsg} from './tinystream' 5 | import WebSocket from 'ws' 6 | import {IncomingMessage} from 'http' 7 | import { CSMsg, SCMsg } from './netmessages'; 8 | 9 | const isProd = process.env.NODE_ENV === 'production' 10 | 11 | export const wrapWebSocket = (socket: WebSocket): [TinyReader, TinyWriter] => { 12 | // TODO: Consider using the code in wsclient instead to convert a socket to a reader/writer pair. 13 | const reader: TinyReader = {buf: [], isClosed: false} 14 | 15 | socket.on("message", data => { 16 | // if (!isProd) console.log('C->S', data) 17 | onMsg(reader, JSON.parse(data as any)) 18 | }) 19 | 20 | // I could just go ahead and make a TinyWriter, but this handles 21 | // backpressure. 22 | const writer = new Writable({ 23 | objectMode: true, 24 | write(data, _, callback) { 25 | // if (!isProd) console.log('S->C', data) 26 | if (socket.readyState === socket.OPEN) { 27 | // TODO: Should this pass the callback? Will that make backpressure 28 | // work? 29 | socket.send(JSON.stringify(data)) 30 | } 31 | callback() 32 | }, 33 | }) 34 | 35 | socket.on('close', () => { 36 | writer.end() // Does this help?? 37 | reader.isClosed = true 38 | reader.onClose && reader.onClose() 39 | }) 40 | 41 | return [reader, wrapWriter(writer)] 42 | } 43 | 44 | export const serveWS = (wsOpts: WebSocket.ServerOptions, store: I.Store | ((ws: WebSocket, msg: IncomingMessage) => I.Store | undefined | null)) => { 45 | const getStore = typeof store === 'function' ? store : () => store 46 | 47 | const wss = new WebSocket.Server(wsOpts) 48 | 49 | wss.on('connection', (socket, req) => { 50 | const [reader, writer] = wrapWebSocket(socket) 51 | const store = getStore(socket, req) 52 | if (store != null) serve(reader, writer, store) 53 | }) 54 | 55 | return wss 56 | } 57 | 58 | export default serveWS -------------------------------------------------------------------------------- /demos/text/public/mdstyle.css: -------------------------------------------------------------------------------- 1 | 2 | * { 3 | box-sizing: border-box; 4 | } 5 | 6 | body { 7 | margin: 0 auto; 8 | /*background-color: #ffd1fa;*/ 9 | max-width: 40em; 10 | } 11 | 12 | #content { 13 | 14 | } 15 | 16 | /*! Typebase.less v0.1.0 | MIT License */ 17 | /* Setup */ 18 | html { 19 | /* Change default typefaces here */ 20 | font-family: serif; 21 | /*font-size: 137.5%;*/ 22 | font-size: 115.5%; 23 | color: #1e1e1e; 24 | -webkit-font-smoothing: antialiased; 25 | } 26 | /* Copy & Lists */ 27 | p { 28 | line-height: 1.5rem; 29 | margin-top: 1.5rem; 30 | margin-bottom: 0; 31 | } 32 | ul, 33 | ol { 34 | margin-top: 1.5rem; 35 | margin-bottom: 1.5rem; 36 | } 37 | ul li, 38 | ol li { 39 | line-height: 1.5rem; 40 | } 41 | ul ul, 42 | ol ul, 43 | ul ol, 44 | ol ol { 45 | margin-top: 0; 46 | margin-bottom: 0; 47 | } 48 | blockquote { 49 | line-height: 1.5rem; 50 | margin-top: 1.5rem; 51 | margin-bottom: 1.5rem; 52 | } 53 | /* Headings */ 54 | h1, 55 | h2, 56 | h3, 57 | h4, 58 | h5, 59 | h6 { 60 | /* Change heading typefaces here */ 61 | font-family: sans-serif; 62 | margin-top: 1.5rem; 63 | margin-bottom: 0; 64 | line-height: 1.5rem; 65 | } 66 | h1 { 67 | font-size: 4.242rem; 68 | line-height: 4.5rem; 69 | margin-top: 3rem; 70 | } 71 | h2 { 72 | font-size: 2.828rem; 73 | line-height: 3rem; 74 | margin-top: 3rem; 75 | } 76 | h3 { 77 | font-size: 1.414rem; 78 | } 79 | h4 { 80 | font-size: 0.707rem; 81 | } 82 | h5 { 83 | font-size: 0.4713333333333333rem; 84 | } 85 | h6 { 86 | font-size: 0.3535rem; 87 | } 88 | /* Tables */ 89 | table { 90 | margin-top: 1.5rem; 91 | border-spacing: 0px; 92 | border-collapse: collapse; 93 | } 94 | table td, 95 | table th { 96 | padding: 0; 97 | line-height: 33px; 98 | } 99 | /* Code blocks */ 100 | code { 101 | vertical-align: bottom; 102 | } 103 | /* Leading paragraph text */ 104 | .lead { 105 | font-size: 1.414rem; 106 | } 107 | /* Hug the block above you */ 108 | .hug { 109 | margin-top: 0; 110 | } 111 | -------------------------------------------------------------------------------- /demos/bp/public/toolpanel.css: -------------------------------------------------------------------------------- 1 | .toolpanel { 2 | position: fixed; 3 | left: 0; 4 | right: 0; 5 | bottom: 0; 6 | text-align: center; 7 | padding-bottom: 26px; 8 | color: white; 9 | 10 | -webkit-user-select: none; 11 | -moz-user-select: none; 12 | -ms-user-select: none; 13 | cursor: default; 14 | /*background-color: hsl(184, 49%, 7%);*/ 15 | pointer-events: none; 16 | } 17 | 18 | .toolpanel > div { 19 | width: 80px; 20 | margin: 0px; 21 | display: inline-block; 22 | text-align: center; 23 | text-transform: uppercase; 24 | border-left: 2px hsl(0, 100%, 50%); 25 | cursor: pointer; 26 | pointer-events: auto; 27 | } 28 | 29 | .toolpanel > div:not(:first-child) { 30 | margin-left: 6px; 31 | } 32 | .toolpanel > div:not(:last-child) { 33 | margin-right: 6px; 34 | } 35 | 36 | .toolpanel > div:not(.selected) { 37 | box-shadow: 0px 8px hsl(0, 0%, 47%); 38 | padding: 20px 0px 20px 0px; 39 | position: relative; 40 | } 41 | .toolpanel > div:not(.selected)::after { 42 | content: ''; 43 | z-index: 1; 44 | position: absolute; 45 | left:0; 46 | top: 0; 47 | width: 80px; 48 | height: 70px; 49 | 50 | /*background-color: hsla(0, 0%, 0%, 0.31);*/ 51 | } 52 | 53 | .toolpanel > div.selected { 54 | /*box-shadow: 0px -8px white;*/ 55 | padding: 20px 0px 20px 0px; 56 | position: relative; 57 | top: 8px; 58 | } 59 | 60 | .toolpanel > #move { color: black; background-color: hsl(44, 87%, 52%); } 61 | 62 | .toolpanel > #nothing { color: black; background-color: white; } 63 | .toolpanel > #thinsolid { color: black; background-color: hsl(0, 0%, 71%); } 64 | .toolpanel > #solid { background-color: hsl(184, 49%, 7%); } 65 | .toolpanel > #bridge { background-color: hsl(208, 78%, 47%); } 66 | 67 | .toolpanel > #positive { background-color: #5ECC5E; } 68 | .toolpanel > #negative { background-color: #D65A2B; } 69 | 70 | .toolpanel > #shuttle { background-color: #9429BF; } 71 | .toolpanel > #thinshuttle { color: black; background-color: hsl(283, 89%, 75%); } 72 | 73 | .toolpanel > #glue { background-color: #af8145; } 74 | .toolpanel > #cut { color: black; background-color: #d2c1ac; } 75 | -------------------------------------------------------------------------------- /lmdb/test.ts: -------------------------------------------------------------------------------- 1 | import 'mocha' 2 | import * as I from '../lib/interfaces' 3 | 4 | import fs from 'fs' 5 | import assert from 'assert' 6 | 7 | import createMock from './prozess-mock' 8 | import {PClient} from 'prozess-client' 9 | import lmdb from '../lib/stores/lmdb' 10 | import prozessOps from '../lib/stores/prozessops' 11 | import runTests from './common' 12 | 13 | const rmdir = (path: string) => { 14 | //console.log('rmdir path', path) 15 | require('child_process').exec('rm -r ' + path) 16 | } 17 | 18 | const pathOfDb = new Map 19 | let _dbid = 1 20 | 21 | process.on('exit', function() { 22 | for (let p of pathOfDb.values()) { 23 | rmdir(p) 24 | } 25 | }) 26 | 27 | const create = async () => { 28 | let path: string 29 | do { 30 | // path = __dirname + '/_test' + _dbid++ 31 | path = '_test' + _dbid++ 32 | } while (fs.existsSync(path)) 33 | 34 | const store = await lmdb(prozessOps(createMock()), path) 35 | pathOfDb.set(store, path) 36 | return store 37 | } 38 | 39 | const teardown = (store: I.Store) => { // teardown. Nuke it. 40 | const path = pathOfDb.get(store) 41 | rmdir(path) 42 | pathOfDb.delete(store) 43 | } 44 | 45 | describe('prozess mock', () => { 46 | beforeEach(function() { 47 | this.client = createMock() 48 | }) 49 | 50 | it('conflicts at the right time', async function() { 51 | const client = this.client as PClient 52 | const base = await client.getVersion() 53 | await client.send("hi", {conflictKeys: ['a']}) 54 | 55 | try { 56 | await client.send("hi", { 57 | conflictKeys: ['a'], 58 | targetVersion: 1 + base, // should conflict with a 1, pass with a 2. 59 | }) 60 | } catch (e) { 61 | assert(e) 62 | } 63 | 64 | // Ok like this. 65 | await client.send("hi", { 66 | conflictKeys: ['a'], 67 | targetVersion: 2 + base, // should conflict with a 1, pass with a 2. 68 | }) 69 | }) 70 | }) 71 | 72 | describe('lmdb on prozess', () => { 73 | it('supports two stores pointed to the same prozess backend') 74 | it('catches up on missing operations from prozess') 75 | 76 | runTests(create, teardown) 77 | }) -------------------------------------------------------------------------------- /demos/monitor/monitor.ts: -------------------------------------------------------------------------------- 1 | import cpuStats from 'cpu-stats' 2 | import net from 'net' 3 | import os from 'os' 4 | 5 | import {stores, setSingle} from '@statecraft/core' 6 | import {serveToSocket} from '@statecraft/net' 7 | 8 | process.on('unhandledRejection', err => { 9 | console.error((err as any).stack) 10 | process.exit(1) 11 | }) 12 | 13 | type CPU = {user: number, sys: number} 14 | type ClientInfo = { 15 | hostname: string, 16 | cpus: CPU[] 17 | } 18 | 19 | ;(async () => { 20 | const port = process.env.PORT || 3003 21 | const host = process.env.HOST || 'localhost' 22 | 23 | const localStore = stores.singlemem(null) 24 | 25 | let listening = false 26 | const listen = async () => { 27 | listening = true 28 | 29 | // We want to keep trying to reconnect here, so the dashboard can be restarted. 30 | while (true) { 31 | console.log('Connecting to monitoring server at', host) 32 | const socket = net.createConnection(+port, host) 33 | serveToSocket(localStore, socket) 34 | 35 | socket.on('connect', () => {console.log('connected')}) 36 | 37 | await new Promise(resolve => { 38 | socket.on('error', err => { 39 | console.warn('Error connecting to monitoring server', err.message) 40 | }) 41 | socket.on('close', hadError => { 42 | // console.log('closed', hadError) 43 | resolve() 44 | }) 45 | }) 46 | 47 | // Wait 3 seconds before trying to connect again. 48 | await new Promise(resolve => setTimeout(resolve, 3000)) 49 | } 50 | } 51 | 52 | const hostname = os.hostname() 53 | 54 | const pollCpu = () => { 55 | cpuStats(1000, async (err, results) => { 56 | if (err) throw err // crash. 57 | 58 | process.stdout.write('.') 59 | 60 | const cpus = results.map(({cpu, user, sys}) => ({user, sys})) 61 | const info = {hostname, cpus} 62 | await setSingle(localStore, info) 63 | 64 | // We'll only connect after we have some data, to save on null checks. 65 | if (!listening) listen() 66 | 67 | pollCpu() 68 | }) 69 | } 70 | 71 | pollCpu() 72 | })() -------------------------------------------------------------------------------- /prozess/playground.ts: -------------------------------------------------------------------------------- 1 | 2 | const connectProzess = (): Promise => new Promise((resolve, reject) => { 3 | const client = reconnecter(9999, 'localhost', err => { 4 | if (err) reject(err) 5 | else resolve(client) 6 | }) as PClient 7 | }) 8 | 9 | const testProzess = async () => { 10 | const client = await connectProzess() 11 | const store = prozessOps(client) 12 | 13 | const sub = store.subscribe({type: I.QueryType.AllKV, q: true}) 14 | ;(async () => { 15 | for await (const cu of sub) { 16 | console.log('cu', cu) 17 | } 18 | }) 19 | 20 | const txn = new Map([['x', {type:'set', data: {x: 10}}]]) 21 | const v = await store.mutate(I.ResultType.KV, txn, [new Uint8Array()]) 22 | console.log('mutate cb', v) 23 | } 24 | 25 | 26 | const retryTest = async () => { 27 | const concurrentWrites = 10 28 | 29 | const client = await connectProzess() 30 | // const localStore = augment(await lmdbStore(prozessOps(client), process.argv[2] || 'testdb')) 31 | const localStore = await kvStore() 32 | 33 | // Using net to give it a lil' latency. 34 | // const s = server(localStore).listen(3334) 35 | // const store = await remoteStore(3334, 'localhost') 36 | 37 | const store = localStore 38 | 39 | await store.mutate(I.ResultType.KV, setSingle('x', 0)) 40 | // await db.set('x', numToBuf(0)) 41 | 42 | let txnAttempts = 0 43 | const sync = await Promise.all(new Array(concurrentWrites).fill(0).map((_, i) => ( 44 | doTxn(store, async txn => { 45 | const val = await txn.get('x') 46 | txn.set('x', val+1) 47 | return txnAttempts++ 48 | }) 49 | ))) 50 | 51 | const result = await doTxn(store, txn => txn.get('x')) 52 | assert.strictEqual(result, concurrentWrites) 53 | 54 | // This doesn't necessarily mean there's an error, but if there weren't 55 | // more attempts than there were increments, the database is running 56 | // serially and this test is doing nothing. 57 | assert(txnAttempts > concurrentWrites) 58 | 59 | console.log('attempts', txnAttempts) 60 | console.log(sync) 61 | 62 | client.close() 63 | store.close() 64 | // s.close() 65 | } 66 | 67 | -------------------------------------------------------------------------------- /demos/bp/browserclient/index.ts: -------------------------------------------------------------------------------- 1 | // This is the main entrypoint for the in-browser viewer of live data. 2 | 3 | import html from 'nanohtml' 4 | import render from './render' 5 | 6 | // Should be able to use an alias here, but its broken in tsify for some reason. 7 | import * as I from '../../../lib/interfaces' 8 | import connect from '../../../lib/stores/wsclient' 9 | import fieldOps from '../../../lib/types/field' 10 | import onekey from '../../../lib/stores/onekey' 11 | 12 | 13 | declare const config: { 14 | mimetype: string, 15 | key: string, 16 | initialValue: any, 17 | initialVersions: I.FullVersion, 18 | } 19 | 20 | const container = document.getElementById('content') 21 | if (container == null) throw Error('Could not find document #content div') 22 | const setObj = (data: any) => { 23 | console.log('setobj', data) 24 | if (typeof data === 'object' && data.type === 'Buffer') { 25 | const blob = new Blob([new Uint8Array(data.data)], {type: config.mimetype}) 26 | data = URL.createObjectURL(blob) 27 | } 28 | 29 | const replace = (elem: HTMLElement | null) => { 30 | while (container.firstChild) container.removeChild(container.firstChild); 31 | if (elem != null) { 32 | container.appendChild(elem) 33 | } 34 | } 35 | const result = data == null ? null : render(config.mimetype, data) 36 | if (result instanceof HTMLImageElement) { 37 | if (result.complete) replace(result) 38 | else result.onload = () => replace(result) 39 | } else { 40 | replace(result) 41 | } 42 | } 43 | 44 | type World = any 45 | 46 | ;(async () => { 47 | const store = onekey(await connect('ws://localhost:2000/'), config.key) 48 | 49 | const sub = store.subscribe({type: I.QueryType.Single, q: true}, { 50 | fromVersion: config.initialVersions, 51 | }) 52 | 53 | let last: any = null 54 | 55 | // const r = new Map() 56 | for await (const update of sub) { // TODO 57 | if (update.replace) { 58 | const val = update.replace.with 59 | setObj(val) 60 | last = val 61 | } 62 | 63 | update.txns.forEach(txn => { 64 | last = fieldOps.apply(last, txn.txn as I.SingleTxn) 65 | setObj(last) 66 | }) 67 | } 68 | })() -------------------------------------------------------------------------------- /net/yarn.lock: -------------------------------------------------------------------------------- 1 | # THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. 2 | # yarn lockfile v1 3 | 4 | 5 | "@types/events@*": 6 | version "3.0.0" 7 | resolved "https://registry.yarnpkg.com/@types/events/-/events-3.0.0.tgz#2862f3f58a9a7f7c3e78d79f130dd4d71c25c2a7" 8 | integrity sha512-EaObqwIvayI5a8dCzhFrjKzVwKLxjoG9T6Ppd5CEo07LRKfQ8Yokw54r5+Wq7FaBQ+yXRvQAYPrHwya1/UFt9g== 9 | 10 | "@types/node@*", "@types/node@^11.13.0": 11 | version "11.13.0" 12 | resolved "https://registry.yarnpkg.com/@types/node/-/node-11.13.0.tgz#b0df8d6ef9b5001b2be3a94d909ce3c29a80f9e1" 13 | integrity sha512-rx29MMkRdVmzunmiA4lzBYJNnXsW/PhG4kMBy2ATsYaDjGGR75dCFEVVROKpNwlVdcUX3xxlghKQOeDPBJobng== 14 | 15 | "@types/ws@^6.0.1": 16 | version "6.0.1" 17 | resolved "https://registry.yarnpkg.com/@types/ws/-/ws-6.0.1.tgz#ca7a3f3756aa12f62a0a62145ed14c6db25d5a28" 18 | integrity sha512-EzH8k1gyZ4xih/MaZTXwT2xOkPiIMSrhQ9b8wrlX88L0T02eYsddatQlwVFlEPyEqV0ChpdpNnE51QPH6NVT4Q== 19 | dependencies: 20 | "@types/events" "*" 21 | "@types/node" "*" 22 | 23 | async-limiter@~1.0.0: 24 | version "1.0.0" 25 | resolved "https://registry.yarnpkg.com/async-limiter/-/async-limiter-1.0.0.tgz#78faed8c3d074ab81f22b4e985d79e8738f720f8" 26 | integrity sha512-jp/uFnooOiO+L211eZOoSyzpOITMXx1rBITauYykG3BRYPu8h0UcxsPNB04RR5vo4Tyz3+ay17tR6JVf9qzYWg== 27 | 28 | isomorphic-ws@^4.0.1: 29 | version "4.0.1" 30 | resolved "https://registry.yarnpkg.com/isomorphic-ws/-/isomorphic-ws-4.0.1.tgz#55fd4cd6c5e6491e76dc125938dd863f5cd4f2dc" 31 | integrity sha512-BhBvN2MBpWTaSHdWRb/bwdZJ1WaehQ2L1KngkCkfLUGF0mAWAT1sQUQacEmQ0jXkFw/czDXPNQSL5u2/Krsz1w== 32 | 33 | typescript@^3.4.2: 34 | version "3.4.2" 35 | resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.4.2.tgz#9ed4e6475d906f589200193be056f5913caed481" 36 | integrity sha512-Og2Vn6Mk7JAuWA1hQdDQN/Ekm/SchX80VzLhjKN9ETYrIepBFAd8PkOdOTK2nKt0FCkmMZKBJvQ1dV1gIxPu/A== 37 | 38 | ws@^6.2.1: 39 | version "6.2.1" 40 | resolved "https://registry.yarnpkg.com/ws/-/ws-6.2.1.tgz#442fdf0a47ed64f59b6a5d8ff130f4748ed524fb" 41 | integrity sha512-GIyAXC2cB7LjvpgMt9EKS2ldqr0MTrORaleiOno6TweZ6r3TKtoFQWay/2PceJ3RuBasOHzXNn5Lrw1X0bEjqA== 42 | dependencies: 43 | async-limiter "~1.0.0" 44 | -------------------------------------------------------------------------------- /demos/monitor/dashboard.ts: -------------------------------------------------------------------------------- 1 | import choo from 'choo' 2 | import html from 'choo/html' 3 | 4 | import {I, subValues} from '@statecraft/core' 5 | import {reconnectingclient, connectToWS} from '@statecraft/net' 6 | 7 | type CPU = {user: number, sys: number} 8 | type ClientInfo = { 9 | ip?: string, 10 | hostname: string, 11 | connected: boolean, 12 | cpus?: CPU[] 13 | } 14 | 15 | const machine = (val: ClientInfo) => { 16 | console.log('mach', val) 17 | return html`
18 | ${val.hostname} 19 | ${val.ip} 20 | ${val.cpus && val.cpus.map(({user, sys}, i) => html` 21 | 22 | 23 | 24 | `)} 25 |
` 26 | } 27 | 28 | const mainView = (state: any) => { 29 | const machines = state.machines as Map 30 | // console.log('state', state) 31 | return html` 32 | 33 |

Statecraft CPU monitoring dashboard

34 |
35 |
36 | ${Array.from(machines.values()).map(machine)} 37 |
38 | 39 | ` 40 | } 41 | 42 | (async () => { 43 | const wsurl = `ws${window.location.protocol.slice(4)}//${window.location.host}/ws/` 44 | console.log('connecting to ws', wsurl, '...') 45 | const [statusStore, storeP] = reconnectingclient(() => connectToWS(wsurl)) 46 | 47 | ;(async () => { 48 | for await (const status of subValues(I.ResultType.Single, statusStore.subscribe({type:I.QueryType.Single, q:true}))) { 49 | console.log('status', status) 50 | } 51 | })() 52 | 53 | const store = await storeP 54 | console.log('connected to', store.storeInfo.uid) 55 | 56 | const app = new choo() 57 | app.use((state, emitter) => { 58 | state.machines = new Map() 59 | ;(async () => { 60 | for await (const val of subValues(I.ResultType.KV, store.subscribe({type: I.QueryType.AllKV, q: true}))) { 61 | console.log('val', val) 62 | state.machines = val 63 | emitter.emit('render') 64 | } 65 | })() 66 | }) 67 | app.route('/', mainView) 68 | app.mount('body') 69 | 70 | })() 71 | -------------------------------------------------------------------------------- /core/lib/stores/singlemem.ts: -------------------------------------------------------------------------------- 1 | // This is a simple single value in-memory store. 2 | import * as I from '../interfaces' 3 | import fieldOps from '../types/field' 4 | import genSource from '../gensource' 5 | import err from '../err' 6 | import {V64, vCmp} from '../version' 7 | import {bitSet} from '../bit' 8 | import makeOpCache from '../opcache' 9 | import makeSubGroup from '../subgroup' 10 | 11 | const capabilities = { 12 | queryTypes: bitSet(I.QueryType.Single), 13 | mutationTypes: bitSet(I.ResultType.Single), 14 | // ops: 'none', 15 | } 16 | 17 | export default function singleStore(initialValue: Val, source: I.Source = genSource(), initialVersionNum: number = 0): I.Store { 18 | let version: number = initialVersionNum 19 | let data = initialValue 20 | 21 | const fetch: I.FetchFn = async (query, opts) => { 22 | if (query.type !== I.QueryType.Single) throw new err.UnsupportedTypeError() 23 | 24 | return { 25 | results: data, 26 | queryRun: query, 27 | versions: [{from:V64(version), to:V64(version)}], 28 | } 29 | } 30 | 31 | const opcache = makeOpCache() 32 | const subGroup = makeSubGroup({initialVersion: [V64(version)], fetch, getOps: opcache.getOps}) 33 | 34 | const store: I.Store = { 35 | storeInfo: { 36 | uid: `single(${source})`, 37 | capabilities, 38 | sources: [source], 39 | sourceIsMonotonic: [true], 40 | }, 41 | 42 | fetch, 43 | getOps: opcache.getOps, 44 | 45 | subscribe: subGroup.create.bind(subGroup), 46 | 47 | async mutate(type, _txn, versions, opts = {}) { 48 | if (type !== I.ResultType.Single) throw new err.UnsupportedTypeError() 49 | const txn = _txn as I.Op 50 | 51 | const expectv = versions && versions[0] 52 | const currentv = V64(version) 53 | if (expectv != null && vCmp(expectv, currentv) < 0) throw new err.VersionTooOldError() 54 | 55 | if (txn) data = fieldOps.apply(data, txn) 56 | const newv = V64(++version) 57 | 58 | opcache.onOp(0, currentv, newv, I.ResultType.Single, txn, opts.meta || {}) 59 | subGroup.onOp(0, currentv, [{txn, meta: opts.meta || {}, versions: [newv]}]) 60 | return [newv] 61 | }, 62 | 63 | close() {}, 64 | } 65 | 66 | return store 67 | } 68 | -------------------------------------------------------------------------------- /net/lib/clientservermux.ts: -------------------------------------------------------------------------------- 1 | // This implements a mux server which symmetrically both exposes and consumes 2 | // a store. 3 | import {I} from '@statecraft/core' 4 | import * as N from './netmessages' 5 | 6 | import createStore from './client' 7 | import serve from './server' 8 | 9 | import {TinyReader, TinyWriter, onMsg} from './tinystream' 10 | 11 | export type BothMsg = N.CSMsg | N.SCMsg 12 | 13 | // Passes invariants inv(0) != 0, inv(inv(x)) == x. 14 | const inv = (x: number) => -x-1 15 | 16 | export default function connectMux(reader: TinyReader, writer: TinyWriter, localStore: I.Store, symmetry: boolean): Promise> { 17 | // To mux them both together I'm going to use the .ref property, which is on 18 | // all net messages except the hello message. 19 | 20 | // There's 4 streams going on here: 21 | // - Local store reader (c->s) / writer (s->c) 22 | // - Remote store reader (s->c) / writer (c->s) 23 | 24 | // One reader/writer pair will have all the .ref properties in messages 25 | // inverted based on symmetry property. If symmetry is true, we invert the 26 | // local store messages. If its false, we invert the remote store messages. 27 | 28 | const localReader = {} as TinyReader 29 | localReader.buf = [] 30 | 31 | // If either writer closes, we'll close the whole tunnel. 32 | const localWriter: TinyWriter = { 33 | write(msg) { 34 | if (msg.a !== N.Action.Hello && symmetry) msg.ref = inv(msg.ref) 35 | writer.write(msg) 36 | }, 37 | close() { writer.close() }, 38 | } 39 | 40 | const remoteReader: TinyReader = {buf: [], isClosed: false} 41 | const remoteWriter: TinyWriter = { 42 | write(msg) { 43 | if (!symmetry) msg.ref = inv(msg.ref) 44 | writer.write(msg) 45 | }, 46 | close() { writer.close() }, 47 | } 48 | 49 | reader.onMessage = msg => { 50 | if (msg.a === N.Action.Hello) onMsg(remoteReader, msg) 51 | else { 52 | const neg = msg.ref < 0 53 | if (neg) msg.ref = inv(msg.ref) 54 | if (neg !== symmetry) onMsg(remoteReader, msg as N.SCMsg) 55 | else onMsg(localReader, msg as N.CSMsg) 56 | } 57 | } 58 | 59 | serve(localReader, localWriter, localStore) 60 | return createStore(remoteReader, remoteWriter) 61 | } 62 | -------------------------------------------------------------------------------- /demos/bidirectional/server.ts: -------------------------------------------------------------------------------- 1 | import * as I from '../../lib/interfaces' 2 | import express from 'express' 3 | import WebSocket from 'ws' 4 | import http from 'http' 5 | 6 | import {wrapWebSocket} from '../../lib/net/wsserver' 7 | import kvMem from '../../lib/stores/kvmem' 8 | // import singleMem, {setSingle} from '../../lib/stores/singlemem' 9 | import connectMux, { BothMsg } from '../../lib/net/clientservermux' 10 | import subValues from '../../lib/subvalues' 11 | import { rmKV, setKV } from '../../lib/kv'; 12 | import serveTCP from '../../lib/net/tcpserver' 13 | 14 | process.on('unhandledRejection', err => { 15 | console.error(err.stack) 16 | process.exit(1) 17 | }) 18 | 19 | ;(async () => { 20 | type Pos = {x: number, y: number} 21 | // type DbVal = {[id: string]: Pos} 22 | 23 | // The store is a kv store mapping from client ID (incrementing numbers) => latest position. 24 | const store = await kvMem() 25 | 26 | const app = express() 27 | app.use(express.static(`${__dirname}/public`)) 28 | 29 | const server = http.createServer(app) 30 | const wss = new WebSocket.Server({server}) 31 | 32 | let nextId = 1000 33 | 34 | wss.on('connection', async (socket, req) => { 35 | const id = `${nextId++}` 36 | 37 | const [reader, writer] = wrapWebSocket(socket) 38 | const remoteStore = await connectMux(reader, writer, store, false) 39 | 40 | // console.log(id, 'info', remoteStore.storeInfo) 41 | console.log(id, 'client connected') 42 | const sub = remoteStore.subscribe({type: I.QueryType.Single, q: true}) 43 | 44 | reader.onClose = () => { 45 | console.log(id, 'client gone') 46 | // delete db[id] 47 | // console.log('db', db) 48 | rmKV(store, id) 49 | } 50 | 51 | for await (const val of subValues(I.ResultType.Single, sub)) { 52 | // console.log(id, 'cu', val) 53 | // await setKV(store, id, {x: val.x, y: val.y}) 54 | await setKV(store, id, val) 55 | // console.log('db', db) 56 | } 57 | }) 58 | 59 | const port = process.env.PORT || 2222 60 | server.listen(port, (err: any) => { 61 | if (err) throw err 62 | console.log('listening on', port) 63 | }) 64 | 65 | 66 | if (process.env.NODE_ENV !== 'production') { 67 | const tcpServer = serveTCP(store) 68 | tcpServer.listen(2002, 'localhost') 69 | console.log('Debugging server listening on tcp://localhost:2002') 70 | } 71 | })() -------------------------------------------------------------------------------- /net/lib/tinystream.ts: -------------------------------------------------------------------------------- 1 | // I'm not using node streams because they add about 150k of crap to the browser 2 | // bundle. 3 | 4 | // This is the simplest possible API around a stream that we can use in server 5 | // & client code. 6 | 7 | // TODO: Consider using streamToIter here, which we're already pulling in to 8 | // the client anyway. 9 | import {Readable, Writable} from 'stream' 10 | 11 | export interface TinyReader { 12 | buf: Msg[] // For messages before onmessage has been set 13 | onMessage?: (msg: Msg) => void 14 | onClose?: () => void 15 | isClosed: boolean, 16 | // In case of error, destroy the stream. 17 | destroy?(e?: Error): void, 18 | } 19 | 20 | export interface TinyWriter { 21 | write(data: Msg): void, 22 | close(): void, 23 | // In case of error, destroy the stream. 24 | destroy?(e?: Error): void, 25 | } 26 | 27 | export const wrapReader = (r: Readable) => { 28 | const reader: TinyReader = {buf: [], isClosed: false} 29 | let destroyed = false 30 | 31 | r.on('data', msg => { 32 | if (destroyed) return 33 | 34 | if (reader.onMessage) reader.onMessage!(msg as any as Msg) 35 | else reader.buf.push(msg) 36 | }) 37 | r.on('end', () => { 38 | reader.isClosed = true 39 | if (reader.onClose) reader.onClose() 40 | }) 41 | r.on('error', err => { 42 | // I think this'll hit end right afterwards, so we don't have to do 43 | // anything here. I think. 44 | console.warn('socket error', err) 45 | }) 46 | reader.destroy = err => { 47 | r.destroy() 48 | destroyed = true 49 | } 50 | 51 | return reader 52 | } 53 | 54 | export const listen = (r: TinyReader, listener: (msg: Msg) => void) => { 55 | r.buf.forEach(listener) 56 | r.buf.length = 0 57 | r.onMessage = listener 58 | } 59 | export const onMsg = (r: TinyReader, msg: Msg) => { 60 | if (r.onMessage) r.onMessage(msg) 61 | else r.buf.push(msg) 62 | } 63 | 64 | export const wrapWriter = (w: Writable, encode?: (msg: Msg) => any) => { 65 | return { 66 | write(data) { 67 | if (w.writable) { 68 | w.write(encode ? encode(data) : data) 69 | 70 | // Work around this bug: 71 | // https://github.com/kawanet/msgpack-lite/issues/80 72 | if ((w as any).encoder) (w as any).encoder.flush() 73 | } 74 | }, 75 | close() { 76 | w.end() 77 | }, 78 | destroy(err) { 79 | // Not passing the error through because if we do it'll emit an error in the 80 | // stream. 81 | w.destroy() 82 | } 83 | } as TinyWriter 84 | } 85 | -------------------------------------------------------------------------------- /demos/bp/rustpng.js: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | var wasm; 3 | 4 | const TextEncoder = require('util').TextEncoder; 5 | 6 | let cachedTextEncoder = new TextEncoder('utf-8'); 7 | 8 | let cachegetUint8Memory = null; 9 | function getUint8Memory() { 10 | if (cachegetUint8Memory === null || cachegetUint8Memory.buffer !== wasm.memory.buffer) { 11 | cachegetUint8Memory = new Uint8Array(wasm.memory.buffer); 12 | } 13 | return cachegetUint8Memory; 14 | } 15 | 16 | function passStringToWasm(arg) { 17 | 18 | const buf = cachedTextEncoder.encode(arg); 19 | const ptr = wasm.__wbindgen_malloc(buf.length); 20 | getUint8Memory().set(buf, ptr); 21 | return [ptr, buf.length]; 22 | } 23 | 24 | function getArrayU8FromWasm(ptr, len) { 25 | return getUint8Memory().subarray(ptr / 1, ptr / 1 + len); 26 | } 27 | 28 | let cachedGlobalArgumentPtr = null; 29 | function globalArgumentPtr() { 30 | if (cachedGlobalArgumentPtr === null) { 31 | cachedGlobalArgumentPtr = wasm.__wbindgen_global_argument_ptr(); 32 | } 33 | return cachedGlobalArgumentPtr; 34 | } 35 | 36 | let cachegetUint32Memory = null; 37 | function getUint32Memory() { 38 | if (cachegetUint32Memory === null || cachegetUint32Memory.buffer !== wasm.memory.buffer) { 39 | cachegetUint32Memory = new Uint32Array(wasm.memory.buffer); 40 | } 41 | return cachegetUint32Memory; 42 | } 43 | /** 44 | * @param {string} arg0 45 | * @param {number} arg1 46 | * @param {number} arg2 47 | * @param {number} arg3 48 | * @returns {Uint8Array} 49 | */ 50 | module.exports.convert = function(arg0, arg1, arg2, arg3) { 51 | const [ptr0, len0] = passStringToWasm(arg0); 52 | const retptr = globalArgumentPtr(); 53 | try { 54 | wasm.convert(retptr, ptr0, len0, arg1, arg2, arg3); 55 | const mem = getUint32Memory(); 56 | const rustptr = mem[retptr / 4]; 57 | const rustlen = mem[retptr / 4 + 1]; 58 | 59 | const realRet = getArrayU8FromWasm(rustptr, rustlen).slice(); 60 | wasm.__wbindgen_free(rustptr, rustlen * 1); 61 | return realRet; 62 | 63 | 64 | } finally { 65 | wasm.__wbindgen_free(ptr0, len0 * 1); 66 | 67 | } 68 | 69 | }; 70 | 71 | const TextDecoder = require('util').TextDecoder; 72 | 73 | let cachedTextDecoder = new TextDecoder('utf-8'); 74 | 75 | function getStringFromWasm(ptr, len) { 76 | return cachedTextDecoder.decode(getUint8Memory().subarray(ptr, ptr + len)); 77 | } 78 | 79 | module.exports.__wbindgen_throw = function(ptr, len) { 80 | throw new Error(getStringFromWasm(ptr, len)); 81 | }; 82 | 83 | wasm = require('./rustpng_bg'); 84 | -------------------------------------------------------------------------------- /core/lib/subvalues.ts: -------------------------------------------------------------------------------- 1 | // This is a helper method for yielding the values out of a subscription, and 2 | // ignoring everything else. 3 | import * as I from './interfaces' 4 | import {resultTypes} from './qrtypes' 5 | 6 | /** 7 | * This is a state machine which can process catchup objects. Each time the 8 | * object transitions, the transition function returns the new state. 9 | * 10 | * TODO: Consider finding a way to return the initial value when the object is 11 | * first created 12 | * 13 | * @param type The expected result type of the incoming catchup data 14 | * @returns Transition function 15 | */ 16 | export const catchupStateMachine = (type: I.ResultType) => { 17 | const rtype = resultTypes[type] 18 | let value = rtype.create() 19 | const versions: I.FullVersionRange = [] 20 | 21 | return (update: I.CatchupData): I.FetchResults => { 22 | if (update.replace) { 23 | // TODO: Should we be pulling update.replace.versions in here? 24 | 25 | // console.log('replace', value, update.replace) 26 | value = rtype.updateResults(value, update.replace.q, update.replace.with) 27 | // console.log('->', value) 28 | update.replace.versions.forEach((v, si) => { 29 | if (v != null) versions[si] = {from:v, to:v} 30 | }) 31 | } 32 | 33 | for (const txn of update.txns) { 34 | // This is like this because I haven't implemented apply for range results 35 | if (rtype.applyMut) rtype.applyMut!(value, txn.txn) 36 | else (value = rtype.apply(value, txn.txn)) 37 | 38 | txn.versions.forEach((v, si) => { 39 | // The version *must* exist already in versions. 40 | if (v) versions[si]!.from = v 41 | }) 42 | } 43 | 44 | update.toVersion.forEach((v, si) => { 45 | if (v != null) { 46 | if (versions[si] == null) versions[si] = {from:v, to:v} 47 | else versions[si]!.to = v 48 | } 49 | }) 50 | 51 | return {results: value!, versions} 52 | } 53 | } 54 | 55 | // TODO: Refactor things so we don't need a ResultType passed in here. 56 | export async function* subResults(type: I.ResultType, sub: I.Subscription) { 57 | const u = catchupStateMachine(type) 58 | for await (const update of sub) { 59 | // console.log('upd', update) 60 | yield {...u(update), raw: update} 61 | } 62 | } 63 | 64 | export default async function* subValues(type: I.ResultType, sub: I.Subscription) { 65 | const u = catchupStateMachine(type) 66 | for await (const update of sub) { 67 | yield u(update).results // A Val if type is single, or a container of Vals 68 | } 69 | } -------------------------------------------------------------------------------- /demos/universalclient/main.ts: -------------------------------------------------------------------------------- 1 | import express from 'express' 2 | import http from 'http' 3 | import net from 'net' 4 | import url from 'url' 5 | import {Console} from 'console' 6 | 7 | import {I, sel, subValues, registerType} from '@statecraft/core' 8 | import {tcpclient, createTCPStreams, wsclient, connectToWS, reconnectingclient, wsserver} from '@statecraft/net' 9 | 10 | // import {type as texttype} from 'ot-text-unicode' 11 | // import {type as jsontype} from 'ot-json1' 12 | 13 | // registerType(texttype) 14 | // registerType(jsontype) 15 | 16 | process.on('unhandledRejection', err => { 17 | console.error((err as any).stack) 18 | process.exit(1) 19 | }) 20 | 21 | global.console = new (Console as any)({ 22 | stdout: process.stdout, 23 | stderr: process.stderr, 24 | inspectOptions: {depth: null} 25 | }) 26 | 27 | ;(async () => { 28 | const urlStr = process.argv[2] 29 | if (urlStr == null) throw Error('Missing URL argument') 30 | 31 | console.log(urlStr) 32 | 33 | const type = (urlStr.startsWith('ws://') || urlStr.startsWith('wss://')) ? 'ws' 34 | : urlStr.startsWith('tcp://') ? 'tcp' 35 | : null 36 | if (type == null) throw Error('URL must start with ws:// or tcp://') 37 | const {hostname, port} = url.parse(urlStr) 38 | if (hostname == null) throw Error('invalid URL') 39 | 40 | // TODO: Pick a default port number for statecraft 41 | // const store = type === 'ws' ? await wsClient(urlStr) 42 | // : await tcpClient(+(port || 3000), host!) 43 | while (true) { 44 | const [status, storeP, uidChanged] = reconnectingclient(type === 'ws' 45 | ? () => connectToWS(urlStr) 46 | : () => createTCPStreams(+(port || 3000), hostname) 47 | ) 48 | const store = await storeP 49 | 50 | // TODO: Listen to status... 51 | 52 | console.log('got store', store.storeInfo) 53 | 54 | // Ok, now host the client app. 55 | const app = express() 56 | app.use(express.static(`${__dirname}/public`)) 57 | 58 | const webServer = http.createServer(app) 59 | 60 | const wss = wsserver({server: webServer}, store) 61 | 62 | const listenPort = process.env.PORT || 3333 63 | webServer.listen(listenPort, () => { 64 | console.log('http server listening on', listenPort) 65 | }) 66 | 67 | await uidChanged.catch(() => {}) 68 | console.log('UID changed. Re-hosting.') 69 | 70 | // Closing the server entirely forces all ws clients to reconnect 71 | await new Promise(resolve => wss.close(resolve)) 72 | await new Promise(resolve => webServer.close(resolve)) 73 | } 74 | 75 | // const sub = store.subscribe({type: 'static range', q: [{low: sel(''), high: sel('\xff')}]}) 76 | // for await (const cu of subValues('range', sub)) { 77 | // console.log('cu', cu) 78 | // } 79 | })() -------------------------------------------------------------------------------- /demos/monitor/server.ts: -------------------------------------------------------------------------------- 1 | // This is the monitoring server / host. 2 | 3 | // It exposes 2 endpoints: 4 | // - TCP client for monitoring daemons to connect 5 | // - HTTP listener + websockets for the monitoring app 6 | 7 | import express from 'express' 8 | import http from 'http' 9 | import net from 'net' 10 | 11 | import {I, stores, subValues, setKV, rmKV} from '@statecraft/core' 12 | import {connectToSocket, wsserver} from '@statecraft/net' 13 | 14 | process.on('unhandledRejection', err => { 15 | console.error((err as any).stack) 16 | process.exit(1) 17 | }) 18 | 19 | ;(async () => { 20 | type CPU = {user: number, sys: number} 21 | type ClientInfo = { 22 | ip?: string, 23 | hostname: string, 24 | connected: boolean, 25 | cpus?: CPU[] 26 | } 27 | 28 | const store = await stores.kvmem() 29 | 30 | // 1. Setup the TCP server to listen for incoming clients 31 | 32 | // We sort of want to do the opposite of what tcpserver does - instead of 33 | // serving a store, we want to consume stores from incoming network 34 | // connections. 35 | const tcpserver = net.createServer(async socket => { 36 | console.log('got client') 37 | const ip = socket.remoteAddress 38 | 39 | // Set to the client's hostname once the client first sends us information 40 | let id: string | null = null 41 | 42 | const remoteStore = await connectToSocket(socket) 43 | const sub = remoteStore.subscribe({type: I.QueryType.Single, q: true}) 44 | 45 | let lastVal: ClientInfo | null = null 46 | for await (const _val of subValues(I.ResultType.Single, sub)) { 47 | const val = _val as ClientInfo 48 | val.ip = ip 49 | if (id == null) id = val.hostname 50 | val.connected = true 51 | // console.log('val', val) 52 | console.log('update from', id) 53 | 54 | lastVal = val 55 | await setKV(store, id, val) 56 | } 57 | 58 | socket.on('close', async () => { 59 | console.log('closed', id) 60 | // if (id != null) await rmKV(store, id) 61 | if (lastVal != null) { 62 | lastVal.connected = false 63 | delete lastVal.cpus 64 | await setKV(store, id!, lastVal) 65 | } 66 | }) 67 | }) 68 | tcpserver.listen(3003) 69 | console.log('tcp server listening on port 3003') 70 | 71 | 72 | 73 | // 2. Set up the HTTP server & websocket listener for the dashboard 74 | const app = express() 75 | app.use(express.static(`${__dirname}/public`)) 76 | 77 | const webServer = http.createServer(app) 78 | 79 | wsserver({server: webServer}, store) 80 | 81 | const port = process.env.PORT || 3000 82 | webServer.listen(+port, ((err: any) => { 83 | if (err) throw err 84 | console.log('http server listening on', port) 85 | }) as any as () => void) 86 | })() -------------------------------------------------------------------------------- /demos/contentfulgraphql/public/style.css: -------------------------------------------------------------------------------- 1 | * { 2 | box-sizing: border-box; 3 | } 4 | 5 | body { 6 | margin: 0; 7 | background-color: #f9f9f9; 8 | } 9 | 10 | body > * { 11 | margin: 0 auto; 12 | max-width: 700px; 13 | } 14 | 15 | .footer { 16 | margin-top: 10em; 17 | border-top: 4px solid #626161; 18 | padding-top: 1em; 19 | font-size: 14px; 20 | /* width: 100%; */ 21 | /* padding: 0 auto; */ 22 | /* max-width: initial; */ 23 | /* height: 300vh; */ 24 | /* padding: 1em; */ 25 | font-family: monospace; 26 | } 27 | 28 | hr { 29 | border: 4px solid #626161; 30 | } 31 | 32 | blockquote { 33 | font-style: italic; 34 | color: #1f1f1f; 35 | background-color: #efefef; 36 | 37 | line-height: 1.5rem; 38 | margin-top: 1.5rem; 39 | margin-bottom: 1.5rem; 40 | margin-left: 0; 41 | border-left: 9px solid #8b8b8b; 42 | padding-left: 2em; 43 | } 44 | blockquote > *:first-child { 45 | margin-top: 0; 46 | } 47 | 48 | img { 49 | max-width: 100%; 50 | } 51 | 52 | /*! Typebase.less v0.1.0 | MIT License */ 53 | /* Setup */ 54 | html { 55 | /* Change default typefaces here */ 56 | font-family: serif; 57 | /* font-size: 137.5%; */ 58 | font-size: 115.5%; 59 | /* color: #6a6969; */ 60 | } 61 | /* Copy & Lists */ 62 | p { 63 | line-height: 1.5rem; 64 | margin-top: 1.5rem; 65 | margin-bottom: 0; 66 | } 67 | ul, 68 | ol { 69 | margin-top: 1.5rem; 70 | margin-bottom: 1.5rem; 71 | } 72 | ul li, 73 | ol li { 74 | line-height: 1.5rem; 75 | } 76 | ul ul, 77 | ol ul, 78 | ul ol, 79 | ol ol { 80 | margin-top: 0; 81 | margin-bottom: 0; 82 | } 83 | /* Headings */ 84 | h1, 85 | h2, 86 | h3, 87 | h4, 88 | h5, 89 | h6 { 90 | /* Change heading typefaces here */ 91 | font-family: sans-serif; 92 | margin-top: 1.5rem; 93 | margin-bottom: 0; 94 | line-height: 1.5rem; 95 | color: #4f4f4f; 96 | } 97 | h1 { 98 | font-size: 4.242rem; 99 | line-height: 4.5rem; 100 | padding-top: 3rem; 101 | margin-top: 0; 102 | } 103 | h2 { 104 | font-size: 2.828rem; 105 | line-height: 3rem; 106 | margin-top: 3rem; 107 | } 108 | h3 { 109 | font-size: 1.414rem; 110 | } 111 | h4 { 112 | font-size: 0.707rem; 113 | } 114 | h5 { 115 | font-size: 0.4713333333333333rem; 116 | } 117 | h6 { 118 | font-size: 0.3535rem; 119 | } 120 | /* Tables */ 121 | table { 122 | margin-top: 1.5rem; 123 | border-spacing: 0px; 124 | border-collapse: collapse; 125 | } 126 | table td, 127 | table th { 128 | padding: 0; 129 | line-height: 33px; 130 | } 131 | /* Code blocks */ 132 | code { 133 | vertical-align: bottom; 134 | } 135 | /* Leading paragraph text */ 136 | .lead { 137 | font-size: 1.414rem; 138 | } 139 | /* Hug the block above you */ 140 | .hug { 141 | margin-top: 0; 142 | } 143 | -------------------------------------------------------------------------------- /demos/universalclient/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | Universal client 3 | 135 | 136 | -------------------------------------------------------------------------------- /demos/bidirectional/client.ts: -------------------------------------------------------------------------------- 1 | import * as I from '../../lib/interfaces' 2 | import {connect} from '../../lib/stores/wsclient' 3 | import singleMem, {setSingle} from '../../lib/stores/singlemem' 4 | import connectMux, { BothMsg } from '../../lib/net/clientservermux' 5 | import subValues from '../../lib/subvalues' 6 | 7 | const wsurl = `ws${window.location.protocol.slice(4)}//${window.location.host}/ws` 8 | 9 | type Pos = { 10 | x: number, 11 | y: number, 12 | gamepad?: { 13 | id: string, 14 | buttons: number[], 15 | axes: readonly number[], 16 | } 17 | } 18 | // type DbVal = {[id: string]: Pos} 19 | 20 | let data = new Map() 21 | 22 | // The local store that holds our mouse location 23 | const localStore = singleMem({x:0, y:0}) 24 | 25 | const canvas = document.getElementsByTagName('canvas')[0] 26 | let ctx: CanvasRenderingContext2D 27 | 28 | const resize = () => { 29 | canvas.width = canvas.clientWidth 30 | canvas.height = canvas.clientHeight 31 | ctx = canvas.getContext('2d', {alpha: false})! 32 | draw() 33 | } 34 | resize() 35 | 36 | const colors = ['red', 'green', 'blue', 'white', 'purple', 'yellow', 'orange'] 37 | 38 | function draw() { 39 | ctx.fillStyle = 'black' 40 | // ctx.fillRect(0, 0, canvas.width, canvas.height) 41 | ctx.fillRect(0, 0, canvas.width, canvas.height) 42 | 43 | ctx.fillStyle = 'red' 44 | for (const [id, {x, y}] of data) { 45 | ctx.fillStyle = colors[parseInt(id) % colors.length] 46 | ctx.fillRect(x-5, y-5, 10, 10) 47 | } 48 | } 49 | 50 | let lastTS: number = -1 51 | let mouse: Pos = {x:0, y:0} 52 | const pollGamepads = () => { 53 | const gamepads = navigator.getGamepads() 54 | const g = gamepads[0] 55 | if (g == null) { 56 | return 57 | } 58 | 59 | console.log(g.timestamp) 60 | if (lastTS != g.timestamp) { 61 | mouse.gamepad = { 62 | id: g.id, 63 | buttons: g.buttons.map(b => b.value), 64 | axes: g.axes, 65 | } 66 | 67 | setSingle(localStore, mouse) 68 | lastTS = g.timestamp 69 | } 70 | 71 | setTimeout(pollGamepads, 16) 72 | // requestAnimationFrame(pollGamepads) 73 | } 74 | 75 | window.addEventListener("gamepadconnected", e => { 76 | console.log('gamepad connected') 77 | pollGamepads() 78 | // requestAnimationFrame(pollGamepads) 79 | }) 80 | 81 | 82 | ;(async () => { 83 | const [reader, writer] = await connect(wsurl) 84 | const remoteStore = await connectMux(reader, writer, localStore, true) 85 | 86 | document.body.onmousemove = e => { 87 | // console.log('setting', e.clientX) 88 | // mouse = {x: e.clientX, y: e.clientY} 89 | mouse.x = e.clientX; mouse.y = e.clientY 90 | setSingle(localStore, mouse) 91 | } 92 | 93 | const sub = remoteStore.subscribe({type: I.QueryType.AllKV, q:true}) 94 | for await (const d of subValues(I.ResultType.KV, sub)) { 95 | // console.log('d', d) 96 | data = d 97 | draw() 98 | } 99 | })() 100 | 101 | -------------------------------------------------------------------------------- /demos/midi/server.ts: -------------------------------------------------------------------------------- 1 | import {I, stores, rmKV, setKV, subValues, registerType, resultTypes, catchupStateMachine} from '@statecraft/core' 2 | import {wrapWebSocket, connectMux, BothMsg, tcpserver} from '@statecraft/net' 3 | import express from 'express' 4 | import WebSocket from 'ws' 5 | import http from 'http' 6 | 7 | // import kvMem from '../../lib/stores/kvmem' 8 | import State from './state' 9 | import {type as jsonType, JSONOp} from 'ot-json1' 10 | registerType(jsonType) 11 | 12 | const {kvmem} = stores 13 | 14 | process.on('unhandledRejection', err => { 15 | console.error((err as any).stack) 16 | process.exit(1) 17 | }) 18 | 19 | ;(async () => { 20 | // The store is a kv store mapping from client ID (incrementing numbers) => latest position. 21 | const store = await kvmem() 22 | 23 | const app = express() 24 | app.use(express.static(`${__dirname}/public`)) 25 | 26 | const server = http.createServer(app) 27 | const wss = new WebSocket.Server({server}) 28 | 29 | let nextId = 1000 30 | 31 | wss.on('connection', async (socket, req) => { 32 | const id = `${nextId++}` 33 | 34 | const [reader, writer] = wrapWebSocket(socket) 35 | const remoteStore = await connectMux(reader, writer, store, false) 36 | 37 | // console.log(id, 'info', remoteStore.storeInfo) 38 | console.log(id, 'client connected') 39 | const sub = remoteStore.subscribe({type: I.QueryType.Single, q: true}) 40 | 41 | reader.onClose = () => { 42 | console.log(id, 'client gone') 43 | // delete db[id] 44 | // console.log('db', db) 45 | rmKV(store, id) 46 | } 47 | 48 | const sm = catchupStateMachine(I.ResultType.Single) 49 | for await (const cu of sub) { 50 | // Most updates are just going to be a single JSON transaction. If thats the case, just port the transaction. 51 | 52 | // TODO: This code is way more complicated than I'd like. A functionally 53 | // equivalent (but much simpler) version is commented below. But, doing it 54 | // this way keeps network traffic down. Consider making a utility method 55 | // out of this sort of thing in the future. 56 | const newVal = sm(cu).results 57 | 58 | // Each transaction in the catchup might be a list or a single item. Throw 59 | // everything away except the ops themselves. 60 | const ops = ([] as I.SingleOp[]).concat(...cu.txns.map(({txn}) => txn as I.SingleTxn)) 61 | if (cu.replace || ops.find(op => op.type !== 'json1')) await setKV(store, id, newVal) 62 | else { 63 | // Flatten ops into a single json1 operation using compose() and send it. 64 | const op = ops.map(({data}) => data as JSONOp).reduce(jsonType.compose) 65 | await store.mutate(I.ResultType.KV, new Map([[id, {type: 'json1', data: op}]])) 66 | } 67 | } 68 | // for await (const val of subValues(I.ResultType.Single, sub)) { 69 | // await setKV(store, id, val) 70 | // } 71 | }) 72 | 73 | const port = process.env.PORT || 2444 74 | server.listen(port, () => { 75 | console.log('listening on', port) 76 | }) 77 | 78 | 79 | if (process.env.NODE_ENV !== 'production') { 80 | const tcpServer = tcpserver(store) 81 | tcpServer.listen(2002, 'localhost') 82 | console.log('Debugging server listening on tcp://localhost:2002') 83 | } 84 | })() -------------------------------------------------------------------------------- /core/lib/version.ts: -------------------------------------------------------------------------------- 1 | import * as I from './interfaces' 2 | 3 | const TWO_32 = Math.pow(2, 32) // Cannot be constructed via bit operations 4 | 5 | // The easiest way to use versions is to use numbers internally and then use 6 | // this version constructor, which makes a byte array out of the big-endian 7 | // encoding of the specified number. The number must be an integer. 8 | export const V64 = (x: number): I.Version => { 9 | if (x > Number.MAX_SAFE_INTEGER) throw Error('Cannot use normal number encoding on version above 2^53') 10 | 11 | const ab = new ArrayBuffer(8) 12 | const dataview = new DataView(ab) 13 | dataview.setUint32(0, x / TWO_32) 14 | dataview.setUint32(4, x & 0xffffffff) 15 | 16 | return new Uint8Array(ab) 17 | } 18 | 19 | export const V_EMPTY = new Uint8Array() 20 | 21 | export const v64ToNum = (v: I.Version): number => { 22 | if (v.length !== 8) throw new Error('Invalid byte length in version') 23 | const dataview = new DataView(v.buffer, v.byteOffset, v.byteLength) 24 | return dataview.getUint32(0) * TWO_32 + dataview.getUint32(4) 25 | } 26 | 27 | export const vIncMut = (v: Uint8Array) => { 28 | let i = v.length 29 | while (i > 0 && v[--i]++ === 0xff); 30 | return v 31 | } 32 | export const vInc = (v: Uint8Array) => vIncMut(new Uint8Array(v)) 33 | export const vDecMut = (v: Uint8Array) => { 34 | let i = v.length 35 | while (i > 0 && v[--i]-- === 0x00); 36 | return v 37 | } 38 | export const vDec = (v: Uint8Array) => vDecMut(new Uint8Array(v)) 39 | 40 | // Lexographical comparison, though in reality versions should probably always 41 | // have the same length. This is included so we don't need to bundle all of 42 | // Buffer in the browser. 43 | export const vCmp = (a: Uint8Array, b: Uint8Array): number => { 44 | if (a === b) return 0 // Shortcut 45 | let i 46 | for (i = 0; i < a.length; i++) { 47 | if (i >= b.length) return 1 48 | const v = a[i] - b[i] 49 | if (v) return v 50 | } 51 | return (i < b.length) ? -1 : 0 52 | } 53 | 54 | export const vMax = (a: Uint8Array, b: Uint8Array) => vCmp(a, b) > 0 ? a : b 55 | export const vMin = (a: Uint8Array, b: Uint8Array) => vCmp(a, b) > 0 ? b : a 56 | export const vEq = (a: Uint8Array, b: Uint8Array) => vCmp(a, b) === 0 57 | 58 | // Modifies and returns dest = intersect(dest, src) if an intersection is valid. 59 | // Otherwise returns null if no intersection exists. 60 | export const vIntersectMut = (dest: I.FullVersionRange, src: I.FullVersionRange) => { 61 | for (let i = 0; i < src.length; i++) if (src[i] != null) { 62 | const {from: fromSrc, to: toSrc} = src[i]! 63 | if (dest[i] == null) dest[i] = {from: fromSrc, to: toSrc} 64 | else { 65 | const {from:fromDest, to:toDest} = dest[i]! 66 | // No intersection! 67 | if (vCmp(fromSrc, toDest) > 0 || vCmp(toSrc, fromDest) < 0) return null 68 | dest[i] = {from: vMax(fromDest, fromSrc), to: vMin(toDest, toSrc)} 69 | } 70 | } 71 | return dest 72 | } 73 | 74 | export const vRangeFrom = (vr: I.FullVersionRange) => vr.map(v => v == null ? null : v.from) 75 | export const vRangeTo = (vr: I.FullVersionRange) => vr.map(v => v == null ? null : v.to) 76 | export const vToRange = (v: I.FullVersion): I.FullVersionRange => v.map(v => v == null ? null : ({from: v, to: v})) 77 | 78 | export const vSparse = (i: number, val: V): (V | null)[] => { 79 | const result = [] 80 | result[i] = val 81 | return result 82 | } -------------------------------------------------------------------------------- /demos/bp/public/editor.css: -------------------------------------------------------------------------------- 1 | @font-face { 2 | font-family: 'Play'; 3 | font-style: normal; 4 | font-weight: 700; 5 | src: local('Play-Bold'), url(/Play.woff) format('woff'); 6 | } 7 | 8 | body, html { 9 | width: 100%; height: 100%; padding: 0; margin: 0; 10 | font-family: 'Play', sans-serif; 11 | box-sizing: border-box; 12 | 13 | /* you see this if you try to scroll out of the module list */ 14 | background-color: hsl(184, 49%, 7%); 15 | } 16 | 17 | * { box-sizing: inherit; } 18 | 19 | a { 20 | color: hsl(203, 6%, 70%); 21 | text-decoration: none; 22 | } 23 | 24 | a:before { 25 | content: '['; 26 | } 27 | a:after { 28 | content: ']'; 29 | } 30 | 31 | .boilerplate { 32 | position: relative; 33 | width: 100%; height: 100%; 34 | } 35 | 36 | .boilerplate canvas { 37 | position: absolute; 38 | top: 0; left: 0; 39 | width: 100%; height: 100%; 40 | cursor: crosshair; 41 | } 42 | 43 | .overlay { 44 | position: fixed; 45 | 46 | padding: 5px; 47 | background-color: hsla(184, 49%, 7%, 0.5); 48 | cursor: default; 49 | color: white; 50 | 51 | -webkit-user-select: none; 52 | -moz-user-select: none; 53 | -ms-user-select: none; 54 | } 55 | 56 | .topleft { top: 0; left: 0; } 57 | /*.topright { top: 0; right: 0; }*/ 58 | .botright { bottom: 0; right: 0; } 59 | 60 | #readonly { 61 | display: none; 62 | color: red; 63 | } 64 | 65 | #worldname { 66 | color: hsl(42, 95%, 69%); 67 | margin: 0; 68 | padding: 0 0 0 8px; 69 | border: 0; 70 | font-size: 1em; 71 | background-color: transparent; 72 | min-width: 20em; 73 | } 74 | 75 | #playpanel { 76 | padding-top: 7px; 77 | } 78 | 79 | #playpanel span { 80 | cursor: pointer; 81 | color: hsl(42, 95%, 69%); 82 | font-size: 1.5em; 83 | margin: 7px; 84 | display: inline-block; 85 | min-width: 1em; 86 | text-align: center; 87 | vertical-align: top; /* needed because play button is tall */ 88 | } 89 | /*#playpanel.running #play { color: hsl(0, 0%, 78%); }*/ 90 | /*#playpanel.stopped #pause { color: white; }*/ 91 | #playpanel.running #step { color: gray; } 92 | 93 | #componentPanel { 94 | top: calc(-50% - 1px); /* No idea why the extra pixel is needed, but it is */ 95 | right: 0; 96 | width: 300px; 97 | /*height: calc(50% + 1em + 2 * 5px);*/ 98 | height: 50%; 99 | transition: top 0.6s; 100 | background-color: rgba(66, 66, 66, 0.61); 101 | padding: 0; 102 | } 103 | 104 | #componentPanel:hover { top: 0%; } 105 | 106 | #moduleList { 107 | height: 100%; 108 | overflow-y: scroll; 109 | } 110 | 111 | #componentPanel > .disclosure { 112 | /*color: hsl(42, 95%, 69%);*/ 113 | color: white; 114 | position: absolute; 115 | right: 0; 116 | top: 100%; 117 | padding: 5px; 118 | width: 50%; 119 | text-align: right; 120 | transition: color 0.6s; 121 | background-color: hsla(184, 49%, 7%, 0.5); 122 | } 123 | 124 | #componentPanel:hover > .disclosure { 125 | /*color: white;*/ 126 | color: hsl(42, 95%, 69%); 127 | } 128 | 129 | #componentPanel .module { 130 | width: 100%; 131 | height: 300px; /* Same as width of #componentPanel */ 132 | padding: 10px 10px 0 10px; 133 | position: relative; 134 | } 135 | 136 | #componentPanel .module:last-child { 137 | padding-bottom: 10px; 138 | } 139 | 140 | .module > canvas { 141 | width: 100%; 142 | height: 100%; 143 | background-color: hsla(184, 47%, 3%, 0.47); 144 | } 145 | 146 | .module.selected > canvas { 147 | background-color: rgba(19, 65, 70, 0.82); 148 | } 149 | 150 | .module > .rm { 151 | font-size: 30px; 152 | color: hsl(42, 95%, 69%); 153 | position: absolute; 154 | top: 10px; 155 | right: 14px; 156 | text-rendering: geometricPrecision; 157 | background-color: rgba(15, 21, 22, 0.51); 158 | } 159 | 160 | #addmod { 161 | font-size: 240px; 162 | text-align: center; 163 | background-color: hsla(184, 47%, 3%, 0.47); 164 | transition: height 0.6s; 165 | } 166 | -------------------------------------------------------------------------------- /core/lib/types/field.ts: -------------------------------------------------------------------------------- 1 | import {QueryType, ResultType, SingleOp, Op, ResultOps} from '../interfaces' 2 | import {typeOrThrow, supportedTypes, typeRegistry} from '../typeregistry' 3 | 4 | function appendMut(a: Op, b: SingleOp) { 5 | const {type} = b 6 | // Rm and set override preceding data in the stream. 7 | if (type === 'rm' 8 | || type === 'set' 9 | || Array.isArray(b) && b.length === 0) return b 10 | 11 | // If we have a type with a compose function, use it. 12 | const last = Array.isArray(a) ? a[a.length - 1] : a 13 | if (type === last.type) { 14 | const t = typeRegistry[type] 15 | if (t && t.compose) { 16 | if (Array.isArray(a)) { 17 | a[a.length - 1] = t.compose(last.data, b.data) 18 | return a 19 | } else { 20 | return t.compose(last.data, b.data) 21 | } 22 | } 23 | } 24 | 25 | // Otherwise just turn it into a composite op. 26 | if (Array.isArray(a)) { 27 | a.push(b) 28 | return a 29 | } else return [a, b] 30 | } 31 | 32 | const id = (x: T) => x 33 | const apply2 = (x: T, fn: (x: T, y: null) => R) => fn(x, null) 34 | 35 | const type: ResultOps> = { 36 | name: 'single', 37 | type: ResultType.Single, 38 | 39 | create(data) { 40 | // Remember, this is creating a document. 41 | return data 42 | }, 43 | 44 | apply(snapshot: Val, op: Op) { 45 | if (Array.isArray(op)) { 46 | // Multi operation. 47 | for (let i = 0; i < op.length; i++) { 48 | // Recurse. 49 | snapshot = type.apply(snapshot, op[i]) 50 | } 51 | return snapshot 52 | } 53 | 54 | const {type:typeName, data} = op 55 | 56 | switch (typeName) { 57 | case 'rm': return undefined 58 | case 'set': return data 59 | default: { 60 | return typeOrThrow(typeName).apply(snapshot, data) 61 | } 62 | } 63 | }, 64 | 65 | // TODO: Add applyMut. 66 | 67 | compose(op1, op2) { 68 | if (Array.isArray(op2)) { 69 | // This is textbook reduce territory. 70 | let comp = op1 71 | for (let i = 0; i < op2.length; i++) { 72 | // Recurse. 73 | appendMut(comp, op2[i]) 74 | } 75 | return comp 76 | } else { 77 | // Now just the single case. 78 | return appendMut(op1, op2) 79 | } 80 | }, 81 | 82 | asOp(snap: any) { 83 | return (snap == null) ? {type:'rm'} : {type:'set', data:snap} 84 | }, 85 | 86 | // from(type, data) { 87 | // switch(type) { 88 | // case 'single': return data 89 | // case 'kv': return data.get('content') 90 | // } 91 | // }, 92 | 93 | checkOp(op, snap) { 94 | type.apply(snap, op) // this will throw if invalid. TODO: use check on subtype if available. 95 | }, 96 | 97 | mapEntries: (v, fn) => { 98 | const v2 = fn(null, v) 99 | return v2 != null ? v2[1] : null 100 | }, 101 | mapEntriesAsync: (v, fn) => fn(null, v).then(x => x == null ? null : x[1]), 102 | 103 | map: apply2, 104 | mapAsync: apply2, 105 | mapTxn: apply2, // TODO: Consider pulling out SingleOp / SingleOp[] in these functions. 106 | mapTxnAsync: apply2, 107 | 108 | mapReplace: apply2, 109 | 110 | 111 | snapToJSON: id, 112 | snapFromJSON: id, 113 | opToJSON: id, 114 | opFromJSON: id, 115 | 116 | 117 | getCorrespondingQuery(data) { return {type: QueryType.Single, q: true} }, 118 | 119 | filterSupportedOps(txn, view, supportedTypes) { 120 | if (Array.isArray(txn)) { 121 | let hasAll = true 122 | for (let i = 0; i < txn.length; i++) { 123 | if (!supportedTypes.has(txn[i].type)) hasAll = false 124 | } 125 | return hasAll ? txn : type.asOp(view) 126 | } else { 127 | return supportedTypes.has(txn.type) ? txn : type.asOp(view) 128 | } 129 | }, 130 | 131 | composeResultsMut(a, b) { return b }, 132 | 133 | updateResults: (s, q, data) => data, 134 | } 135 | 136 | export default type -------------------------------------------------------------------------------- /net/lib/netmessages.ts: -------------------------------------------------------------------------------- 1 | import {I, ErrJson} from '@statecraft/core' 2 | 3 | export type Ref = /*string |*/ number 4 | 5 | export type NetKVTxn = [I.Key, I.Op][] 6 | export type NetTxn = I.SingleTxn | NetKVTxn 7 | 8 | export type NetQuery = I.QueryType.Single | I.QueryType.AllKV | [I.QueryType, any] 9 | 10 | export type NetVersion = number[] // This makes me really sad. Not needed with msgpack; only for json. 11 | export type NetVersionRange = [NetVersion, NetVersion] 12 | export type NetFullVersion = (null | NetVersion)[] 13 | export type NetFullVersionRange = (null | NetVersionRange)[] 14 | 15 | export type SubscribeOpts = { 16 | // TODO: Add all the rest!!! 17 | st?: string[] 18 | fv?: NetFullVersion | 'c', // known at versions 19 | } 20 | 21 | export const enum Action { 22 | // Many of these are used by both the server and client. 23 | Hello = 0, 24 | Err = 1, 25 | 26 | Fetch = 2, 27 | GetOps = 3, 28 | Mutate = 4, 29 | SubCreate = 5, 30 | SubClose = 6, 31 | SubUpdate = 7, 32 | } 33 | 34 | // **************** Client -> Server messages 35 | 36 | export interface FetchRequest { 37 | a: Action.Fetch, 38 | ref: Ref, 39 | query: NetQuery, 40 | opts: I.FetchOpts, 41 | } 42 | 43 | export interface GetOpsRequest { 44 | a: Action.GetOps, 45 | ref: Ref, 46 | query: NetQuery, 47 | v: NetFullVersionRange, 48 | opts: I.GetOpsOptions, 49 | } 50 | 51 | export interface MutateRequest { 52 | a: Action.Mutate, 53 | ref: Ref, 54 | mtype: I.ResultType, 55 | txn: any, 56 | v: NetFullVersion, 57 | opts: I.MutateOptions, 58 | } 59 | 60 | export interface SubCreate { 61 | a: Action.SubCreate, 62 | ref: Ref, 63 | query: NetQuery, 64 | opts: SubscribeOpts 65 | } 66 | 67 | export interface SubClose { 68 | a: Action.SubClose, 69 | ref: Ref, // Ref of subscription 70 | } 71 | 72 | export type CSMsg = 73 | FetchRequest 74 | | MutateRequest 75 | | GetOpsRequest 76 | | SubCreate 77 | | SubClose 78 | 79 | 80 | // **************** Server -> Client messages 81 | 82 | export interface HelloMsg { 83 | a: Action.Hello, 84 | p: 'statecraft', 85 | pv: number, 86 | uid: string, 87 | sources: I.Source[], // ??? TODO: Still not sure whether to allow unknown sources. 88 | m: boolean[], // This is kinda wasteful over the wire, but probably not a big deal. Bitvector would be better. 89 | capabilities: number[] 90 | } 91 | 92 | export interface FetchResponse { 93 | a: Action.Fetch, 94 | ref: Ref, 95 | results: any, // Dependant on query. 96 | 97 | bakedQuery?: NetQuery, 98 | versions: NetFullVersionRange, // Range across which version is valid. 99 | } 100 | 101 | // txn, version, meta. 102 | export type NetTxnWithMeta = [any, NetFullVersion, any] 103 | 104 | export interface GetOpsResponse { 105 | a: Action.GetOps, 106 | ref: Ref, 107 | ops: NetTxnWithMeta[], 108 | v: NetFullVersionRange 109 | } 110 | 111 | export interface MutateResponse { 112 | a: Action.Mutate, 113 | ref: Ref, 114 | v: NetFullVersion, 115 | } 116 | 117 | export interface ResponseErr { // Used for fetch, mutate and getops 118 | a: Action.Err, 119 | ref: Ref, 120 | err: ErrJson, 121 | } 122 | 123 | export interface SubUpdate { 124 | a: Action.SubUpdate, 125 | ref: Ref, 126 | 127 | // Replace data. If r exists, q must exist too. 128 | q?: NetQuery, // active query diff 129 | r?: any, // replacement 130 | rv?: NetFullVersion, 131 | u: boolean, 132 | 133 | txns: NetTxnWithMeta[], // updates on top of replacement 134 | 135 | tv: NetFullVersion, // version diff 136 | } 137 | 138 | // The subscription has closed 139 | export interface SubRet { 140 | a: Action.SubClose, 141 | ref: Ref, 142 | } 143 | 144 | 145 | export type SCMsg = 146 | HelloMsg 147 | | FetchResponse 148 | | GetOpsResponse 149 | | MutateResponse 150 | | ResponseErr 151 | | SubUpdate 152 | | SubRet 153 | 154 | // export const flatten1 = (data: any) => (data instanceof Map || data instanceof Set) ? Array.from(data) : data 155 | -------------------------------------------------------------------------------- /core/lib/opcache.ts: -------------------------------------------------------------------------------- 1 | import * as I from './interfaces' 2 | import * as err from './err' 3 | import {queryTypes} from './qrtypes' 4 | 5 | import binsearch from 'binary-search' 6 | import {vCmp, vEq} from './version' 7 | 8 | export interface OpCacheOpts { 9 | readonly qtype?: I.QueryType, 10 | readonly maxTime?: number, // in milliseconds. 0 = ignore. 11 | readonly maxNum?: number, // Max number of ops kept for each source. 0 = ignore. Defaults to 100. 12 | } 13 | 14 | interface OpsEntry { 15 | fromV: I.Version, 16 | toV: I.Version, 17 | txn: I.Txn, 18 | meta: I.Metadata, 19 | ctime: number, 20 | } 21 | const cmp = (item: OpsEntry, v: I.Version) => vCmp(item.toV, v) 22 | 23 | /** 24 | * Opcache is a helper object which keeps a set of recent operations in memory and can use it to respond to getOps calls. 25 | * 26 | * At the moment opcache only works with monotonically increasing operations. 27 | */ 28 | const opcache = (opts: OpCacheOpts = {}): { 29 | onOp(sourceIdx: number, fromV: I.Version, toV: I.Version, type: I.ResultType, txn: I.Txn, meta: I.Metadata): void, 30 | getOps: I.GetOpsFn, 31 | } => { 32 | const maxNum = opts.maxNum || 100 33 | const maxTime = opts.maxTime || 0 34 | // List is sorted in order and accessed using binary search. 35 | 36 | // source => list of ops entries sorted by version 37 | const opsForSource: OpsEntry[][] = [] 38 | 39 | const getOpsForSource = (si: number) => { 40 | let ops = opsForSource[si] 41 | if (ops == null) opsForSource[si] = ops = [] 42 | return ops 43 | } 44 | 45 | return { 46 | onOp(source, fromV, toV, type, txn, meta) { 47 | const ops = getOpsForSource(source) 48 | if (ops.length && !vEq(ops[ops.length - 1].toV, fromV)) throw Error('Emitted versions don\'t match') 49 | ops.push({fromV, toV, txn, meta, ctime: Date.now()}) 50 | 51 | while (maxNum !== 0 && ops.length > maxNum) ops.shift() 52 | 53 | const now = Date.now() 54 | while (maxTime !== 0 && ops.length && ops[0].ctime < now - maxTime) ops.shift() 55 | }, 56 | 57 | getOps(query, versions, options = {}) { 58 | const qtype = query.type 59 | const qops = queryTypes[qtype] 60 | if (qops == null) throw Error('Missing qops for type ' + qtype) 61 | 62 | let limitOps = options.limitOps || -1 63 | 64 | const vOut: I.FullVersionRange = [] 65 | 66 | const result: I.TxnWithMeta[] = [] 67 | for (let si = 0; si < opsForSource.length; si++) { 68 | // This is a bit inefficient - if there's lots of sources 69 | // we're looking through all of them even if the user only 70 | // wants one. But that shouldn't happen much in practice (right?) 71 | const ops = opsForSource[si] 72 | if (ops == null) continue 73 | const vs = versions[si] //|| versions._other 74 | if (vs == null) continue 75 | 76 | const {from, to} = vs 77 | let fromidx: number 78 | if (from.length === 0) fromidx = 0 // From version known. 79 | else { 80 | const searchidx = binsearch(ops, from, cmp) 81 | fromidx = searchidx < 0 ? ~searchidx : searchidx + 1 82 | } 83 | 84 | if (fromidx >= ops.length) continue 85 | 86 | // Figure out the actual returned version range. 87 | const vFrom = ops[fromidx].fromV 88 | let vTo = vFrom 89 | 90 | for (let i = fromidx; i < ops.length; i++) { 91 | const item = ops[i] 92 | if (to.length && vCmp(item.toV, to) > 0) break 93 | 94 | // The transaction will be null if the operation doesn't match 95 | // the supplied query. 96 | const txn = qops.adaptTxn(item.txn, query.q) 97 | const versions = [] 98 | versions[si] = item.toV 99 | if (txn != null) result.push({versions, txn, meta: item.meta}) 100 | 101 | vTo = item.toV 102 | if (limitOps > 0 && --limitOps === 0) break 103 | } 104 | /*if (vTo !== vFrom)*/ vOut[si] = {from: vFrom, to: vTo} 105 | 106 | if (limitOps === 0) break 107 | } 108 | 109 | // console.log('opcache', result, vOut) 110 | return Promise.resolve({ops: result, versions: vOut}) 111 | }, 112 | } 113 | } 114 | 115 | export default opcache -------------------------------------------------------------------------------- /demos/midi/client.ts: -------------------------------------------------------------------------------- 1 | import {I, stores, subValues, setSingle, registerType} from '@statecraft/core' 2 | import {connectToWS, connectMux, BothMsg} from '@statecraft/net' 3 | import State, {MIDIPort, MIDIInput} from './state' 4 | 5 | import {type as jsonType, insertOp, replaceOp, JSONOp, removeOp} from 'ot-json1' 6 | registerType(jsonType) 7 | 8 | const {singlemem} = stores 9 | 10 | const wsurl = `ws${window.location.protocol.slice(4)}//${window.location.host}/ws` 11 | console.log('wsurl', wsurl) 12 | 13 | // let data = new Map() 14 | 15 | // The local store that holds our mouse location 16 | 17 | ;(async () => { 18 | const access = await navigator.requestMIDIAccess() 19 | // Get lists of available MIDI controllers 20 | 21 | const inputs = Array.from(access.inputs.values()) 22 | const outputs = Array.from(access.outputs.values()) 23 | 24 | const portData = (port: WebMidi.MIDIPort): MIDIPort => ({ 25 | id: port.id, // These IDs are really silly. 26 | manufacturer: port.manufacturer, 27 | name: port.name, 28 | version: port.version, 29 | state: port.state, 30 | }) 31 | const inputPortData = (port: WebMidi.MIDIPort): MIDIInput => ({ 32 | ...portData(port), 33 | keys: {},//new Array(128).fill(null), 34 | pots: new Array(8).fill(NaN), //[], 35 | sliders: [NaN], 36 | pitch: 64, 37 | modulation: 0, 38 | }) 39 | 40 | let data: State = { 41 | timeOrigin: performance.timeOrigin, 42 | inputs: inputs.map(inputPortData), 43 | outputs: outputs.map(portData), 44 | } 45 | 46 | console.log('state', data) 47 | console.log(inputs[0].connection, inputs[0].state) 48 | const localStore = singlemem(data) 49 | 50 | const [reader, writer] = await connectToWS(wsurl) 51 | const remoteStore = await connectMux(reader, writer, localStore, true) 52 | 53 | const apply = (op: JSONOp) => { 54 | console.log('before', data, 'op', op) 55 | data = jsonType.apply(data, op) 56 | console.log('after', data) 57 | return localStore.mutate(I.ResultType.Single, {type: 'json1', data: op}) 58 | } 59 | 60 | const subscribeToInput = (input: WebMidi.MIDIInput, i: number) => { 61 | // console.log(i) 62 | input.onmidimessage = m => { 63 | // console.log(input.name, m.data, m.timeStamp) 64 | 65 | const [mtype, oper1, oper2] = m.data 66 | const inputData = data.inputs[i] 67 | const {keys, pots, sliders} = inputData 68 | switch (mtype) { 69 | case 0x90: // Note ON 70 | apply(replaceOp(['inputs', i, 'keys', ''+oper1], keys[oper1], {held: true, pressure: oper2, timestamp: m.timeStamp})) 71 | break 72 | case 0x80: // Note OFF 73 | apply(replaceOp(['inputs', i, 'keys', ''+oper1], keys[oper1], {held: false, pressure: 0, timestamp: m.timeStamp})) 74 | break 75 | case 176: { // Pots and sliders 76 | const field = oper1 === 1 ? ['modulation'] 77 | : (oper1 >= 0x15 && oper1 <= 0x1c) ? ['pots', oper1 - 0x15] 78 | : (oper1 === 0x7) ? ['sliders', 0] 79 | : null 80 | if (field == null) console.log('unknown CC /', oper1, oper2) 81 | else apply(replaceOp(['inputs', i, ...field], true, oper2)) 82 | break 83 | } 84 | case 224: // Pitch slider 85 | apply(replaceOp(['inputs', i, 'pitch'], inputData.pitch, oper2)) 86 | break 87 | default: 88 | console.log('unknown message', mtype, oper1, oper2) 89 | break 90 | } 91 | } 92 | } 93 | inputs.forEach(subscribeToInput) 94 | 95 | access.onstatechange = e => { 96 | // Print information about the (dis)connected MIDI controller 97 | const {port} = e 98 | const set = port.type === 'input' ? data.inputs : data.outputs 99 | const deviceIdx = set.findIndex(({id}) => id === port.id) 100 | if (deviceIdx >= 0) { 101 | set[deviceIdx].state = port.state 102 | } else { 103 | set.push(port.type === 'input' ? inputPortData(port) : portData(port)) 104 | if (port.type === 'input') subscribeToInput(port as WebMidi.MIDIInput, set.length-1) 105 | } 106 | setSingle(localStore, data) 107 | 108 | // console.log('onstatechange', e.port.name, e.port.manufacturer, e.port.state); 109 | // console.log(e.port.type) 110 | } 111 | })() 112 | 113 | -------------------------------------------------------------------------------- /demos/contentfulgraphql/server.ts: -------------------------------------------------------------------------------- 1 | import * as fs from 'fs' 2 | import http from 'http' 3 | import express from 'express' 4 | import * as gql from 'graphql' 5 | import * as gqlTools from 'graphql-tools' 6 | 7 | import * as I from '../../lib/interfaces' 8 | import createContentful from '../../lib/stores/contentful'; 9 | import kvMem from '../../lib/stores/kvmem' 10 | import gqlmr, {Ctx} from '../../lib/stores/graphqlmapreduce' 11 | import {Console} from 'console' 12 | 13 | // For debugging 14 | import router, {ALL} from '../../lib/stores/router' 15 | import serveTCP from '../../lib/net/tcpserver' 16 | 17 | import Post from './post' 18 | import {renderToString} from 'react-dom/server' 19 | import fresh from 'fresh' 20 | import { vRangeFrom } from '../../lib/version'; 21 | 22 | 23 | process.on('unhandledRejection', err => { 24 | console.error(err.stack) 25 | process.exit(1) 26 | }) 27 | 28 | global.console = new (Console as any)({ 29 | stdout: process.stdout, 30 | stderr: process.stderr, 31 | inspectOptions: {depth: null} 32 | }) 33 | 34 | // Eventually this will be autogenerated from the schema. 35 | const schema = gqlTools.makeExecutableSchema({ 36 | typeDefs: ` 37 | type Post { 38 | title: String, 39 | content: String, 40 | author: Author!, 41 | slug: [String!], 42 | updatedAt: String, 43 | } 44 | 45 | type Author { 46 | fullName: String, 47 | } 48 | 49 | type Query { 50 | postById(id: String!): Post 51 | foo: String, 52 | 53 | selfPost: Post, 54 | selfAuthor: Author, 55 | } 56 | `, 57 | resolvers: { 58 | Post: { 59 | author: (post, args, {txn}: Ctx) => ( 60 | txn.get('author/' + post.author) 61 | ) 62 | }, 63 | 64 | Query: { 65 | postById: (_, {id}: {id: string}, {txn}: Ctx) => ( 66 | txn.get('post/' + id) 67 | ), 68 | 69 | selfPost: (_self, _args, {Post}) => Post, 70 | } 71 | } 72 | }) 73 | 74 | const genEtag = (versions: I.FullVersion): string => { 75 | return versions.map(v => v == null ? '' : Buffer.from(v).toString('base64')).join('.') 76 | } 77 | 78 | ;(async () => { 79 | const keys = JSON.parse(fs.readFileSync('keys.json', 'utf8')) 80 | const syncStore = await kvMem() 81 | const ops = createContentful(syncStore, { 82 | space: keys.space, 83 | accessToken: keys.contentAPI, 84 | }) 85 | // Ideally this wouldn't return until after we get the initial sync. 86 | const cfstore = await kvMem(undefined, {inner: ops}) 87 | 88 | const store = await gqlmr(cfstore, { 89 | schema, 90 | mapReduces: [ 91 | { 92 | type: 'Post', 93 | queryStr: `{selfPost {updatedAt, title, content, slug, author {fullName}}}`, 94 | reduce: ({selfPost}) => ( 95 | renderToString(Post(selfPost)) 96 | ), 97 | getOutputKey: (k, {selfPost}) => (`${selfPost.slug[0]}`) 98 | } 99 | ], 100 | prefixForCollection: new Map([ 101 | // This is mapping graphql types -> backend key prefixes. 102 | ['Post', 'post/'], 103 | ['Author', 'author/'], 104 | ]), 105 | }) 106 | 107 | const app = express() 108 | app.use(express.static(`${__dirname}/public`)) 109 | 110 | app.get('/post/:slug', async (req, res, next) => { 111 | const r = await store.fetch({type: I.QueryType.KV, q: new Set([req.params.slug])}) 112 | const html = r.results.get(req.params.slug) 113 | if (html == null) return next() 114 | else { 115 | // TODO: ETag, etc. 116 | const headers = { 117 | 'content-type': 'text/html', 118 | 'etag': genEtag(vRangeFrom(r.versions)), 119 | } 120 | if (fresh(req.headers, headers)) return res.sendStatus(304) 121 | 122 | res.set(headers) 123 | res.send('\n' + html) 124 | } 125 | }) 126 | 127 | const server = http.createServer(app) 128 | const port = process.env['PORT'] || 2004 129 | server.listen(port, () => { 130 | console.log('listening on port', port) 131 | }) 132 | 133 | if (process.env.NODE_ENV !== 'production') { 134 | const all = router() 135 | all.mount(cfstore, 'cf/', null, '', false) 136 | all.mount(store, 'rendered/', null, '', false) 137 | const tcpServer = serveTCP(all) 138 | tcpServer.listen(2002, 'localhost') 139 | console.log('Debugging server listening on tcp://localhost:2002') 140 | } 141 | })() -------------------------------------------------------------------------------- /core/lib/types/map.ts: -------------------------------------------------------------------------------- 1 | import * as I from '../interfaces' 2 | import fieldOps from './field' 3 | 4 | const mapMapEntry = (input: Map, fn: (key: I.Key, val: In) => [I.Key, Out] | null) => { 5 | const result = new Map() 6 | for (const [k, val] of input) { 7 | const newEntry = fn(k, val) 8 | if (newEntry != null) result.set(newEntry[0], newEntry[1]) 9 | } 10 | return result 11 | } 12 | 13 | // Could just write this in terms of mapMapEntry above. 14 | const mapMapVal = (input: Map, fn: (val: In, key: I.Key) => Out) => { 15 | const result = new Map() 16 | for (const [k, val] of input) result.set(k, fn(val, k)) 17 | return result 18 | } 19 | 20 | const mapEntryAsync = (input: Map, fn: (key: I.Key, val: In) => Promise<[I.Key, Out] | null>) => { 21 | const entries = Array.from(input.entries()) 22 | const mapped = entries.map(([k, v]) => fn(k, v)) 23 | return Promise.all(mapped).then((results) => new Map(results.filter(e => e != null) as [I.Key, Out][])) 24 | } 25 | 26 | const mapAsync = (input: Map, fn: (val: In, key: I.Key) => Promise) => ( 27 | mapEntryAsync(input, (k, v) => fn(v, k).then(v2 => ([k, v2] as [I.Key, Out]))) 28 | ) 29 | 30 | type Val = any 31 | const type: I.ResultOps, I.KVTxn> = { 32 | name: 'kv', 33 | type: I.ResultType.KV, 34 | 35 | create(data) { 36 | return data instanceof Map ? data : new Map(data) 37 | }, 38 | 39 | applyMut(snap, op) { 40 | for (var [k, docop] of op) { 41 | const oldval = snap.get(k) 42 | const newval = fieldOps.apply(oldval, docop) 43 | 44 | // Statecraft considers null / undefined to be the same as a document not existing. 45 | if (newval == null) snap.delete(k) 46 | else snap.set(k, newval) 47 | } 48 | }, 49 | 50 | apply(snap, op) { 51 | const newdata = new Map(snap) 52 | type.applyMut!(newdata, op) 53 | return newdata 54 | }, 55 | 56 | composeMut(txn, other) { 57 | for (const [k, op] of other) { 58 | const orig = txn.get(k) 59 | if (orig == null) txn.set(k, op) 60 | else { 61 | txn.set(k, fieldOps.compose(orig, op)) 62 | } 63 | } 64 | }, 65 | 66 | compose(a, b) { 67 | const result = new Map(a) 68 | type.composeMut!(result, b) 69 | return result 70 | }, 71 | 72 | composeResultsMut(dest, src) { 73 | // For maps this is the same as copyInto. 74 | return type.copyInto!(dest, src) 75 | }, 76 | 77 | copyInto(dest, src) { 78 | for (const [k, v] of src) dest.set(k, v) 79 | return dest 80 | }, 81 | 82 | filter(snap: Map, query: Set): Map { 83 | const result = new Map() 84 | for (const k of query) { 85 | const v = snap.get(k) 86 | if (v !== undefined) result.set(k, v) 87 | } 88 | return result 89 | }, 90 | 91 | // from(type, data) { 92 | // switch(type) { 93 | // case 'single': return new Map([['content', data]]) 94 | // case 'resultmap': return data 95 | // } 96 | // }, 97 | 98 | mapEntries: mapMapEntry, 99 | mapEntriesAsync: mapEntryAsync, 100 | map: mapMapVal, 101 | mapAsync, 102 | mapTxn: mapMapVal, 103 | mapTxnAsync: mapAsync, 104 | 105 | mapReplace: mapMapVal, 106 | 107 | snapToJSON(snap) { return Array.from(snap) }, 108 | snapFromJSON(data) { return new Map(data) }, 109 | opToJSON(op) { return Array.from(op) }, 110 | opFromJSON(data) { return new Map(data) }, 111 | 112 | getCorrespondingQuery(snap) { 113 | return {type: I.QueryType.KV, q: new Set(snap.keys())} 114 | }, 115 | 116 | filterSupportedOps(op, values: Map, supportedTypes) { 117 | // console.log('fso', op, values) 118 | return mapMapVal(op, (o, k) => ( 119 | fieldOps.filterSupportedOps(o, values.get(k), supportedTypes)) 120 | ) 121 | }, 122 | 123 | updateResults(s: Map, q: I.ReplaceQuery, data: Map) { 124 | if (q.type === I.QueryType.KV) { 125 | for (const k of q.q) { 126 | if (data.has(k)) s.set(k, data.get(k)) 127 | else s.delete(k) 128 | } 129 | return s 130 | } else return data // allkv. 131 | // I'm not sure if we should look at q.q for this ?? 132 | // } else return q.q ? data : s // allkv. 133 | }, 134 | } 135 | export default type 136 | -------------------------------------------------------------------------------- /core/lib/stores/onekey.ts: -------------------------------------------------------------------------------- 1 | // This is a simple store that re-exposes a single key from the underlying 2 | // store as a single value. 3 | 4 | import * as I from '../interfaces' 5 | import err from '../err' 6 | import { bitSet, bitHas } from '../bit'; 7 | 8 | const capabilities = { 9 | queryTypes: bitSet(I.QueryType.Single), 10 | mutationTypes: bitSet(I.ResultType.Single), 11 | // ops: 'none', 12 | } 13 | 14 | const onekey = (innerStore: I.Store, key: I.Key): I.Store => { 15 | const canMutate = bitHas(innerStore.storeInfo.capabilities.mutationTypes, I.ResultType.KV) 16 | // console.log('cm', canMutate, innerStore.storeInfo) 17 | 18 | const innerQuery: I.Query = {type: I.QueryType.KV, q: new Set([key])} 19 | if (!bitHas(innerStore.storeInfo.capabilities.queryTypes, I.QueryType.KV)) throw new err.UnsupportedTypeError('Inner store must support KV queries') 20 | 21 | const unwrapTxns = (txns: I.TxnWithMeta[]): I.TxnWithMeta[] => ( 22 | txns.map(txn => { 23 | if(!(txn.txn instanceof Map)) throw new err.InvalidDataError() 24 | return { 25 | ...txn, 26 | txn: txn.txn.get(key), 27 | } as I.TxnWithMeta 28 | }) 29 | ) 30 | 31 | const unwrapCatchupData = (data: I.CatchupData): I.CatchupData => { 32 | let hasReplace = false 33 | if (data.replace) { 34 | const replaceQ = data.replace.q 35 | if (replaceQ.type !== I.QueryType.KV) throw new err.InvalidDataError() 36 | if (replaceQ.q.has(key)) hasReplace = true 37 | } 38 | 39 | return { 40 | ...data, 41 | replace: hasReplace ? { 42 | q: {type: I.QueryType.Single, q: true}, 43 | with: data.replace!.with.get(key), 44 | versions: data.replace!.versions, 45 | } : undefined, 46 | txns: unwrapTxns(data.txns), 47 | } 48 | } 49 | 50 | return { 51 | storeInfo: { 52 | uid: `onekey(${innerStore.storeInfo.uid},${key})`, 53 | sources: innerStore.storeInfo.sources, 54 | sourceIsMonotonic: innerStore.storeInfo.sourceIsMonotonic, 55 | capabilities: { 56 | mutationTypes: canMutate ? capabilities.mutationTypes : new Set(), 57 | ...capabilities, 58 | } 59 | }, 60 | 61 | async fetch(query, opts) { 62 | if (query.type !== I.QueryType.Single) throw new err.UnsupportedTypeError() 63 | 64 | const results = await innerStore.fetch(innerQuery, opts) 65 | // if (results.type !== 'kv') throw new err.InvalidDataError() 66 | 67 | return { 68 | results: results.results.get(key), 69 | queryRun: query, 70 | versions: results.versions, 71 | } 72 | }, 73 | 74 | async mutate(type, txn, versions, opts = {}) { 75 | if (!canMutate) throw new err.UnsupportedTypeError('Underlying store is read only') 76 | if (type !== I.ResultType.Single) throw new err.UnsupportedTypeError() 77 | 78 | const innerTxn = new Map([[key, txn as I.Op]]) 79 | return await innerStore.mutate(I.ResultType.KV, innerTxn, versions, { 80 | ...opts, 81 | // conflictKeys: opts.conflictKeys && opts.conflictKeys.includes(key) ? [''] : undefined, 82 | }) 83 | }, 84 | 85 | // Should probably pass a parameter on whether we take ownership of the underlying store or not. 86 | close() { innerStore.close() }, 87 | 88 | async getOps(query, reqVersions, opts) { 89 | if (query.type !== I.QueryType.Single) throw new err.UnsupportedTypeError() 90 | 91 | const {ops, versions} = await innerStore.getOps(innerQuery, reqVersions, opts) 92 | 93 | // Just going to mutate the ops in place. 94 | 95 | return { 96 | ops: unwrapTxns(ops), 97 | versions, 98 | } 99 | }, 100 | 101 | catchup: innerStore.catchup ? async (query, fromVersion, opts) => ( 102 | unwrapCatchupData(await innerStore.catchup!(innerQuery, fromVersion, opts)) 103 | ) : undefined, 104 | 105 | // TODO: Map catchup if it exists in the underlying store. 106 | 107 | subscribe(query, opts = {}) { 108 | if (query.type !== I.QueryType.Single) throw new err.UnsupportedTypeError() 109 | 110 | const innerSub = innerStore.subscribe(innerQuery, opts) 111 | 112 | // TODO: It'd be great to have a SubscriptionMap thing that lets you map 113 | // the resulting data you get back from a subscription. That'd be useful 114 | // in a few places and this is really wordy. 115 | return (async function*() { 116 | for await (const innerUpdates of innerSub) { 117 | yield unwrapCatchupData(innerUpdates) 118 | } 119 | // TODO: Does this pass through return() correctly? 120 | })() as I.AsyncIterableIteratorWithRet> 121 | }, 122 | 123 | } 124 | } 125 | 126 | export default onekey -------------------------------------------------------------------------------- /prozess/prozess-mock.ts: -------------------------------------------------------------------------------- 1 | // This is a simple in-memory mock for prozess. This is useful so we don't 2 | // have to spin up & down lots of prozess instances to run tests. 3 | 4 | // This can't be used outside of tests because its used as the central source 5 | // of truth, and it loses all content on restart. So if you back an LMDB store 6 | // with this, it'll correctly fail to restart. 7 | 8 | import { 9 | PClient, 10 | VersionConflictError, 11 | Event, 12 | } from 'prozess-client' 13 | import genSource from '../lib/gensource' 14 | 15 | const clamp = (x: T, a: T, b: T) => x < a ? a : (x > b ? b : x) 16 | 17 | export default function createMock(): PClient { 18 | interface MockEvent extends Event { 19 | conflictKeys: string[], 20 | } 21 | 22 | const base = 0xff01 // Pretend the mock starts at some offset in the file 23 | const events: MockEvent[] = [] 24 | const nextVersion = () => base + events.length 25 | 26 | let subscribed = false 27 | 28 | const addEvent = ( 29 | data: Buffer | string, 30 | targetVersion: number = events.length, 31 | conflictKeys: string[] = []) => { 32 | 33 | // debugger 34 | // This isn't super efficient - it'd be better to sort and walk together. 35 | // But this is just for testing so it shouldn't matter. 36 | const ckSet = new Set(conflictKeys) 37 | 38 | // Version of the event 39 | const ev = nextVersion() 40 | 41 | for (targetVersion = clamp(targetVersion, base, ev); 42 | targetVersion < ev; 43 | targetVersion++) { 44 | const e = events[targetVersion - base] 45 | for (const k of e.conflictKeys) { 46 | if (ckSet.has(k)) return null // Write conflict. 47 | } 48 | } 49 | 50 | const event = { 51 | version: ev, 52 | crc32: 0, // dummy. 53 | batch_size: 1, 54 | flags: 0, 55 | data: typeof data === 'string' ? Buffer.from(data) : data, 56 | conflictKeys: conflictKeys || [], 57 | } 58 | 59 | events.push(event) 60 | return event 61 | } 62 | 63 | return { 64 | source: genSource(), 65 | onevents: undefined, 66 | onclose: undefined, 67 | onunsubscribe: undefined, 68 | 69 | subscribe(from, opts, callback) { 70 | process.nextTick(() => { 71 | if (from == null || from < 0 || from > nextVersion()) from = nextVersion() 72 | 73 | // First forward a bunch of events 74 | const idx = from - base 75 | const evts = events.slice(idx) 76 | this.onevents && this.onevents(evts, from) 77 | subscribed = true 78 | 79 | callback && callback(null, { 80 | v_start: from, 81 | v_end: nextVersion(), 82 | size: evts.reduce((sum, e) => e.data.length + sum, 0), 83 | complete: false, 84 | oneshot: false, 85 | current: true, 86 | events: evts, 87 | }) 88 | }) 89 | }, 90 | 91 | // This gets ops in the range [from, to]. 92 | getEventsRaw(from, to, opts, callback) { 93 | process.nextTick(() => { 94 | const f = clamp(from - base, 0, events.length) // Allowed to fall off the end. 95 | const t = to === -1 ? undefined : clamp(to - base, 0, events.length-1)+1 96 | const evts = events.slice(f, t) 97 | callback && callback(null, { 98 | v_start: f+base, 99 | v_end: f+base + evts.length, 100 | complete: true, 101 | current: false, 102 | oneshot: true, 103 | size: evts.reduce((sum, e) => e.data.length + sum, 0), 104 | events: evts, 105 | }) 106 | }) 107 | }, 108 | 109 | getEvents(from, to, opts) { 110 | return new Promise((resolve, reject) => { 111 | this.getEventsRaw(from, to, opts, (err, data) => { 112 | if (err) reject(err) 113 | else resolve(data) 114 | }) 115 | }) 116 | }, 117 | 118 | sendRaw(data, opts, callback) { 119 | const {targetVersion, conflictKeys} = opts 120 | if (callback) process.nextTick(() => { 121 | const event = addEvent(data, targetVersion, conflictKeys) 122 | if (event == null) callback(new VersionConflictError('Rejected event due to conflict')) 123 | else { 124 | if (subscribed && this.onevents) this.onevents([event], event.version) 125 | callback(null, event.version) 126 | } 127 | }) 128 | }, 129 | 130 | send(data, opts = {}) { 131 | return new Promise((resolve, reject) => { 132 | this.sendRaw(data, opts, (err, version) => { 133 | if (err) reject(err) 134 | else resolve(version) 135 | }) 136 | }) 137 | }, 138 | 139 | getVersion() { 140 | // base + events.length is next version. 141 | return Promise.resolve(nextVersion() - 1) 142 | }, 143 | 144 | close() {}, 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /core/lib/stores/ot.ts: -------------------------------------------------------------------------------- 1 | // This is a simple passthrough store which catches write conflicts and tries 2 | // to do operational tranformation to recover them. 3 | import * as I from '../interfaces' 4 | import err from '../err' 5 | import {queryTypes, resultTypes} from '../qrtypes' 6 | import {typeOrThrow} from '../typeregistry' 7 | 8 | const mapTxnWithPair = (type: I.ResultType, a: I.Txn, b: I.Txn, fn: (a: I.SingleTxn, b: I.SingleTxn) => I.SingleTxn) => { 9 | if (a instanceof Map) { 10 | const result = new Map>() 11 | if (!(b instanceof Map)) throw new err.InvalidDataError('Transactions are different types') 12 | for (const [k, av] of a.entries()) { 13 | const bv = b.get(k) 14 | if (bv != null) result.set(k, fn(av, bv)) 15 | else result.set(k, av) 16 | } 17 | return result 18 | } else if (type === I.ResultType.Single) { 19 | return fn(a as I.Op, b as I.Op) 20 | } else { 21 | throw new err.UnsupportedTypeError('Type ' + type + ' not supported by ot store') 22 | // throw new err.InvalidDataError('Transactions are different types') 23 | } 24 | } 25 | 26 | const eachOp = (op: I.Op, fn: (op: I.SingleOp) => void) => { 27 | if (Array.isArray(op)) op.forEach(fn) 28 | else fn(op) 29 | } 30 | 31 | const otStore = (inner: I.Store /*, filter: (key: I.Key) => boolean */): I.Store => { 32 | 33 | return { 34 | ...inner, 35 | 36 | async mutate(type, txn, versions, opts = {}) { 37 | // First try to just submit the operation directly. 38 | while (true) { 39 | try { 40 | const result = await inner.mutate(type, txn, versions, opts) 41 | return result 42 | } catch (e) { 43 | // console.error('Caught error in mutate', e.stack) 44 | if (!(e instanceof err.WriteConflictError) 45 | || versions == null) { 46 | throw e 47 | } 48 | 49 | const uid = opts.meta ? opts.meta.uid : undefined 50 | 51 | // console.log('wowooo got a write conflict') 52 | 53 | // We don't know what conflicted, and the transaction could have a 54 | // lot of stuff in it. 55 | const q = resultTypes[type].getCorrespondingQuery(txn) 56 | 57 | // This is a terrible error message for a complex error. Should be 58 | // pretty rare in practice - basically the mutation type has to 59 | // match the type of the corresponding query. Which will basically 60 | // always be kv or single. 61 | if (queryTypes[q.type].resultType.type !== type) throw Error(`Mismatched query types unsupported ${queryTypes[q.type].resultType.name} != ${type}`) 62 | 63 | // empty to version == all. 64 | const catchupVersions: I.FullVersionRange = versions.map(v => v == null ? null : {from: v, to: new Uint8Array()}) 65 | const {ops} = await inner.getOps(q, catchupVersions) 66 | if (ops.length === 0) throw e // Can't make progress. 67 | 68 | let madeProgress = false 69 | for (let i = 0; i < ops.length; i++) { 70 | if (uid != null && ops[i].meta.uid === uid) { 71 | // The transaction has actually already been applied - we just 72 | // found it. We don't need to do any more work; just return as 73 | // if this txn was the txn you tried to submit. 74 | return ops[i].versions 75 | } 76 | 77 | txn = mapTxnWithPair(type, txn, ops[i].txn, (a, b) => { 78 | // The mutation we recieved has not been collapsed together. 79 | // This requires an M*N transform, which is not implemented. 80 | if (Array.isArray(a)) throw e 81 | if (a.type === 'set' || a.type === 'rm') throw e 82 | const type = typeOrThrow(a.type) 83 | if (type.transform == null) throw e 84 | 85 | let {data} = a 86 | eachOp(b, (bop) => { 87 | // Can't transform if they're a different type 88 | if (a.type !== bop.type) throw e 89 | // console.log('xf', data, bop.data) 90 | data = type.transform!(data, bop.data, 'left') 91 | // console.log('->', data) 92 | madeProgress = true 93 | }) 94 | return {type: a.type, data} 95 | }) 96 | 97 | ops[i].versions.forEach((v, si) => { 98 | if (v != null && versions[si] != null) versions[si] = v 99 | }) 100 | } 101 | 102 | // If we didn't make progress, abort to avoid an infinite loop. 103 | // The store is generating a conflict for some other reason. Let it be. 104 | if (!madeProgress) throw e 105 | 106 | // console.log('retrying with', txn) 107 | } 108 | } 109 | } 110 | } 111 | } 112 | 113 | export default otStore 114 | -------------------------------------------------------------------------------- /prozess/prozessops.ts: -------------------------------------------------------------------------------- 1 | // This is a wrapper 'store' around prozess. 2 | // 3 | // The store type is raw. The 'document' is empty, and you can't 4 | // query anything. 5 | 6 | import {PClient} from 'prozess-client' 7 | import assert from 'assert' 8 | 9 | import * as I from '../interfaces' 10 | import err from '../err' 11 | import {decodeTxn, decodeEvent, sendTxn} from '../prozess' 12 | import {queryTypes} from '../qrtypes' 13 | import {V64, v64ToNum, vEq} from '../version' 14 | import { bitSet } from '../bit' 15 | import SubGroup from '../subgroup' 16 | 17 | // const codec = msgpack.createCodec({usemap: true}) 18 | 19 | // const doNothing = () => {} 20 | const capabilities = { 21 | // ... Except you can't fetch. :/ 22 | queryTypes: bitSet(I.QueryType.AllKV, I.QueryType.KV, I.QueryType.StaticRange, I.QueryType.Range), 23 | mutationTypes: bitSet(I.ResultType.KV), 24 | } 25 | 26 | // TODO: pick a better port. 27 | const prozessStore = (conn: PClient): I.Store => { 28 | const source = conn.source! 29 | 30 | const getOps: I.GetOpsFn = async (query, versions, opts) => { 31 | // We don't need to wait for ready here because the query is passed straight back. 32 | if (query.type !== I.QueryType.AllKV && query.type !== I.QueryType.KV && query.type !== I.QueryType.StaticRange) throw new err.UnsupportedTypeError() 33 | const qops = queryTypes[query.type] 34 | 35 | // We need to fetch ops in the range of (from, to]. 36 | const vs = versions[0] 37 | if (!vs || vEq(vs.from, vs.to)) return {ops: [], versions: vs ? [{from:vs.from, to:vs.to}] : []} 38 | 39 | const {from, to} = vs 40 | 41 | const data = await conn.getEvents(from.length ? v64ToNum(from) + 1 : 0, to.length ? v64ToNum(to) : -1, {}) 42 | // console.log('client.getEvents', from+1, to, data) 43 | 44 | // Filter events by query. 45 | let ops = data!.events.map(event => decodeEvent(event, source)) 46 | .filter((data) => { 47 | const txn = qops.adaptTxn(data.txn, query.q) 48 | if (txn == null) return false 49 | else { 50 | data.txn = txn 51 | return true 52 | } 53 | }) 54 | 55 | return { 56 | ops, 57 | versions: [{from:V64(data!.v_start - 1), to: V64(data!.v_end - 1)}] 58 | } 59 | } 60 | 61 | const subGroup = new SubGroup({ 62 | getOps, 63 | start(v) { 64 | let currentVersion = -1 65 | conn.onevents = (events, nextVersion) => { 66 | // console.log('conn.onevents', events, nextVersion) 67 | const txns: I.TxnWithMeta[] = [] 68 | if (events.length == 0) return 69 | 70 | let fromV: I.Version | null = null 71 | events.forEach(event => { 72 | if (currentVersion !== -1) assert.strictEqual( 73 | currentVersion, event.version - 1, 74 | 'Error: Prozess version consistency violation. This needs debugging' 75 | ) 76 | 77 | if (fromV == null) fromV = V64(event.version - 1) 78 | 79 | // TODO: Pack & unpack batches. 80 | const [txn, meta] = decodeTxn(event.data) 81 | 82 | const nextVersion = event.version - 1 + event.batch_size 83 | 84 | txns.push({txn, meta, versions: [V64(nextVersion)]}) 85 | 86 | // store.onTxn!(source, V64(event.version - 1), V64(nextVersion), 'kv', txn, meta) 87 | 88 | currentVersion = nextVersion 89 | }) 90 | 91 | subGroup.onOp(0, fromV!, txns) 92 | } 93 | 94 | return new Promise((resolve, reject) => { 95 | const vs = v == null ? null : v[0] 96 | // Should this be vs or vs+1 or something? 97 | conn.subscribe(vs == null ? -1 : v64ToNum(vs) + 1, {}, (err, subdata) => { 98 | if (err) { 99 | // I'm not sure what to do here. It'll depend on the error 100 | console.error('Error subscribing to prozess store') 101 | return reject(err) 102 | } 103 | 104 | // console.log('prozess ops from', V64(subdata!.v_end + 1), subdata) 105 | // The events will all be emitted via the onevent callback. We'll do catchup there. 106 | resolve([V64(subdata!.v_end - 1)]) // +1 ??? 107 | }) 108 | }) 109 | } 110 | }) 111 | 112 | const store: I.Store = { 113 | storeInfo: { 114 | uid: `prozess(${source})`, 115 | sources: [source], 116 | capabilities, 117 | }, 118 | 119 | async mutate(type, _txn, versions, opts = {}) { 120 | if (type !== I.ResultType.KV) throw new err.UnsupportedTypeError() 121 | const txn = _txn as I.KVTxn 122 | 123 | const version = await sendTxn(conn, txn, opts.meta || {}, (versions && versions[0]) || new Uint8Array(), {}) 124 | return [version!] 125 | }, 126 | 127 | fetch() { 128 | throw new err.UnsupportedTypeError() 129 | }, 130 | getOps, 131 | subscribe: subGroup.create.bind(subGroup), 132 | 133 | close() { 134 | conn.close() 135 | }, 136 | } 137 | 138 | return store 139 | } 140 | 141 | export default prozessStore -------------------------------------------------------------------------------- /demos/bp/server.ts: -------------------------------------------------------------------------------- 1 | import * as I from '../../lib/interfaces' 2 | import lmdbStore from '../../lib/stores/lmdb' 3 | import kvStore from '../../lib/stores/kvmem' 4 | import mapStore from '../../lib/stores/map' 5 | import router, {ALL} from '../../lib/stores/router' 6 | import createWss from '../../lib/net/wsserver' 7 | 8 | import render from './browserclient/render' 9 | // import createHttp from '../net/httpserver' 10 | import http from 'http' 11 | 12 | import fs from 'fs' 13 | import url from 'url' 14 | 15 | import express from 'express' 16 | import bodyParser from 'body-parser' 17 | import jsesc from 'jsesc' 18 | 19 | import mod from './rustpng' 20 | import opmem from '../../lib/stores/opmem'; 21 | import { vRangeTo } from '../../lib/version'; 22 | 23 | process.on('unhandledRejection', err => { throw err }) 24 | 25 | // const store = augment(lmdbStore( 26 | 27 | type BPWorld = { 28 | v: 2, 29 | offx: number, 30 | offy: number, 31 | img: string 32 | } 33 | 34 | const data = new Map( 35 | fs.readFileSync('bp.json', 'utf8') 36 | .split('\n') 37 | .filter(x => x !== '') 38 | .map(data => JSON.parse(data)) 39 | .map(([key, val]) => [key, val.data] as [string, BPWorld]) 40 | ) 41 | // console.log(data) 42 | 43 | ;(async () => { 44 | const rootStore = await kvStore(data, {inner: opmem({source: 'rootstore'})}) 45 | const imgStore = mapStore(rootStore, (v, k) => { 46 | console.log('v', v) 47 | return Buffer.from(mod.convert(v.img, 8, 1, 2) as any) 48 | }) 49 | 50 | const store = router() 51 | store.mount(rootStore, 'world/', ALL, '', true) 52 | store.mount(imgStore, 'img/', ALL, '', false) 53 | 54 | // const s2 = augment(kvStore()) 55 | // s2.storeInfo.sources[0] = 's2' 56 | // store.mount(s2, 's2/', ALL, '', true) 57 | 58 | // const s3 = augment(kvStore()) 59 | // s3.storeInfo.sources[0] = 's3' 60 | // // s3.storeInfo.sources.push(s2.storeInfo.sources[0]) 61 | // // s3.storeInfo.sources.push(imgStore.storeInfo.sources[0]) 62 | // store.mount(s3, 's3/', ALL, '', true) 63 | // console.log(store.storeInfo.sources) 64 | 65 | // store.subscribe({type: 'kv', q: new Set(['world/asdf', 'img/adf', 's3/asdf', 's2/sfd'])}, {}) 66 | 67 | 68 | const app = express() 69 | app.use(express.static(`${__dirname}/public`)) 70 | app.use(bodyParser.json()) 71 | 72 | 73 | // TODO: Do all this work in another map() function. 74 | app.get('*', async (req, res, next) => { 75 | // Try to answer request using a rendered statecraft store 76 | const key = url.parse(req.url).pathname!.slice(1) 77 | // console.log('key', key) 78 | const result = await store.fetch({type: I.QueryType.KV, q: new Set([key])}) 79 | // console.log('result', result) 80 | let value = result.results.get(key) 81 | if (value == null) return next() 82 | 83 | res.setHeader('content-type', 'text/html') 84 | res.setHeader('x-sc-version', JSON.stringify(result.versions)) 85 | // TODO: generate an etag based off the version, and parse it back to pass to fetch. 86 | 87 | const versionRange = result.versions 88 | const versions: I.FullVersion = vRangeTo(versionRange) 89 | 90 | const mimetype = Buffer.isBuffer(value) ? 'image/png' : 'application/json' 91 | // if (Buffer.isBuffer(value)) value = value.toString('base64') 92 | if (Buffer.isBuffer(value)) value = `data:${mimetype};base64,${value.toString('base64')}` 93 | 94 | // const v = Buffer.isBuffer(value) ? value.toString('base64') : value 95 | res.send(` 96 |
${render(mimetype, value).toString()}
97 | 106 | 107 | `) 108 | }) 109 | 110 | app.get('/:user/:key.json', async (req, res, next) => { 111 | const {user, key} = req.params 112 | const k = `world/${user}/${key}` 113 | const result = await store.fetch({type: I.QueryType.KV, q: new Set([k])}) 114 | const value = result.results.get(k) 115 | if (value == null) return next() 116 | res.setHeader('x-sc-version', JSON.stringify(result.versions)) 117 | res.setHeader('content-type', 'application/json') 118 | res.json({ 119 | v: 0, 120 | data: value, 121 | readonly: false, 122 | }) 123 | }) 124 | 125 | app.put('/:user/:key.json', async (req, res, next) => { 126 | const {user, key} = req.params 127 | const txn = new Map([[`world/${user}/${key}`, {type:'set', data: req.body.data}]]) 128 | const result = await store.mutate(I.ResultType.KV, txn) 129 | console.log(result) 130 | res.end() 131 | }) 132 | 133 | app.get('/:user/:key', (req, res) => { 134 | res.sendFile('public/editor.html', {root: __dirname}) 135 | }) 136 | 137 | const server = http.createServer(app) 138 | 139 | const wss = createWss({server}, store) 140 | 141 | server.listen(2000) 142 | console.log('http server listening on port 2000') 143 | })() -------------------------------------------------------------------------------- /core/lib/stores/map.ts: -------------------------------------------------------------------------------- 1 | import * as I from '../interfaces' 2 | import err from '../err' 3 | import {queryTypes} from '../qrtypes' 4 | import iterGuard from '../iterGuard' 5 | import {vRangeTo} from '../version' 6 | 7 | const supportedOpTypes = new Set(['rm', 'set']) 8 | 9 | export type MapFn = (v: In, k: I.Key | null, version: I.FullVersion) => Out 10 | 11 | const mapValues = (map: Map, fn: (v: In, k: K) => Out): Map => { 12 | const result = new Map() 13 | for (const [key, val] of map.entries()) { 14 | result.set(key, fn(val, key)) 15 | } 16 | return result 17 | } 18 | 19 | const mapSingleOp = (op: I.SingleOp, key: I.Key | null, version: I.FullVersion, fn: MapFn): I.SingleOp => { 20 | switch (op.type) { 21 | case 'rm': return op as I.SingleOp as I.SingleOp 22 | case 'set': return {type: 'set', data: fn(op.data, key, version)} 23 | default: throw Error('Map function cannot process operation with type ' + op.type) 24 | } 25 | } 26 | 27 | const mapTxnWithMetas = (type: I.ResultOps>, txn: I.TxnWithMeta[], fn: MapFn): I.TxnWithMeta[] => ( 28 | txn.map(({versions, txn, meta}) => ({ 29 | txn: type.mapTxn(txn, (op: I.Op, k) => ( 30 | Array.isArray(op) 31 | ? op.map(singleOp => mapSingleOp(singleOp, k, versions, fn)) 32 | : mapSingleOp(op, k, versions, fn) 33 | )), 34 | versions, 35 | meta, 36 | })) 37 | ) 38 | 39 | export interface MapOpts { 40 | fnUid?: string, // This is a UID that should change each time the function is edited. 41 | } 42 | 43 | // Syncronous map fn 44 | const map = (inner: I.Store, mapfn: MapFn, opts: MapOpts = {}): I.Store => { 45 | const sources = inner.storeInfo.sources 46 | return { 47 | storeInfo: { 48 | uid: `map(${inner.storeInfo.uid},${opts.fnUid || '_'})`, 49 | sources, 50 | sourceIsMonotonic: inner.storeInfo.sourceIsMonotonic, 51 | 52 | capabilities: { 53 | // TODO: Filter these capabilities by the ones we support locally. 54 | queryTypes: inner.storeInfo.capabilities.queryTypes, 55 | mutationTypes: 0, // You cannot mutate through the mapping. 56 | }, 57 | }, 58 | 59 | async fetch(query, opts) { 60 | const qtype = queryTypes[query.type] 61 | if (qtype == null) throw Error('unknown query type') 62 | 63 | const innerResults = await inner.fetch(query, opts) 64 | 65 | const version = vRangeTo(innerResults.versions) 66 | const outerResults: I.FetchResults = { 67 | // In the noDocs case, inner.fetch will have already stripped the documents. 68 | results: (opts && opts.noDocs) 69 | ? innerResults.results 70 | : qtype.resultType.map(innerResults.results, (v, k) => mapfn(v as In, k, version)), 71 | bakedQuery: innerResults.bakedQuery, 72 | versions: innerResults.versions, 73 | } 74 | 75 | return outerResults 76 | }, 77 | 78 | mutate(mtype, txn, versions, opts) { 79 | return Promise.reject(new err.UnsupportedTypeError('You cannot modify through a map fn')) 80 | }, 81 | 82 | async getOps(query, versions, opts): Promise> { 83 | const r = await inner.getOps(query, versions, { 84 | ...opts, 85 | supportedTypes: supportedOpTypes, 86 | }) 87 | 88 | return { 89 | ops: mapTxnWithMetas(queryTypes[query.type].resultType, r.ops, mapfn), 90 | versions: r.versions, 91 | } 92 | }, 93 | 94 | // TODO: catchup. 95 | 96 | subscribe(q, opts) { 97 | const qtype = queryTypes[q.type] 98 | 99 | const innerSub = inner.subscribe(q, { 100 | ...opts, 101 | supportedTypes: supportedOpTypes, 102 | trackValues: true, 103 | }) 104 | 105 | const version: I.FullVersion = [] 106 | 107 | return iterGuard((async function*() { 108 | for await (const innerUpdates of innerSub) { 109 | // Note that version contains the version *after* the whole catchup 110 | // is applied. I'm updating it here, but note that it is not used by 111 | // txns. 112 | for (let i = 0; i < sources.length; i++) { 113 | if (innerUpdates.toVersion[i] != null) version[i] = innerUpdates.toVersion[i] 114 | } 115 | 116 | // TODO: It'd be better to avoid calling the map function so often 117 | // here. We should really only call it at most once per object per 118 | // iteration. 119 | const result = { 120 | ...innerUpdates, 121 | txns: mapTxnWithMetas(qtype.resultType, innerUpdates.txns, mapfn), 122 | replace: innerUpdates.replace ? { 123 | q: innerUpdates.replace.q, 124 | with: qtype.resultType.mapReplace(innerUpdates.replace.with, (v: In, k) => mapfn(v, k, version)), 125 | versions: innerUpdates.replace.versions, 126 | } : undefined, 127 | } 128 | // console.log('map yield', result) 129 | 130 | yield result 131 | } 132 | })(), innerSub.return) as I.AsyncIterableIteratorWithRet> 133 | }, 134 | 135 | close() {}, 136 | } 137 | } 138 | 139 | export default map 140 | -------------------------------------------------------------------------------- /demos/bp/public/editor.html: -------------------------------------------------------------------------------- 1 | 2 | Steamdance 3 | 4 | 5 | 6 | 9 | 176 | 177 |
178 | 179 |
180 | ◀Browse 181 | CHANGES WILL NOT BE SAVED 182 | 183 |
184 | ► 185 | 186 | >| 187 |
188 |
189 | 190 |
191 |
192 |
+
193 |
194 |
▼ Modules
195 |
196 | 197 |
198 | github 199 |
200 | 201 |
202 |
Move 203 | 204 |
Empty 205 |
grill 206 |
Solid 207 | 208 |
bridge 209 | 210 |
pos 211 |
neg 212 | 213 |
shuttle 214 |
ts 215 |
S Glue 216 |
S Cut ✂ 217 |
218 |
219 | 220 | 221 | -------------------------------------------------------------------------------- /net/lib/reconnectingclient.ts: -------------------------------------------------------------------------------- 1 | import {I, err, setSingle, stores} from '@statecraft/core' 2 | import {TinyReader, TinyWriter} from './tinystream' 3 | import * as N from './netmessages' 4 | import createStore, {NetStore, ClientOpts} from './client' 5 | // import singleMem, {setSingle} from './singlemem' 6 | import resolvable from '@josephg/resolvable' 7 | 8 | const {singlemem} = stores 9 | const wait = (timeout: number) => new Promise(resolve => setTimeout(resolve, timeout)) 10 | 11 | // Connection states: 12 | // - connecting 13 | // - connected 14 | // - waiting 15 | // - stopped 16 | type Status = string 17 | 18 | /** 19 | * Create a new client reconnector. 20 | * 21 | * This is the main entrypoint for clients to connect to remote stores, and 22 | * automatically reconnect when they are disconnected. 23 | * 24 | * @param connect A function which creates the network streams. This function 25 | * will be called to connect initially, and then which each reconnect. Most 26 | * users should use `connectToSocket` or `connectToWS`. Eg `() => net.connectToWS(wsurl)` 27 | * 28 | * @returns a 3-tuple with 3 values: 29 | * 30 | * - A single value store with connection status information. This will have a 31 | * string with values `'waiting'`, `'connecting'`, `'connected'` and 32 | * `'stopped'`. 33 | * - A promise to the store itself. The promise will resolve once the store 34 | * connects for the first time. 35 | * - A promise to an exception which will throw if the connected store's UID 36 | * changes. If the server's architecture changes, its UID will change and 37 | * we can no longer re-establish our subscriptions because any version 38 | * numbers are no longer necessarily valid. On the web You should probably 39 | * reload the page when this happens. 40 | */ 41 | const reconnector = (connect: (() => Promise<[TinyReader, TinyWriter]>)): [I.Store, Promise>, Promise] => { 42 | // This is a tiny store that the client can use to track & display whether 43 | // or not we're currently connected. Ite tempting to make a metastore be a 44 | // default feature. 45 | const status = singlemem('waiting') 46 | let innerStore: NetStore | null = null 47 | let shouldReconnect = true 48 | const uidChanged = resolvable() 49 | 50 | // const initialStoreP = 51 | // initialStoreP.then(store => { innerStore = store; }) 52 | 53 | const ready: Promise> = new Promise((resolve, reject) => { 54 | const opts: ClientOpts = { 55 | preserveState: true, 56 | onClose() { 57 | // We don't clear innerStore yet - all requests will still go there 58 | // until we change the guard. 59 | setSingle(status, 'waiting') 60 | 61 | // This is pretty rough. 62 | ;(async () => { 63 | while (shouldReconnect) { 64 | console.log('... trying to reconnect ...') 65 | 66 | let r: TinyReader, w: TinyWriter 67 | let netConnected = false 68 | try { 69 | setSingle(status, 'connecting') 70 | ;[r, w] = await connect() 71 | netConnected = true 72 | } catch (e) { 73 | console.warn('Reconnection failed', e.message) 74 | } 75 | 76 | if (netConnected) try { 77 | console.log('createStore') 78 | await createStore(r!, w!, { 79 | ...opts, 80 | restoreFrom: innerStore!, 81 | syncReady(store) { 82 | // Ok, we've reconnected & gotten our hello message. 83 | 84 | // We have to initialize here to avoid an event loop frame 85 | // where innerStore is set incorrectly. 86 | innerStore = store 87 | setSingle(status, 'connected') 88 | console.warn('Reconnected') 89 | } 90 | }) 91 | break 92 | } catch (e) { 93 | // TODO: Consider calling reject instead of resolve here - so the 94 | // error makes an exception by default. 95 | if (e instanceof err.StoreChangedError) { 96 | console.log('uid changed') 97 | w!.close() 98 | uidChanged.reject(e) 99 | break 100 | } else throw e 101 | } 102 | 103 | setSingle(status, 'waiting') 104 | await wait(5000) 105 | console.log('done waiting') 106 | } 107 | })() 108 | }, 109 | } 110 | 111 | 112 | setSingle(status, 'connecting') 113 | connect() 114 | .then(([r, w]) => createStore(r, w, opts)) 115 | .then(initialStore => { 116 | innerStore = initialStore 117 | setSingle(status, 'connected') 118 | 119 | // This is basically a proxy straight to innerStore. 120 | const s: I.Store = { 121 | storeInfo: initialStore.storeInfo, 122 | fetch(...args) { return innerStore!.fetch(...args) }, 123 | getOps(...args) { return innerStore!.getOps(...args) }, 124 | mutate(...args) { return innerStore!.mutate(...args) }, 125 | subscribe(...args) { return innerStore!.subscribe(...args) }, 126 | close() { 127 | shouldReconnect = false 128 | innerStore!.close() 129 | innerStore = null // Error if we get subsequent requests 130 | setSingle(status, 'closed') 131 | // ... And stop reconnecting. 132 | } 133 | } 134 | resolve(s) 135 | }, reject) 136 | }) 137 | 138 | return [status, ready, uidChanged] 139 | } 140 | 141 | export default reconnector -------------------------------------------------------------------------------- /core/lib/stores/opmem.ts: -------------------------------------------------------------------------------- 1 | // This is a trivial implementation of an in-memory op store. This is designed 2 | // for use with kvmem when it has no other inner store to use as a source of 3 | // truth. You should never use this with lmdbstore (or any other persistent 4 | // store). 5 | 6 | import * as I from '../interfaces' 7 | import err from '../err' 8 | import {queryTypes} from '../qrtypes' 9 | import {V64, v64ToNum, vInc, vEq, vCmp, V_EMPTY, vRangeTo} from '../version' 10 | import genSource from '../gensource' 11 | import { bitSet } from '../bit' 12 | import makeSubGroup from '../subgroup' 13 | 14 | // TODO: Shouldn't need binsearch. 15 | import binsearch from 'binary-search' 16 | 17 | export interface Opts { 18 | readonly initialVersion?: I.Version, 19 | readonly source?: I.Source, 20 | readonly readOnly?: boolean, 21 | readonly maxOps?: number, // Max number of ops kept for each source. 0 = ignore 22 | } 23 | 24 | export interface Trigger { 25 | // Apply and notify that a change happened in the database. This is useful 26 | // so if the memory store wraps an object that is edited externally, you can 27 | // update listeners. 28 | internalDidChange(type: I.ResultType, txn: I.Txn, meta: I.Metadata, toVersion?: I.Version): I.Version 29 | } 30 | 31 | interface OpsEntry { 32 | fromV: I.Version, 33 | toV: I.Version, 34 | txn: I.Txn, 35 | meta: I.Metadata, 36 | // ctime: number, 37 | } 38 | 39 | const cmp = (item: OpsEntry, v: I.Version) => vCmp(item.toV, v) 40 | 41 | // TODO: Consider inlining opcache in here. 42 | const opmem = (opts: Opts = {}): I.Store & Trigger => { 43 | const source = opts.source || genSource() 44 | const initialVersion = opts.initialVersion || V64(0) 45 | const isReadonly = opts.readOnly 46 | 47 | const ops: OpsEntry[] = [] 48 | const maxOps = opts.maxOps || 0 49 | 50 | let version = initialVersion 51 | 52 | const getOpsSync = (query: I.Query, versions: I.FullVersionRange, options: I.GetOpsOptions = {}): I.GetOpsResult => { 53 | // TODO: This is awful - I've copy+pasta'ed this big function from opcache. 54 | // Figure out a decent way to do this - maybe opcache should be another store too? 55 | const qtype = query.type 56 | const qops = queryTypes[qtype] 57 | if (qops == null) throw Error('Missing qops for type ' + qtype) 58 | 59 | let limitOps = options.limitOps || -1 60 | 61 | const result: I.TxnWithMeta[] = [] 62 | 63 | const vs = versions[0] 64 | if (vs == null) return {ops: result, versions: []} 65 | 66 | const {from, to} = vs 67 | let fromidx: number 68 | if (from.length === 0) fromidx = 0 // From start 69 | else { 70 | // TODO: The version numbers here are sequential, so we should be able to remove binsearch. 71 | const searchidx = binsearch(ops, from, cmp) 72 | fromidx = searchidx < 0 ? ~searchidx : searchidx + 1 73 | } 74 | 75 | if (fromidx >= ops.length) return {ops: result, versions: [{from:version, to:version}]} 76 | 77 | // Figure out the actual returned version range. 78 | const vFrom = ops[fromidx].fromV 79 | let vTo = vFrom 80 | 81 | for (let i = fromidx; i < ops.length; i++) { 82 | const item = ops[i] 83 | if (to.length && vCmp(item.toV, to) > 0) break 84 | 85 | // The transaction will be null if the operation doesn't match 86 | // the supplied query. 87 | const txn = qops.adaptTxn(item.txn, query.q) 88 | if (txn != null) result.push({versions: [item.toV], txn, meta: item.meta}) 89 | 90 | vTo = item.toV 91 | if (limitOps > 0 && --limitOps === 0) break 92 | } 93 | 94 | // console.log('opcache', result, vOut) 95 | return {ops: result, versions: [{from: vFrom, to: vTo}]} 96 | } 97 | const getOps: I.GetOpsFn = (q, v, opts) => Promise.resolve(getOpsSync(q, v, opts)) 98 | 99 | const subgroup = makeSubGroup({getOps, initialVersion: [version]}) 100 | 101 | const store: I.Store & Trigger = { 102 | storeInfo: { 103 | uid: `opmem(${source})`, 104 | sources: [source], 105 | sourceIsMonotonic: [true], 106 | capabilities: { 107 | queryTypes: 0, 108 | // Is this right? Maybe we should just set all the bits. 109 | mutationTypes: opts.readOnly ? 0 : bitSet(I.ResultType.Single, I.ResultType.KV) 110 | } 111 | }, 112 | 113 | fetch() { 114 | throw new err.UnsupportedTypeError() 115 | }, 116 | 117 | getOps, 118 | 119 | subscribe(q, opts = {}) { 120 | return subgroup.create(q, opts) 121 | }, 122 | 123 | internalDidChange(type, txn, meta, toV) { 124 | const fromV = version 125 | if (!toV) toV = vInc(version) 126 | else { 127 | if (vCmp(version, toV) >= 0) throw Error('opmem requires monotonically increasing changes') 128 | } 129 | 130 | ops.push({ fromV: version, toV, txn, meta }) 131 | while (maxOps !== 0 && ops.length > maxOps) ops.shift() 132 | 133 | version = toV 134 | // console.log('internalDidChange', toV) 135 | subgroup.onOp(0, fromV, [{txn, meta, versions: [toV]}]) 136 | return toV 137 | }, 138 | 139 | async mutate(type, txn, versions = [], opts = {}) { 140 | if (isReadonly) throw new err.UnsupportedTypeError('Cannot write to readonly store') 141 | const reqV = versions[0] 142 | 143 | // Opmem is blind wrt operation contents. 144 | if (reqV != null && !vEq(reqV, version)) throw new err.VersionTooOldError() 145 | 146 | return [this.internalDidChange(type, txn, opts.meta || {})] 147 | }, 148 | 149 | close() {}, 150 | 151 | } 152 | 153 | return store 154 | } 155 | 156 | export default opmem -------------------------------------------------------------------------------- /net/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | /* Basic Options */ 4 | "target": "ES2017", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019' or 'ESNEXT'. */ 5 | "module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ 6 | // "lib": [], /* Specify library files to be included in the compilation. */ 7 | // "allowJs": true, /* Allow javascript files to be compiled. */ 8 | // "checkJs": true, /* Report errors in .js files. */ 9 | // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */ 10 | "declaration": true, /* Generates corresponding '.d.ts' file. */ 11 | // "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */ 12 | // "sourceMap": true, /* Generates corresponding '.map' file. */ 13 | // "outFile": "./", /* Concatenate and emit output to single file. */ 14 | "outDir": "./dist", /* Redirect output structure to the directory. */ 15 | // "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ 16 | // "composite": true, /* Enable project compilation */ 17 | // "incremental": true, /* Enable incremental compilation */ 18 | // "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */ 19 | // "removeComments": true, /* Do not emit comments to output. */ 20 | // "noEmit": true, /* Do not emit outputs. */ 21 | // "importHelpers": true, /* Import emit helpers from 'tslib'. */ 22 | // "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */ 23 | // "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */ 24 | 25 | /* Strict Type-Checking Options */ 26 | "strict": true, /* Enable all strict type-checking options. */ 27 | // "noImplicitAny": true, /* Raise error on expressions and declarations with an implied 'any' type. */ 28 | // "strictNullChecks": true, /* Enable strict null checks. */ 29 | // "strictFunctionTypes": true, /* Enable strict checking of function types. */ 30 | // "strictBindCallApply": true, /* Enable strict 'bind', 'call', and 'apply' methods on functions. */ 31 | // "strictPropertyInitialization": true, /* Enable strict checking of property initialization in classes. */ 32 | // "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */ 33 | // "alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */ 34 | 35 | /* Additional Checks */ 36 | // "noUnusedLocals": true, /* Report errors on unused locals. */ 37 | // "noUnusedParameters": true, /* Report errors on unused parameters. */ 38 | // "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */ 39 | // "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */ 40 | 41 | /* Module Resolution Options */ 42 | // "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */ 43 | // "baseUrl": "./", /* Base directory to resolve non-absolute module names. */ 44 | // "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */ 45 | // "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */ 46 | // "typeRoots": [], /* List of folders to include type definitions from. */ 47 | // "types": [], /* Type declaration files to be included in compilation. */ 48 | // "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */ 49 | "esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */ 50 | // "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */ 51 | 52 | /* Source Map Options */ 53 | // "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */ 54 | // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ 55 | // "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */ 56 | // "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */ 57 | 58 | /* Experimental Options */ 59 | // "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */ 60 | // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /demos/text/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | /* Basic Options */ 4 | "target": "ES2017", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019' or 'ESNEXT'. */ 5 | "module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ 6 | // "lib": [], /* Specify library files to be included in the compilation. */ 7 | // "allowJs": true, /* Allow javascript files to be compiled. */ 8 | // "checkJs": true, /* Report errors in .js files. */ 9 | // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */ 10 | "declaration": true, /* Generates corresponding '.d.ts' file. */ 11 | // "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */ 12 | // "sourceMap": true, /* Generates corresponding '.map' file. */ 13 | // "outFile": "./", /* Concatenate and emit output to single file. */ 14 | // "outDir": "./dist", /* Redirect output structure to the directory. */ 15 | // "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ 16 | // "composite": true, /* Enable project compilation */ 17 | // "incremental": true, /* Enable incremental compilation */ 18 | // "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */ 19 | // "removeComments": true, /* Do not emit comments to output. */ 20 | // "noEmit": true, /* Do not emit outputs. */ 21 | // "importHelpers": true, /* Import emit helpers from 'tslib'. */ 22 | // "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */ 23 | // "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */ 24 | 25 | /* Strict Type-Checking Options */ 26 | "strict": true, /* Enable all strict type-checking options. */ 27 | // "noImplicitAny": true, /* Raise error on expressions and declarations with an implied 'any' type. */ 28 | // "strictNullChecks": true, /* Enable strict null checks. */ 29 | // "strictFunctionTypes": true, /* Enable strict checking of function types. */ 30 | // "strictBindCallApply": true, /* Enable strict 'bind', 'call', and 'apply' methods on functions. */ 31 | // "strictPropertyInitialization": true, /* Enable strict checking of property initialization in classes. */ 32 | // "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */ 33 | // "alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */ 34 | 35 | /* Additional Checks */ 36 | // "noUnusedLocals": true, /* Report errors on unused locals. */ 37 | // "noUnusedParameters": true, /* Report errors on unused parameters. */ 38 | // "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */ 39 | // "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */ 40 | 41 | /* Module Resolution Options */ 42 | // "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */ 43 | // "baseUrl": "./", /* Base directory to resolve non-absolute module names. */ 44 | // "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */ 45 | // "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */ 46 | // "typeRoots": [], /* List of folders to include type definitions from. */ 47 | // "types": [], /* Type declaration files to be included in compilation. */ 48 | // "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */ 49 | "esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */ 50 | // "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */ 51 | 52 | /* Source Map Options */ 53 | // "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */ 54 | // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ 55 | // "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */ 56 | // "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */ 57 | 58 | /* Experimental Options */ 59 | // "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */ 60 | // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /lmdb/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | /* Basic Options */ 4 | "target": "ES2017", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019' or 'ESNEXT'. */ 5 | "module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ 6 | // "lib": [], /* Specify library files to be included in the compilation. */ 7 | // "allowJs": true, /* Allow javascript files to be compiled. */ 8 | // "checkJs": true, /* Report errors in .js files. */ 9 | // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */ 10 | "declaration": true, /* Generates corresponding '.d.ts' file. */ 11 | // "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */ 12 | // "sourceMap": true, /* Generates corresponding '.map' file. */ 13 | // "outFile": "./", /* Concatenate and emit output to single file. */ 14 | "outDir": "./dist", /* Redirect output structure to the directory. */ 15 | // "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ 16 | // "composite": true, /* Enable project compilation */ 17 | // "incremental": true, /* Enable incremental compilation */ 18 | // "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */ 19 | // "removeComments": true, /* Do not emit comments to output. */ 20 | // "noEmit": true, /* Do not emit outputs. */ 21 | // "importHelpers": true, /* Import emit helpers from 'tslib'. */ 22 | // "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */ 23 | // "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */ 24 | 25 | /* Strict Type-Checking Options */ 26 | "strict": true, /* Enable all strict type-checking options. */ 27 | // "noImplicitAny": true, /* Raise error on expressions and declarations with an implied 'any' type. */ 28 | // "strictNullChecks": true, /* Enable strict null checks. */ 29 | // "strictFunctionTypes": true, /* Enable strict checking of function types. */ 30 | // "strictBindCallApply": true, /* Enable strict 'bind', 'call', and 'apply' methods on functions. */ 31 | // "strictPropertyInitialization": true, /* Enable strict checking of property initialization in classes. */ 32 | // "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */ 33 | // "alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */ 34 | 35 | /* Additional Checks */ 36 | // "noUnusedLocals": true, /* Report errors on unused locals. */ 37 | // "noUnusedParameters": true, /* Report errors on unused parameters. */ 38 | // "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */ 39 | // "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */ 40 | 41 | /* Module Resolution Options */ 42 | // "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */ 43 | "baseUrl": "./", /* Base directory to resolve non-absolute module names. */ 44 | "paths": { /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */ 45 | "node-lmdb": ["./deps"] 46 | }, 47 | // "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */ 48 | // "typeRoots": [], /* List of folders to include type definitions from. */ 49 | // "types": [], /* Type declaration files to be included in compilation. */ 50 | // "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */ 51 | "esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */ 52 | // "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */ 53 | 54 | /* Source Map Options */ 55 | // "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */ 56 | // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ 57 | // "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */ 58 | // "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */ 59 | 60 | /* Experimental Options */ 61 | // "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */ 62 | // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ 63 | } 64 | } 65 | --------------------------------------------------------------------------------