├── .gitignore ├── docs ├── .nojekyll ├── README.md ├── _sidebar.md ├── best-practices │ └── README.md ├── case-studies │ └── README.md ├── guides │ ├── README.md │ ├── graphql-api │ │ └── README.md │ ├── multi-device-sync │ │ └── README.md │ ├── quickstart │ │ └── README.md │ ├── schema-migrations │ │ └── README.md │ ├── storage-middleware │ │ └── README.md │ ├── storage-modules │ │ └── README.md │ ├── storage-operations │ │ └── README.md │ └── storage-registry │ │ └── README.md ├── index.html ├── resources │ └── README.md ├── storex-hub │ ├── README.md │ ├── api-reference │ │ └── README.md │ ├── contact │ │ └── README.md │ ├── getting-started │ │ ├── README.md │ │ └── README.old.md │ ├── guides │ │ ├── README.md │ │ ├── memex │ │ │ └── README.md │ │ ├── plugin-dev-guide │ │ │ └── README.md │ │ ├── remote-apps │ │ │ └── README.md │ │ ├── settings │ │ │ └── README.md │ │ └── storing-data │ │ │ └── README.md │ └── roadmap │ │ └── README.md └── use-cases │ └── README.md ├── notes └── journeys.md ├── package.json ├── tools └── maintainable-docs │ ├── api-reference-generation │ └── index.ts │ ├── check-coverage.ts │ ├── check-links.ts │ ├── main.ts │ └── utils │ └── typedoc.ts ├── tsconfig.json └── yarn.lock /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | private 3 | -------------------------------------------------------------------------------- /docs/.nojekyll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WorldBrain/storex-docs/824bda78bd3c40eae1f31036735f7164b1d5d652/docs/.nojekyll -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # What is Storex 2 | 3 | Storex is a minimal storage layer as a foundation for easing common problems around storing and moving data around. Allowing you to describe your data layout in a standardized way and providing different plugins, it helps you interact with (No)SQL databases, data migration, offline first applications architecture, creating and consuming REST/GraphQL APIs, permission management, finding optimization opportunaties and more. 4 | 5 | The aim is to provide a minimalistic common ground/language for working with your data, providing packages for solving the most common problems around data, while giving you easy access to the underlying machinery to do the things that are specific to your application. Everything together that means that every problem you encounter while rapidly iterating towards a serious product, from choosing a suitable DB to suddenly realizing you need to migrate your data model, or even switch DBs, will get a ton easier because you don't have to solve them for the 109th time yourself. 6 | 7 | _This project started as the storage layer for Memex, a tool to organize your web-research for yourself and collaboratively, in collaboration with [YouAPT](https://www.youapt.eu/). Download it [here](https://worldbrain.io/), and check out our vision [here](https://worldbrain.io/vision)._ 8 | 9 | # Why does Storex exist? 10 | 11 | Over the years, many storage technologies and architectures have been researched and developed. Each of these technologies consists different trade-offs, specializations and ideas. As an industry, we've seen that the way these technologies invent re-combine different building blocks has lead to many ongoing heated debates, rewrites of products and a continuous reinventing of the same building blocks that are needed in real-world applications hindering user-driven innovation. 12 | 13 | Storex aims to identify the building blocks that have been re-combined in various ways in different storage technologies and architectures, and provide the industry with modules providing a common way to speak about these issues (live schema migration, access control, offline-first, live-debugging micro-service architectures, etc.) in ways that they can be re-combined according to different use-cases. 14 | 15 | At the simplest level, this allows for applications to adapt to new storage architectures with drastically reduced effort, reducing the amount of re-writes (read: wasted energy and money that could have been used to produce value for society) and allowing for more research technologies which as a result have a greater chance to be adopted. Also, it allows for technologies to be chosen based on requirements emerging from high-fidelity prototyping and measurements, instead of up-front research when requirements are still vague. This enables a whole new level of user-driven development, and potentially changes the choice of technology from an (often not entirely rational) debate to data-driven application architecting. With this, and by creating a common tool-chain for traditionally conflicted communities (centralized vs. decentralized, SQL vs. NoSQL, etc.) we hope to increase the amount of unity in the tech industry, while reducing wasted resources. 16 | 17 | # How does Storex work? 18 | 19 | ```mermaid 20 | graph LR; 21 | storage_modules[Storage modules . .]; 22 | storage_manager[Storage manager . .]; 23 | storage_registry[Storage registry . .]; 24 | storage_middleware[Storage middleware . .]; 25 | storage_backend[Storage backend . .]; 26 | 27 | style storage_modules stroke-dasharray:5,5; 28 | 29 | storage_modules-.->storage_manager; 30 | storage_manager---storage_registry; 31 | storage_manager-->storage_middleware; 32 | storage_middleware-->storage_backend; 33 | storage_registry---storage_backend; 34 | ``` 35 | 36 | Storex at its core does two things: 37 | 38 | 1. Allow you to describe your data model in a standard way using the `StorageRegistry` 39 | 1. Allow you to execute standardized and custom operations on your data using a `StorageBackend` (currently backends are available for IndexedDB, SQL databases, Firebaase and experimentally MongoDB.) 40 | 41 | The `StorageManager` class combines both of these classes in a single class that the rest of your application can interact with, possibly sending each operation you request to execute on your data through various `StorageMiddleware`. 42 | 43 | ```js 44 | import StorageManager, { StorageBackend, StorageRegistry } from "@worldbrain/storex"; 45 | import { DexieStorageBackend } from "@worldbrain/storex-backend-dexie"; 46 | import inMemory from "@worldbrain/storex-backend-dexie/lib/in-memory"; 47 | 48 | export async function demo() { 49 | const backend = new DexieStorageBackend({ dbName: 'demo', idbImplementation: inMemory() }) 50 | const storageManager = new StorageManager({ backend: clientStorageBackend }) 51 | storageManager.registry.registerCollections({ 52 | user: { ... } // More details in the storage registry guide 53 | }) 54 | await storageManager.finishInitialization() 55 | 56 | storageManager.setMiddleware([ 57 | { 58 | async process({ operation, next }) { 59 | console.log('executing operation', operation) 60 | return next.process({ operation }) 61 | } 62 | } 63 | ]) 64 | 65 | const { object } = await storageManager.operation('createObject', 'user', { displayName: 'Jane' }) 66 | await storageManager.operation('updateObject', 'user', { id: object.id }, { displayName: 'Jane Doe' }) 67 | } 68 | ``` 69 | 70 | On top of this, we can optionally organize different parts of our storage logic into storage module, which do nothing more than providing a convenient way to expose information about a logical part of your storage logic in a coherent way: 71 | 72 | ```js 73 | import { 74 | StorageModule, 75 | StorageModuleConfig, 76 | registerModuleMapCollections 77 | } from "@worldbrain/storex-pattern-modules"; 78 | 79 | class TodoListStorage extends StorageModule { 80 | getConfig(): StorageModuleConfig { 81 | // More details in the storage modules guide 82 | return { 83 | // What you'd pass to StorageRegistry.registerCollections(), but with a bit more info 84 | collections: { ... }, 85 | 86 | // Templates of operations passed to StorageManager.operation() 87 | operations: { 88 | createList: { ... }, 89 | }, 90 | 91 | // Info about exposed methods, which you can use to generate REST/GraphQL endpoints for example 92 | methods: { 93 | createTodoList: { ... }, 94 | }, 95 | 96 | // Access and valdation rules determining who can execute what operations on what data 97 | accessRules: { ... } 98 | }; 99 | } 100 | 101 | async createTodoList(list: { name: string }) { 102 | return (await this.operation('createList', list)).object 103 | } 104 | } 105 | 106 | export async function demo() { 107 | const backend = new DexieStorageBackend({ dbName: 'demo', idbImplementation: inMemory() }) 108 | const storageManager = new StorageManager({ backend: clientStorageBackend }) 109 | const todoLists = new TodoListStorage({ storageManager }) 110 | registerModuleMapCollections(storageManager.registry, { todoLists }) 111 | await storageManager.finishInitialization() 112 | } 113 | ``` 114 | 115 | Using this higher-level organization, you can: 116 | 117 | - Automatically register the collections for all the different parts of your program with the storage registry in one standard place 118 | - Automatically generate API providers/consumers, so whether you run all this in the browser while testing, or on a server, becomes an implementation detail you can code out in one isolated place instead of having implementation details like REST vs GraphQL spread throughout your application. 119 | - Generate useful reports about what kind of operations you're doing on your data to look for optimization opportunities. 120 | - Use the access rules the encforce access control in different ways, whether it's using a storage middleware that enforces these rules, compiling them to [Google Cloud Firestore security rules](https://firebase.google.com/docs/firestore/security/get-started) or one could even imagine compiling them into an Ethereum smart contract (although this'd a bit more work in exposing more information about storage module methods.) 121 | - Get rid of Storex entirely if you wish so, because the rest of your application the UI and business logic only talk to the methods you've exposed on your storage modules, which have nothing to with Storex per ṡé. 122 | 123 | All of this together, you can create rapidly iterate on highly flexible applications that run across a variety of architectures. In fact, when buidling a new application it's totally feasible to start with a completely in-memory store until you get the requirements and UI right, after which you use the information you gathered and the kind of operations you execute to either move the entire storage to PostgreSQL, Firestore or DynamoDB, which'd normally be radical rewrites of the storage layer. The `storex-sync` package uses this to provide a multi-device sync that can use Firestore as it's backend, a custom cloud solution, or decentralized solutions. 124 | 125 | # What next? 126 | 127 | If you're a developer and here for the first time, start exploring the [quickstart](/guides/quickstart/) and the rest of the [guides](/guides/). 128 | 129 | 130 | 131 | If you're a decision maker trying to figure out how Storex might make sense for your organization, check out the [case studies](/case-studies/) and [use cases](/use-cases/). 132 | -------------------------------------------------------------------------------- /docs/_sidebar.md: -------------------------------------------------------------------------------- 1 | - [Home](/) 2 | 3 | - Storex Hub 4 | 5 | - [Overview](/storex-hub/) 6 | - [Getting started](/storex-hub/getting-started/) 7 | - [Plugin Development Guide](/storex-hub/guides/plugin-dev-guide/) 8 | - [Integrating with Memex](/storex-hub/guides/memex/) 9 | - [API reference](/storex-hub/api-reference/) 10 | - [Roadmap](/storex-hub/roadmap/) 11 | - [Get in touch](/storex-hub/contact/) 12 | 13 | 14 | - Storex 15 | 16 | - [Quickstart](/guides/quickstart/) 17 | - [Storage registry](/guides/storage-registry/) 18 | - [Storage operations](/guides/storage-operations/) 19 | - [Storage middleware](/guides/storage-middleware/) 20 | - [Storage modules](/guides/storage-modules/) 21 | - [Schema migrations](/guides/schema-migrations/) 22 | - [GraphQL API](/guides/graphql-api/) 23 | - [Multi-device sync](/guides/multi-device-sync/) 24 | 25 | 26 | 27 | 34 | -------------------------------------------------------------------------------- /docs/best-practices/README.md: -------------------------------------------------------------------------------- 1 | # Best practices 2 | 3 | TBD 4 | -------------------------------------------------------------------------------- /docs/case-studies/README.md: -------------------------------------------------------------------------------- 1 | # Case studies 2 | 3 | TBD 4 | -------------------------------------------------------------------------------- /docs/guides/README.md: -------------------------------------------------------------------------------- 1 | # Guides 2 | 3 | TBD 4 | -------------------------------------------------------------------------------- /docs/guides/graphql-api/README.md: -------------------------------------------------------------------------------- 1 | # GraphQL API 2 | 3 | Using Storex, you can create a clean interface between your storage logic and the rest of your application, such as your business logic and UI code. As a consequence of this, you can start developing your application entirely client-side and move your entire storage logic server-side when you need it. This was a clear design goal from the beginning for Storex, as in the early stages of a product you should be concentrating on prototyping functionality, not on choosing and managing implementation details like what kind of database your application uses or the layout of your REST/GraphQL API before you even know what your product is about! 4 | 5 | ## Set up 6 | 7 | Remember [storage module method descriptions](/guides/storage-modules/?id=storage-module-methods)? If you don't, do that now. Using those descriptions, Storex has a package that can automatically generate a GraphQL API for you, and a client that mimicks the orginal interface of your storage modules, but uses GraphQL under the hood to communicate with the back-end under the hood. This way, a lot of your application code can stay under the hood. Let's say we're using the `UserAdminModule` described in the guide above. Setting up the GraphQL API using [Apollo Server](https://www.apollographql.com/docs/apollo-server/) might look like this: 8 | 9 | ```js 10 | import * as graphql from "graphql"; 11 | import { ApolloServer } from "apollo-server"; 12 | import { createStorexGraphQLSchema } from "@worldbrain/storex-graphql-schema/lib/modules"; 13 | 14 | import StorageManager from "@worldbrain/storex"; 15 | import { DexieStorageBackend } from "@worldbrain/storex-backend-dexie"; 16 | import inMemory from "@worldbrain/storex-backend-dexie/lib/in-memory"; 17 | import { 18 | StorageModule, 19 | registerModuleMapCollections 20 | } from "@worldbrain/storex-pattern-modules/lib"; 21 | 22 | export function createServer(storageModules: {}): { 23 | start: () => Promise 24 | } { 25 | return { 26 | start: async () => { 27 | const schema = createStorexGraphQLSchema(application.storage.modules, { 28 | storageManager: application.storage.manager, 29 | autoPkType: "int", 30 | graphql 31 | }); 32 | const server = new ApolloServer({ schema }); 33 | const { url } = await server.listen(); 34 | console.log("Server is running on ", url); 35 | } 36 | }; 37 | } 38 | 39 | export async function main() { 40 | const backend = new DexieStorageBackend({ 41 | dbName: "my-app", 42 | idbImplementation: inMemory() 43 | }); 44 | const manager = new StorageManager({ backend }); 45 | const storage = { 46 | manager, 47 | modules: { 48 | userAdmin: new UserAdminModule({ 49 | storageManager: manager, 50 | autoPkType: "int" 51 | }) 52 | } 53 | }; 54 | registerModuleMapCollections(storage.manager.registry, storage.modules); 55 | await storage.manager.finishInitialization(); 56 | 57 | await storage.manager.backend.migrate(); 58 | const server = createServer(application); 59 | await server.start(); 60 | } 61 | ``` 62 | 63 | Now, you can access and use the GraphQL API as you normally would, including through the GraphQL playground. But, most GraphQL tutorials recommend you to directly embed the code interacting with GraphQL directly in your UI (React) code, meaning that your UI code would be tangled up with the code responsible for talking with your back-end, which it shouldn't. Instead, use the GraphQL client provided by Storex which lets you code as if Storex and GraphQL didn't even exist: 64 | 65 | ```js 66 | export async function setup() { 67 | // We're not actually storing any data on the client-side 68 | const manager = new StorageManager({ backend: null }); 69 | const storage = { 70 | manager, 71 | modules: { 72 | userAdmin: new UserAdminModule({ 73 | storageManager: manager, 74 | autoPkType: "int" 75 | }) 76 | } 77 | }; 78 | registerModuleMapCollections(storage.manager.registry, storage.modules); 79 | await storage.manager.finishInitialization(); 80 | 81 | const graphQLClient = new StorexGraphQLClient({ 82 | endpoint: options.graphQLEndpoint, 83 | modules: { 84 | userAdmin 85 | }, 86 | storageRegistry 87 | }); 88 | return { 89 | sharedSyncLog: graphQLClient.getModule("userAdmin") as UserAdminModule 90 | }; 91 | } 92 | 93 | export async function main() { 94 | // This would be your main application code, which'd normally some layers on 95 | // top of this before being used in your UI code. 96 | // Notice nothing inherently Storex or GraphQL here. 97 | const modules = await setup(); 98 | const user = await modules.userAdmin.byName({ name: "Joe" }); 99 | await modules.userAdmin.setAgeByName({ name: "Joe", age: 30 }); 100 | } 101 | ``` 102 | 103 | ## State of this functionality 104 | 105 | By design, the initial GraphQL functionality doesn't provide the typical GraphQL API where clients can query and manipulate the data in any way they want. This is because you normally want to be able to predict the queries you need to optimize, and you don't want your clients to be tightly coupled to the internals of your database structure. As a matter of fact, some more work needs to be done on the GraphQL API to allow for even further decoupling between how you want your data to be accessed and manipulated, and your internal database structure. However, it would be fairly trivial to implement the Storex equivalent of [Prisma](https://www.prisma.io/), allow your clients to flexibly query/manipulate the database in terms of Storex operations. 106 | 107 | Since there were some weird Node.js dependency problems with Apollo, it hasn't been possible to get integration tests with real-world functionality based on the GraphQL functionality to run, like the [multi-device sync](/guides/multi-device-sync/) integration tests which run on different architectures. As such, weird bugs might be present. 108 | 109 | Also, no authentication is implemented yet. This could be provided by creating a [storage middleware](/guides/storage-middleware/) enforcing Storex [access rules](/guides/storage-modules/?id=access-rules), or by creating higher-level access rules that work on a method, not operation level. 110 | 111 | ## What's next? 112 | 113 | Congratulations, you've gone through all the basics that make Storex what it is! If you're creating client-side applications like productivity and knowledge management applications that don't require a full-fledged back-end, but need to work across multiple devices by the same user, check out the [multi-device sync guide](/guides/multi-device-sync/). If not, you're all set with the basics you need to start producing some cool stuff :) 114 | -------------------------------------------------------------------------------- /docs/guides/multi-device-sync/README.md: -------------------------------------------------------------------------------- 1 | # Multi device sync 2 | 3 | Sometimes you want to keep data client-side, but still want to synchronize data between multiple user device to provide a seamless user experience. The [storex-sync package](https://github.com/WorldBrain/storex-sync/) provides this functionality in a unique, modular and surprisingly easy to understand way. That being said, synchronizing data is a really hard problem, and the default synchronization algorithm might not be for you. As such, you're encouraged to dive and fully understand the source code of this package before integrating it into your own application. For this, you'll need to understand the contents in the [quickstart](/guides/quickstart/), [storage registry](/guides/storage-registry/), [storage operations](/guides/storage-operations/), [storage middleware](/guides/storage-middleware/) and [storage modules](/guides/storage-modules/) guides. 4 | 5 | **IMPORTANT:** This functionality is still being developed and the most recent versions are not released yet. Therefore, we'll only explain how it works here and update the documentation as the codebase stabilizes enough (battle-testing it in [Memex](https://worldbrain.io/)) to be properly released. 6 | 7 | ## What's included? 8 | 9 | The Sync functionality was designed with the following requirements in mind: 10 | 11 | - It should be database-agnostic. We shouldn't have to switch databases just to have sync functionality. At WorldBrain, we ran our entire application on IndexedDB in a browser extension when we decided to also create a mobile app using SQLite. This should just work, without any major rewrites. 12 | - It should work for exisiting datasets. Since sync is a feature one might add later on in the product (which was the case for Memex), we should have a way to introduce this feature without restructuring the entire data model. 13 | - In the future, **not implemented yet**, it should be generalizable enough to also work for offline-first cloud application. That is, all changes are made locally first, then sent to the cloud if/when there's an internet connection. Wouldn't it be great if all your productivity apps just worked quickly and reliably, whether you're online, offline or have a spotty internet connection? 14 | 15 | ## How it works 16 | 17 | At the highest level, there are two ways of synchronizing databases using `storex-sync`. The first is by logging every change made to the database, sending them to a space shared between multiple devices and processing them on other devices using a reconciliation algorithm, which we call the continuous sync. The second one is to pump all data from one device to another, which we call the initial sync. At WorldBrain, we use the initial sync for users that have an existing data set, after which we switch to the continuous sync. 18 | 19 | ### Initial sync 20 | 21 | The initial sync is simple and brute-force. You first open a channel between two devices, after which the entire database contents are sent from once device to the other, effectively merging the database contents of the first device into the other one (which optionally can be done both ways.) It's set up in such a way, that you can implement the data channel in any way you want. At WorldBrain, we use WebRTC for direct communication between the two devices, with Firebase to negotiate how this connection will be set up. But if you want, you could also let the devices communicate over a WebSocket connection through a proxy. 22 | 23 | ### Continuous sync 24 | 25 | When you set up Storex, you: 26 | 27 | - create client sync log, where all the changes to the client database will be written. 28 | - install a [storage middleware](/guides/storage-middleware/) that intercept your operations and both executes and logs them in one automical operation 29 | 30 | There's one problem here though. If we just send object around which have sequentially incrementing primary keys, data will end up being overwritten. To prevent this, you can either make sure your data doesn't have these kinds of primary keys (choosing other fields for your primary keys), or generate globally unique UUIDs for objects with automatically generated primary keys. This can be done using the `CustomAutoPkMiddleware` and for example the [uuid/v4](https://www.npmjs.com/package/uuid) package. 31 | 32 | After this, you establish a connection with a `SharedSyncLog`, a place where all devices send the changes they made locally and retrieve changes made on other devices. A device sync consists of sending and receiving changes to and from this log and integrating the newly received changes on the device. When and how often you do this, is up to the needs of your application. You could sync on startup and every 10 minutes, or whenever a change is made locally and/or in the `SharedSyncLog` to create a live sync. 33 | 34 | ## How you can use it 35 | 36 | Since this functionality is being finalized while deployed in a real product, documentation will be writtten once the API is fully stabilized. That being said, the code is written to be easy to understand, so you shouldn't have any trouble starting to read through the integration tests which show in detail how you'd use sync. These are the [tests for the initial sync](https://github.com/WorldBrain/storex-sync/blob/develop/ts/fast-sync/index.test.ts), and these are the [tests for the continuous sync](https://github.com/WorldBrain/storex-sync/blob/develop/ts/index.test.ts). Also, there are [convenience classes](https://github.com/WorldBrain/storex-sync/tree/develop/ts/integration) to integrate these in your application the initial sync and the continuous sync. 37 | -------------------------------------------------------------------------------- /docs/guides/quickstart/README.md: -------------------------------------------------------------------------------- 1 | # Quickstart 2 | 3 | We're going to start getting familiar with Storex showing a single-user client-side application, like a note taking app, or a knowledge management tool, that you'll be able to distribute as a static website (through S3 for example), a Web Extension or a React Native app. We'll start by using an in-memory database, so we can experiment freely without having to create, modify or delete test data repeatedly while iterating on our code. After that, we'll start using IndexedDB for the browser, and SQLite on React Native, so your data can be stored both in a web browser, and in a mobile app. 4 | 5 | From there, you can we'll show you a number of directions you can take depending on what you're building, like running your storage code server-side in an SQL/MongoDB database, using Google Cloud Firestore as a back-end, or using sync if you want a single-user application that can sync between multiple devices. 6 | 7 | ## Installation 8 | 9 | Start by creating a project in your favorite framework, such as React, Angular or Vue.js. (Storex only provides storage logic, thus being totally UI-framework agnostic.) After that, install the Storex package we're going to use in this guide. 10 | 11 | ```bash 12 | npm install storex storex-backend-dexie storex-pattern-modules 13 | ``` 14 | 15 | ## Set up 16 | 17 | Now somewhere in our application, we'll write some code to set up Storex and describe the data we're going to work with. 18 | 19 | ```js 20 | // storage.ts 21 | 22 | import StorageManager, { 23 | StorageBackend, 24 | StorageRegistry 25 | } from "@worldbrain/storex"; 26 | import { DexieStorageBackend } from "@worldbrain/storex-backend-dexie"; 27 | import inMemory from "@worldbrain/storex-backend-dexie/lib/in-memory"; 28 | 29 | export async function createStorage(options: { 30 | backend: "in-memory" | "indexeddb" 31 | }) { 32 | // The StorageBacken is responsible for interacting with the database 33 | // In this case, it's IndexedDB using the Dexie.libray, but 34 | // this could also be Firestore or on a server PostgreSQL or MongoDB 35 | const backend = new DexieStorageBackend({ 36 | dbName: "todo-app", 37 | idbImplementation: options.backend === "in-memory" ? inMemory() : undefined 38 | }); 39 | 40 | const storageManager = new StorageManager({ backend }); 41 | storageManager.registry.registerCollections({ 42 | user: { 43 | version: new Date("2019-10-10"), 44 | fields: { 45 | displayName: { type: "string" }, 46 | age: { type: "int", optional: true } 47 | } 48 | }, 49 | todoList: { 50 | version: new Date("2019-10-10"), 51 | fields: { 52 | title: { type: "string" } 53 | }, 54 | relationships: [ 55 | { childOf: "user" } // creates one-to-many relationship 56 | ] 57 | }, 58 | todoListEntry: { 59 | version: new Date("2019-10-10"), 60 | fields: { 61 | label: { type: "text" }, 62 | done: { type: "boolean" } 63 | }, 64 | relationships: [{ childOf: "todoList" }] 65 | } 66 | }); 67 | await storageManager.finishInitialization(); 68 | 69 | return { 70 | storage: { mananger: storageManager } 71 | }; 72 | } 73 | ``` 74 | 75 | Here, we 76 | 77 | - create a place to storage the data, the `storageBackend`, in this case in-memory using [Dexie.js](https://dexie.org/). 78 | - register our data model and the relationships between them, which can be one-to-many (`childOf`, like a user having many to do lists), one-to-one (`singleChildOf`, like a user having a single user profile), or many-to-many (`connects`, like a subscription connecting many users to many newletters.) More in the [storage registry guide](/guides/storage-registry/). 79 | - finish initialzing the storage, which allows the storage registry to connect all relationships and the `StorageBackend` to establish any connections or set up any internal models it needs in order to work. 80 | 81 | Each collection is versioned with a Date, and you can pass an array of collection definitions for each version, which can be used to generate [schema migrations](/guides/schema-migrations/). 82 | 83 | ## Manipulating data 84 | 85 | Now that we've set up the storage, we can execute basic operations on it: 86 | 87 | ```js 88 | export async function demo() { 89 | const storage = await createStorage({ backend: "in-memory" }); 90 | const { object: user } = await storage.manager.operation( 91 | "createObject", 92 | "user", 93 | { displayName: "Bob" } 94 | ); 95 | const { object: list } = await storage.manager.operation( 96 | "createObject", 97 | "list", 98 | { user: user.id, title: "My todo list" } 99 | ); 100 | const { object: list } = await storage.manager.operation( 101 | "updateObject", 102 | "list", 103 | { id: list.id }, // filter 104 | { tille: "Updated title" } // updates 105 | ); 106 | await storage.manager.operation( 107 | "deleteObject", 108 | "list", 109 | { id: list.id } // filter 110 | ); 111 | } 112 | ``` 113 | 114 | For the full list of standard storage operations, see [the storage operations guide](/guides/storage-operations/). To see you can implement custom operations, see [custom storage operations](/guides/storage-operations/?id=custom-operations). 115 | 116 | ## Organizing your application into storage modules 117 | 118 | The storage manager provides you with an API to interact with your data and inspects its structure. But a common anti-pattern is weaving frameworks, libraries and other implementation details throughout the entire application. Examples of this is using an ORM directly from UI code, or using a REST/GraphQL API directly from UI code. A best practice we encourage though, is to set things up so the rest of your business logic and UI code knows nothing about Storex, so you can swap it out at any time if desired, and you have a clear separation of concerns. 119 | 120 | Storex helps you with this by providing the pattern of [storage modules](/guides/storage-modules/), which serve two purposes: 121 | 122 | - They provide you with a clear boundry into your storage logic which the rest of your application can use. 123 | - They allow you to organize information about how your storage logic is laid out, so you can do interesting things like automatically generating GraphQL APIs and managing access control to your data. 124 | 125 | ```js 126 | import { 127 | StorageModule, 128 | StorageModuleConfig, 129 | StorageModuleConstructorArgs, 130 | registerModuleMapCollections 131 | } from "@worldbrain/storex-pattern-modules"; 132 | 133 | import { 134 | StorageModule, 135 | StorageModuleConfig 136 | } from "@worldbrain/storex-pattern-modules"; 137 | 138 | export class TodoListStorage extends StorageModule { 139 | getConfig(): StorageModuleConfig { 140 | return { 141 | collections: { 142 | todoList: { 143 | version: new Date("2018-03-04"), 144 | fields: { 145 | label: { type: "text" }, 146 | default: { type: "boolean" } 147 | } 148 | }, 149 | todoItem: { 150 | version: new Date("2018-03-03"), 151 | fields: { 152 | label: { type: "text" }, 153 | done: { type: "boolean" } 154 | }, 155 | relationships: [ 156 | { alias: "list", reverseAlias: "items", childOf: "todoList" } 157 | ] 158 | } 159 | }, 160 | operations: { 161 | createList: { 162 | operation: "createObject", 163 | collection: "todoList" 164 | }, 165 | findAllLists: { 166 | operation: "findObjects", 167 | collection: "todoList", 168 | args: {} 169 | }, 170 | createItem: { 171 | operation: "createObject", 172 | collection: "todoItem" 173 | }, 174 | findListItems: { 175 | operation: "findObjects", 176 | collection: "todoItem", 177 | args: { 178 | list: "$list:pk" 179 | } 180 | }, 181 | } 182 | }); 183 | } 184 | 185 | async getOrCreateDefaultList(options: { 186 | defaultLabel: string 187 | }): Promise { 188 | const defaultList = await this.getDefaultList(); 189 | if (defaultList) { 190 | return defaultList; 191 | } 192 | 193 | const { object: list }: { object: TodoList } = await this.operation( 194 | "createList", 195 | { label: options.defaultLabel, default: true } 196 | ); 197 | const items: TodoItem[] = [ 198 | await this.addListItem({ label: "Cook spam", done: true }, { list }), 199 | await this.addListItem({ label: "Buy eggs", done: false }, { list }) 200 | ]; 201 | return { ...list, items }; 202 | } 203 | 204 | async getDefaultList(): Promise { 205 | const allLists = await this.operation("findAllLists", {}); 206 | if (!allLists.length) { 207 | return null; 208 | } 209 | 210 | const defaultList = allLists.filter((list: TodoList) => list.default)[0]; 211 | const items = await this.operation("findListItems", { 212 | list: defaultList.id 213 | }); 214 | return { ...defaultList, items }; 215 | } 216 | 217 | async addListItem(item : { label: string, done: boolean }, options : { list : TodoList }) { 218 | return (await this.operation('createItem', { ...item, list: options.list.id })).object; 219 | } 220 | } 221 | 222 | export function createStorageModules(storageManager: StorageManager) { 223 | const todoLists = new TodoListStorage({ storageManager}); 224 | return { todoLists } 225 | } 226 | 227 | ``` 228 | 229 | Now, instead of calling `storageManager.registry.registerCollections()` directly in the `createStorage()` function, you can replace that line with: 230 | 231 | ```js 232 | const modules = createStorageModules(storageManager); 233 | registerModuleMapCollections(storageRegistry, { todoLists }); 234 | ``` 235 | 236 | The rest of your business logic and UI only interacts with the methods you've exposed like `todoLists.getDefaultList()`, which doesn't say anything inherently Storex. Also, because you can inject the storage manager in its constructor, it becomes trivial to unit test these modules using different storage set ups, like an in-memory database, an SQL database, etc. 237 | 238 | ## What about mobile? 239 | 240 | At this point, your application stores data locally in the browser. If you want your application to run on mobile, you have to options. 241 | 242 | If you don't need any functionality not provided by [Web APIs](https://developer.mozilla.org/en-US/docs/Web/API) (which are quite a lot including storage, WebRTC, notifications, etc.) the easiest and most user-friendly way is to convert your app into a [Progresive Web App](https://developers.google.com/web/progressive-web-apps), allowing your users to install your website as app with only one click. No app stores, no download time, leading to much higher conversion rates. 243 | 244 | But, if you do need access to native APIs, you can choose to create a React Native app using the TypeORM storage backend to store user data in SQLite. Currenlty, there aren't any official boilerplates for this, but you can take a look at a real application [here](https://github.com/WorldBrain/Memex-Mobile), with the storage set up located here [here](https://github.com/WorldBrain/Memex-Mobile/blob/c109a27486505c27296d76989ac5951d8a7d2461/app/src/storage/index.ts). 245 | 246 | ## Next steps 247 | 248 | As a reference, you can find a boilerplate demonstating Storex using React in a few different setups [here](https://github.com/WorldBrain/storex-frontend-boilerplate/). 249 | 250 | Now, you might want to do a few things, depending on your use cases. Modern application are expected to automatically sync between multiple devices. If you want to remain offline-first and are buidling a single-user application, but do want to sync between multiple devices, check out the [multi-device sync guide](/guides/multi-device-sync/). 251 | 252 | If, however, you're making a platform and thus need multiple users to interact with each other, you'll need a back-end. You can either do this with the [Google Cloud Firestore](https://github.com/WorldBrain/storex-backend-firestore) storage backend, or if you need more control, by moving some of your storage code server-side using an SQL/MongoDB database, or any other database for which storage backends are available (or you want to contribute.) A guide still need to be written for Firestore, but you can follow the [GraphQL API guide](/guides/graphql-api/) for moving your storage logic server-side. 253 | -------------------------------------------------------------------------------- /docs/guides/schema-migrations/README.md: -------------------------------------------------------------------------------- 1 | # Schema migrations 2 | 3 | As your application evolves, so does your data model. And, each application architecture requires a different a different approach to that. Live schema migrations on a heavily-used SQL-based back-end are very different beasts than offline, single-user applications. This is why the schema migration system in Storex allows you to describe how your data changes, rather than having you code it out in commands. This way, you can choose different strategies of executing these migrations. This whole system is inspired by the article [evolutionary database design](https://www.martinfowler.com/articles/evodb.html) and experiences using different database system and abstraction layers. 4 | 5 | The `@worldbrain/storex-schema-migrations` [package](https://github.com/WorldBrain/storex-schema-migrations) contains these different sub-packages, which can be recombined according to your needs: 6 | 7 | - `schema-diff` takes your `StorageRegistry`, a base version and a target version, returning a `SchemaDiff` telling you which collections and fields have been added, changed and removed. 8 | - `migration-generator` takes a `SchemaDiff` and a declaration of what kind of operations you need to be done (like concatenating a firstName and lastName into a displayName), returing a `SchemaMigration` which a sequence of steps to execute, like adding a field, writing a value to a field, removing a field, etc. The `SchemaMigration` is divided into the `prepare`, `data` (manipulation) and `finish` phases, which is useful so you can execute these steps in different moments during your application update. 9 | - `migration-schema` combines two version a storage migrations to include collections and fields of a schema both pre- and post-migration. This is useful when you want to execute a migration and need to read from the old fields and write to the new fields. 10 | - `migration-runner` is one way of executing these migrations in a synchronous way, which can be useful from data sets that are small enough. 11 | 12 | Some examples of how these can be re-combined (none of these are coded out yet): 13 | 14 | - **Using IndexDB client-side with the Dexie storage back-end** there is no need to explictly add and remove fields, since Dexie does that automatically based on the versions you've defined. However, you can use the `migration-generator` package to describe the migration in a standard way and execute it synchronously with the `migration-runner`. Or, you might choose to take that migration and perform these migration live as you read and write data from the database. 15 | - **Migrating the schema of an SQL database with zero downtime** requires us to take into account that old versions of our application operating on the old version of a schema, will co-exist with new versions of our application operating against the new version of the schema. For a lot of database refactorings, it would not be hard to automate the process described [here](https://www.martinfowler.com/articles/evodb.html#TransitionPhase) while you route your database writes to the new version of your application, while routing reads to the old version of your application. 16 | - **Firestore migrations** could be implemented using Cloud Functions that run in the background responding to writes by old client versions and automatically migrates them to the new data structure, meaning that you can accomodate the fact that when you deploy a new version of your application, there might be a lot of users still running your old code for a while. 17 | - **Generating an SQL script to be reviewed and executed by a DBA** might be something required by some organizations. This would mean feeding the output of the `migration-generator` to an SQL generator. 18 | - **Live migrating serialized data that was exported using an older version of your application** is also possible and experimented with [here](https://github.com/WorldBrain/storex-data-tools/blob/f9e3ad205d5800a15dee855cff089b77c7bb1a8d/ts/data-migration/index.test.ts). This is very useful for import/export of data that will be stored for a long time, and will be integrated into [multi-device sync](/guides/multi-device-sync/) to allow dealing with synchronization between clients running different versions of your application. 19 | 20 | ## Status of this package 21 | 22 | Although the base of this package is there, there's still some work needed for this to be ready for production, after which more detailed documentation will be available. This includes some cleaning up of the naming of the types in the package, and the addition of functionality needed for the cases where you want to split out subcollections (a big user collection into a separate user profile collection for example), or join small ones into bigger ones. 23 | 24 | ## What next? 25 | 26 | If you haven't yet checked it out, take a look at the [GraphQL API generation guide](/guides/graphql-api/) to see how Storex allows you to quickly develop your application in memory without having to spin up an application server or database instances, after which when you're ready you can use the same code and move it server-side with mininal effort. Or, you might want to check out the [multi-device sync guide](/guides/multi-device-sync/) describing how you can make client-side single-user applications that a user can use across multiple devices without a full-fledged back-end. 27 | -------------------------------------------------------------------------------- /docs/guides/storage-middleware/README.md: -------------------------------------------------------------------------------- 1 | # Storage middleware 2 | 3 | Whenever you execute a [storage operation](/guides/storage-operations/), you have the chance to do something with that operation before it arrives at the storage backend which uses the underlying storage technology to execute that operation. This is done through middleware, and it's used by the [multi-device sync](/guides/multi-device-sync/) to log all modification operations in order to replicate them to other devices. Other things you might want to do is to debug all operations done by your application. This is what a storage middleware looks like, and how to use it: 4 | 5 | ```js 6 | import { StorageMiddleware } from "@worldbrain/storex/lib/types/middleware"; 7 | 8 | export class LoggingMiddleware implements StorageMiddleware { 9 | // public log : Array<{ operation : any, result : any }> = []; 10 | 11 | async process({ 12 | operation, 13 | next 14 | }: { 15 | operation: any[], 16 | next: { process: Function } 17 | }) { 18 | const result = await next.process({ operation }); 19 | this.log.append({ operation, result }); 20 | return result; 21 | } 22 | } 23 | 24 | export async function demo() { 25 | const loggingMiddleware = new LoggingMiddleware(); 26 | storageManager.setMiddleware([loggingMiddleware]); 27 | await storageManager.collection("user").createObject({ displayName: "Joe" }); 28 | expect(loggingMiddleware.log).toEqual([ 29 | { 30 | operation: ["createObject", "user", { displayName: "Joe" }], 31 | result: { object: { id: expect.any(Number), displayName: "Joe" } } 32 | } 33 | ]); 34 | } 35 | ``` 36 | 37 | ## What's next? 38 | 39 | If you're here for the first time, you're now up to speed on the very basics of Storex. As a next step, check out [storage modules](/guides/storage-modules/) which allows your to organize your storage logic into logical blocks with some extra information attached to them that'll help your application to be more flexible, including generating a GraphQL API with very little extra effort. 40 | -------------------------------------------------------------------------------- /docs/guides/storage-modules/README.md: -------------------------------------------------------------------------------- 1 | # Storage modules 2 | 3 | As seen in the [quickstart](/guides/quickstart/), storage modules provide a standard pattern to group your storage logic into logical chunks and provide some extra information about them. Using this kinds of organization, you can do things like automatically [generating GraphQL APIs](/guides/graphql-api/) or scan your program automatically for optimization opertunities such as expensive operations you're executing without having the right indices in place to make them fastter. 4 | 5 | Summerized, storage modules provide: 6 | 7 | - a standard place to put your [collection definitions](/guides/storage-registry/) and organize their previous versions. 8 | - a standard place to expose which [storage operations](/guides/storage-operations/) you're using inside your higher-level methods, so they can be introspected by other tools. 9 | - a standard way to expose higher-level methods that operate on your your data (like fetching suggesting, inserting something into a list, etc.), and exposing meta-data about those methods. This way, the rest of your application only interacts with these methods, without caring about the fact you're using Storex underneath, nor implementation details like whether you're using REST or GraphQL to move data around. 10 | - a standard place to expose operation-level access rules, which can be enforced in different ways, like compiling them to Firestore security rules, or enforcing them manually server-side. 11 | 12 | ## Basic usage 13 | 14 | The `@worldbrain/storex-pattern-modules` package provides the `StorageModule` abstract base class, which requires you to implement the `getConfig()` method on which you use to expose info about your storage modules. Aside from that, this base class does nothing more than provide you with an internal convenience method to execute operations (more about that later.) 15 | 16 | ```js 17 | import { 18 | StorageModule, 19 | StorageModuleConfig, 20 | registerModuleMapCollections 21 | } from "@worldbrain/storex-pattern-modules"; 22 | 23 | class TodoListStorage extends StorageModule { 24 | getConfig(): StorageModuleConfig { 25 | return { 26 | // What you'd pass to StorageRegistry.registerCollections(), but with a bit more info 27 | collections: { 28 | todoList: { ... } 29 | }, 30 | 31 | // Templates of operations passed to StorageManager.operation() 32 | operations: { 33 | createList: { ... }, 34 | }, 35 | 36 | // Info about exposed methods, which you can use to generate REST/GraphQL endpoints for example 37 | methods: { 38 | createTodoList: { ... }, 39 | }, 40 | 41 | // Access and valdation rules determining who can execute what operations on what data 42 | accessRules: { ... } 43 | }; 44 | } 45 | 46 | async createTodoList(list: { name: string }) { 47 | return (await this.operation('createList', list)).object 48 | } 49 | } 50 | ``` 51 | 52 | ## Defining collections 53 | 54 | As seen above, you return your collection definitions in the `StorageModuleConfig.collections` property which you return in your `getConfig()` implementation. The only difference is that in the collections you return, instead of including previous versions of your collections in the same place, you may choose to include them in the `history` property of your collection definitions. This way, you have programmatic access to how your data schema evolved over time (so you can generate [schema migrations](/guides/schema-migrations/)), while keeping your current code readable and clean with only the current version of your data schema in your sight. 55 | 56 | ```js 57 | import { 58 | StorageModule, 59 | StorageModuleConfig, 60 | registerModuleMapCollections 61 | } from "@worldbrain/storex-pattern-modules"; 62 | 63 | class TodoListStorage extends StorageModule { 64 | getConfig(): StorageModuleConfig { 65 | return { 66 | collections: { 67 | todoList: { 68 | history: [ 69 | // this would normally live in a separate file 70 | { 71 | version: new Date("2019-10-10"), 72 | fields: { 73 | title: { type: "text" } 74 | } 75 | } 76 | ], 77 | version: new Date("2019-10-11"), 78 | fields: { 79 | title: { type: "text" }, 80 | category: { type: "string" } 81 | } 82 | } 83 | } 84 | }; 85 | } 86 | } 87 | ``` 88 | 89 | Additionally, the `@worldbrain/storex-pattern-modules` package provides a `withHistory()` helper function, that you can use to separate your entire schema history in a more convenient way. See [this](https://github.com/WorldBrain/storex-frontend-boilerplate/blob/2bf0ca5ecdcfdae3abbe2e2ded619a6f4f109a30/src/storage/modules/todo-list.ts) and [this](https://github.com/WorldBrain/storex-frontend-boilerplate/blob/2bf0ca5ecdcfdae3abbe2e2ded619a6f4f109a30/src/storage/modules/todo-list.history.ts) file of the [Storex front-end boilerplate](https://github.com/WorldBrain/storex-frontend-boilerplate) for example usage. 90 | 91 | Collection versioning works by versioning your collection with the schema version of you application. When you have two collection with version `2019-10-10`, and you add a third one with version `2019-10-20`, you'll have two application schema versions: 1) `2019-10-10` containing two collections, and 2) `2019-10-20` containing three collections. But when want to package storage modules for inclusion in other applcations, the collections in that package will have their independent versioning. For this, there's the `mapCollectionVersions()` helper function, which can map module collection versions to versions of the application that uses them. See [here](https://github.com/WorldBrain/Memex/blob/dd66472feb73af86e2952d343937988f9b25771a/src/sync/background/storage.ts) for an example of Memex using this function to integrate [multi-device sync](/guides/multi-device-sync/). 92 | 93 | ## Executing operations 94 | 95 | Storage modules allow you to specify which operations your storage methods are using in a discoverable way. This allows you for example to write an automatic tool that scans your entire program for inefficient access patterns, or diagrams about where you're interacting with your data in which ways. 96 | 97 | ```js 98 | import { 99 | StorageModule, 100 | StorageModuleConfig, 101 | registerModuleMapCollections 102 | } from "@worldbrain/storex-pattern-modules"; 103 | 104 | class TodoListStorage extends StorageModule { 105 | getConfig(): StorageModuleConfig { 106 | return { 107 | collections: { 108 | todoList: { 109 | version: new Date("2019-10-11"), 110 | fields: { 111 | title: { type: "text" } 112 | } 113 | } 114 | }, 115 | operations: { 116 | findListById: { 117 | operation: "findObject", 118 | collection: "todoList", 119 | 120 | // The rest of the arguments passed to the StorageManager.operation() 121 | // String starting with $ represent placeholders filled in when 122 | // executing the operation. 123 | args: [{ id: "$id" }] 124 | }, 125 | createList: { 126 | // Since createObject always takes the same, predictable `args`, 127 | // they are filled in automatically 128 | operation: "createObject", 129 | collection: "todoList" 130 | } 131 | } 132 | }; 133 | } 134 | 135 | async getList(id: string | number) { 136 | // The second argument to `this.operation()` are the substitutions 137 | // that'll be used to fill in the placeholder defined above in `args` 138 | return this.operation("findListById", { id }); 139 | } 140 | 141 | async createList(list: { title: string }) { 142 | return this.operation("createList", list); 143 | } 144 | } 145 | ``` 146 | 147 | One other advantage of this thin abstraction is that you can specify how the operations are executed for each storage module. [This](https://github.com/WorldBrain/storex-pattern-modules/blob/877909b37c81e59b7743b0ee3c3160dfa5fe69dd/ts/index.ts#L83) is the implementation of the default `operationExecutor`, but you can pass in a custom one in [the constructor](https://github.com/WorldBrain/storex-pattern-modules/blob/877909b37c81e59b7743b0ee3c3160dfa5fe69dd/ts/index.ts#L23) of a storage module. One application of this would be to detect slow operation executions and logging them, knowing which storage module executed it and what the operation name was. Also, there is an experimental [module spy](https://github.com/WorldBrain/storex-pattern-modules/blob/877909b37c81e59b7743b0ee3c3160dfa5fe69dd/ts/spy.test.ts) that allows the `storageExecutor` to detect from which method the operation was executed. (NOTE: the module spy class is not stable yet, so feel free to contribute to it for that to happen.) 148 | 149 | ## Storage module methods 150 | 151 | In the example above, we have the `getList()` and `createList()` methods of the storage module. These are the methods that the rest of your application uses to interact with your data instead of directly using the storage manager. In addition to allowing you in this way to remove Storex entirely from your program if you wish to do so, with a little bit of extra description of these methods we seemlessly move these storage module server-side and communicate with it using an automatically generated [GraphQL API](/guides/graphql-api/), or any other communication protocol you might want to implement (REST, WebSockets, TCP sockets, WebRTC, etc.) The underlying idea is that the transport layer you use to let systems communicate should be an implementation detail, not something that's dominant throughout your entire application (like most GraphQL tutorials that teach people to directly embed GraphQL in their UI code.) 152 | 153 | A description of your methods looks like this: 154 | 155 | ```js 156 | class UserAdminModule extends StorageModule { 157 | getConfig = (): StorageModuleConfig => ({ 158 | collections: { 159 | user: { 160 | version: new Date("2019-01-01"), 161 | fields: { 162 | name: { type: "string" }, 163 | age: { type: "int" } 164 | } 165 | } 166 | }, 167 | operations: { 168 | findByName: { 169 | operation: "findObject", 170 | collection: "user", 171 | args: { name: "$name:string" } 172 | }, 173 | updateAgeByName: { 174 | operation: "updateObjects", 175 | collection: "user", 176 | args: [{ name: "$name:string" }, { age: "$age:int" }] 177 | } 178 | }, 179 | methods: { 180 | byName: { 181 | type: "query", 182 | args: { name: "string" }, 183 | returns: { collection: "user" } 184 | }, 185 | setAgeByName: { 186 | type: "mutation", 187 | args: { name: "string", age: "int" }, 188 | returns: { collection: "user" } 189 | } 190 | } 191 | }); 192 | 193 | async byName(args: { name: string }) { 194 | return this.operation("findByName", args); 195 | } 196 | 197 | async setAgeByName(args: { name: string, age: number }) { 198 | await this.operation("updateAgeByName", args); 199 | return this.byName(args); 200 | } 201 | } 202 | ``` 203 | 204 | More examples can be found in the [GraphQL schema tests](https://github.com/WorldBrain/storex-graphql-schema/blob/81611b84d480629ab22963f85452b281b4461c80/ts/modules.test.ts). Currently, GraphQL is the only transport protocol implemented, for which you can find usage instruction [here](/guides/graphql-api/). 205 | 206 | ## Access rules 207 | 208 | As soon as you're creating multi-user systems, whether that means a back-end for your web application, or a P2P system, you'll need to manage who can do what. For this, Storex provides a technology-independent way of describing who is allowed to execute what operations on what data. This system is still in it's early beginnings, and for now only supports [compilation](https://github.com/WorldBrain/storex-backend-firestore/blob/fad40f3701268543b48b6e0e1976fcd651599243/ts/security-rules/index.test.ts) to [Firestore access rules](https://firebase.google.com/docs/firestore/security/get-started). An `operationExecutor` or a storage middleware to enforce these rules in Node.js environments is planned. Also planned are access rules based on methods, rather than operations. 209 | 210 | The end goal of access rules is to create a common ground where possible for access control, portable to multiple system architectures and maybe most important of all, sharing a common toolset to work with access control, including automated testing for unforeseen scenarios, visualization and manual testing. 211 | 212 | ## What's next? 213 | 214 | Now that you have a grasp of the all the basics that together allow for a very flexible application architecture, you'll probably want to check out the [schema migrations guide](/guides/schema-migrations/). 215 | -------------------------------------------------------------------------------- /docs/guides/storage-operations/README.md: -------------------------------------------------------------------------------- 1 | # Storage operations 2 | 3 | If you've read the [quickstart guide](/guides/quickstart/), you've seen that data is queried and manipulated with operations, either directly though the `StorageManager.operation()` method, or through [storage modules](/guides/storage-modules/). There is a set of standard operations that is universal enough to work across every kind storage backend (CRUD and batch operations), but every storage technology has its own characteristics and capabilities. The goal of Storex is not to make these storage technologies invisible, but rather provide a common toolchain that gives you easy ways to do the boring stuff, and ways of cleanly interacting with and inspecting the underlying storage technology. 4 | 5 | ## Standard operations 6 | 7 | Whatever storage backend you'll be working with, you'll be creating, updating, deleting and querying data. Respectively, these are the `createObject`, `updateObjects`, `deleteObjects` and `findObject(s)` operations. You'll find examples of how to use these in the [quickstart guide](/guides/quickstart/). What isn't shown there however, is how the filters used by the update, delete and find operations work. These filters are inspired by MongoDB filters, with the following supported operators: 8 | 9 | ```js 10 | // builds on quickstart setup 11 | 12 | export async function demo() { 13 | const storage = await createStorage({ backend: "in-memory" }); 14 | await storage.manager.operation("findObjects", "user", { age: 30 }); 15 | await storage.manager.operation("findObjects", "user", { age: { $gt: 30 } }); 16 | await storage.manager.operation("findObjects", "user", { age: { $ge: 30 } }); 17 | await storage.manager.operation("findObjects", "user", { age: { $lt: 30 } }); 18 | await storage.manager.operation("findObjects", "user", { age: { $le: 30 } }); 19 | } 20 | ``` 21 | 22 | ## Feature detection 23 | 24 | Not all storage technologies have the same features, and if you want your code to work on multiple storage technologies, or assert the features of a new storage technology if you're switching, you'll have to be able to detect in your code whether the storage technology you're using supports certain standardized features. This is done by using the `StorageBackend.supports(feature: string): boolean` method: 25 | 26 | ```js 27 | export async function demo() { 28 | const storage = await createStorage({ backend: "in-memory" }); 29 | expect(storage.manager.backend.supports("executeBatch")).toEqual(true); 30 | } 31 | ``` 32 | 33 | Since the beginning of Storex until now, the number of features a storage backend support has been growing. Therefore, this API will be changed in the future to allow for more fine-grained feature detection in a cleaner way, before documenting more detectable features. 34 | 35 | ### Feature: `executeBatch` operation 36 | 37 | Since you'll want certain operations to happen atomically, `executeBatch` allows you to execute `createObject`, `updateObjects` and `deleteObjects` operations together as an alternative to transaction, which are not as easy to implement in a consistent way across different technologies. As of now, this operation is supported for IndexedDB through the Dexie backend, MySQL, PostgreSQL and SQLite through the TypeORM backend, and Firestore. 38 | 39 | ```js 40 | export async function demo() { 41 | const storage = await createStorage({ backend: "in-memory" }); 42 | expect(storage.manager.backend.supports("executeBatch")).toEqual(true); 43 | await storage.manager.operation("executeBatch", [ 44 | { 45 | operation: 'createObject', 46 | collection: 'todoList', 47 | placeholder: 'myList' 48 | object: { 49 | title: 'my todo list' 50 | } 51 | }, 52 | { 53 | operation: 'createObject', 54 | collection: 'todoListEntry', 55 | object: { 56 | title: 'write document' 57 | }, 58 | 59 | // Here we insert the ID of the created list 60 | // into the `list` property of the create list entry 61 | replace: [{ 62 | path: 'list', 63 | placeholder: 'myList', 64 | }] 65 | }, 66 | ]) 67 | } 68 | 69 | ``` 70 | 71 | ## Custom operations 72 | 73 | As you're developing a real-world application, there will come a time the standard operations don't cut it. Therefore, Storex allows you to register custom operations on a storage backend, so that you can directly use the underlying storage technology, while still having that operation run through Storex, so you can do things with them in [storage middleware](/guides/storage-middleware/) (like logging or timing the operations you're executing) and use them in [storage modules](/guides/storage-modules/). 74 | 75 | ```js 76 | export async function demo() { 77 | const storage = await createStorage({ backend: "in-memory" }); 78 | storage.manager.backend.registerOperation("my-app:fetchSuggestions", async (options: { 79 | collection: string, 80 | field: string, 81 | prefix: string 82 | }) => { 83 | const dexieInstance = (storage.manager.backend as DexieStorageBackend).dexieInstance 84 | const table = dexieInstance.table(options.collection) 85 | return table.where(options.field).startsWith(options.prefix).toArray() 86 | }) 87 | const objects = await storageManager.operation("my-app:fetchSuggestions", { 88 | collection: 'todoList', 89 | field: 'title', 90 | prefix: 'te' 91 | }) 92 | } 93 | ``` 94 | 95 | In order to prevent operation naming conflicts, operations have a standard format. If you're registering an operation that is part of your application or a package not part of the storage backend, you must use the `:` character to namespace your operation, e.g. `todo-list:fetchOverview` or `suggestion-package:fetchSuggestions`. Aside from that, storage backends may ship with custom, storage backend specific operations, using the `.` character to namespace the operations, like (a hypothethical example) `dexie.bulkPut`. 96 | 97 | ## What's next? 98 | 99 | If you're checking out Storex for the first time, check out the [storage middleware guide](/guides/storage-middleware/) which will complete your grasp of the most fundamental principles of Storex, before moving on to higher-level concepts such as [storage modules](/guides/storage-modules/) or [schema migrations](/guides/schema-migrations/). 100 | -------------------------------------------------------------------------------- /docs/guides/storage-registry/README.md: -------------------------------------------------------------------------------- 1 | # Storage registry 2 | 3 | Data models are deined in Storex through the storage registry, which is automatically constructed for you when you create a storage manager: 4 | 5 | ```js 6 | import StorageManager from "@worldbrain/storex"; 7 | import { DexieStorageBackend } from "@worldbrain-storex-backend-dexie"; 8 | import inMemory from "@worldbrain-storex-backend-dexie/lib/in-memory"; 9 | 10 | export function createStorageManager(): StorageManager { 11 | const backend = new DexieStorageBackend({ 12 | dbName: "test", 13 | idbImplementation: inMemory() 14 | }); 15 | return new StorageManager({ backend }); 16 | } 17 | 18 | export async function main() { 19 | const storageManager = createStorageManager(); 20 | storageManager.registry.registerCollections({ 21 | note: { 22 | version: new Date("2019-11-02"), 23 | fields: { 24 | // Fields are required by default, and must be null (not undefined) when not provided. 25 | title: { type: "text", optional: true } 26 | body: { type: "text" } 27 | } 28 | } 29 | }); 30 | await storageManager.finishInitialization(); 31 | 32 | // You can find the latest versions of all collections here 33 | expect(storageManager.registry.collections['note']).toEqual(expect.any(Object)) 34 | } 35 | ``` 36 | 37 | ## Versioning 38 | 39 | Since your data models are going to change as your product evolves, every registered collection has a version, and you can register multiple versions of the same model. These versions are application-wide version, described as a `Date` object. As a result, you have access to the version history of all the collections in your applications through either `StorageRegistry.getSchemaHistory()` or `StorageRegistry.getCollectionsByVersion(version: Date)`: 40 | 41 | ```js 42 | export async function main() { 43 | const storageManager = createStorageManager(); 44 | storageManager.registry.registerCollections({ 45 | user: { 46 | version: new Date("2019-11-03"), 47 | fields: { 48 | displayName: { type: "text" } 49 | } 50 | }, 51 | note: [ 52 | { 53 | version: new Date("2019-11-02"), 54 | fields: { 55 | body: { type: "text" } 56 | } 57 | }, 58 | { 59 | version: new Date("2019-11-03"), 60 | fields: { 61 | title: { type: "text" }, 62 | body: { type: "text" } 63 | } 64 | } 65 | ] 66 | }); 67 | await storageManager.finishInitialization(); 68 | 69 | expect(storageManager.registry.getSchemaHistory()).toEqual([ 70 | { 71 | version: new Date("2019-11-02"), 72 | collections: { 73 | note: expect.any(Object) // first version of note 74 | } 75 | }, 76 | { 77 | version: new Date("2019-11-03"), 78 | collections: { 79 | user: expect.any(Object), // first version of user 80 | note: expect.any(Object) // second version of note 81 | } 82 | } 83 | ]); 84 | expect( 85 | storageManager.registry.getCollectionsByVersion(new Date("2019-11-03")) 86 | ).toEqual({ 87 | user: expect.any(Object), 88 | note: expect.any(Object) 89 | }); 90 | 91 | // You can find the latest versions of all collections here 92 | expect(storageManager.registry.collections["note"]).toEqual( 93 | expect.any(Object) 94 | ); 95 | } 96 | ``` 97 | 98 | This information can be used for different purposes. For example the [schema migrations][/guides/schema-migrations/] package can use this info to determine which collections/fields you've removed and generate a migration plan which you can execute in various ways (synchronously, through a Lamda function performing live migrations, etc.) 99 | 100 | ## Field types 101 | 102 | Currently supported field types are `string`, `text`, `json`, `datetime`, `timestamp`, `boolean`, `float`, `int`, `blob` and `binary`. 103 | 104 | **`string` and `text` fields** both bold strings, but the `text` field is meant for fields that are meant to be full-text searched. If you place an index on a `text` field, whatever form of full-text support your using will create a full-text index. The Dexie (IndexedDB) back-end has built-in support for this, which is the fastest and most scalable way to provide client-side full-text search using only Web technologies at this moment. Different methods, like SQLite full-text search plugins, ElasticSearch and AWS CloudSearch, are not developed yet, but planned for. 105 | 106 | **`json` fields** hold JSON-serializable objects, which will be stored by the back-end in the fastest, and most queryable way possible depending on the storage backend you're using (PostgreSQL has a native JSON field for example, while MongoDB can store and query any JSON-serializable object). Serialization and deserialization happens automatically on store/retrieval if needed. 107 | 108 | **`datetime` and `timestamp` fields** both store date-times on which in the future (not yet) you'll be able to perform date-specific operations (like filtering by month.) Although functionally the same, the `timestamp` field type exposes datetimes as milisecond-based floating-point number (Javascript `number` type), while `datetime` works with Javascript `Date` object. These types will probably be unified in the future, but for now you can choose based on your preference. 109 | 110 | **`blob` and `binary` fields** both store binary data, but the `blob` type works with Javascript `Blob` objects (thus storing also the mimetype), while the `binary` field works with `ArrayBuffer` objects in browsers and `Buffer` objects in Node.js. 111 | 112 | 120 | 121 | ## Indices and primary keys 122 | 123 | You can set indices of single fields and combining multiple fields when registering a collection. As always, you should be careful to use the right indices when constructing your data model with the right trade-off between read speed, write speed and space consuption for your application. This will vary a lot depending on what you're trying to build, so it's your job to know your data model and storage technology underlying the storage backend you're using. 124 | 125 | ```js 126 | export async function main() { 127 | const storageManager = createStorageManager(); 128 | storageManager.registry.registerCollections({ 129 | note: { 130 | version: new Date("2019-11-02"), 131 | fields: { 132 | createdWhen: { type: "timestamp" }, 133 | title: { type: "text" }, 134 | slug: { type: "string" }, 135 | body: { type: "text" } 136 | }, 137 | indices: [ 138 | { field: "slug" }, // single-field index 139 | { field: ["createdWhen", "slug"] } // multi-field index 140 | ] 141 | } 142 | }); 143 | await storageManager.finishInitialization(); 144 | 145 | // Insertion will probably be slowser, since 3 indices are affected 146 | const { object } = await storageManager.operation("createObject", "note", { 147 | createdWhen: Date.now(), 148 | title: "First note", 149 | slug: "first-note", 150 | body: "Something important to remember" 151 | }); 152 | 153 | // This will be fast, because the single-field index is used 154 | expect( 155 | await storageManager.operation("findObject", "note", { 156 | slug: object.slug 157 | }) 158 | ).toEqual(object); 159 | 160 | // This will be fast, because the multi-field index is used 161 | expect( 162 | await storageManager.operation("findObject", "note", { 163 | slug: object.slug, 164 | createdWhen: { $lt: Date.now() } 165 | }) 166 | ).toEqual(object); 167 | } 168 | ``` 169 | 170 | By default, if you don't specify a primary key, an auto-generated primary key field called `id` is added to the collection. How this primary key is generated depends on the storage backend, but on SQL databases and IndexedDB this will typically be an incrementing integer, while on Firebase/Firestore this will be a (semi-)random string. You can specify a custom primary key field as follows: 171 | 172 | ```js 173 | export async function main() { 174 | const storageManager = createStorageManager(); 175 | storageManager.registry.registerCollections({ 176 | note: { 177 | version: new Date("2019-11-02"), 178 | fields: { 179 | createdWhen: { type: "timestamp" }, 180 | title: { type: "text" }, 181 | slug: { type: "string" }, 182 | body: { type: "text" } 183 | }, 184 | indices: [ 185 | { field: "slug", pk: true }, // this is now the primary key 186 | { field: ["createdWhen", "slug"] } 187 | ] 188 | } 189 | }); 190 | await storageManager.finishInitialization(); 191 | 192 | expect(storageManager.registry.collections["note"].pkField).toEqual("slug"); 193 | } 194 | ``` 195 | 196 | ## Relationships 197 | 198 | All data you'll store will probably be connected with each other. Storex, for the time being allows you to express three kinds of relationships: 199 | 200 | - `singleChildOf` which in relational databases is called a one-to-one relationship, meaning that there will exist only one object of the collection you're defining for the parent you're pointing to. Think of a user having only one profile. 201 | - `childOf` which in relational databases is called a one-to-many relationship, meaning that there might be zero or more objects of the collection you're defining for the parent you're pointing to. Think of a user having potentially multiple e-mail addresses. 202 | - `connects` which is in relational databases is called a many-to-many relationship, meaning that an object of this collection connects two other objects. Think of a newsletter subscription connecting a user to a newsletter, while optionally containing information about the connection, like when the subscription was created. 203 | 204 | ```js 205 | export async function main() { 206 | const storageManager = createStorageManager(); 207 | storageManager.registry.registerCollections({ 208 | user: { 209 | version: new Date("2019-11-02"), 210 | fields: { 211 | displayName: { type: "string" } 212 | } 213 | }, 214 | userProfile: { 215 | version: new Date("2019-11-02"), 216 | fields: { 217 | city: { type: "string" } 218 | }, 219 | relationships: [{ singleChildOf: "user" }] 220 | }, 221 | userEmail: { 222 | version: new Date("2019-11-02"), 223 | fields: { 224 | address: { type: "string" } 225 | }, 226 | relationships: [{ childOf: "user" }] 227 | }, 228 | newsletter: { 229 | version: new Date("2019-11-02"), 230 | fields: { 231 | title: { type: "string" } 232 | } 233 | }, 234 | newsletterSubscription: { 235 | version: new Date("2019-11-02"), 236 | fields: { 237 | subscribedWhen: { type: "timestamp" } 238 | }, 239 | relationships: [{ connects: ["user", "newsletter"] }] 240 | } 241 | }); 242 | await storageManager.finishInitialization(); 243 | 244 | const { object: user } = await storageManager.operation( 245 | "createObject", 246 | "user", 247 | { displayName: "Brian" } 248 | ); 249 | const { object: profile } = await storageManager.operation( 250 | "createObject", 251 | "userProfile", 252 | { user: user.id, hometown: "London" } 253 | ); 254 | const { object: email } = await storageManager.operation( 255 | "createObject", 256 | "userEmail", 257 | { user: user.id, address: "life@brian.com" } 258 | ); 259 | const { object: newsletter } = await storageManager.operation( 260 | "createObject", 261 | "newsletter", 262 | { title: "Life of Brian" } 263 | ); 264 | const { 265 | object: subscription 266 | } = await storageManager.operation("createObject", "newsletterSubscription", { 267 | user: user.id, 268 | newsletter: newsletter.id, 269 | subscribedWhen: Date.now() 270 | }); 271 | } 272 | ``` 273 | 274 | Notice the ID(s) of the target object(s) of the child or connection, like the user of the profile, or the user and newsletters of the subscription, are passed in as the name of the target collection. You can change this by setting the `alias` property on a `singleChildOf` or `childOf` relationship, the `aliases: [string, string]` propery on a `connects` relationship. 275 | 276 | ## What next? 277 | 278 | If you're exploring Storex for the first time, now that you know how to define your data models, you'll probably want to visit the [storage operations guide](/guides/storage-operations/) to how to query and manipulate your data. 279 | -------------------------------------------------------------------------------- /docs/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Storex Documentation 6 | 7 | 8 | 12 | 13 | 17 | 18 | 19 | 20 |
21 | 46 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /docs/resources/README.md: -------------------------------------------------------------------------------- 1 | # Resources 2 | 3 | TBD 4 | -------------------------------------------------------------------------------- /docs/storex-hub/README.md: -------------------------------------------------------------------------------- 1 | # Storex Hub 2 | 3 | Everyone has different knowledge workflows, uses different apps and communicates differently. 4 | The systems around us tend to lock us in or are inflexible to adapt to our needs. 5 | Often, the features or integrations we want are not on the developers roadmap. 6 | 7 | Storex Hub was built to change that. 8 | With it you can connect any combination of apps, and do custom data processing - without a cloud on your local computer. All processed data is also stored locally, so you have permanent access to it to reuse or innovate. 9 | 10 | ##### Examples on what you can do with this: 11 | 12 | - Download all your Pocket and Memex bookmarks, do a content analysis and then publish those about `COVID-19` to Twitter and IPFS 13 | - Automatically add all the pages, notes or calendar entries tagged with `TODO` to your task management tool. 14 | - Search over all your stored data and find all content with the words `climate change` 15 | - Write a backup application that backs up the data of all your applications at once, instead of each application having its own backup solution. 16 | - Write a task management application where you could link tasks to Tweets, people, e-mails, etc. 17 | - Develop a specialized application for managing tags of different types of content stored by different applications, like web pages stored by Memex, Tweets stored by a Twitter integration, articles coming from your RSS reeds, etc. instead of creating such a tag management UI for each application individually. 18 | 19 | ## Current state 20 | 21 | Currently, Storex Hub is a local server that multiple local and cloud applications can connect to from their using either REST/Websocket and 1) query/manipulate the data of other applications, and 2) process requests by other application to query/manipulate their data. This means that one application can store its data in IndexedDB running on Javacript, while another application could be based on Python and store its data in some service in the cloud, while communicating in a simple and standardized way. 22 | 23 | ## Feature Roadmap 24 | 25 | - Permission model & access control to ensure apps don't have unauthorized access to other apps data 26 | - Offline-first multi-DEVICE sync that Storex Hub plugins can use out of the box 27 | - Offline-first multi-USER sync for sharing & collaboration that Storex Hub plugins can use out of the box 28 | 29 | Help us accelerating this roadmap and support us on [OpenCollective](https://opencollective.com/worldbrain). 30 | 31 | ## Want to get started? 32 | 33 | Check out the [getting started guide](/storex-hub/getting-started/?id=getting-started) to start playing around with Storex Hub. 34 | 35 | 39 | -------------------------------------------------------------------------------- /docs/storex-hub/api-reference/README.md: -------------------------------------------------------------------------------- 1 | # Storex Hub API reference 2 | 3 | For now, we don't have the resources to develop well-polished documentation. This, combined with the fact that we're still fleshing out the APIs, means that the best compromise for now is to have up-to-date API tests and examples that accurately show how to use Storex Hub. If you didn't read the [quickstart](/storex-hub/getting-started/) yet, that's the best place to start. After that, these are some useful resources: 4 | 5 | - The Socket.io wrapper gives you back the Storex API, handling all the websocket stuff under the hood. That API is the `StorexHubApi_v0` type defined [here](https://github.com/WorldBrain/storex-hub/blob/master/ts/public-api/server.ts). 6 | - The callacks you can register are defined as the `StorexHubCallbacks_v0` type [here](https://github.com/WorldBrain/storex-hub/blob/master/ts/public-api/client.ts). 7 | - There's also a REST-like API expose, where each method is assigned a URL, takes its argument as a JSON object in a POST body, and returns the result as a JSON object. The URL for each method can be found as the `STOREX_HUB_API_v0` constant [here](https://github.com/WorldBrain/storex-hub/blob/master/ts/public-api/server.ts). 8 | - All demos maintained as part of Storex Hub are [here](https://github.com/WorldBrain/storex-hub/tree/master/demos) 9 | -------------------------------------------------------------------------------- /docs/storex-hub/contact/README.md: -------------------------------------------------------------------------------- 1 | # Getting in touch about Storex Hub 2 | 3 | Do you like where we're heading with Storex and Storex Hub? There's many ways to help the development of Storex Hub. We want to hear about your use cases, could use help developing, researching or with funds to develop features enabling new use cases for the whole ecosystem. Drop by on our [team chat](http://join-worldbrain.herokuapp.com/) in the #storex channel and let us know! 4 | -------------------------------------------------------------------------------- /docs/storex-hub/getting-started/README.md: -------------------------------------------------------------------------------- 1 | # Getting started 2 | 3 | Storex Hub is an offline-first Zapier-like API platform. With it you can connect any app in custom workflows, by building Wordpress-like plugins. 4 | You can also work with Memex data more efficiently. 5 | 6 | # What's your use case? 7 | 8 | 1. [I want to run Storex Hub & use plugins](/storex-hub/getting-started/?id=_1-i-want-to-run-storex-hub-amp-play-around-with-my-memex-data) 9 | 2. [I want to import/export Memex data](/storex-hub/getting-started/?id=_2-i-want-to-importexport-memex-data-in-other-apps) 10 | 3. [I want to develop my own plugin](/storex-hub/getting-started/?id=_3-i-want-to-develop-my-own-plugin) 11 | 4. [I want to access the Storex Hub / Memex API from an external application](/storex-hub/getting-started/?id=_4-i-want-to-access-the-storex-hub-memex-api-from-an-external-application) 12 | 13 | --- 14 | 15 | ### 1. I want to run Storex Hub & play around with my Memex data 16 | 17 | Currently, we're figuring out how to make Storex Hub easy to install for end-users, but we're not there yet. In the meanwhile, you're going to need some knowledge of the command line and Git, and [have Node.js/NPM installed](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm). 18 | 19 | 1. On the command line, clone the Storex Hub repository: `git clone https://github.com/WorldBrain/storex-hub.git` 20 | 1. Inside the cloned repository, install the dependencies: `npm install` 21 | 1. Run Storex Hub: `DB_PATH='./dev/db' PLUGINS_DIR='./dev/db/plugins npm start'` 22 | 1. Connect Memex to Storex Hub: 23 | 1. Open the background console of the Memex extension 24 | - Firefox: about:devtools-toolbox?type=extension&id=info%40worldbrain.io 25 | - Chrome: chrome://extensions > enable developer mode > Memex > background page > console 26 | 1. In the background console, tell Memex to connect to Storex Hub: `await bgModules.storexHub.connect({ development: true })` 27 | 1. Access your Memex data through the CLI: 28 | 1. In the cloned repository, execute `yarn cli operations:execute --remote io.worldbrain.memex '["findObjects", "customLists", {}]'` 29 | 1. For more ways to interact with your data, see [Integrating with Memex](/storex-hub/guides/memex/) 30 | 31 | ### 2. I want to import/export Memex data in other apps 32 | 33 | 1. [Follow the plugin development guide](/storex-hub/guides/plugin-dev-guide/) 34 | 2. [Understand the Memex API endpoints](/storex-hub/guides/memex/) 35 | 3. [Bundle & load it as a Storex Hub plugin](storex-hub/guides/plugin-dev-guide/?id=_4-bundling-and-installing-a-plugin) 36 | 37 | ### 3. I want to develop my own plugin 38 | 39 | [Follow the plugin development guide](/storex-hub/guides/plugin-dev-guide/) 40 | 41 | ### 4. I want to access the Storex Hub / Memex API from an external application 42 | 43 | 1. [Understand the Memex API endpoints](/storex-hub/guides/memex/) 44 | 2. [Understand the Storex Hub API](/storex-hub/api-reference/) 45 | 3. [Understand the plugin boilerplate](https://github.com/WorldBrain/storex-hub-boilerplate) 46 | 4. [Read into the Plugin Development Guide](/storex-hub/guides/plugin-dev-guide/) 47 | -------------------------------------------------------------------------------- /docs/storex-hub/getting-started/README.old.md: -------------------------------------------------------------------------------- 1 | # Getting started with Storex Hub 2 | 3 | Storex Hub is a Node.js server meant to run on the device(s) of its users, providing a REST/Socket.io API for other applications to connect to. As such, first we'll download and run Storex Hub, after which we'll write and run separate apps connecting to it. 4 | 5 | ## Install and run 6 | 7 | First, clone the Storex Hub repository, install the dependencies with `npm install`, then run this (on Linux or Mac): 8 | 9 | ``` 10 | DB_PATH=./db.sqlite3 yarn start 11 | ``` 12 | 13 | This will run Storex Hub and store all its data in the database file you specified. If you'd rather experiment without any data being persisted, you can omit the DB_PATH and it'll run entirely from memory. 14 | 15 | ## Connecting an app 16 | 17 | Storex Hub mediates between different apps which expose their data and allow you to interact with it. We differentiate between normal apps and remote apps, with remote apps running in a different process, storing their data themselves instead of letting Storex Hub take care of its storage. This allows you to store the data wherever you want and allow access to it by other apps however you want. 18 | 19 | Since only remote apps are supported for now, we'll start by creating one of those. Storex Hub is designed so you can connect to it in different ways, but for now only REST and Socket.io are supported. In Typescript/Javascript, there are convenience classes that wrap the Storex Hub API, while not tying you down to a specific protocol, so either: 20 | 21 | 1. In the Storex Hub repo you just cloned, just copy the `/demos/gist-sharer` directory to something like `/demo/getting-started`, modify the in `main.ts` and run it with `yarn ts-node demos/getting-started/main.ts`. 22 | 2. Set up your own Node.js project and install the `@worldbrain/storex-hub` and `socket.io-client` dependencies. The `@worldbrain/storex-hub` package contains both the entire Storex Hub application (which we won't at the start) and the client, which gives you a convenient wrapper around the API and will in the future be available as a separate package. 23 | 24 | Now, it's time to connect to Storex Hub and register our new app: 25 | 26 | ```js 27 | import io from "socket.io-client"; 28 | import { StorexHubApi_v0 } from "@worldbrain/storex-hub/lib/public-api"; 29 | import { createStorexHubSocketClient } from "@worldbrain/storex-hub/lib/client"; 30 | 31 | const APP_NAME = "getting-started"; 32 | 33 | interface Application { 34 | // All the stuff making up our application, to be filled out later 35 | } 36 | 37 | async function setupApplication(): Promise { 38 | // We'll fill this in later to set up our own database 39 | return {}; 40 | } 41 | 42 | async function setupClientCallbacks( 43 | application: Application 44 | ): Promise { 45 | // We'll fill this in later to allow Storex Hub to interact with our application 46 | return {}; 47 | } 48 | 49 | async function setupClient(application: Application) { 50 | // Storex Hub run on port 50483 in development, and 50482 when 51 | // started with NODE_ENV=production 52 | const socket = io(`http://localhost:50483`); 53 | const client = await createStorexHubSocketClient(io, { 54 | callbacks: await setupClientCallbacks(application) 55 | }); 56 | // At this point, the connection is ready to be used 57 | return client; 58 | } 59 | 60 | async function subscribeToEvents(client: StorexHubApi_v0) { 61 | // We'll fill this in later 62 | } 63 | 64 | async function register(client: StorexHubApi_v0) { 65 | const registrationResult = await client.registerApp({ 66 | name: APP_NAME, 67 | remote: true, // we're a remote app 68 | identify: true // not only register, but also identify in one go 69 | }); 70 | if (!registrationResult.status !== "success") { 71 | throw new Error( 72 | `Couldn't register app '${APP_NAME}'": ${registrationResult.errorText}` 73 | ); 74 | } 75 | 76 | // Save this access token somewhere so you can use it later to identify 77 | return registrationResult.accessToken; 78 | } 79 | 80 | async function identify(client: StorexHubApi_v0, accessToken: string) { 81 | // This is how you identify on next startup 82 | const identificationResult = await client.identifyApp({ 83 | name: APP_NAME, 84 | accessToken 85 | }); 86 | if (identificationResult.status !== "success") { 87 | throw new Error( 88 | `Couldn't identify app '${APP_NAME}': ${identificationResult.status}` 89 | ); 90 | } 91 | } 92 | 93 | async function doStuff(client: StorexHubApi_v0) { 94 | // We'll show how to execute remote operations here later 95 | } 96 | 97 | async function main() { 98 | const application = await setupApplication(); 99 | const client = await setupClient(application); 100 | const accessToken = await register(client); 101 | await subscribeToEvents(client); 102 | await doStuff(client); 103 | } 104 | ``` 105 | 106 | ## Accepting remote operations from other applications 107 | 108 | At this point, you can start doing useful things with the `client`. One of them, since we've told Storex Hub we're a remote application, is to accept remote operations. Normally, remote operations are standard [Storex operations](/guides/storage-operations/) like `createObject`, `findObjects`, etc. that you can directly feed into your own Storex storage set up (like an SQLite or Firestore database), but really you could implement whatever operations you see fit. This is an example with an in-memory Dexie database, for which we'll change a few things: 109 | 110 | ```js 111 | import StorageManager, { 112 | StorageBackend, 113 | StorageRegistry 114 | } from "@worldbrain/storex"; 115 | import { DexieStorageBackend } from "@worldbrain/storex-backend-dexie"; 116 | import inMemory from "@worldbrain/storex-backend-dexie/lib/in-memory"; 117 | 118 | interface Application { 119 | storageManager: StorageManager; 120 | } 121 | 122 | async function createStorage() { 123 | const backend = new DexieStorageBackend({ 124 | dbName: APP_NAME, 125 | idbImplementation: inMemory() 126 | }); 127 | 128 | const storageManager = new StorageManager({ backend }); 129 | storageManager.registry.registerCollections({ 130 | todoItem: { 131 | version: new Date("2020-03-03"), 132 | fields: { 133 | label: { type: "text" }, 134 | done: { type: "boolean" } 135 | } 136 | } 137 | }); 138 | await storageManager.finishInitialization(); 139 | 140 | return storageManager; 141 | } 142 | 143 | async function setupApplication() { 144 | return { 145 | storageManager: await createStorage() 146 | }; 147 | } 148 | 149 | async function setupClientCallbacks( 150 | application: Application 151 | ): Promise { 152 | return { 153 | handleRemoteOperation: async event => { 154 | // event.operation will be an operation array, like: 155 | // ['createObject', 'todoItem', { label: 'Test', done: false }] 156 | // or 157 | // ['findObjects', 'todoItem', { done: false }] 158 | 159 | // Normally you'd do some kind of validation or access control here 160 | return { 161 | result: await this.dependencies.storageManager.operation( 162 | event.operation[0], 163 | ...event.operation.slice(1) 164 | ) 165 | }; 166 | } 167 | }; 168 | } 169 | ``` 170 | 171 | Now, other applications can query your in-memory database! Normally, you'd plug in a real database here, probably SQLite for local storage. As mentioned, you can handle storage operations however you want, so one might imagine a graph storage app for example that can handle SPARQL queries. Just remember to properly namespace your non-standard operations to prevent confusion, like `my-app:specialQuery`. 172 | 173 | ## Interacting with other remote applications 174 | 175 | Once you have other applications identified with Storex Hub, you can interact with their data: 176 | 177 | ```js 178 | async function doStuff(client: StorexHubApi_v0) { 179 | // Get everything from Memex tagged with 'share-example' 180 | const tagsResponse = await options.client.executeRemoteOperation({ 181 | app: "memex", 182 | operation: ["findObjects", "tags", { name: "share-example" }] 183 | }); 184 | if (tagsResponse.status !== "success") { 185 | throw new Error(`Error while fetching URLs for tag '${SHARE_TAG_NAME}'`); 186 | } 187 | const pageUrls = (tagsResponse.result as Array<{ url: string }>).map( 188 | tag => tag.url 189 | ); 190 | // do something with pageUrls 191 | } 192 | ``` 193 | 194 | Also, since not every application will be running all the time, you can listen for availibility changes: 195 | 196 | ```js 197 | async function subscribeToEvents(client: StorexHubApi_v0) { 198 | await client.subscribeToEvent({ 199 | request: { 200 | type: "app-availability-changed" 201 | } 202 | }); 203 | } 204 | 205 | async function setupClientCallbacks( 206 | application: Application 207 | ): Promise { 208 | return { 209 | handleEvent: async ({ event }) => { 210 | if (event.type === "storage-change" && event.app === "memex") { 211 | handleMemexStorageChange(event.info, { 212 | client: client, 213 | settings: { 214 | githubToken 215 | } 216 | }); 217 | } else if ( 218 | event.type === "app-availability-changed" && 219 | event.app === "memex" 220 | ) { 221 | const message = 222 | "Changed Memex availability: " + (event.availability ? "up" : "down"); 223 | } 224 | } 225 | }; 226 | } 227 | ``` 228 | 229 | ## Listening for storage changes 230 | 231 | We can also signal changes to our storage, and listen for changes in the storage of other applications. This is useful for example to share pages tagged with a certain tag from Memex right when a user tags the page. 232 | 233 | This is how we can signal changes: 234 | 235 | ```js 236 | import { StorageOperationEvent } from "@worldbrain/storex-middleware-change-watcher/lib/types"; 237 | 238 | async function doStuff(client: StorexHubApi_v0) { 239 | // See https://github.com/WorldBrain/storex-middleware-change-watcher/blob/master/ts/index.test.ts to see what these changes can look like 240 | await client.emitEvent({ 241 | event: { 242 | type: "storage-change", 243 | info: { 244 | originalOperation: [ 245 | "createObject", 246 | "todoListItem", 247 | { label: "Test", done: false } 248 | ], 249 | info: { 250 | changes: [ 251 | { 252 | type: "create", 253 | collection: "user", 254 | pk: 1, 255 | values: { label: "Test", done: false } 256 | } 257 | ] 258 | } 259 | } 260 | } 261 | }); 262 | } 263 | 264 | async function setupClientCallbacks( 265 | application: Application 266 | ): Promise { 267 | let subscriptionCount = 0; 268 | return { 269 | handleSubscription: async ({ request }) => { 270 | // request.collection is which collection to subscribe to 271 | 272 | // You can use this returned ID to handle unsubscriptions 273 | return { subscriptionId: (++subscriptionCount).toString() }; 274 | }, 275 | handleUnsubscription: async ({ subscriptionId }) => {} 276 | }; 277 | } 278 | ``` 279 | 280 | This is how we can listen for changes: 281 | 282 | ```js 283 | async function subscribeToEvents(client: StorexHubApi_v0) { 284 | const subscriptionResult = await client.subscribeToEvent({ 285 | request: { 286 | type: "storage-change", 287 | app: "memex", 288 | collections: ["tags"] 289 | } 290 | }); 291 | if (subscriptionResult.status === "success") { 292 | console.log("Successfuly subscribed to Memex storage changes"); 293 | } else { 294 | console.log( 295 | "Could not subscribe to Memex storage changes (yet?):", 296 | subscriptionResult.status 297 | ); 298 | } 299 | } 300 | 301 | async function setupClientCallbacks( 302 | application: Application 303 | ): Promise { 304 | return { 305 | handleEvent: async ({ event }) => { 306 | if (event.type === "storage-change" && event.app === "memex") { 307 | // do something with event.info 308 | } 309 | } 310 | }; 311 | } 312 | ``` 313 | 314 | ## Testing Storex Hub integrations 315 | 316 | Remember how we've said earlier that the `@worldbrain/storex-hub` package contains the entire Storex Hub application which you can run from memory? You can use this to easily integration test your applications with Storex Hub. Using Mocha/Jest: 317 | 318 | ```ts 319 | import { createMultiApiTestSuite } from "@worldbrain/storex-hub/lib/tests/api/index.tests"; 320 | 321 | // Creates a test suite that tests both in-memory IndexedDB database and 322 | // with an on-disk SQLite database 323 | createMultiApiTestSuite("Integration Memex + Backup", ({ it }) => { 324 | it("should work", async ({ createSession }) => { 325 | const memex = await createSession({ 326 | type: "websocket", 327 | callbacks: {} 328 | }); 329 | await memex.registerApp({ name: "memex", identify: true }); 330 | // ... do things here 331 | 332 | const backup = await createSession({ 333 | type: "websocket", 334 | callbacks: {} 335 | }); 336 | await backup.registerApp({ name: "backup", identify: true }); 337 | // ... do things here 338 | }); 339 | }); 340 | ``` 341 | 342 | ## Next steps 343 | 344 | Check out the [demos](https://github.com/WorldBrain/storex-hub/tree/master/demos) for more full-fledged example combining the steps above. Also, the full Storex Hub and callback APIs have type definitions [here](https://github.com/WorldBrain/storex-hub/tree/master/ts/public-api). Finally, the [API tests](https://github.com/WorldBrain/storex-hub/tree/master/ts/tests/api) are the best way for now to see up-to-date usage examples of Storex Hub. Gradually, as the API becomes more stable, everything will be documented in a nicer way, 345 | -------------------------------------------------------------------------------- /docs/storex-hub/guides/README.md: -------------------------------------------------------------------------------- 1 | - [Store your own data](/storex-hub/guides/storing-data/) 2 | - [Interacting with other external applications](/storex-hub/guides/remote-apps/) 3 | - [Storing your user-configurable settings](/storex-hub/guides/settings/) 4 | - [Integrating with Memex](/storex-hub/guides/memex/) 5 | -------------------------------------------------------------------------------- /docs/storex-hub/guides/memex/README.md: -------------------------------------------------------------------------------- 1 | # Integrating with Memex 2 | 3 | Integrating with Memex allows you to: 4 | 5 | 1. Listen to data changes in Memex and process them 6 | 2. Index urls on demand 7 | 3. Export & import data on demand 8 | 9 | ## Setting up the environment. 10 | 11 | ### Enable feature that Memex can talk to outside applications 12 | 13 | By default Memex does not connect to Storex Hub. To open the connection open the background console of Memex and type: 14 | 15 | ##### Go to the background console of the extension 16 | 17 | - Firefox: [about:devtools-toolbox?type=extension&id=info%40worldbrain.io](about:devtools-toolbox?type=extension&id=info%40worldbrain.io) 18 | - Chrome: `chrome://extensions > enable developer mode > Memex > background page > console` 19 | 20 | ##### Enter these commands into the console 21 | 22 | - When you're developing a plugin using the development version of Storex Hub: `await bgModules.storexHub.connect({ development: true })` 23 | - To connect to your production, standalone version of Storex Hub: `await bgModules.storexHub.connect({})` 24 | 25 | ### Setup your Storex Hub developer environment 26 | 27 | Follow the [Storex Hub development guide](/storex-hub/guides/plugin-dev-guide/) 28 | 29 | ### Create an plugin or connect an external app 30 | 31 | - Create your client (in an external app) or get the API (in a plugin) with a `handleEvent` callback. ([Example](https://github.com/WorldBrain/storex-hub-integration-memex-arweave/blob/878bf121bfba36ddf734dead9eba9e1272b61764/ts/application.ts#L44)) 32 | - You listen to when Memex is started again ([Example 1](https://github.com/WorldBrain/storex-hub-integration-memex-arweave/blob/878bf121bfba36ddf734dead9eba9e1272b61764/ts/application.ts#L162), [Example 2](https://github.com/WorldBrain/storex-hub-integration-memex-arweave/blob/878bf121bfba36ddf734dead9eba9e1272b61764/ts/application.ts#L47)) 33 | 34 | ## Use Cases: 35 | 36 | ### Use Case 1: Listen and process data changes in Memex 37 | 38 | With this method you can listen to every change in Memex and process it individually. 39 | Memex & your browser needs to be running to receive and process changes. 40 | 41 | An example implementation can be found [here](https://github.com/WorldBrain/storex-hub-integration-memex-arweave/blob/878bf121bfba36ddf734dead9eba9e1272b61764/ts/application.ts#L43). 42 | 43 | **Note:** Memex/Storex Hub do not buffer changes yet. Meaning in order for the connections to work, both Storex Hub and Memex need to run. Get in touch with us [via the chat](https://worldbrain.slack.com/join/shared_invite/enQtOTcwMjQxNTgyNTE4LTRhYTAzN2QwNmM3ZjQwMGE5MzllZDM3N2E5OTFjY2FmY2JmOTY3ZWJhNGEyNWRiMzU5NTZjMzU0MWJhOTA2ZDA) if you like to contribute to improving this. 44 | 45 | ### Use Case 2: Index urls on demand 46 | 47 | You can use Memex internal indexing process to add new urls to your Memex history/bookmarks. 48 | 49 | ```js 50 | interface IndexPageArgs { 51 | url: string; 52 | visitTime?: number; 53 | bookmark?: true | { creationTime: number }; 54 | tags?: string[]; 55 | } 56 | ``` 57 | 58 | ```js 59 | const { status } = await api.executeRemoteCall({ 60 | app: "io.worldbrain.memex", 61 | call: "indexPage", 62 | args: { 63 | url: "https://en.wikipedia.org/wiki/Memex", 64 | visitTime: Date.now(), // Is set to Date.now() if omitted 65 | bookmark: true, 66 | tags: ["my-tag", "my-tag2"], 67 | customLists: ["", ""], // You find a list's ID via the background console (see data explorer guide in Use Case 3) 68 | }, 69 | }); 70 | if (status === "success") { 71 | // do something 72 | } 73 | ``` 74 | 75 | ### Use Case 3: Import/Export data on demand: 76 | 77 | With this method you can query Memex data or save new things to the database. An example can be found [here](https://github.com/WorldBrain/storex-hub-integration-memex-arweave/blob/878bf121bfba36ddf734dead9eba9e1272b61764/ts/application.ts#L76). The storage operations you can execute with `executeRemoteOperation()` are the standard Storex operations described [here](/guides/storage-operations/) and [here](/guides/quickstart/?id=manipulating-data). 78 | 79 | If you want to inspect the data model of Memex, you can do so via the database explorer of the broswer you're running Memex in: 80 | 81 | - Firefox: [about:devtools-toolbox?type=extension&id=info%40worldbrain.io](about:devtools-toolbox?type=extension&id=info%40worldbrain.io) `storage > IndexedDB > memex` 82 | - Chrome: `chrome://extensions > enable developer mode > Memex > background page > Application > IndexedDB > memex` 83 | 84 | #### Examples: 85 | 86 | ##### Get all pages tagged with `share-test`: 87 | 88 | ```js 89 | const { status, result: pages } = await api.executeRemoteOperation({ 90 | app: "io.worldbrain.memex", 91 | operation: ["findObjects", "tags", { name: "share-test" }], 92 | }); 93 | ``` 94 | 95 | ##### Get all pages visited in the last 3 hours: 96 | 97 | ```js 98 | const { status: result: visits } = await api.executeRemoteOperation({ 99 | app: "io.worldbrain.memex", 100 | operation: [ 101 | "findObjects", 102 | "visits", 103 | { time: { $gt: Date.now() - 1000 * 60 * 3 } }, 104 | ], 105 | }); 106 | ``` 107 | 108 | ##### Get page details by URL 109 | 110 | With the above commands you will only get the respective url back (and tag/visit time respectively). 111 | To get all page data, like `title`, `text` or other metadata, you need to then loop through your previous results and fetch the page data with the following command. 112 | 113 | ```js 114 | const { status, result: page } = await api.executeRemoteOperation({ 115 | app: "io.worldbrain.memex", 116 | operation: ["findObjects", "pages", { url: "test.com" }], 117 | }); 118 | ``` 119 | 120 | ##### Add a new page object 121 | 122 | You can either full-text index a `url` via the Memex indexing function (2.2) or manually add a page object if you have all necessary data already. 123 | Note: This is not indexing the page, just populating the database with the data you entered. 124 | 125 | ```js 126 | const { status, result: page } = await api.executeRemoteOperation({ 127 | app: "io.worldbrain.memex", 128 | operation: [ 129 | "createObject", 130 | "pages", 131 | { 132 | canonicalUrl: undefined, 133 | domain: "mozilla.org", 134 | fullTitle: 135 | "Source Code Upload :: WorldBrain's Memex :: Add-ons for Firefox", 136 | fullUrl: "https://addons.mozilla.org/en-GB/developers/addon/worldbrain", 137 | hostname: "addons.mozilla.org", 138 | text: "lorem ipsum", 139 | url: "addons.mozilla.org/en-GB/developers/addon/worldbrain", 140 | }, 141 | ], 142 | }); 143 | ``` 144 | 145 | ##### Add a tag to an existing page 146 | 147 | Make sure the respective PAGE object (4. or 2.2) does already exist otherwise the tag can't be displayed 148 | 149 | ```js 150 | const { status, result: page } = await api.executeRemoteOperation({ 151 | app: "io.worldbrain.memex", 152 | operation: ["createObject", "tags", { url: "test.com", name: "my-tag" }], 153 | }); 154 | ``` 155 | 156 | ##### Create a new list 157 | 158 | You can either add a page to an existing list, or adding a new list. But in order to add an item to a list (6.), the list needs to exist beforehand. 159 | 160 | ```js 161 | const { status, result: page } = await api.executeRemoteOperation({ 162 | app: "io.worldbrain.memex", 163 | operation: [ 164 | "createObject", 165 | "customLists", 166 | { 167 | createdAt: Date.now(), 168 | isDeletable: 1, // 0 for lists that can't be removed like the "saved from mobile" list 169 | isNestable: 1, // non-used parameter preparing us for nested lists 170 | name: "Great tools for thought", 171 | }, 172 | ], 173 | }); 174 | ``` 175 | 176 | ##### Get all Mememx lists 177 | 178 | ```js 179 | const { status, result: page } = await api.executeRemoteOperation({ 180 | app: "io.worldbrain.memex", 181 | operation: ["findObjects", "customLists"], 182 | }); 183 | ``` 184 | 185 | ##### Find a specific Memex list with its name 186 | 187 | ```js 188 | const { status, result: page } = await api.executeRemoteOperation({ 189 | app: "io.worldbrain.memex", 190 | operation: ["findObject", "customLists", { name: "Great tools for thought" }], 191 | }); 192 | ``` 193 | 194 | ##### Add a new item to a list 195 | 196 | To add a new item to a list make sure the PAGE object does already exist otherwise the entry won't appear in the list 197 | 198 | ```js 199 | const { status, result: page } = await api.executeRemoteOperation({ 200 | app: "io.worldbrain.memex", 201 | operation: [ 202 | "createObject", 203 | "pageListEntries", 204 | { 205 | createdAt: Date.now(), 206 | fullUrl: "https://www.test.com/about-us", 207 | listId: 12345678, //existing or newly created through process "create new list" 208 | pageUrl: "test.com/about-us", // normalised format without http(s) and www. 209 | }, 210 | ], 211 | }); 212 | ``` 213 | 214 | ##### Add a bookmark to an existing page 215 | 216 | Make sure the PAGE object does already exist otherwise the bookmark status can't be displayed. It will be saved though. 217 | 218 | ```js 219 | const { status, result: page } = await api.executeRemoteOperation({ 220 | app: "io.worldbrain.memex", 221 | operation: [ 222 | "createObject", 223 | "bookmarks", 224 | { url: "test.com/about-us", time: Date.now() }, 225 | ], 226 | }); 227 | ``` 228 | 229 | ## Understanding Memex' data model 230 | 231 | ### How to read this 232 | 233 | - Optional fields are marked with `?` 234 | - Generally the ID of each data class is the first field (mostly `url`) 235 | - Exceptions: 236 | - annotations: The `url` + `createdWhen` field ID in the format: `http://www.page.com#creationTimestamp` 237 | - annotBookmarks: The respective annotation's url in the format: `http://www.page.com#creationTimestamp` 238 | - pageListEntries: The customLists ID + `url` as an array the format: `[1, 'http://www.page.com']` 239 | - `url` fields in `pages`, `bookmarks`, `visits` and `pageListEntries` are normalized URLs. 240 | - All the data classes with an arrow to another class can't be displayed without their existence. (e.g. `bookmarks` status without having a `pages` object) 241 | - If you want to inspect the data model of Memex, you can do so via the browser database explorer. 242 | - **This is how to find it:** 243 | - Firefox: [about:devtools-toolbox?type=extension&id=info%40worldbrain.io](about:devtools-toolbox?type=extension&id=info%40worldbrain.io) `storage > IndexedDB > memex` 244 | - Chrome: `chrome://extensions > enable developer mode > Memex > background page > Application > IndexedDB > memex` 245 | 246 | ```mermaid 247 | classDiagram 248 | pages <|-- visits 249 | pages <|-- bookmarks 250 | pages <|-- tags 251 | pages <|-- annotations 252 | pages <|-- pageListEntries 253 | customLists <|-- pageListEntries 254 | customLists <|-- annotListEntries 255 | annotations <|-- annotListEntries 256 | annotations <|-- annotBookmarks 257 | 258 | class pages { 259 | url: string 260 | fullUrl: text 261 | domain: string 262 | hostname: string 263 | fullTitle?: text 264 | text?: text 265 | lang?: string 266 | canonicalUrl?: string 267 | description?: text 268 | } 269 | class visits { 270 | url: string 271 | time: timestamp 272 | } 273 | class bookmarks { 274 | url: string 275 | time: timestamp 276 | } 277 | class tags { 278 | url: string 279 | name: string 280 | } 281 | class customLists { 282 | id: int 283 | name: string 284 | isDeletable?: boolean 285 | isNestable?: boolean 286 | createdAt: datetime 287 | } 288 | class pageListEntries { 289 | listId: int 290 | pageUrl: string 291 | fullUrl: string 292 | createdAt: datetime 293 | } 294 | class annotations { 295 | url: string 296 | createdWhen: datetime 297 | lastEdited: datetime 298 | pageTitle: text 299 | pageUrl: string 300 | body?: text 301 | comment?: text 302 | selector?: json 303 | } 304 | class annotBookmarks { 305 | url: string 306 | createdAt: datetime 307 | } 308 | ``` 309 | -------------------------------------------------------------------------------- /docs/storex-hub/guides/plugin-dev-guide/README.md: -------------------------------------------------------------------------------- 1 | # Plugin Development Guide 2 | 3 | ## Setting up the developer environment 4 | 5 | Type the following commands in a new Terminal window. 6 | 7 | ``` 8 | $ git clone git@github.com:WorldBrain/storex-hub.git 9 | $ cd storex-hub 10 | $ npm install 11 | $ DB_PATH='./dev/db' PLUGINS_DIR='./dev/db/plugins' npm start 12 | ``` 13 | 14 | With this command Storex Hub runs in development mode on port `50483`. (Production runs on `50842`) 15 | 16 | Access the running Storex Hub Interface with 17 | `http://localhost:50483/management` 18 | 19 | With these steps, you manually specified another database directory and plugins directory. These things ensure you can safely develop your apps without messing up important data in your main Storex Hub, and can isolate the development environment of different apps you may be writing. 20 | 21 | Your DB location on disk in the folder you've specified with DB_PATH. You can use a separate one for each app you're developing, so you have isolated environments. This way your data is preserved between restarts and if you have an external application that stores the access token, you can re-use that access token. If you run Storex Hub in memory, it will forget those access token, so you also need to forget it and recreate your app from your external application. To run Storex Hub in memory, just omit the DB_PATH. 22 | 23 | ## External applications vs. plugins 24 | 25 | You can interact with Storex Hub from external applications, or by writing a plugin. 26 | 27 | An external application connects through Storex Hub through the WebSocket or HTTP API, can be written use any programming language, and can be started and stopped independently of Storex Hub. External applications register themselves through the API and get an access token back they can use for subsequent sessions. An example of an external application could be an existing product like Memex (running as a web extension) or Evernote. Also, it is useful to run plugins as external applications during development, so you can restart the plugin quickly when you change it instead of compiling your plugin on every change and reloading Storex Hub. 28 | 29 | Plugins on the other hand, are a bundle of packaged code with a manifest telling Storex Hub how to display the plugin to the user and how to run it. They get loaded as soon as Storex Hub starts, and you don't have to manage your own access tokens. 30 | 31 | Both external applications and plugins register apps, which have their own ID and can have settings, data and functionality associated with them. This is why an app can run both as an external application and a plugin. 32 | 33 | We'll start by running our plugin as an external application, then package it when it's ready. 34 | 35 | ## Understanding the structure of Storex Hub Plugins 36 | 37 | As a starting point, let's clone the [Storex Hub plugin boilerplate](https://github.com/WorldBrain/storex-hub-boilerplate): 38 | 39 | ``` 40 | git clone git@github.com:WorldBrain/storex-hub-boilerplate.git 41 | cd storex-hub-boilerplate 42 | npm install 43 | ``` 44 | 45 | To start using the boilerplate, we'll have to change two things: 46 | 47 | - `manifest.json`: the `identifier` field to your own reverse-domain name identifier, like `com.example.test` 48 | - `ts/constants.ts`: change the `APP_NAME` to the same identifier. 49 | 50 | For this guide, we'll use the `com.example.guide` identifier. 51 | 52 | ### The manifest 53 | 54 | In the boilerplate, you'll find the `manifest.json` file, describing a few things Storex Hub needs to know about the plugin before running it: 55 | 56 | | Key | Description | Expected Value | 57 | | ------------- | -------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | 58 | | identifer | ID of your application. | Reverse domain name convention with the reversed domain name of your organization, followed the plugin name, like `io.worldbrain.twitter` for a Twitter integration written by WorldBrain. The boilerplate comes with a default app identifier, so change the `APP_NAME` in `ts/constant.ts` to something like: `const APP_NAME = "com.test.example";` | 59 | | version | Version Number of your application | Follow the [SemVer](https://semver.org/) convention. | 60 | | siteUrl | Url to docs or website your plugin links in the Storex Hub dashboard | Any URL you want | 61 | | mainPath | Path relative to the plugin root contain the plugin code | A relative path, like plugin.js | 62 | | entryFunction | Which function in the main JS file to call to initialize the plugin | A function name, like `main` | 63 | 64 | ### The scaffolding 65 | 66 | Since we have two ways of running the code (as an external application, or a plugin), we also have two different entry points into our program in the boilerplate: 67 | 68 | - `ts/main.ts`: the entry point when running the plugin as an external application. In here, we have to manually take care of establishing the connection to Storex Hub, registering or identifying our app, and storing its access token. The standard Storex Hub client (found in `@worldbrain/storex-hub/lib/client) exposes the standard Storex Hub API while hiding the Websocket or HTTP API details. 69 | - `ts/plugin.ts`: the entry point when Storex Hub loads the code as a plugin (as defined in `mainPath` of the `manifest.json`), which should contain a function with the name you specified in `entryFunction` of the `manifest.json` (`main` in the case of the boilerplate. This function gets called with a single parameter containing a `getApi(options)` function which gives you direct access to the standard Storex Hub API. 70 | 71 | ### The actual functionality of the plugin 72 | 73 | Shared between `ts/main.ts` and `ts/plugin.ts` is `ts/application.ts` which contains a class in which we can do things like listening for storage changes in other apps, define our own data schema and store data in it, and expose and consume functionality to and from other apps. 74 | 75 | The communication between your application happens in two directions: 76 | 77 | - Your code calls Storex Hub, for example to store data/settings, call functionality exposed by other apps, subscribe to events, etc. 78 | - Storex Hub calls into your code through callbacks, for example to deliver an event you registered to, to use consume a call you've exposed, etc. 79 | 80 | Callbacks are passed in when you get access to the API, whether that's through the [WebSocket/HTTP client](https://github.com/WorldBrain/storex-hub-boilerplate/blob/7f5f0d7b86d144e85a59e0f1c57ab27d9214a54e/ts/main.ts#L14), or [using the `getApi(options)` function](https://github.com/WorldBrain/storex-hub-boilerplate/blob/7f5f0d7b86d144e85a59e0f1c57ab27d9214a54e/ts/plugin.ts#L6) you get passed into your plugin entry function. 81 | 82 | As an example, we'll expose a remote call to be used by other applications or integration recipes. There will be guides how to use other functionality exposed by Storex Hub. 83 | 84 | ## Exposing a remote call 85 | 86 | As a simple example, we'll a remote call that you can pass in a name, and returns `Hello !` back. In a real app, this could be retrieving something from Twitter, sending an e-mail, storing something on IPFS, etc. You can see that in `application.ts`, in the `getCallbacks()` method, we already have some callbacks: 87 | 88 | ```js 89 | return { 90 | handleEvent: async ({ event }) => { 91 | if (event.type === "storage-change" && event.app === "memex") { 92 | this.handleMemexStorageChange(event.info); 93 | } else if ( 94 | event.type === "app-availability-changed" && 95 | event.app === "memex" 96 | ) { 97 | this.logger( 98 | "Changed Memex availability:", 99 | event.availability ? "up" : "down" 100 | ); 101 | if (event.availability) { 102 | this.tryToSubscribeToMemex(); 103 | } 104 | } 105 | }, 106 | }; 107 | ``` 108 | 109 | Change that to this: 110 | 111 | ```js 112 | return { 113 | handleRemoteCall: async ({ call, args }) => { 114 | if (call === "sayHello") { 115 | return { status: "success", result: `Hello ${args.name}!` }; 116 | } else { 117 | return { status: "call-not-found" }; 118 | } 119 | }, 120 | }; 121 | ``` 122 | 123 | Start your app by executing this in the plugin boilerplate: 124 | 125 | ``` 126 | $ yarn start 127 | ``` 128 | 129 | Now, there's two ways of consuming your call. The first is through the CLI, which you can access by running this in the Storex Hub repository you cloned before. We're using the `com.example.guide` identifier here, but be sure to use the one you configured earlier in this guide: 130 | 131 | ``` 132 | $ yarn cli calls:execute com.example.guide sayHello '{"name": "Vincenzo"}' 133 | ``` 134 | 135 | Other apps can also consume calls through the API: 136 | 137 | ```js 138 | const callResult = await pocket.executeRemoteCall({ 139 | app: "com.example.guide", 140 | call: "sayHello", 141 | args: { 142 | name: "Vincenzo", 143 | }, 144 | }); 145 | expect(callResult).toEqual({ 146 | status: "success", 147 | result: "Hello Vincenzo!", 148 | }); 149 | ``` 150 | 151 | You'll notice a pattern here of every call returning a single object with a status of `success`, or an error. Every call, both into Storex Hub and callbacks, implement this pattern 152 | 153 | ## Bundling and installing the app as a plugin 154 | 155 | Now that our app is working, we can bundle it as a plugin to distribute it to users (although we're still working on making it accessible to non-technical users.) 156 | 157 | In the boilerplate, run this command, which generates the bundle plugin in the `build/` directory: 158 | 159 | ``` 160 | yarn build:prod 161 | ``` 162 | 163 | Then, you can install the plugin in two ways. The first is through the CLI, again in the Storex Hub repository 164 | 165 | ``` 166 | cd 167 | yarn cli plugins:install /build 168 | ``` 169 | 170 | Or, you can put the `build/` directory in the plugins directory you selected in the beginning with `PLUGINS_DIR`, renaming it to `com.example.guide` and install it through the management interface found at: 171 | 172 | ``` 173 | http://localhost:50483/management/ 174 | ``` 175 | 176 | ## Next steps 177 | 178 | Now you know the basics of interacting with Storex Hub, you can dive into specific functionality you need. Unfortunately, we don't have the resources yet to make nice guides for those, but you can refer to the [API integration tests](https://github.com/WorldBrain/storex-hub/tree/master/ts/tests/api) in the meanwhile. 179 | -------------------------------------------------------------------------------- /docs/storex-hub/guides/remote-apps/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WorldBrain/storex-docs/824bda78bd3c40eae1f31036735f7164b1d5d652/docs/storex-hub/guides/remote-apps/README.md -------------------------------------------------------------------------------- /docs/storex-hub/guides/settings/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WorldBrain/storex-docs/824bda78bd3c40eae1f31036735f7164b1d5d652/docs/storex-hub/guides/settings/README.md -------------------------------------------------------------------------------- /docs/storex-hub/guides/storing-data/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WorldBrain/storex-docs/824bda78bd3c40eae1f31036735f7164b1d5d652/docs/storex-hub/guides/storing-data/README.md -------------------------------------------------------------------------------- /docs/storex-hub/roadmap/README.md: -------------------------------------------------------------------------------- 1 | # Storex Hub Status & Roadmap 2 | 3 | The vision of Storex Hub is a software ecosystem where users have full data ownership, experience no vendor lock-ins and can adapt knowledge tools to their ever changing workflow needs. 4 | For example by providing plugins that import/export data in custom data formats, use novel transport & storage protocols (like IPFS) or connect applications with each other, like with Zapier, or build many different apps that work with the same core data. 5 | 6 | For this to happen, a lot needs to be done, and we're still figuring out in which order to do these things. 7 | As a first step we want to provide the ability for developers to work with their Memex data. 8 | This way we can focus on getting the basics right, without having to invest significantly into making this easy to use for non-developers, maening more work into security, app distribution, UX, etc. 9 | 10 | ## Status 11 | 12 | Currently Storex Hub has 3 functionalities: 13 | 14 | 1. Register a new remote application (with no persistent storage) 15 | 2. Listen to Memex data changes 16 | 3. Query Memex database 17 | 4. Write to Memex database 18 | 19 | This enables use cases like: 20 | 21 | - Post all urls tagged with a specific tag as a Gist [See demo](https://twitter.com/worldbrain/status/1235279061624250369?s=20) 22 | - Import/export data between other services (& Memex), like Hypothes.is. 23 | 24 | ## Potential Roadmap Projects 25 | 26 | #### Making Storex Hub ready for an ecosystem of early developers 27 | 28 | - DONE: Allow different apps to expose their storage to other apps. 29 | - DONE: Allow apps to signal changes to their storage to other apps. 30 | - DONE: Plugin packaging and installation mechanism 31 | 32 | #### Separation of data from apps 33 | 34 | - DONE: Storing data inside of Storex Hub to separate data from apps 35 | - Schema migrations to standardize how data changes as its (explicit or implicit) schema evolves using [storex-schema-migrations](/guides/schema-migrations/) 36 | - Backward compatibility for apps written to interact with old data models of other apps 37 | - Media storage 38 | 39 | #### Inter-app connectivity 40 | 41 | - Discovery of data apps Storex Hub different apps can work with (give me everyhing that's tagged, give me all contacts, etc. ) 42 | - Inter-app data relations (where any apps can attach different information to a stored web page for example) 43 | - Adapt schema migrations to account for inter-app data relations 44 | 45 | #### Wider interoperability 46 | 47 | - Adaptable import/export mechanism to map internal data structures to and from other data formats (web annotations, csv, etc.) 48 | - Standard mechanisms to work with external APIs and datastores, both centralized (cloud databases) and non-centralized (IPFS, Ethereum, etc.) 49 | 50 | #### Multi-device usage 51 | 52 | - Real-time communication between multiple devices 53 | - Message bus for asynchronous communication between multiple devices 54 | - Media storage for usage across devices 55 | 56 | #### Multi-user usage 57 | 58 | - Aggregated identities that allow us to construct a local representation of identities of other users across a variety of protocols and platforms (e-mail, DID, Twitter, etc.) 59 | - Sync-based collaboration protocol for use in small, trusted environments with relatively small data sets 60 | - File-sharing for trusted environments 61 | - Federated collaboration protocol for use in untrusted environments with bigger data sets 62 | - File-sharing for untrusted, larger environments 63 | - Real-time messaging between users 64 | 65 | #### Distribution to non-developers 66 | 67 | - DONE: Easy, cross-platform installation method for Storex Hub 68 | - App distribution and runtime security model 69 | - User-centered data security model explaining in plain terms what different apps might do to your data 70 | - Unified GUI to manage settings and permissions of different apps (DONE: settings) 71 | 72 | #### Notifications 73 | 74 | - A user-centered notification system allowing people to interact with notification in a non-intrusive way 75 | - A mobile client to receive user-centered notifications 76 | 77 | #### Convenience for users 78 | 79 | - Automated backups 80 | - Data history 81 | - Multi-device sync with multiple providers 82 | 83 | ## Interested in collaborating? 84 | 85 | There's many ways to help the development of Storex Hub. [Here](/storex-hub/contact/) you can discover what you could do and how to get in touch with us. 86 | -------------------------------------------------------------------------------- /docs/use-cases/README.md: -------------------------------------------------------------------------------- 1 | # Use cases 2 | 3 | TBD 4 | -------------------------------------------------------------------------------- /notes/journeys.md: -------------------------------------------------------------------------------- 1 | Personas: 2 | 3 | - Software architect 4 | - Software engineer 5 | - Product designer 6 | - Decision maker 7 | 8 | User journeys: 9 | 10 | - Persona: Software architect 11 | 12 | - Journey: I want to see whether Storex fits my requirements 13 | 14 | - Software engineer 15 | 16 | - Journey: I want to learn to use Storex 17 | - Journey: I want to refresh my mind on something 18 | 19 | - Product designer 20 | 21 | - Journey: I want to improve our product development processes 22 | 23 | - Decision maker 24 | 25 | - Journey: One of our engineers wants to use Storex 26 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "storex-docs", 3 | "repository": "git@github.com:WorldBrain/storex-docs.git", 4 | "private": true, 5 | "scripts": { 6 | "start": "docsify serve docs", 7 | "maintainable-docs": "ts-node tools/maintainable-docs/main.ts" 8 | }, 9 | "devDependencies": { 10 | "@types/colors": "^1.2.1", 11 | "@types/glob": "^7.1.1", 12 | "@types/marked": "^0.6.5", 13 | "@types/node": "^12.11.7", 14 | "@types/node-fetch": "^2.5.5", 15 | "docsify-cli": "^4.3.0", 16 | "node-fetch": "^2.6.0", 17 | "ts-node": "^8.4.1", 18 | "typedoc": "^0.15.0", 19 | "typedoc-neo-theme": "^1.0.4", 20 | "typescript": "^3.6.4" 21 | }, 22 | "dependencies": { 23 | "colors": "^1.4.0", 24 | "glob": "^7.1.5", 25 | "marked": "^0.7.0" 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /tools/maintainable-docs/api-reference-generation/index.ts: -------------------------------------------------------------------------------- 1 | import { generateDocumentationObject } from "../utils/typedoc"; 2 | 3 | export default async function generateApiReference(rootPath: string) { 4 | const documentationObject = await generateDocumentationObject(rootPath) 5 | if (!documentationObject) { 6 | return 7 | } 8 | } 9 | 10 | -------------------------------------------------------------------------------- /tools/maintainable-docs/check-coverage.ts: -------------------------------------------------------------------------------- 1 | import { findDocumentationModule, generateDocumentationObject } from "./utils/typedoc"; 2 | 3 | export default async function checkCoverage(rootPath: string) { 4 | const documentationObject = await generateDocumentationObject(rootPath) 5 | if (!documentationObject) { 6 | return 7 | } 8 | console.log(documentationObject) 9 | console.log(findDocumentationModule(documentationObject, 'types/relationships')) 10 | } 11 | -------------------------------------------------------------------------------- /tools/maintainable-docs/check-links.ts: -------------------------------------------------------------------------------- 1 | import fs from 'fs' 2 | import path from 'path' 3 | import glob from 'glob' 4 | import marked from 'marked' 5 | import colors from 'colors/safe' 6 | import fetch, { Response, FetchError } from 'node-fetch' 7 | 8 | interface MarkdownLink { 9 | href: string 10 | title: string 11 | text: string 12 | } 13 | 14 | type DocumentError = { absDocumentPath: string } & ( 15 | { type: 'internal-link.target-absent', link: MarkdownLink } | 16 | { type: 'internal-link.no-trailing-slash', link: MarkdownLink } | 17 | { type: 'internal-link.relative-url-forbidden', link: MarkdownLink } | 18 | { type: 'external-link.not-found', link: MarkdownLink } | 19 | { type: 'external-link.not-ok', link: MarkdownLink } | 20 | { type: 'external-link.error', link: MarkdownLink, error: FetchError } 21 | ) 22 | 23 | async function getMarkdownDocumentPaths(args: { rootDir: string }): Promise { 24 | return glob.sync(`${args.rootDir}/**/*.md`) 25 | } 26 | 27 | async function checkMarkdownDocument(args: { rootDir: string, absDocumentPath: string, content: string }): Promise<{ errors: DocumentError[] }> { 28 | const { links } = await parseMarkdownDocument(args) 29 | 30 | const errors: DocumentError[] = [] 31 | for (const link of links) { 32 | const error = await checkMarkdownLink({ 33 | ...args, 34 | link, 35 | }) 36 | if (error) { 37 | errors.push(error) 38 | } 39 | } 40 | 41 | return { errors } 42 | } 43 | 44 | async function parseMarkdownDocument(args: { rootDir: string; absDocumentPath: string; content: string; }): Promise<{ 45 | links: Array 46 | }> { 47 | const links: Array = [] 48 | const renderer = new marked.Renderer() 49 | renderer.link = (href: string, title: string, text: string) => { 50 | links.push({ href, title, text }) 51 | return '' 52 | } 53 | marked(args.content, { renderer }) 54 | 55 | return { links } 56 | } 57 | 58 | async function checkMarkdownLink(args: { rootDir: string, absDocumentPath: string, link: MarkdownLink }): Promise { 59 | const href = args.link.href 60 | const originalHost = 'worldbrain.github.io' 61 | const parsedUrl = new URL(href, `https://${originalHost}/storex-docs/`) 62 | 63 | if (parsedUrl.host !== originalHost) { 64 | return checkExternalLink(args) 65 | } 66 | 67 | if (parsedUrl.pathname.charAt(0) !== '/') { 68 | return { type: 'internal-link.relative-url-forbidden', ...args } 69 | } 70 | if (parsedUrl.pathname.substr(-1) !== '/') { 71 | return { type: 'internal-link.no-trailing-slash', ...args } 72 | } 73 | 74 | const targetDirWithTrailingSlash = path.join(args.rootDir, parsedUrl.pathname.substr(1)) 75 | const targetFilePath = path.join(targetDirWithTrailingSlash, 'README.md') 76 | const targetExists = fs.existsSync(targetFilePath) 77 | if (!targetExists) { 78 | return { type: 'internal-link.target-absent', ...args } 79 | } 80 | 81 | return null 82 | } 83 | 84 | async function checkExternalLink(args: { absDocumentPath: string, link: MarkdownLink }): Promise { 85 | console.log(`Check external link:`, args.link.href) 86 | 87 | let response: Response 88 | try { 89 | response = await fetch(args.link.href) 90 | } catch (error) { 91 | return { type: 'external-link.error', error, ...args } 92 | } 93 | if (!response.ok) { 94 | if (response.status === 400) { 95 | return { type: 'external-link.not-found', ...args } 96 | } else { 97 | return { type: 'external-link.not-ok', ...args } 98 | } 99 | } 100 | return null 101 | } 102 | 103 | function getHumanReadableError(error: DocumentError): string { 104 | if (error.type === 'internal-link.no-trailing-slash') { 105 | return `${error.absDocumentPath} - found link without trailing slash: ${error.link.href}` 106 | } else if (error.type === 'internal-link.target-absent') { 107 | return `${error.absDocumentPath} - found link to non-existent document: ${error.link.href}` 108 | } else if (error.type === 'internal-link.relative-url-forbidden') { 109 | return `${error.absDocumentPath} - found relative link, but all links must be absolute: ${error.link.href}` 110 | } else { 111 | return `${error.absDocumentPath} - non-human-readable error checking this link: ${error.link.href}` 112 | } 113 | } 114 | 115 | export default async function checkLinks() { 116 | const rootDir = path.join(__dirname, '../../docs') 117 | 118 | const docPaths = await getMarkdownDocumentPaths({ rootDir }) 119 | const errors: DocumentError[][] = await Promise.all(docPaths.map(async absDocumentPath => { 120 | const content = fs.readFileSync(absDocumentPath).toString() 121 | const checkResult = await checkMarkdownDocument({ 122 | rootDir, 123 | absDocumentPath, 124 | content 125 | }) 126 | return checkResult.errors 127 | })) 128 | 129 | for (const errorSet of errors) { 130 | for (const error of errorSet) { 131 | console.error(colors.red('ERROR:'), getHumanReadableError(error)) 132 | } 133 | } 134 | } -------------------------------------------------------------------------------- /tools/maintainable-docs/main.ts: -------------------------------------------------------------------------------- 1 | import generateApiReference from './api-reference-generation'; 2 | import checkLinks from './check-links'; 3 | import checkCoverage from './check-coverage'; 4 | 5 | function usageError(message: string) { 6 | console.error(message) 7 | process.exit(1) 8 | } 9 | 10 | export async function main() { 11 | const command = process.argv[2] 12 | if (!command) { 13 | return usageError('Error: no command specified (must be either check-links or check-coverage)') 14 | } 15 | if (['check-links', 'check-coverage', 'generate-reference'].indexOf(command) === -1) { 16 | return usageError(`Error: unknown command '${command}'`) 17 | } 18 | 19 | if (command === 'check-coverage') { 20 | const rootPath = process.argv[3] 21 | await checkCoverage(rootPath) 22 | } else if (command === 'check-links') { 23 | await checkLinks() 24 | } else if (command === 'generate-reference') { 25 | const rootPath = process.argv[3] 26 | await generateApiReference(rootPath) 27 | } 28 | } 29 | 30 | main() 31 | -------------------------------------------------------------------------------- /tools/maintainable-docs/utils/typedoc.ts: -------------------------------------------------------------------------------- 1 | import fs from 'fs' 2 | import * as TypeDoc from 'typedoc' 3 | 4 | export interface DocumentationObject { 5 | children: any[] 6 | } 7 | 8 | export async function generateDocumentationObject(rootPath: string): Promise { 9 | const app = new TypeDoc.Application({ 10 | mode: 'Modules', 11 | logger: 'console', 12 | target: 'ES5', 13 | module: 'CommonJS', 14 | experimentalDecorators: true, 15 | typeRoots: [ 16 | rootPath + "/node_modules/@types" 17 | ], 18 | }) 19 | 20 | const project = app.convert(app.expandInputFiles([rootPath + '/ts'])) 21 | 22 | if (project) { // Project may not have converted correctly 23 | const tempFilePath = '/tmp/documentation.json' 24 | 25 | // Alternatively generate JSON output 26 | app.generateJson(project, tempFilePath) 27 | 28 | return JSON.parse(fs.readFileSync(tempFilePath).toString()) 29 | } else { 30 | return null 31 | } 32 | } 33 | 34 | export function findDocumentationModule(documentationObject: DocumentationObject, moduleName: string) { 35 | for (const child of documentationObject.children) { 36 | if (child.kindString === 'External module' && child.name === `"${moduleName}"`) { 37 | return child 38 | } 39 | } 40 | 41 | return null 42 | } 43 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "3.6", 3 | "compilerOptions": { 4 | "target": "es5", 5 | "module": "commonjs", 6 | "moduleResolution": "node", 7 | "strict": true, 8 | "esModuleInterop": true, 9 | "allowSyntheticDefaultImports": true, 10 | "removeComments": true, 11 | "lib": ["es2017", "dom", "esnext.asynciterable"], 12 | "noLib": false, 13 | "downlevelIteration": true, 14 | "preserveConstEnums": true, 15 | "declaration": true, 16 | "sourceMap": true, 17 | "suppressImplicitAnyIndexErrors": true, 18 | "typeRoots": ["./node_modules/@types"], 19 | "outDir": "lib" 20 | }, 21 | "filesGlob": ["./tools/**/*.ts", "!./node_modules/**/*.ts"] 22 | } 23 | --------------------------------------------------------------------------------