├── .dockerignore ├── .gitignore ├── architecture.png ├── Dockerfile ├── .github └── workflows │ └── tests.yml ├── tools └── touch.js ├── lib ├── error.js ├── abstract-swarm.js ├── util.js ├── subscription.js ├── ldp.js ├── swarm.js └── server.js ├── bin └── server.js ├── LICENSE ├── package.json ├── test └── util.js └── README.md /.dockerignore: -------------------------------------------------------------------------------- 1 | node_modules -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | /dist 3 | -------------------------------------------------------------------------------- /architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hyperwell/gateway/HEAD/architecture.png -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:12 2 | 3 | WORKDIR /usr/src/app 4 | COPY package*.json ./ 5 | RUN npm ci 6 | 7 | COPY . . 8 | 9 | EXPOSE 8080 10 | CMD ["./bin/server.js", "-p", "8080"] 11 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: tests 2 | on: [push] 3 | jobs: 4 | run: 5 | name: npm test 6 | runs-on: ubuntu-latest 7 | steps: 8 | - uses: actions/checkout@v1 9 | - uses: actions/setup-node@v1 10 | with: 11 | node-version: '12.x' 12 | - run: npm ci 13 | - run: npm test 14 | -------------------------------------------------------------------------------- /tools/touch.js: -------------------------------------------------------------------------------- 1 | const {Repo} = require('hypermerge') 2 | const Hyperswarm = require('hyperswarm') 3 | const {encodeDocUrl} = require('../lib/util') 4 | 5 | const repo = new Repo({memory: true}) 6 | repo.setSwarm(Hyperswarm(), { 7 | lookup: false, 8 | announce: true, 9 | }) 10 | 11 | const url = repo.create({annotations: []}) 12 | const handle = repo.watch(url, () => {}) 13 | 14 | console.log(`New notebook created. 15 | Document URL: ${url} 16 | Encoded URL: ${encodeDocUrl(url)}`) 17 | -------------------------------------------------------------------------------- /lib/error.js: -------------------------------------------------------------------------------- 1 | const ERROR_NOT_FOUND = 1 2 | const ERROR_BAD_DOC = 2 3 | const ERROR_BAD_REQUEST = 3 4 | 5 | class SwarmError extends Error { 6 | constructor(code, message) { 7 | super(message) 8 | this.code = code 9 | } 10 | 11 | static badRequest(message) { 12 | return new SwarmError(ERROR_BAD_REQUEST, message) 13 | } 14 | 15 | static notFound(message) { 16 | return new SwarmError(ERROR_NOT_FOUND, message) 17 | } 18 | 19 | static badDoc(message) { 20 | return new SwarmError(ERROR_BAD_DOC, message) 21 | } 22 | } 23 | 24 | module.exports = { 25 | SwarmError, 26 | ERROR_NOT_FOUND, 27 | ERROR_BAD_DOC, 28 | ERROR_BAD_REQUEST, 29 | } 30 | -------------------------------------------------------------------------------- /bin/server.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | const parseArgs = require('minimist') 3 | const {createServer} = require('../lib/server') 4 | const HyperwellSwarm = require('../lib/swarm') 5 | 6 | async function main() { 7 | const argv = parseArgs(process.argv.slice(2), { 8 | string: ['port', 'host'], 9 | boolean: ['ssl'], 10 | alias: { 11 | port: ['p'], 12 | ssl: ['s'], 13 | }, 14 | default: { 15 | port: '3000', 16 | host: 'localhost:3000', 17 | ssl: false, 18 | }, 19 | }) 20 | 21 | await createServer(new HyperwellSwarm(), Number.parseInt(argv.port), { 22 | ssl: argv.ssl, 23 | host: argv.host, 24 | }) 25 | } 26 | 27 | main() 28 | -------------------------------------------------------------------------------- /lib/abstract-swarm.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert') 2 | 3 | class AbstractSwarm { 4 | async getAnnotations(docUrl) { 5 | assert.fail('`swarm.getAnnotations needs to be implemented.') 6 | } 7 | 8 | async getAnnotation(docUrl, annotationId) { 9 | assert.fail('`swarm.getAnnotation needs to be implemented.') 10 | } 11 | 12 | async createAnnotation(docUrl) { 13 | assert.fail('`swarm.createAnnotation needs to be implemented.') 14 | } 15 | 16 | async updateAnnotation(docUrl, annotation) { 17 | assert.fail('`swarm.updateAnnotation needs to be implemented.') 18 | } 19 | 20 | async deleteAnnotation(docUrl, annotation) { 21 | assert.fail('`swarm.deleteAnnotation needs to be implemented.') 22 | } 23 | 24 | async subscribeToAnnotations(docUrl) { 25 | assert.fail('`swarm.subscribeToAnnotations needs to be implemented.') 26 | } 27 | 28 | async destroy() { 29 | assert.fail('`swarm.destroy needs to be implemented.') 30 | } 31 | } 32 | 33 | module.exports = AbstractSwarm 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2019-2020 Jan Kaßel 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@hyperwell/gateway", 3 | "version": "0.2.0", 4 | "author": "Jan Kaßel ", 5 | "description": "A P2P system that leverages collaboration, local-first principles, and more on W3C Web Annotations", 6 | "private": true, 7 | "bin": { 8 | "hyperwell-gateway": "./bin/server.js" 9 | }, 10 | "scripts": { 11 | "start": "node ./bin/server.js", 12 | "test": "tape test/*.js" 13 | }, 14 | "license": "MIT", 15 | "dependencies": { 16 | "@hapi/boom": "^8.0.1", 17 | "@hapi/hapi": "^18.4.1", 18 | "browserify-package-json": "^1.0.1", 19 | "debug": "^4.1.1", 20 | "etag": "^1.8.1", 21 | "hapi-plugin-websocket": "^2.3.5", 22 | "hypermerge": "github:automerge/hypermerge#63182f", 23 | "hyperswarm": "^2.15.3", 24 | "minimist": "^1.2.5", 25 | "node-cache": "^5.1.0", 26 | "uuid": "^3.4.0" 27 | }, 28 | "devDependencies": { 29 | "husky": "^3.1.0", 30 | "lint-staged": "^9.5.0", 31 | "prettier": "^1.19.1", 32 | "tape": "^4.13.2" 33 | }, 34 | "engines": { 35 | "node": ">=12.0.0" 36 | }, 37 | "husky": { 38 | "hooks": { 39 | "pre-commit": "lint-staged" 40 | } 41 | }, 42 | "lint-staged": { 43 | "*.{js,css,json,md}": [ 44 | "prettier --write", 45 | "git add" 46 | ] 47 | }, 48 | "prettier": { 49 | "trailingComma": "es5", 50 | "tabWidth": 2, 51 | "semi": false, 52 | "singleQuote": true, 53 | "bracketSpacing": false 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /lib/util.js: -------------------------------------------------------------------------------- 1 | const encodeDocUrl = docUrl => Buffer.from(docUrl).toString('hex') 2 | const decodeDocUrl = encodedDocUrl => 3 | Buffer.from(encodedDocUrl, 'hex').toString() 4 | 5 | class NotebookInfo { 6 | constructor(host, ssl, docUrl) { 7 | this.host = host 8 | this.ssl = ssl 9 | this.docUrl = docUrl 10 | } 11 | 12 | getContainerUrl() { 13 | return `${this.ssl ? 'https' : 'http'}://${ 14 | this.host 15 | }/annotations/${encodeDocUrl(this.docUrl)}` 16 | } 17 | } 18 | 19 | function normalizeId(host, docUrl, annotationId, opts = {}) { 20 | const ssl = opts.ssl || false 21 | const pattern = new RegExp( 22 | `^${ssl ? 'https' : 'http'}:\/\/${host}\/annotations\/${encodeDocUrl( 23 | docUrl 24 | )}\/([0-9a-z-]+)$` 25 | ) 26 | // FIXME: validate annotation schema prior to normalizing ID 27 | const matches = annotationId.match(pattern) 28 | return !matches ? null : matches[1] 29 | } 30 | 31 | function normalizeAnnotation(host, docUrl, annotation, opts = {}) { 32 | return { 33 | ...annotation, 34 | id: normalizeId(host, docUrl, annotation.id, opts), 35 | } 36 | } 37 | 38 | const denormalizeAnnotation = (host, docUrl, annotation, opts = {}) => { 39 | const ssl = opts.ssl || false 40 | // FIXME: validate annotation schema prior to denormalizing ID 41 | return { 42 | ...annotation, 43 | id: `${ssl ? 'https' : 'http'}://${host}/annotations/${encodeDocUrl( 44 | docUrl 45 | )}/${annotation.id}`, 46 | } 47 | } 48 | 49 | module.exports = { 50 | NotebookInfo, 51 | normalizeId, 52 | normalizeAnnotation, 53 | denormalizeAnnotation, 54 | encodeDocUrl, 55 | decodeDocUrl, 56 | } 57 | -------------------------------------------------------------------------------- /lib/subscription.js: -------------------------------------------------------------------------------- 1 | const crypto = require('crypto') 2 | const EventEmitter = require('events') 3 | const Cache = require('node-cache') 4 | const uuid = require('uuid/v1') 5 | const debug = require('debug')('hyperwell:gateway:subscription') 6 | 7 | const hashAnnotation = annotation => 8 | crypto 9 | .createHash('sha256') 10 | .update(JSON.stringify(annotation)) 11 | .digest() 12 | 13 | class Subscription extends EventEmitter { 14 | constructor() { 15 | super() 16 | this.id = uuid() 17 | this.cache = new Cache() 18 | } 19 | 20 | init(annotations) { 21 | debug(`initializing subscription ${this.id}`) 22 | this.cache.mset( 23 | annotations.reduce( 24 | (pairs, annotation) => [ 25 | ...pairs, 26 | { 27 | key: annotation.id, 28 | val: { 29 | hash: hashAnnotation(annotation), 30 | annotation, 31 | }, 32 | }, 33 | ], 34 | [] 35 | ) 36 | ) 37 | } 38 | 39 | diff(annotations) { 40 | const inserted = [] 41 | const changed = [] 42 | const deleted = [] 43 | 44 | const ids = annotations.map(({id}) => id) 45 | for (const id of this.cache.keys()) { 46 | if (!ids.includes(id)) { 47 | deleted.push(this.cache.get(id).annotation) 48 | this.cache.del(id) 49 | } 50 | } 51 | 52 | for (const annotation of annotations) { 53 | const hash = hashAnnotation(annotation) 54 | if (!this.cache.has(annotation.id)) { 55 | inserted.push(annotation) 56 | this.cache.set(annotation.id, {hash, annotation}) 57 | } else { 58 | if (!this.cache.get(annotation.id).hash.equals(hash)) { 59 | changed.push(annotation) 60 | this.cache.set(annotation.id, {hash, annotation}) 61 | } 62 | } 63 | } 64 | 65 | return {inserted, changed, deleted} 66 | } 67 | 68 | close() { 69 | debug(`closing subscription ${this.id}`) 70 | this.emit('close') 71 | this.removeAllListeners() 72 | this.cache.close() 73 | } 74 | } 75 | 76 | module.exports = Subscription 77 | -------------------------------------------------------------------------------- /test/util.js: -------------------------------------------------------------------------------- 1 | const test = require('tape') 2 | const { 3 | normalizeId, 4 | normalizeAnnotation, 5 | denormalizeAnnotation, 6 | encodeDocUrl, 7 | decodeDocUrl, 8 | } = require('../lib/util') 9 | 10 | const docUrl = 'foo-container' 11 | const annotationId = '1c76c270-11fb-11ea-b65a-d9e5ad101414' 12 | const fixture = { 13 | type: 'Annotation', 14 | body: [ 15 | { 16 | type: 'TextualBody', 17 | value: 'foobar', 18 | }, 19 | ], 20 | target: { 21 | selector: [ 22 | { 23 | type: 'TextQuoteSelector', 24 | exact: 'baz', 25 | }, 26 | { 27 | type: 'TextPositionSelector', 28 | start: 98, 29 | end: 104, 30 | }, 31 | ], 32 | }, 33 | '@context': 'http://www.w3.org/ns/anno.jsonld', 34 | id: `https://www.example.com/annotations/${encodeDocUrl( 35 | docUrl 36 | )}/${annotationId}`, 37 | } 38 | 39 | test('normalizeId', t => { 40 | const fixtures = [ 41 | [ 42 | `https://www.example.com/annotations/${encodeDocUrl( 43 | docUrl 44 | )}/${annotationId}`, 45 | 'www.example.com', 46 | docUrl, 47 | annotationId, 48 | ], 49 | [ 50 | `https://www.example.com/annotations/${encodeDocUrl( 51 | docUrl 52 | )}/${annotationId}`, 53 | 'www.example2.com', 54 | docUrl, 55 | null, 56 | ], 57 | [ 58 | `https://www.example.com/annotations/${encodeDocUrl( 59 | docUrl 60 | )}/${annotationId}`, 61 | 'www.example.com', 62 | 'bar-container', 63 | null, 64 | ], 65 | [ 66 | `https://www.example.com:80/annotations/${encodeDocUrl( 67 | docUrl 68 | )}/${annotationId}`, 69 | 'www.example.com:80', 70 | docUrl, 71 | annotationId, 72 | ], 73 | [ 74 | `http://www.example.com:80/annotations/${encodeDocUrl( 75 | docUrl 76 | )}/${annotationId}`, 77 | 'www.example.com:80', 78 | docUrl, 79 | annotationId, 80 | false, 81 | ], 82 | [ 83 | `https://www.example.com:80/annotations/${encodeDocUrl( 84 | docUrl 85 | )}/${annotationId}`, 86 | 'www.example.com:80', 87 | docUrl, 88 | null, 89 | false, 90 | ], 91 | ] 92 | t.plan(fixtures.length) 93 | 94 | for (const [id, hostname, docUrl, expectedId, ssl = true] of fixtures) { 95 | t.equal( 96 | normalizeId(hostname, docUrl, id, {ssl}), 97 | expectedId, 98 | 'Annotation IDs do match' 99 | ) 100 | } 101 | }) 102 | 103 | test('normalizeAnnotation', t => { 104 | t.plan(1) 105 | const normalizedAnnotation = normalizeAnnotation( 106 | 'www.example.com', 107 | docUrl, 108 | fixture, 109 | {ssl: true} 110 | ) 111 | t.deepEqual( 112 | normalizedAnnotation, 113 | { 114 | ...fixture, 115 | id: annotationId, 116 | }, 117 | 'Normalized annotation does match' 118 | ) 119 | }) 120 | 121 | test('denormalizeAnnotation', t => { 122 | t.plan(1) 123 | const annotation = denormalizeAnnotation( 124 | 'www.example.com', 125 | docUrl, 126 | {...fixture, id: annotationId}, 127 | {ssl: true} 128 | ) 129 | t.deepEqual( 130 | annotation, 131 | { 132 | ...fixture, 133 | id: `https://www.example.com/annotations/${encodeDocUrl( 134 | docUrl 135 | )}/${annotationId}`, 136 | }, 137 | 'Normalized annotation does match' 138 | ) 139 | }) 140 | 141 | test('denormalizeAnnotation') 142 | -------------------------------------------------------------------------------- /lib/ldp.js: -------------------------------------------------------------------------------- 1 | const Boom = require('@hapi/boom') 2 | const etag = require('etag') 3 | 4 | class PagedCollection { 5 | constructor(getPage, notebookInfo, total, pageSize) { 6 | if (Array.isArray(getPage)) { 7 | const items = getPage 8 | this._getPage = () => items 9 | this.total = items.length 10 | this.pageSize = Infinity 11 | } else { 12 | this._getPage = getPage 13 | this.total = total 14 | this.pageSize = pageSize 15 | } 16 | this.notebookInfo = notebookInfo 17 | } 18 | 19 | getPage(pageNumber) { 20 | return this._getPage(pageNumber, this.pageSize) 21 | } 22 | 23 | get lastPage() { 24 | return this.pageSize === Infinity 25 | ? 0 26 | : Math.floor(this.total / this.pageSize) 27 | } 28 | } 29 | 30 | function createPage(h, collection, pageNumber, iris) { 31 | if (pageNumber > collection.lastPage) { 32 | return Boom.notFound() 33 | } 34 | 35 | const items = collection.getPage(pageNumber) 36 | const containerUrl = collection.notebookInfo.getContainerUrl() 37 | const page = { 38 | '@context': 'http://www.w3.org/ns/anno.jsonld', 39 | id: `${containerUrl}/?page=${pageNumber}&iris=${iris ? 1 : 0}`, 40 | type: 'AnnotationPage', 41 | partOf: { 42 | id: `${containerUrl}/?iris=${iris ? 1 : 0}`, 43 | total: collection.total, 44 | modified: '2016-07-20T12:00:00Z', 45 | }, 46 | startIndex: pageNumber === 0 ? 0 : collection.pageSize * pageNumber, 47 | items: iris ? items.map(item => item.id) : items, 48 | } 49 | 50 | const response = h.response(page) 51 | response.type( 52 | 'application/ld+json; profile="http://www.w3.org/ns/anno.jsonld"' 53 | ) 54 | response.header('allow', 'HEAD, GET, OPTIONS') 55 | 56 | return response 57 | } 58 | 59 | function createContainer(h, collection, iris) { 60 | const containerUrl = collection.notebookInfo.getContainerUrl() 61 | const container = { 62 | '@context': [ 63 | 'http://www.w3.org/ns/anno.jsonld', 64 | 'http://www.w3.org/ns/ldp.jsonld', 65 | ], 66 | id: `${containerUrl}/?iris=${iris ? 1 : 0}`, 67 | type: ['BasicContainer', 'AnnotationCollection'], 68 | total: collection.total, 69 | modified: '2016-07-20T12:00:00Z', 70 | label: 'tbd', 71 | first: `${containerUrl}/?iris=${iris ? 1 : 0}&page=0`, 72 | ...(collection.lastPage > 0 && { 73 | last: `${containerUrl}/?iris=${iris ? 1 : 0}&page=${collection.lastPage}`, 74 | }), 75 | } 76 | 77 | const response = h.response(container) 78 | response.header('link', [ 79 | '; rel="type"', 80 | '; rel="http://www.w3.org/ns/ldp#constrainedBy"', 81 | ]) 82 | response.header( 83 | 'Accept-Post', 84 | 'application/ld+json; profile="http://www.w3.org/ns/anno.jsonld"' 85 | ) 86 | response.type( 87 | 'application/ld+json; profile="http://www.w3.org/ns/anno.jsonld"' 88 | ) 89 | response.header('allow', 'HEAD, GET, POST, OPTIONS') 90 | response.etag(etag(JSON.stringify(container))) 91 | 92 | return response 93 | } 94 | 95 | function wrapResource(h, annotation) { 96 | const response = h.response(annotation) 97 | response.etag(etag(JSON.stringify(annotation))) 98 | response.type( 99 | 'application/ld+json; profile="http://www.w3.org/ns/anno.jsonld"' 100 | ) 101 | response.header('allow', 'OPTIONS,HEAD,GET,PUT,DELETE') 102 | response.header('link', '; rel="type"') 103 | 104 | return response 105 | } 106 | 107 | module.exports = {PagedCollection, createPage, createContainer, wrapResource} 108 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Hyperwell Gateway 2 | 3 | [![DOI](https://zenodo.org/badge/208063924.svg)](https://zenodo.org/badge/latestdoi/208063924) 4 | 5 | Proof-of-concept implementation of a peer-to-peer (P2P) system that leverages collaboration, local-first principles, and more on [W3C Web Annotations](https://www.w3.org/TR/annotation-model/). The system provides software for local nodes that store annotations, as well as a gateway server that implements the [Web Annotation Protocol](https://www.w3.org/TR/annotation-protocol/). 6 | 7 | For establishing an environment of [local-first applications](https://www.inkandswitch.com/local-first.html) for collaborative annotation, we store collections of annotations called **notebooks** in [Hypermerge](https://github.com/automerge/hypermerge) documents. These documents are distributed via the [hyperswarm](https://github.com/hyperswarm/hyperswarm) decentralized network and merged without conflicts via [automerge](https://github.com/automerge/automerge). 8 | 9 | The Hyperwell gateway aims to bridge the decentralized network and the web, offering collaborative annotation to Linked Data systems and web-based annotation environments alike by implementing the W3C Web Annotation specifications. For users, the gateway aims to offer institutional affiliation and archiving. 10 | 11 | ![Hyperwell architecture](architecture.png) 12 | 13 | We laid out the motivation behind the decentralized annotation architecture of Hyperwell in our 2020 paper, [‘From Me to You: Peer-to-Peer Collaboration with Linked Data‘](https://zenodo.org/record/3750243). For more on Hyperwell and the journey behind it: https://kassel.works/hyperwell 14 | 15 | **Important**: This is alpha software for research. Your annotations will be available publicly and accessible without authentication. This software has not been professionally audited. 16 | 17 | ## Usage 18 | 19 | Run a gateway server via `npm start` or `./bin/server.js`. The CLI accepts the following arguments: 20 | 21 | - `--port ` (`-p`): The port number to listen on. Defaults to `3000`. Example: `--port 8080`. 22 | - `--host `: The public hostname. Defaults to `localhost:3000`. Example: `--host www.example.com:8080` 23 | - `--ssl` (`-s`): Whether the gateway is served via SSL. This will not make the gateway actually terminate HTTPS requests (it listens for standard HTTP requests), but will transform annotation IDs accordingly, using the `https:` scheme. Defaults to `false` (not set). 24 | 25 | ## API 26 | 27 | The gateway exposes a web-based API as a superset of the [Web Annotation Protocol](https://www.w3.org/TR/annotation-protocol/), including support for batch operations as well as subscribing to real-time updates on notebooks via the WebSocket protocol. In the following, the `` identifier corresponds to the notion of [‘containers’](https://www.w3.org/TR/ldp/#ldpc) on the LDP. We simply use a hexadecimal encoding of the respective Hypermerge document URL (`hypermerge://abc123...`) for URL safety. `` corresponds to the ID of an annotation within a notebook, which commonly is a [UUID](https://tools.ietf.org/html/rfc4122) string. 28 | 29 | - `/annotations/`. REST endpoint for operations on an entire notebook. This endpoint supports retrieval of all of its annotations (`GET`) and creation of new a new annotation (`POST`). 30 | - `/annotations//`. REST endpoint for operations on a particular annotation within a notebook. This endpoint supports retrieval (`GET`), editing (`PUT`), and deletion (`DELETE`). 31 | - `/annotations/batch/`. REST endpoint for batch operations on a notebook. This endpoint supports batch creation (`POST`), batch edits (`PUT`), and batch deletions (`DELETE`). 32 | - `/annotations/subscribe/`. WebSocket endpoint for subscribing to changes on a notebook. Upon initiating a connection via the standard WebSocket protocol, the gateway will send messages as soon as the respective notebooks receives changes. 33 | 34 | (tbc). 35 | 36 | ## License 37 | 38 | MIT License, see [`./LICENSE`](/LICENSE). 39 | -------------------------------------------------------------------------------- /lib/swarm.js: -------------------------------------------------------------------------------- 1 | const uuid = require('uuid/v1') 2 | const {Repo} = require('hypermerge') 3 | const Hyperswarm = require('hyperswarm') 4 | const Cache = require('node-cache') 5 | const {SwarmError} = require('./error') 6 | const AbstractSwarm = require('./abstract-swarm') 7 | const debug = require('debug')('hyperwell:gateway:swarm') 8 | 9 | const cacheTTL = 60 * 10 10 | 11 | // FIXME: validate with JSON schema 12 | const validDoc = doc => Array.isArray(doc.annotations) 13 | const findAnnotation = (doc, annotationId) => { 14 | if (!validDoc(doc)) { 15 | throw SwarmError.badDoc() 16 | } 17 | 18 | const index = doc.annotations.findIndex(({id}) => id === annotationId) 19 | return index > -1 20 | ? [true, doc.annotations[index], index] 21 | : [false, index, null] 22 | } 23 | 24 | class HyperwellSwarm extends AbstractSwarm { 25 | constructor() { 26 | super() 27 | 28 | this.repo = new Repo({ 29 | memory: true, 30 | }) 31 | this.repo.setSwarm(Hyperswarm(), { 32 | lookup: true, 33 | announce: false, 34 | }) 35 | 36 | this.cache = new Cache({ 37 | stdTTL: cacheTTL, 38 | checkPeriod: 60, 39 | }) 40 | this.cache.on('expired', this._handleDocExpired) 41 | this.cache.on('del', this._handleDocExpired) 42 | } 43 | 44 | async getAnnotations(docUrl) { 45 | const doc = await this.repo.doc(docUrl) 46 | 47 | if (!validDoc(doc)) { 48 | throw SwarmError.badDoc() 49 | } 50 | 51 | this.cache.set(docUrl, true) 52 | return doc.annotations 53 | } 54 | 55 | async getAnnotation(docUrl, annotationId) { 56 | const doc = await this.repo.doc(docUrl) 57 | const [found, annotation] = findAnnotation(doc, annotationId) 58 | if (!found) { 59 | throw SwarmError.notFound() 60 | } 61 | 62 | this.cache.set(docUrl, true) 63 | return annotation 64 | } 65 | 66 | async createAnnotation(docUrl, annotation) { 67 | // FIXME: validate with JSON schema 68 | // if (annotation.id || !annotation.body) { 69 | if (annotation.id) { 70 | throw SwarmError.badRequest() 71 | } 72 | 73 | const id = uuid() 74 | await new Promise((resolve, reject) => 75 | this.repo.change(docUrl, doc => { 76 | // FIXME: validate with JSON schema 77 | if (!validDoc(doc)) { 78 | reject(SwarmError.badDoc()) 79 | } 80 | 81 | doc.annotations.push({ 82 | ...annotation, 83 | id, 84 | }) 85 | 86 | resolve() 87 | }) 88 | ) 89 | 90 | this.cache.set(docUrl, true) 91 | return id 92 | } 93 | 94 | async createAnnotationsBatch(docUrl, annotations) { 95 | // FIXME: validate with JSON schema 96 | if (annotations.some(annotation => annotation.id)) { 97 | throw SwarmError.badRequest() 98 | } 99 | 100 | const newAnnotations = annotations.map(annotation => ({ 101 | ...annotation, 102 | id: uuid(), 103 | })) 104 | await new Promise((resolve, reject) => 105 | this.repo.change(docUrl, doc => { 106 | // FIXME: validate with JSON schema 107 | if (!validDoc(doc)) { 108 | reject(SwarmError.badDoc()) 109 | } 110 | 111 | for (const newAnnotation of newAnnotations) { 112 | doc.annotations.push(newAnnotation) 113 | } 114 | 115 | resolve() 116 | }) 117 | ) 118 | 119 | this.cache.set(docUrl, true) 120 | return newAnnotations 121 | } 122 | 123 | async updateAnnotation(docUrl, annotation) { 124 | // FIXME: validate with JSON schema 125 | // if (annotation.id || !annotation.body) { 126 | if (!annotation.id) { 127 | throw SwarmError.badRequest() 128 | } 129 | 130 | await new Promise((resolve, reject) => 131 | this.repo.change(docUrl, doc => { 132 | const [found, previousAnnotation, index] = findAnnotation( 133 | doc, 134 | annotation.id 135 | ) 136 | if (!found) { 137 | reject(SwarmError.notFound()) 138 | } 139 | 140 | doc.annotations.splice(index, 1, annotation) 141 | resolve() 142 | }) 143 | ) 144 | 145 | this.cache.set(docUrl, true) 146 | return annotation 147 | } 148 | 149 | async updateAnnotationsBatch(docUrl, changedAnnotations) { 150 | // FIXME: validate with JSON schema 151 | if (changedAnnotations.some(annotation => !annotation.id)) { 152 | throw SwarmError.badRequest() 153 | } 154 | 155 | await new Promise((resolve, reject) => 156 | this.repo.change(docUrl, doc => { 157 | // FIXME: validate with JSON schema 158 | if (!validDoc(doc)) { 159 | reject(SwarmError.badDoc()) 160 | } 161 | 162 | const changedIds = changedAnnotations.map(({id}) => id) 163 | for (let i = 0; i < doc.annotations.length; i++) { 164 | var j = changedIds.indexOf(doc.annotations[i].id) 165 | if (j > -1) { 166 | doc.annotations.splice(i, 1, changedAnnotations[j]) 167 | } 168 | } 169 | resolve() 170 | }) 171 | ) 172 | this.cache.set(docUrl, true) 173 | return changedAnnotations 174 | } 175 | 176 | async deleteAnnotation(docUrl, annotationId) { 177 | await new Promise((resolve, reject) => 178 | this.repo.change(docUrl, doc => { 179 | const [found, annotation, index] = findAnnotation(doc, annotationId) 180 | if (!found) { 181 | reject(SwarmError.notFound()) 182 | } 183 | 184 | doc.annotations.splice(index, 1) 185 | resolve() 186 | }) 187 | ) 188 | 189 | this.cache.set(docUrl, true) 190 | } 191 | 192 | async deleteAnnotationsBatch(docUrl, deletedAnnotations) { 193 | // FIXME: validate with JSON schema 194 | if (deletedAnnotations.some(annotation => !annotation.id)) { 195 | throw SwarmError.badRequest() 196 | } 197 | 198 | const deletedAnnotationIds = deletedAnnotations.map(({id}) => id) 199 | await new Promise(resolve => 200 | this.repo.change(docUrl, doc => { 201 | for (const annotationId of deletedAnnotationIds) { 202 | // FIXME: validate if all deleted annotations exist 203 | const [found, annotation, index] = findAnnotation(doc, annotationId) 204 | if (found) { 205 | doc.annotations.splice(index, 1) 206 | } 207 | } 208 | resolve() 209 | }) 210 | ) 211 | 212 | this.cache.set(docUrl, true) 213 | } 214 | 215 | async subscribeToAnnotations(docUrl, subscription) { 216 | const annotations = await this.getAnnotations(docUrl) 217 | subscription.init(annotations) 218 | 219 | const handle = this.repo.watch(docUrl, doc => { 220 | if (!validDoc(doc)) { 221 | return 222 | } 223 | 224 | const diff = subscription.diff(doc.annotations) 225 | if (diff !== null) { 226 | subscription.emit('change', diff) 227 | } 228 | }) 229 | 230 | subscription.on('close', () => handle.close()) 231 | } 232 | 233 | _handleDocExpired = docUrl => { 234 | debug('document expired:', docUrl) 235 | // FIXME: `repo.close()` and `repo.destroy()` will render the gateway unable 236 | // to re-open/retrieve the document later on. 237 | // this.repo.close(docUrl) 238 | } 239 | 240 | async destroy() { 241 | this.cache.close() 242 | } 243 | } 244 | 245 | module.exports = HyperwellSwarm 246 | -------------------------------------------------------------------------------- /lib/server.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert') 2 | const Hapi = require('@hapi/hapi') 3 | const Boom = require('@hapi/boom') 4 | const WebSocketPlugin = require('hapi-plugin-websocket') 5 | const { 6 | PagedCollection, 7 | createContainer, 8 | createPage, 9 | wrapResource, 10 | } = require('./ldp') 11 | const { 12 | SwarmError, 13 | ERROR_NOT_FOUND, 14 | ERROR_BAD_DOC, 15 | ERROR_BAD_REQUEST, 16 | } = require('./error') 17 | const { 18 | NotebookInfo, 19 | encodeDocUrl, 20 | decodeDocUrl, 21 | normalizeAnnotation, 22 | denormalizeAnnotation, 23 | } = require('./util') 24 | const Subscription = require('./subscription') 25 | const debug = require('debug')('hyperwell:gateway:server') 26 | 27 | let server 28 | 29 | const handleError = err => { 30 | debug(err) 31 | if (err instanceof SwarmError) { 32 | if (err.code === ERROR_NOT_FOUND || err.code === ERROR_BAD_DOC) { 33 | return Boom.notFound() 34 | } else if (err.code === ERROR_BAD_REQUEST) { 35 | return Boom.badRequest() 36 | } 37 | } 38 | 39 | console.error(err) 40 | return Boom.internal() 41 | } 42 | 43 | async function createServer(backendSwarm, port, {host, ssl}) { 44 | assert(Number.isInteger(port), 'Not a valid port number provided.') 45 | server = Hapi.server({ 46 | port, 47 | router: { 48 | stripTrailingSlash: true, 49 | }, 50 | routes: { 51 | cors: { 52 | origin: ['*'], 53 | exposedHeaders: ['link', 'allow', 'etag'], 54 | }, 55 | }, 56 | }) 57 | await server.register(WebSocketPlugin) 58 | 59 | server.route({ 60 | method: 'GET', 61 | path: '/', 62 | handler: () => { 63 | return "Hi! I'm hyperwell, a Web Annotation P2P gateway." 64 | }, 65 | }) 66 | 67 | server.route({ 68 | method: 'GET', 69 | path: '/annotations/{container}', 70 | handler: async (request, h) => { 71 | const docUrl = decodeDocUrl(request.params.container) 72 | const pageNumber = request.query.page 73 | ? Number.parseInt(request.query.page) 74 | : null 75 | const iris = request.query.iris === '1' 76 | const notebookInfo = new NotebookInfo(host, ssl, docUrl) 77 | 78 | try { 79 | const annotations = await backendSwarm.getAnnotations(docUrl) 80 | const collection = new PagedCollection( 81 | annotations.map(annotation => 82 | denormalizeAnnotation(host, docUrl, annotation, {ssl}) 83 | ), 84 | notebookInfo 85 | ) 86 | 87 | if (pageNumber !== null) { 88 | return createPage(h, collection, pageNumber, iris) 89 | } 90 | return createContainer(h, collection, iris) 91 | } catch (err) { 92 | return handleError(err) 93 | } 94 | }, 95 | }) 96 | 97 | server.route({ 98 | method: 'POST', 99 | path: '/annotations/subscribe/{container}', 100 | config: { 101 | plugins: { 102 | websocket: { 103 | only: true, 104 | initially: true, 105 | autoping: 30 * 1000, 106 | }, 107 | }, 108 | }, 109 | handler: async request => { 110 | let {ws, initially} = request.websocket() 111 | if (!initially) { 112 | return Boom.badRequest() 113 | } 114 | 115 | const docUrl = decodeDocUrl(request.params.container) 116 | const subscription = new Subscription() 117 | subscription.on('change', ({inserted, changed, deleted}) => { 118 | ws.send( 119 | JSON.stringify({ 120 | inserted: inserted.map(annotation => 121 | denormalizeAnnotation(host, docUrl, annotation, {ssl}) 122 | ), 123 | changed: changed.map(annotation => 124 | denormalizeAnnotation(host, docUrl, annotation, {ssl}) 125 | ), 126 | deleted: deleted.map(annotation => 127 | denormalizeAnnotation(host, docUrl, annotation, {ssl}) 128 | ), 129 | }) 130 | ) 131 | }) 132 | 133 | ws.on('close', () => subscription.close()) 134 | 135 | await backendSwarm.subscribeToAnnotations(docUrl, subscription) 136 | await new Promise(resolve => subscription.on('close', resolve)) 137 | return '' 138 | }, 139 | }) 140 | 141 | server.route({ 142 | method: 'GET', 143 | path: '/annotations/{container}/{id}', 144 | handler: async (request, h) => { 145 | const docUrl = decodeDocUrl(request.params.container) 146 | 147 | try { 148 | const annotation = await backendSwarm.getAnnotation( 149 | docUrl, 150 | request.params.id 151 | ) 152 | const resource = denormalizeAnnotation(host, docUrl, annotation, {ssl}) 153 | return wrapResource(h, resource) 154 | } catch (err) { 155 | return handleError(err) 156 | } 157 | }, 158 | }) 159 | 160 | server.route({ 161 | method: 'POST', 162 | path: '/annotations/{container}', 163 | handler: async (request, h) => { 164 | const docUrl = decodeDocUrl(request.params.container) 165 | const annotation = request.payload 166 | 167 | try { 168 | const annotationId = await backendSwarm.createAnnotation( 169 | docUrl, 170 | annotation 171 | ) 172 | return h.redirect( 173 | `${ssl ? 'https' : 'http'}://${host}/annotations/${encodeDocUrl( 174 | docUrl 175 | )}/${annotationId}` 176 | ) 177 | } catch (err) { 178 | return handleError(err) 179 | } 180 | }, 181 | }) 182 | 183 | server.route({ 184 | method: 'PUT', 185 | path: '/annotations/{container}/{id}', 186 | handler: async (request, h) => { 187 | const docUrl = decodeDocUrl(request.params.container) 188 | const annotation = request.payload 189 | try { 190 | return await backendSwarm.updateAnnotation( 191 | docUrl, 192 | // FIXME: add error handling for invalid annotations (e.g., wrong ID) 193 | normalizeAnnotation(host, docUrl, annotation, {ssl}) 194 | ) 195 | } catch (err) { 196 | return handleError(err) 197 | } 198 | }, 199 | }) 200 | 201 | server.route({ 202 | method: 'DELETE', 203 | path: '/annotations/{container}/{id}', 204 | handler: async (request, h) => { 205 | const docUrl = decodeDocUrl(request.params.container) 206 | try { 207 | await backendSwarm.deleteAnnotation(docUrl, request.params.id) 208 | return h.response().code(204) 209 | } catch (err) { 210 | return handleError(err) 211 | } 212 | }, 213 | }) 214 | 215 | server.route({ 216 | method: 'POST', 217 | path: '/annotations/batch/{container}', 218 | handler: async (request, h) => { 219 | const docUrl = decodeDocUrl(request.params.container) 220 | const annotations = request.payload 221 | 222 | try { 223 | const insertedAnnotations = await backendSwarm.createAnnotationsBatch( 224 | docUrl, 225 | annotations 226 | ) 227 | return insertedAnnotations.map(annotation => 228 | denormalizeAnnotation(host, docUrl, annotation, {ssl}) 229 | ) 230 | } catch (err) { 231 | return handleError(err) 232 | } 233 | }, 234 | }) 235 | 236 | server.route({ 237 | method: 'PUT', 238 | path: '/annotations/batch/{container}', 239 | handler: async (request, h) => { 240 | const docUrl = decodeDocUrl(request.params.container) 241 | const annotations = request.payload 242 | 243 | try { 244 | const changedAnnotations = await backendSwarm.updateAnnotationsBatch( 245 | docUrl, 246 | annotations.map(annotation => 247 | normalizeAnnotation(host, docUrl, annotation, {ssl}) 248 | ) 249 | ) 250 | return changedAnnotations.map(annotation => 251 | denormalizeAnnotation(host, docUrl, annotation, {ssl}) 252 | ) 253 | } catch (err) { 254 | return handleError(err) 255 | } 256 | }, 257 | }) 258 | 259 | server.route({ 260 | method: 'DELETE', 261 | path: '/annotations/batch/{container}', 262 | handler: async (request, h) => { 263 | const docUrl = decodeDocUrl(request.params.container) 264 | const annotations = request.payload 265 | 266 | try { 267 | await backendSwarm.deleteAnnotationsBatch( 268 | docUrl, 269 | annotations.map(annotation => 270 | normalizeAnnotation(host, docUrl, annotation, {ssl}) 271 | ) 272 | ) 273 | return { 274 | deleted: true, 275 | error: null, 276 | } 277 | } catch (err) { 278 | return handleError(err) 279 | } 280 | }, 281 | }) 282 | 283 | server.route({ 284 | method: 'PUT', 285 | path: '/annotations/announce', 286 | handler: async request => { 287 | if (typeof request.payload !== 'object' || !request.payload.docUrl) { 288 | return Boom.badRequest() 289 | } 290 | return `/annotations/${encodeDocUrl(request.payload.docUrl)}/` 291 | }, 292 | }) 293 | 294 | await server.start() 295 | console.log('Server running on %s', server.info.uri) 296 | 297 | return server 298 | } 299 | 300 | process.on('unhandledRejection', err => { 301 | console.error(err) 302 | process.exit(1) 303 | }) 304 | 305 | module.exports = {createServer} 306 | --------------------------------------------------------------------------------