├── .gitignore
├── README.md
├── clock.js
├── docs
    └── mergeHandler.md
├── examples
    ├── Readme.md
    └── example.js
├── index.js
├── mergeHandler.js
├── package.json
├── test
    ├── constants.js
    ├── helpers.js
    ├── hyperswarm.test.js
    ├── multi-hyperbee.test.js
    └── peerRestoreAndCleanup.js
└── timestamp.js


/.gitignore:
--------------------------------------------------------------------------------
1 | node_modules/
2 | .DS_Store
3 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # Multi-writer hyperbee
  2 | This repository owes origins of its design to [@RangerMauve](https://github.com/RangerMauve)'s awesome [multi-hyperdrive](https://github.com/RangerMauve/multi-hyperdrive), and of course, the very awesome [Hyperbee](https://github.com/mafintosh/hyperbee) and the [whole of Hypercore](https://hypercore-protocol.org).
  3 | 
  4 | ## About
  5 | A LevelUP compatible leaderless multi-master database with eventual consistency, using hyperbee + CRDT + HLC.  Similarly CockroachDB achieves replication on top of RocksDB, but here it is a pure P2P [streaming](https://github.com/tradle/why-hypercore/blob/master/FAQ.md#what-is-the-usp-unique-selling-proposition-of-hypercore) database, with zero central management. LevelDB compatibility allows to use Dynalite on top to achieve [DynamoDB compatibility](https://aws.amazon.com/dynamodb/) with multiple tables, auto-updated secondary indexes, and fairly complex queries combining those indexes. Work on @mhart's [Dynalite](https://github.com/tradle/dynalite) is almost completed to remove the HTTP server, to make this combination perfect as an embedded database and for serverless scenarios.
  6 | 
  7 | ## The need
  8 | [Hyperbee](https://github.com/mafintosh/hyperbee) is one-of-a-kind steaming database that can change the way we work with databases. But like all other low-level components of hypercore ecosystem it is a single-writer data structure. Multi-writer is a higher-level abstraction, hence the name multi-hyperbee.
  9 | 
 10 | ## Use cases
 11 | - Multi-device support. One or more devices are personal cloud peers.
 12 | - Later we will consider a shared DB for a team 
 13 | 
 14 | ## Usage
 15 | ``` js
 16 | const MultiHyperbee = require('multi-hyperbee')
 17 | 
 18 | const hyperbeeOpts = { keyEncoding: 'utf-8', valueEncoding: 'json' }
 19 | const multiHyperbee = new MultiHyperbee(storage, hyperbeeOpts)
 20 | 
 21 | // Each app usually has its own key exchange mechanism with remote peers. So after exchange is completed, 
 22 | // we will know the keys of the peer's diff feeds. To receive updates from them, you need to add them here. Repeat for all remote peers.
 23 | {
 24 |   await multiHyperbee.addPeer(peerDiffFeedKey)
 25 | }  
 26 | ```
 27 | ## Example
 28 | 
 29 | See example [here](https://github.com/tradle/multi-hyperbee/tree/master/examples). 
 30 | 
 31 | ## API
 32 | ### const db = new MultiHyperbee(storage, [options], [customMergeHandler])
 33 | 
 34 | creates a new MultiHyperbee with two single-writer hypercores: 
 35 | - **Store** - a hyperbee into which we will store objects created/changed locally or received from peers. This hyperbee is not replicated to peers. Multi-hyperbee's main goal is to achieve convergence, that is to keep this store in exactly the same state as store on other peers. This can't happen synchronously as peers are not expected to be only all the time, but eventually.
 36 | - **Diff** - here we store all changes made locally. Other peers replicate this and merge each diff into their own store.
 37 | Options included:
 38 | ``` js
 39 | {
 40 |   keyEncoding: 'utf-8' | 'binary' | 'ascii', // or some abstract encoding
 41 |   valueEncoding: <same as above>
 42 | }
 43 | ```
 44 | **customMergeHandler** - CRDT handler to apply changes to the Object. 
 45 | If not using default, it should be implemented in a following way
 46 | ```
 47 | class MergeHandler {
 48 |   constructor(store) {
 49 |     this.store = store
 50 |   }
 51 |   // It'll apply diff to the correct version of an object 
 52 |   merge(diff) {  
 53 |   }
 54 |   // That will generate diff based on the last version of the object in the store. 
 55 |   genDiff(oldValue, newValue) {
 56 |     return diff
 57 |   }
 58 | }
 59 | ```
 60 | 
 61 | ### await db.put(key, storeValue)
 62 | 
 63 | ```storeValue``` - should be a JSON object
 64 | 
 65 | Put will write two objects at the same time to Store and to Diff hyperbee.
 66 | Put will add to each of the objects following properties:
 67 | 
 68 | to Store object:
 69 | - _objectId - which is the same as key
 70 | - _timestamp - HLC timestamp
 71 | - _prevTimestamp - if the objct is not new
 72 | 
 73 | to Diff object
 74 | - _timestamp
 75 | - _prevTimestamp to the Diff.obj property if the Store object is not new
 76 | 
 77 | Diff object will be put with the key ```key/_timestamp``` 
 78 | 
 79 | Diff object can be set as a property of the storeValue or it will be generated by MultiHyperbee based on the previous version of the resource in the Store (that still needs to ).
 80 | If Diff is set as a property of the value it should be added to Object as property **_diff**. It will be deleted from the Store object and won't be a part of the Store object
 81 | 
 82 | Check the diff format [here](https://github.com/tradle/multi-hyperbee/blob/master/test/constants.js)
 83 | 
 84 | Diff object that is written to diffHyperbee will have a key that is consists of 2 parts:
 85 | - storeValue key
 86 | - storeValue timestamp
 87 | 
 88 | ### const replicaPeer = await db.addPeer(replicaKey)
 89 | 
 90 | Created replica Hyperbee using replica key 
 91 | 
 92 | ### const replicaPeer = db.removePeer(replicaKey)
 93 | 
 94 | removes replica Hyperbee
 95 | 
 96 | ### const stream = db.createUnionStream(key)
 97 | 
 98 | Use it for writing your own custom merge handler.
 99 | It creates a union stream from all the replica hyperbees where all the entries are the Diff objects. 
100 | It runs on each replica hyperbee.
101 | ``` js
102 | // key has timestamp in it. To limit the search we define the top limit as a key without the timestamp
103 | 
104 | let lte = key.split('/').slice(0, -1).join('/')
105 | createReadStream({gte: key, lte })
106 | ```
107 | It is used by mergeHandler for applying changes to the Store object when for example:
108 | - Say device was offline for some time, 
109 | - User made changes on his other devices.
110 | - When device comes online again it needs to catch up with all other devices
111 | 
112 | the rest of API is the same as [Hyperbee](https://github.com/mafintosh/hyperbee)
113 | 
114 | ## Roadmap
115 | 
116 | - MH - generate diff for insert/remove to array changes
117 | - **batch** is not yet supported
118 | - Tighten the non-atomic failure modes when process dies after writing to `diff` and before writing to `store`, or after reading from `feed` and applying to `store'.
119 | - Support multiple bees, tries. We invision that peers will use one replication log to establish multi-writer for any number of shared data structures, that is for data structures local and remote peers can write into simultaneously. Using one replication log can help support atomic changes across multiple data structures.
120 | 
121 | ### Extend support to Hyperdrive
122 | In this version we only add multi-writer to Hyperbee. But we can extend it to Trie and Drive. Here are our thoughts on how this might work.
123 | 
124 | Previous version of the design did not have a `diff` feed and thus followed multi-hyperdrive's design more closely. Multi-hyperdrive does not apply updates to the primary, which we did even in the initial version of multi-hyperbee. Instead on each read it checks on the fly which file is fresher and returns that file (It checks in primary and in all the replicas from peers). It also supports on-the-fly merging of the directory listings, without changing any structures on disk. In this design we deviated even further as we needed to support CRDT merging.
125 | 
126 | To implement CRDT for Hyperdrive files, we might need to change multi-hyperdrive's design to use the `diff` feed:
127 | 
128 | - CRDT diff must apply to the local (primary) hyperdrive. Multi-hyperdrive does not do that, keeping all changed files in the replica.
129 | - file diff in CRDT format is 3x the size of the changed data, so might warrant a second `diff` feed, to avoid slowing down DB replication. Hyperdrive uses 2 structures, hypertrie for directory and file metadata and hypercore for data. So changes in hypertrie can be propagated via `diff` and. 
130 | 
131 | 
132 | ## Algorithm
133 | *This is a third interation of the design, previous is described and implemented in the release tagged v0.1.*
134 | 
135 | In the prior design we had a primary hyperbee and sparse replicas of other peers' primary hyperbees. 
136 | In the new design the full object is not replicated to the peers, only its diff (this design eliminated the ping-pong problem of the prior design as the store is not replicated).
137 | 
138 | In this design we have 2 hyperbees into which we write, `store` and `diff`. `Store` contains a full set of fresh objects (and their older versions, as does any hypercore). `Diff` contains only the specially formatted  objects (our custom CRDT format) that represent modifications to local objects. Every peer replicates other peers' `diff` hyperbees (but not the `store`).
139 | Upon `diff` hyperbee getting an update() event, we apply the received diff object to the store using the algo below: 
140 | 
141 | For the CRDT algorithm to do its magic, we first rewind to the proper object version and then apply local diffs and a newly arrived remote diff:
142 | 
143 | - remote diff refers to the version of the object that was nodified on remote peer
144 | - we find the same version of the object in store
145 | - we find all diffs that were applied locally since that version
146 | - we sort all local diffs and the remote diff by time, and apply them in that order
147 | - new version of the object is put() into store
148 | 
149 | This algorithm ensures that all peers have the store in exactly the same state. 
150 | 
151 | ## Cost and future optimizations
152 | **Read performance**: equals normal hyperbee performance
153 | **Write performance**: quite expensive:
154 | - Diff coming from replica is written to disk
155 | - Union range query across all diff replicas and primary diff to find diffs since a particualr HLC time
156 | - Reed matching version of the object in store
157 | - Merge in memory and Write new version to store
158 | 
159 | ## Failure modes 
160 | ### Done
161 | - HLC clock needs to be restored on restart
162 |   - the HLC clock is restored from the **timestamp** of the last record in Store hyperbee
163 | - Recovery after restart - persistent storage only
164 |   - Multihyperbee has a manifest that contains the list of peers keys. 
165 |   - The key to the manifest is saved in the header block of the MultiHyperbee Store. 
166 |   - Peer keys are used to restore the peers in case of the restart. 
167 | - update() event on replica occured and computer died before we applied it to store. (Will it arrive again? - it does not)
168 |   - this needs a test that simulates crash before Diff(s) processed and checks that thei are processed on restart.
169 | 
170 | 


--------------------------------------------------------------------------------
/clock.js:
--------------------------------------------------------------------------------
 1 | const { Timestamp, MutableTimestamp } = require('./timestamp')()
 2 | 
 3 | class Clock {
 4 |   constructor(clock, merkle = {}) {
 5 |     this._clock = this.makeClock(clock, merkle);
 6 |   }
 7 | 
 8 |   getClock() {
 9 |     return this._clock;
10 |   }
11 | 
12 |   makeClock(timestamp, merkle = {}) {
13 |     return { timestamp: MutableTimestamp.from(timestamp), merkle };
14 |   }
15 | 
16 |   serializeClock(clock) {
17 |     return JSON.stringify({
18 |       timestamp: clock.timestamp.toString(),
19 |       merkle: clock.merkle
20 |     });
21 |   }
22 | 
23 |   deserializeClock(clock) {
24 |     const data = JSON.parse(clock);
25 |     return {
26 |       timestamp: Timestamp.from(Timestamp.parse(data.timestamp)),
27 |       merkle: data.merkle
28 |     };
29 |   }
30 | 
31 |   makeClientId() {
32 |     return uuidv4()
33 |       .replace(/-/g, '')
34 |       .slice(-16);
35 |    }
36 | }
37 | module.exports = Clock


--------------------------------------------------------------------------------
/docs/mergeHandler.md:
--------------------------------------------------------------------------------
 1 | # Merge Handler
 2 | 
 3 | All the calls to Merge Handler are done by MultiHyperbee. That means that if you want to use your own Merge Handler, it needs to be passed as a parameter to MultiHyperbee like this:
 4 | 
 5 | ```
 6 | const multiHyperbee = new MultiHyperbee(storage, [options], customMergeHandler)
 7 | ```
 8 | 
 9 | MyltiHyperbee calls MergeHandler in these cases:
10 | 
11 | 1. the **diff** object hits one of the replica peer and the **store** object needs to be updated with the new changes. In this case the call would be:
12 | 
13 | ```
14 | await mergeHandler.merge(diff)
15 | ```
16 | 
17 | 2. `multiHyperbee.put(key, object)` will call `mergeHandler.genDiff(oldValue, newValue)` if the **diff** object was not found in the **object**. In this case **diff** object will be generated based on differences between the **object** and it's last version in store.
18 | The **diff** object will have a key `key/_timestamp`. This way it'll be easy to find the **store** and/or **diff** object that correspond to the same **_timestamp**
19 | 
20 | _Note_ **diff** object could be passed as **_diff** property of the **object**. It will be deleted from **object** before put of the **object** is executed.
21 | 
22 | 
23 | All the calls to Merge Handler are done by MultiHyperbee. Which means that if you write your own Merge Handler, it is need to be passed as a parameter to MultiHyperbee like this:
24 | 
25 | ```
26 | const multihyperbee = new MultiHyperbee(s, [options], customMergeHandler)
27 | ```
28 | 
29 | ## API
30 | 
31 | #### `const mergeHandle = new MergeHandler(store)`
32 | Creates an instance of the Merge Handler for a particular multiHyperbee.
33 | `store` is a MultiHyperbee instance
34 | 
35 | #### `await mergeHandler.merge(diff)`
36 | Finds the object corresponding to **__objectId** in **diff** object and performs the merge. Algorithm below
37 | 
38 | #### `const diffObject = genDiff(oldValue, newValue)`
39 | Generates **diff** object when multi-hyperbee **put** is called and no **_diff** object was passed with the **store** object
40 | 
41 | ## Algorithm for the default Merge Handler
42 | 
43 | 1. find the last version of the **store** object corresponding to the **diff** by _objectId.
44 | 2. if the timestamp of the **diff** object is bigger than the one of the **store** object
45 |     - merge the **diff** to the **store** object
46 | 3. Otherwise:
47 |     - find all the **diff** objects on all the peers from the **diff** object timestamp
48 |     - finds the version of the store **object** with the same timestamp as **diff** object
49 |     - merge all found **diff(s)** to the found **store** object
50 |     - creates new **store** objects with each applied **diff**
51 | 
52 | This creates a fork from the previous sequence of changes of the store objects
53 | 
54 | ## Diff object schema
55 | 
56 | _Note_ the `property` in schema below corresponds to the `property` that is changed in store **object**.
57 | 
58 | ``` js
59 |   const diffSchema = {
60 |     _timestamp: 'string', // _timestamp has the same value as the store object the diff corresponds to
61 |     obj: {
62 |       _objectId: 'string',
63 |       _prevTimestamp: 'string'
64 |     },
65 |     list: {
66 |       add: {
67 |         // value of the property can be primitive JSON types or JSON object or any arrays
68 |         property: 'any type',
69 |       },
70 |       remove: {
71 |         // value of the property can be any string but it's value is not used in any way
72 |         property: ''
73 |       },
74 |       insert: {
75 |         // could be insert in some value like object or array,
76 |         // otherwise will work the same way as add on top level
77 |         add: {
78 |           property: 'object',
79 |           // ARRAY
80 |           property: [
81 |             {
82 |               before: 'some value in array',
83 |               after: 'some value in array',
84 |               index: 'number'
85 |             }
86 |           ]
87 |         },
88 |         remove: {
89 |           property: 'JSON object or Array'
90 |         }
91 |       }
92 |     }
93 |   }
94 | ```
95 | 
96 | 


--------------------------------------------------------------------------------
/examples/Readme.md:
--------------------------------------------------------------------------------
 1 | ### To run this example
 2 |  - open 2 or more tabs in the terminal
 3 |  - in each of them run command 
 4 | ```
 5 |  node examples/example.js -s [some storage name]
 6 | ```
 7 |  using different storage name for different tabs. It will create directory structure for MultiHyperbee and print the **key** of the Diff Hyperbee.
 8 |  - run command 
 9 |  ```
10 |  node examples/example.js -s [some storage name] -k [array of the keys of the Diff Hyperbees from other tabs. Use comma as a delimiter]
11 |  ```
12 |  
13 | For example if you want to test for 3 devices:
14 | - Open 3 terminals
15 | - On the first run you got all 3 keys key1, key2, key3. 
16 | - To run this example for the key1 as the main key, the command will be
17 | ```
18 |  node examples/example.js -s [key1 storage name] -k key2,key3
19 | ```
20 | You can then enter some data (since the example uses stdin) and it'll create the object(s) from the entered data which are going to be replicated.
21 | 
22 | You can check the replication results in **data** files in the directory structure
23 |  
24 |  
25 | 


--------------------------------------------------------------------------------
/examples/example.js:
--------------------------------------------------------------------------------
 1 | const MultiHyperbee = require('../')
 2 | const hyperswarm = require('hyperswarm')
 3 | const crypto = require('crypto')
 4 | const pump = require('pump')
 5 | const { promisify } = require('util')
 6 | const auth = require('hypercore-peer-auth')
 7 | const Protocol = require('hypercore-protocol')
 8 | 
 9 | const { keys, storage } = require('minimist')(process.argv.slice(2), {
10 |   alias: {
11 |     k: 'keys',
12 |     s: 'storage'
13 |   }
14 | })
15 | if (!storage) {
16 |   printUsage()
17 |   process.exit(0)
18 | }
19 | 
20 | const topicHex = crypto.createHash('sha256')
21 |   .update('imdb')
22 |   .digest()
23 | 
24 | // console.log(`topic: ${topicHex.toString('hex')}`)
25 | let data = {
26 |   firstName: 'J',
27 |   lastName: 'S',
28 |   someField: Math.random(),
29 |   friends: ['Claire', 'Martha', 'Jake', 'Sean']
30 | }
31 | 
32 | const OPTIONS = { keyEncoding: 'utf-8', valueEncoding: 'json' }
33 | 
34 | start()
35 | .then((started) => started  &&  console.log('Please enter some data'))
36 | 
37 | async function start() {
38 |   const db = new MultiHyperbee(storage, OPTIONS )
39 |   await db.ready()
40 |   const diffHyperbee = await db.getDiff() // after db.ready() there is no need to await for diffFeed
41 | 
42 |   let diffFeed = diffHyperbee.feed
43 |   console.log(`${storage} diff key: ${diffFeed.key.toString('hex')}`)
44 |   if (!keys || !keys.length) {
45 |     return false
46 |   }
47 | 
48 |   let keysArr = keys.split(',')
49 | 
50 | 
51 |   for (let i=0; i<keysArr.length; i++)
52 |     await db.addPeer(keysArr[i].trim())
53 |   process.stdin.on('data', async (data) => {
54 |     let text = data.toString('utf-8').trim()
55 |     await db.put(`${storage}_${text.replace(/[^a-zA-Z]/g, '')}`, { text })
56 |   })
57 | 
58 |   let rkey = `${storage}_123`
59 |   await db.put(rkey, data)
60 |   await startSwarm(db, topicHex)
61 |   return true
62 | }
63 | 
64 | async function startSwarm(db, topic) {
65 |   var swarm = hyperswarm()
66 |   swarm.join(topic, {
67 |     lookup: true,
68 |     announce: true
69 |   })
70 | 
71 |   // swarm.on('connection', (socket, info) => db.onConnection(socket, info))
72 |   swarm.on('connection', async (socket, info) => {
73 |     let stream = await db.replicate(info.client, {stream: socket, live: true})
74 |     pump(socket, stream, socket)
75 |   })
76 | }
77 | 
78 | function printUsage () {
79 |   console.log(function () {
80 |   /*
81 |   Usage:
82 |   Options:
83 |       -k, --key              print usage
84 |       -s, --storage          file path where the model resides
85 |   */
86 |   }.toString()
87 |   .split(/\n/)
88 |   .slice(2, -2)
89 |   .join('\n'))
90 | 
91 |   process.exit(0)
92 | }
93 | 


--------------------------------------------------------------------------------
/index.js:
--------------------------------------------------------------------------------
  1 | const Hyperbee = require('hyperbee')
  2 | const hypercore = require('hypercore')
  3 | const auth = require('hypercore-peer-auth')
  4 | const Protocol = require('hypercore-protocol')
  5 | const Union = require('sorted-union-stream')
  6 | const pump = require('pump')
  7 | const { isEqual, size, extend } = require('lodash')
  8 | 
  9 | const Clock = require('./clock')
 10 | const MergeHandler = require('./mergeHandler')
 11 | const { Timestamp, MutableTimestamp } = require('./timestamp')()
 12 | 
 13 | const RELATED_FEEDS = '__peers'
 14 | // This implementation uses HLC Clock implemented by James Long in his crdt demo app
 15 | class MultiHyperbee extends Hyperbee {
 16 |   constructor(storage, options, customMergeHandler) {
 17 |     let { valueEncoding, name, metadata } = options
 18 |     let feed = hypercore(storage)
 19 |     let peersListKey
 20 |     if (!metadata)
 21 |       options.metadata = metadata = {}
 22 |     else
 23 |       peersListKey = metadata.contentFeed
 24 |     // Save the key to peers in the head block
 25 |     if (!peersListKey)
 26 |       extend(options.metadata, {contentFeed: RELATED_FEEDS})
 27 | 
 28 |     super(feed, options) // this creates the store
 29 | 
 30 |     this.peerListKey = metadata.contentFeed
 31 |     this.storage = storage
 32 |     this.options = options
 33 |     this.mergeHandler =  customMergeHandler && customMergeHandler || new MergeHandler(this)
 34 |     this.sources = {}
 35 |     this.deletedSources = {}
 36 |     this.name = name || ''
 37 |     this._init = this.init()
 38 |   }
 39 | 
 40 |   async init() {
 41 |     await this.ready()
 42 | 
 43 |     let diffStorage
 44 |     let presistentStorage = typeof this.storage === 'string'
 45 |     if (presistentStorage)
 46 |       diffStorage = `${this.storage}_diff` // place diffHyperbee in the same directory
 47 |     else
 48 |       diffStorage = this.storage // storage function chosen by user: could be ram, ras3, etc.
 49 | 
 50 |     this.diffFeed = hypercore(diffStorage)
 51 | 
 52 |     let options = { ...this.options }
 53 |     options.metadata = {
 54 |       contentFeed: 'multi-hyperbee-diff'
 55 |     }
 56 | 
 57 |     this.diffHyperbee = new Hyperbee(this.diffFeed, options)
 58 | 
 59 |     await this.diffHyperbee.ready()
 60 | 
 61 |     let peers = presistentStorage  &&  await this._restorePeers()
 62 | 
 63 |     this.clock = await this._createClock(peers)
 64 |   }
 65 |   async _createClock(hasPeers) {
 66 |     let keyString = this.feed.key.toString('hex')
 67 |     let clock = new Clock(new Timestamp(0, 0, keyString))
 68 |     if (!hasPeers)
 69 |       return clock
 70 | 
 71 |     let hs = this.createHistoryStream({gte: -1, limit: 1})
 72 |     let entries
 73 |     try {
 74 |       entries = await this._collect(hs)
 75 |     } catch (err) {
 76 |       console.log('No entries found', err)
 77 |     }
 78 |     if (!entries  ||  !entries.length || !entries[0].value._timestamp)
 79 |       return clock
 80 | 
 81 |     let timestamp = entries[0].value._timestamp
 82 |     let idx = timestamp.length - 5
 83 |     let millis = new Date(timestamp.slice(0, idx)).getTime()
 84 |     let counter = parseInt(timestamp.slice(idx + 1))
 85 |     return new Clock(new Timestamp(millis, counter, keyString))
 86 |   }
 87 |   async get(key) {
 88 |     await this._init
 89 |     return this._get(key)
 90 |   }
 91 |   async del(key) {
 92 |     await this._init
 93 |     await super.del(key)
 94 |   }
 95 |   async put(key, value, noDiff, isNew) {
 96 |     await this._init
 97 |     if (key === this.peerListKey) {
 98 |       super.put(key, value)
 99 |       return
100 |     }
101 | 
102 |     if (!value)
103 |       throw new Error('multi-hyperbee: value parameter is required')
104 |     if (!noDiff  &&  (typeof value !== 'object'  || value.constructor !== Object))
105 |       throw new Error('multi-hyperbee: value expected to be JSON object')
106 | 
107 |     if (!value._objectId)
108 |       value._objectId = key
109 |     let timestamp = value._timestamp
110 |     if (!timestamp)
111 |       timestamp = Timestamp.send(this.clock.getClock()).toString().slice(0, 29)
112 |     let diff = value._diff
113 |     delete value._diff
114 | 
115 |     let cur = !isNew  && await this.get(key)
116 | 
117 |     value._timestamp = timestamp
118 |     let prevTimestamp, prevSeq
119 |     if (cur) {
120 |       prevTimestamp = cur.value._timestamp
121 |       prevSeq = cur.seq
122 |     }
123 |     if (prevTimestamp)
124 |       value._prevTimestamp = prevTimestamp
125 |     if (prevSeq)
126 |       value._prevSeq = prevSeq
127 | 
128 |     await super.put(key, value)
129 |     if (diff) {
130 |       diff._timestamp = timestamp
131 |       if (prevTimestamp)
132 |         diff.obj._prevTimestamp = prevTimestamp
133 |       await this.diffHyperbee.put(`${key}/${timestamp}`, diff)
134 |       return
135 |     }
136 |     if (noDiff) return
137 | 
138 |     diff = this.mergeHandler.genDiff(value, cur  &&  cur.value)
139 |     if (prevTimestamp)
140 |       diff.obj._prevTimestamp = prevTimestamp
141 |     await this.diffHyperbee.put(`${key}/${timestamp}`, diff)
142 |   }
143 |   async peek() {
144 |     await this._init
145 |     return await super.peek([options])
146 |   }
147 |   async getDiff() {
148 |     await this._init
149 |     return this.diffHyperbee
150 |   }
151 | 
152 |   async addPeer(key, allPeersKeyStrings) {
153 |     await this._init
154 |     return await this._addPeer(key)
155 |   }
156 | 
157 |   async replicate(isInitiator, options) {
158 |     await this._init
159 |     const { stream } = options
160 |     if (!stream)
161 |       return this.diffFeed.replicate(isInitiator, options)
162 | 
163 |     return await this._joinMainStream(isInitiator, stream)
164 |   }
165 |   async _joinMainStream(isInitiator, stream) {
166 |     const protocol = new Protocol(isInitiator)
167 | 
168 |     // pump(stream, protocol, stream)
169 |     const { key, secretKey } = this.diffFeed
170 | 
171 |     let peer
172 |     let self = this
173 |     auth(protocol, {
174 |       authKeyPair: {
175 |         publicKey: key,
176 |         secretKey
177 |       },
178 |       onauthenticate (peerAuthKey, cb) {
179 |         let peerAuthKeyStr = peerAuthKey.toString('hex')
180 |         const { sources } = self
181 |         for (const key in sources) {
182 |           if (key === peerAuthKeyStr) {
183 |             peer = sources[key]
184 |             return cb(null, true)
185 |           }
186 |         }
187 |         cb(null, false)
188 |       },
189 |       onprotocol: async (protocol) => {
190 |         await self.diffFeed.replicate(isInitiator, {stream: protocol, live: true})
191 |         peer.feed.replicate(!isInitiator, {stream: protocol, live: true})
192 |       }
193 |     })
194 |     return protocol
195 |   }
196 | 
197 |   async _restorePeers() {
198 |     let hs = this.feed.createReadStream({start: 0, end: 1})
199 |     let entries = await this._collect(hs)
200 |     let peerListKey = entries[0].toString().split('\n')[2]
201 |     this.peerListKey = peerListKey.replace(/[^a-zA-Z0-9_]/g, '')
202 |     // debugger
203 | 
204 |     let peerList = await this._get(this.peerListKey)
205 |     if (!peerList || !peerList.value ||  !peerList.value.length)
206 |       return
207 |     let peersHB = []
208 |     let keys = peerList.value
209 |     let keyStrings = keys.map(key => key.toString('hex'))
210 |     for (let i=0; i<keys.length; i++) {
211 |       let key = keys[i]
212 |       let keyString = keyStrings[i]
213 | 
214 |       let peer = await this._addPeer(key, keyStrings)
215 |       peersHB.push(peer)
216 |       this.sources[keyString] = peer
217 |     }
218 |     return peersHB
219 |   }
220 |   async removePeer (key) {
221 |     await this._init
222 | 
223 |     const keyString = key.toString('hex')
224 |     const hyperbee = this.sources[keyString]
225 | 
226 |     if (!hyperbee) return false
227 | 
228 |     delete this.sources[keyString]
229 |     this.deletedSources[keyString] = hyperbee
230 | 
231 |     return hyperbee
232 |   }
233 |   async getPeers() {
234 |     await this._init
235 |     return Object.values(this.sources)
236 |   }
237 |   createUnionStream(key) {
238 |     if (!key)
239 |       throw new Error('Key is expected')
240 |     let sortedStreams = []
241 |     let lte = `${key.split('/').slice(0, -1).join('/')}/\uffff`
242 |     let query = { gte: key, lte }
243 |     for (let s in this.sources) {
244 |       let hb  = this.sources[s]
245 |       sortedStreams.push(hb.createReadStream(query))
246 |     }
247 |     sortedStreams.push(this.diffHyperbee.createReadStream(query))
248 |     if (sortedStreams.length === 1)
249 |       return sortedStreams[0]
250 |     let union
251 |     for (let i=1; i<sortedStreams.length; i++)
252 |       union = new Union(union || sortedStreams[i-1], sortedStreams[i], (a, b) => {
253 |         return a._timestamp > b._timestamp
254 |       })
255 |     return union
256 |   }
257 |   batch() {
258 |     throw new Error('Not supported yet')
259 |   }
260 | 
261 |     // need this for restore peers on init stage
262 |   async _get(key) {
263 |     return super.get(key)
264 |   }
265 |   async _put(key, value, isNew) {
266 |     await this.put(key, value, true, isNew)
267 |   }
268 |   async _addPeer(key, allPeersKeyStrings) {
269 |     const keyString = key.toString('hex')
270 |     let peer = this.sources[keyString]
271 |     if (peer)
272 |       return peer
273 |     let peerStorage
274 |     if (typeof this.storage === 'string')
275 |       peerStorage = `${this.storage}_peer_${size(this.sources) + 1}`
276 |     else
277 |       peerStorage = this.storage
278 |     let { valueEncoding } = this.options
279 |     let peerFeed = hypercore(peerStorage, key)
280 | 
281 |     peer = new Hyperbee(peerFeed, this.options)
282 |     await peer.ready()
283 | 
284 |     this.sources[keyString] = peer
285 | 
286 |     if (!allPeersKeyStrings || allPeersKeyStrings.indexOf(keyString) === -1)
287 |       await this._addRemovePeer(keyString, true)
288 | 
289 |     if (this.deletedSources[keyString])
290 |       delete this.deletedSources[keyString]
291 | 
292 |     await this._update(keyString)
293 | 
294 |     return peer
295 |   }
296 |   async _addRemovePeer(keyString, isAdd) {
297 |     let peersList
298 |     try {
299 |       peersList = await this.get(this.peerListKey)
300 |     } catch (err) {
301 |       debugger
302 |     }
303 | 
304 |     if (!peersList)
305 |       peersList = []
306 |     else
307 |       peersList = peersList.value
308 |     if (isAdd)
309 |       peersList.push(keyString)
310 |     else {
311 |       let idx = peersList.indexOf(keyString)
312 |       if (idx === -1) {
313 |         debugger
314 |         console.log('Something went wrong. The key was not found in peers list')
315 |         return
316 |       }
317 |       peersList.splice(idx, 1)
318 |     }
319 | 
320 |     await this._put(this.peerListKey, peersList)
321 |   }
322 |   _parseTimestamp(timestamp) {
323 |     let tm = timestamp.split('-')
324 |     return {
325 |       millis: new Date(tm[0]).getTime(),
326 |       counter: parseInt(tm[1])
327 |     }
328 |   }
329 |   async _update(keyString, seqs) {
330 |     if (this.deletedSources[keyString])
331 |       return
332 |     const peer = this.sources[keyString]
333 |     const peerFeed = peer.feed
334 |     peerFeed.update(() => {
335 |       if (peer.name)
336 |         console.log(`UPDATE: ${peer.name}`)
337 |  // console.log(`UPDATE: ${peerFeed.key.toString('hex')}`)
338 |       let rs = peer.createHistoryStream({ gte: -1 })
339 |       let newSeqs = []
340 |       let values = []
341 |       rs.on('data', async (data) => {
342 |         let { key, seq, value } = data
343 |         newSeqs.push(seq)
344 |         if (!value) //  &&  key.trim().replace(/[^a-zA-Z0-9_]/g, '') === RELATED_FEEDS)
345 |           return
346 | 
347 |         let {millis, counter, node} = this._parseTimestamp(value._timestamp)
348 | 
349 |         let tm = new Timestamp(millis, counter, node)
350 |         tm = Timestamp.recv(this.clock.getClock(), tm)
351 |         if (seqs  &&  seqs.indexOf(seq) !== -1)
352 |           return
353 |         values.push(value)
354 |         // await this.mergeHandler(this, {...value, _replica: true})
355 |       })
356 |       rs.on('end', async (data) => {
357 |         for (let i=0; i<values.length; i++)
358 |           await this.mergeHandler.merge(values[i])
359 |       })
360 |       this._update(keyString, newSeqs)
361 |     })
362 |   }
363 |   async _collect(stream) {
364 |     return new Promise((resolve, reject) => {
365 |       const entries = []
366 |       stream.on('data', d => entries.push(d))
367 |       stream.on('end', () => resolve(entries))
368 |       stream.on('error', err => reject(err))
369 |       stream.on('close', () => reject(new Error('Premature close')))
370 |     })
371 |   }
372 | }
373 | module.exports = MultiHyperbee
374 | 


--------------------------------------------------------------------------------
/mergeHandler.js:
--------------------------------------------------------------------------------
  1 | const Automerge = require('automerge')
  2 | const { cloneDeep, size, extend, isEqual } = require('lodash')
  3 | 
  4 | // MergeHandler must use store._put to notify MultiHyperbee that it does not need to
  5 | // generate diff object in this case
  6 | 
  7 | class MergeHandler {
  8 |   constructor(store) {
  9 |     this.store = store
 10 |   }
 11 |   async merge (diff) {
 12 |     let { obj, list, _timestamp } = diff
 13 | 
 14 |     let { _objectId: rkey, _prevTimestamp } = obj
 15 | 
 16 |     // let rkey = _objectId
 17 |     let prevObject = await this.store.get(rkey)
 18 | 
 19 |     prevObject = prevObject  &&  prevObject.value
 20 |     // Not working when there is no object but there is _prevTimestamp
 21 |     let isNewHere = !prevObject // &&  !_prevTimestamp
 22 |     let isNewThere = !_prevTimestamp
 23 | 
 24 |     if ((isNewHere && isNewThere) ||  (prevObject  &&  prevObject._timestamp === _prevTimestamp)) {
 25 |       const val = this._doMerge(prevObject, diff, isNewHere)
 26 |       if (prevObject  &&  isEqual(val, prevObject)) {
 27 |         debugger
 28 |         return
 29 |       }
 30 |       await this.store._put(rkey, val)
 31 | // console.log('New object: ' + JSON.stringify(val, null, 2))
 32 |       return
 33 |     }
 34 |     let { tm, needEarlierObject } = this.getTimestampForUpdateQuery(prevObject, diff)
 35 | 
 36 |     let ukey = `${rkey}/${tm}`
 37 |     let unionStream = this.store.createUnionStream(ukey)
 38 | 
 39 |     let entries
 40 |     try {
 41 |       entries = await this.collect(unionStream)
 42 |     } catch (err) {
 43 |       console.log(`Error updating with ${JSON.stringify(diff, null, 2)}`, err)
 44 |       return
 45 |     }
 46 |     let origPrevObject
 47 |     if (needEarlierObject) {
 48 |       if (!entries.length) {
 49 |         await this.handleEarlierObject(entries, prevObject, diff)
 50 |         return
 51 |       }
 52 |       origPrevObject = prevObject
 53 |       prevObject = await this.findStartingObject(prevObject, entries)
 54 |     }
 55 | 
 56 |     entries = entries.map(e => e.value)
 57 |     let idx = entries.find(e => e._timestamp === diff._timestamp)
 58 |     if (idx === -1)
 59 |       entries.push(diff)
 60 |     entries.sort((a, b) => this.getTime(a._timestamp) - this.getTime(b._timestamp))
 61 | 
 62 |     for (let i=0; i<entries.length; i++) {
 63 |       let value = entries[i]
 64 |       let updatedValue = this._doMerge(prevObject, value)
 65 |       if (prevObject  &&  isEqual(updatedValue, prevObject)) {
 66 |         debugger
 67 |         // continue
 68 |       }
 69 | // console.log('Updated object: ' + JSON.stringify(updatedValue, null, 2))
 70 |       await this.store._put(rkey, updatedValue)
 71 |       prevObject = updatedValue
 72 |     }
 73 |   }
 74 |   getTime(timestamp) {
 75 |     let idx = timestamp.length - 5
 76 |     return new Date(timestamp.slice(0, idx)).getTime()
 77 |   }
 78 |   getTimestampForUpdateQuery(prevObject, diff) {
 79 |     let { _timestamp, obj } = diff
 80 |     let { _prevTimestamp } = obj
 81 | 
 82 |     if (!prevObject)
 83 |       return { tm: '' }
 84 |     // Diff for update
 85 |     if (_prevTimestamp) {
 86 |       if (prevObject._timestamp < _prevTimestamp)
 87 |         return { tm: prevObject._timestamp }
 88 |       if (prevObject._prevTimestamp  &&  prevObject._prevTimestamp === _prevTimestamp) {
 89 |         if (prevObject._timestamp < _timestamp)
 90 |           return { tm: _prevTimestamp }
 91 | 
 92 |         let needEarlierObject = true
 93 |         if (_timestamp < prevObject._prevTimestamp)
 94 |           return { tm: _timestamp, needEarlierObject }
 95 |         return { tm: _prevTimestamp, needEarlierObject }
 96 |       }
 97 |       else
 98 |         return { tm: _prevTimestamp }
 99 |     }
100 |     // Diff for creating a new Object
101 |     if (prevObject._timestamp < _timestamp)
102 |       return { tm: prevObject._timestamp }
103 | 
104 |     let needEarlierObject = true
105 |     if (_timestamp < prevObject._prevTimestamp)
106 |       return { tm: _timestamp, needEarlierObject }
107 |     return { tm: _prevTimestamp, needEarlierObject }
108 |   }
109 |   async findStartingObject(prevObject, entries) {
110 |     let { value } = entries[0]
111 |     let entryTimestamp = value.obj._prevTimestamp || value._timestamp
112 |     while (true) {
113 |       let prevSeq = prevObject._prevSeq
114 |       if (!prevSeq)
115 |         break
116 |       let hist = this.store.createHistoryStream({gte: prevSeq, lte: prevSeq})
117 |       let objects = await this.collect(hist)
118 |       if (!objects.length) {
119 |         debugger
120 |         break
121 |       }
122 |       let objTimestamp = objects[0].value._timestamp
123 |       prevObject = objects[0].value
124 |       if (objTimestamp === entryTimestamp)
125 |         break
126 | 
127 |       if (objTimestamp > entryTimestamp)
128 |         continue
129 |       else {
130 |         debugger
131 |         break
132 |       }
133 |     }
134 |     return prevObject
135 |   }
136 |   async handleEarlierObject(entries, prevObject, diff) {
137 |     debugger
138 | 
139 |     let { _objectId: rkey, _prevTimestamp } = diff.obj
140 |     let isNewThere = !_prevTimestamp
141 |     if (prevObject) {
142 |       if (isNewThere) {
143 |         const newThere = this._doMerge(null, diff, true)
144 |         await this.store._put(rkey, newThere, true)
145 |         const pdiff = this.genDiff(prevObject, newThere)
146 |         const updatedValue = this._doMerge(newThere, pdiff)
147 |         await this.store._put(rkey, updatedValue)
148 |         return
149 |       }
150 |       debugger
151 |       return
152 |     }
153 |     if (isNewThere) {
154 |       debugger
155 |       // create object from the received diff
156 |       const val = this._doMerge(prevObject, diff, true)
157 |       await this.store._put(rkey, val)
158 |       return
159 |     }
160 |     debugger
161 |     throw new Error('This should not have happen')
162 |   }
163 | 
164 |   /*
165 |   Generates diff object according to diffSchema
166 |    */
167 |   genDiff(newV, oldV) {
168 |     let oldValue = oldV ? cloneDeep(oldV) : {}
169 |     let newValue = newV
170 |     let add = {}
171 |     let insert = {}
172 |     let remove = {}
173 | 
174 |     for (let p in newValue) {
175 |       if (p.charAt(0) === '_')
176 |         continue
177 |       let oldVal = oldValue[p]
178 |       let newVal = newValue[p]
179 |       delete oldValue[p]
180 | 
181 |       if (!oldVal) {
182 |         add[p] = newVal
183 |         continue
184 |       }
185 | 
186 |       if (oldVal === newVal  || (typeof oldVal === 'object' && isEqual(oldVal, newVal)))
187 |         continue
188 |       if (Array.isArray(oldVal)) {
189 |         this._insertArray(insert, p, newVal, oldVal)
190 |         continue
191 |       }
192 |       if (typeof oldVal === 'object') {
193 |         this._insertObject(insert, p, newVal, oldVal)
194 |         continue
195 |       }
196 |       add[p] = newVal
197 |     }
198 |     for (let p in oldValue) {
199 |       if (p.charAt(0) === '_')
200 |         continue
201 |       remove[p] = ''
202 |     }
203 |     let list = {}
204 |     if (size(add))
205 |       list.add = add
206 |     if (size(remove))
207 |       list.remove = remove
208 |     if (size(insert))
209 |       list.insert = insert
210 |     let diff = {
211 |       _timestamp: newValue._timestamp,
212 |       obj: {
213 |         _objectId: newValue._objectId
214 |       },
215 |       list
216 |     }
217 |     if (newValue._prevTimestamp)
218 |       diff.obj._prevTimestamp = newValue._prevTimestamp
219 |     return diff
220 |   }
221 |   _insertObject(insert, p, newVal, oldVal) {
222 |     let result = this._diff(oldVal, newVal)
223 |     for (let pp in result) {
224 |       if (typeof result[pp] === 'undefined') {
225 |         if (!insert.remove)
226 |           insert.remove = {}
227 |         if (!insert.remove[p])
228 |           insert.remove[p] = {}
229 | 
230 |         extend(insert.remove[p], {
231 |           [pp]: oldVal[pp]
232 |         })
233 |       }
234 |       else {
235 |         if (!insert.add) {
236 |           insert.add = {}
237 |           insert.add[p] = {}
238 |         }
239 |         insert.add[p] = {
240 |           ... insert.add[p],
241 |           [pp]: newVal[pp]
242 |         }
243 |       }
244 |     }
245 |   }
246 |   _insertArray(insert, prop, newVal, oldVal) {
247 |     let newVal1 = newVal.slice()
248 |     for (let i=0; i<oldVal.length; i++) {
249 |       let idx = newVal1.indexOf(oldVal[i])
250 |       if (idx !== -1) {
251 |         newVal1.splice(idx, 1)
252 |         continue
253 |       }
254 |       if (!insert.remove)
255 |         insert.remove = {}
256 |       if (!insert.remove[prop])
257 |         insert.remove[prop] = [{value: oldVal[i]}]
258 |     }
259 |     if (newVal1.length) {
260 |       if (!insert.add)
261 |         insert.add = {}
262 |       insert.add = []
263 |       newVal1.forEach(value => {
264 |         let idx = newVal.indexOf(value)
265 |         insert.add.push({after: newVal[idx - 1], value})
266 |       })
267 |     }
268 |   }
269 | 
270 |   _diff(obj1, obj2) {
271 |     const result = {};
272 |     if (Object.is(obj1, obj2))
273 |       return result
274 | 
275 |     if (!obj2 || typeof obj2 !== 'object')
276 |       return obj2
277 | 
278 |     Object.keys(obj1 || {}).concat(Object.keys(obj2 || {})).forEach(key => {
279 |       if(obj2[key] !== obj1[key] && !Object.is(obj1[key], obj2[key]))
280 |         result[key] = obj2[key]
281 | 
282 |       if(typeof obj2[key] === 'object' && typeof obj1[key] === 'object') {
283 |         const value = this._diff(obj1[key], obj2[key]);
284 |         if (value !== undefined)
285 |           result[key] = value;
286 |       }
287 |     })
288 |     return result;
289 |   }
290 | 
291 |   async collect(stream) {
292 |     return new Promise((resolve, reject) => {
293 |       const entries = []
294 |       stream.on('data', d => {
295 |         entries.push(d)
296 |       })
297 |       stream.on('end', () => {
298 |         resolve(entries)
299 |       })
300 |       stream.on('error', err => {
301 |         reject(err)
302 |       })
303 |       stream.on('close', () => {
304 |         reject(new Error('Premature close'))
305 |       })
306 |     })
307 |   }
308 |   _doMerge(resource, diff, isNew) {
309 |     if (isNew)
310 |       resource = diff.obj
311 | 
312 |     let { _timestamp, obj, list } = diff
313 | 
314 |     let { _objectId, seq, _t } = obj
315 |     let { add={}, remove={}, insert={} } = list
316 | 
317 |     let doc = Automerge.from(resource)
318 |     let updatedDoc = Automerge.change(doc, doc => {
319 |       doc._timestamp = _timestamp
320 |       for (let p in remove)
321 |         delete doc[p]
322 | 
323 |       for (let p in add) {
324 |         let value = add[p]
325 |         let oldValue = doc[p]
326 |         if (!oldValue || (typeof value !== 'object')) {
327 |           doc[p] = value
328 |           continue
329 |         }
330 |         if (!Array.isArray(value)) {
331 |           doc[p] = value
332 |           continue
333 |         }
334 |         for (let i=0; i<value.length; i++) {
335 |           let elm = value[i]
336 |           let oldIdx = oldValue.findIndex(oelm => {
337 |             if (typeof elm === 'object')
338 |               return deepEquals(elm, oelm)
339 |             return elm === oelm
340 |           })
341 |           // let oldIdx = oldValue.findIndex(oelm => elm === oelm)
342 |           if (oldIdx === -1)
343 |             doc[p].splice(i, 0, elm)
344 |         }
345 |       }
346 |       for (let action in insert) {
347 |         let actionProps = insert[action]
348 |         for (let prop in actionProps)
349 |           this.handleInsert({ prop, doc, value: actionProps[prop], isAdd: action === 'add' })
350 |       }
351 |     })
352 | 
353 |     return JSON.parse(JSON.stringify(updatedDoc))
354 |   }
355 |   handleInsert({ prop, doc, value, isAdd }) {
356 |     let to = doc[prop]
357 |     if (!Array.isArray(value)) {
358 |       if (typeof value === 'object') {
359 |         if (isAdd)
360 |           extend(to, value)
361 |         else
362 |           this.deleteFrom(to, value)
363 |       }
364 |       else if (isAdd)
365 |         to = value
366 |       else
367 |         delete to[prop]
368 |       return
369 |     }
370 |     // debugger
371 |     value.forEach(({index, before, after, value}) => {
372 |       if (before)
373 |         index = to.indexOf(before)
374 |       else if (after) {
375 |         index = to.indexOf(after)
376 |         if (index !== -1)
377 |           index++
378 |       }
379 |       if (isAdd) {
380 |         if (!index  ||  index === -1)
381 |           to.push(value)
382 |         else
383 |           to.insertAt(index, value)
384 |       }
385 |       else {
386 |         if (!index)
387 |           index = to.indexOf(value)
388 |         if (index !== -1)
389 |           to.deleteAt(index)
390 |       }
391 |     })
392 |   }
393 |   deleteFrom(from, obj) {
394 |     for (let p in obj) {
395 |       if (typeof obj[p] !== 'object') {
396 |         delete from[p]
397 |       }
398 |       else if (!size(obj[p]))
399 |         delete from[p]
400 |       else
401 |         this.deleteFrom(from[p], obj[p])
402 |     }
403 |   }
404 | }
405 | module.exports = MergeHandler
406 | 
407 | /*
408 |   getTimestampForUpdateQuery(prevObject, diff) {
409 |     let { obj, _timestamp } = diff
410 |     let { _prevTimestamp } = obj
411 | 
412 |     if (!prevObject)
413 |       return { timestamp: '' }
414 |     // Diff for update
415 |     let needEarlierObject = true
416 |     if (_prevTimestamp) {
417 |       if (prevObject._timestamp < _prevTimestamp)
418 |         return { timestamp: prevObject._timestamp }
419 |       // prevObject is a new Object here, so get all diffs
420 |       if (!prevObject._prevTimestamp)
421 |         return { timestamp: '', needEarlierObject }
422 | 
423 |       if (prevObject._prevTimestamp > _prevTimestamp)
424 |         return { timestamp: _prevTimestamp, needEarlierObject }
425 | 
426 |       if (prevObject._prevTimestamp < _prevTimestamp)
427 |         return { timestamp: prevObject._prevTimestamp }
428 | 
429 |       if (prevObject._prevTimestamp === _prevTimestamp) {
430 |         if (prevObject._timestamp > _timestamp)
431 |           return { timestamp: _prevTimestamp, needEarlierObject }
432 |         else
433 |           return { timestamp: _prevTimestamp }
434 |       }
435 |       else
436 |         return { timestamp: _prevTimestamp, needEarlierObject }
437 |     }
438 |     // Diff for creating a new Object
439 |     if (prevObject._timestamp < _timestamp)
440 |       return { timestamp: prevObject._timestamp }
441 |     else
442 |       return { timestamp: _timestamp, needEarlierObject }
443 |   }
444 | 
445 |  */


--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "multi-hyperbee",
 3 |   "version": "1.0.0",
 4 |   "description": "",
 5 |   "main": "index.js",
 6 |   "scripts": {
 7 |     "test": "tape test/*.test.js && tape test/peerRestoreAndCleanup.js"
 8 |   },
 9 |   "repository": {
10 |     "type": "git",
11 |     "url": "git+https://github.com/tradle/multi-hyperbee.git"
12 |   },
13 |   "keywords": [
14 |     "hypercore",
15 |     "hyperbee",
16 |     "crdt"
17 |   ],
18 |   "author": "",
19 |   "license": "MIT",
20 |   "bugs": {
21 |     "url": "https://github.com/tradle/multi-hyperbee/issues"
22 |   },
23 |   "homepage": "https://github.com/tradle/multi-hyperbee#readme",
24 |   "dependencies": {
25 |     "automerge": "^0.14.1",
26 |     "hyperbee": "0.0.18",
27 |     "hypercore": "^9.6.0",
28 |     "lodash": "^4.17.20",
29 |     "pump": "^3.0.0",
30 |     "sorted-union-stream": "^3.0.1"
31 |   },
32 |   "devDependencies": {
33 |     "crypto": "^1.0.1",
34 |     "hypercore-peer-auth": "git+https://github.com/Frando/hypercore-peer-auth.git",
35 |     "hyperswarm": "^2.15.2",
36 |     "minimist": "^1.2.5",
37 |     "random-access-memory": "^3.1.1",
38 |     "rimraf": "^3.0.2",
39 |     "tape": "^5.0.1",
40 |     "why-is-node-running": "^2.2.0"
41 |   }
42 | }
43 | 


--------------------------------------------------------------------------------
/test/constants.js:
--------------------------------------------------------------------------------
  1 | module.exports = {
  2 |   object0: {
  3 |     _objectId: 'Contact/r1',
  4 |     firstName: 'J',
  5 |     lastName: 'S',
  6 |     friends: ['Claire', 'Martha', 'Jake', 'Sean']
  7 |   },
  8 | 
  9 |   diff0: {
 10 |     obj: {
 11 |       _objectId: 'Contact/r1'
 12 |     },
 13 |     list: {
 14 |       add: {
 15 |         firstName: 'J',
 16 |         lastName: 'S',
 17 |         friends: ['Claire', 'Martha', 'Jake', 'Sean']
 18 |       }
 19 |     }
 20 |   },
 21 |   object1: {
 22 |     _objectId: 'Contact/r1',
 23 |     firstName: 'Jane',
 24 |     lastName: 'Smith',
 25 |     gender: 'F',
 26 |     friends: ['Claire', 'Martha', 'Jake', 'Sean'],
 27 |     country: {
 28 |       name: 'United States'
 29 |     }
 30 |   },
 31 |   diff1: {
 32 |     obj: {
 33 |       _objectId: 'Contact/r1'
 34 |     },
 35 |     list: {
 36 |       add: {
 37 |         lastName: 'Smith',
 38 |         gender: 'F',
 39 |         firstName: 'Jane',
 40 |         country: {
 41 |           name: 'United States'
 42 |         }
 43 |       }
 44 |     }
 45 |   },
 46 |   object1_1: {
 47 |     _objectId: 'Contact/r1',
 48 |     firstName: 'Jane',
 49 |     lastName: 'Smith',
 50 |     gender: 'F',
 51 |     country: {
 52 |       name: 'United States',
 53 |       code: 'US'
 54 |     },
 55 |     nickname: 'Jenny',
 56 |     friends: ['Claire', 'Kate', 'Martha', 'Maggie', 'Jake', 'Sean']
 57 |   },
 58 |   diff1_1: {
 59 |     obj: {
 60 |       _objectId: 'Contact/r1'
 61 |     },
 62 |     list: {
 63 |       add: {
 64 |         nickname: 'Jenny'
 65 |       },
 66 |       insert: {
 67 |         add: {
 68 |           friends: [
 69 |             {after: 'Claire', value: 'Kate'},
 70 |             {after: 'Martha', value: 'Maggie'}
 71 |           ],
 72 |           country: {
 73 |             code: 'US'
 74 |           }
 75 |         }
 76 |       }
 77 |     }
 78 |   },
 79 | 
 80 |   object2: {
 81 |     _objectId: 'Contact/r1',
 82 |     firstName: 'Jane',
 83 |     lastName: 'Smith',
 84 |     nickname: 'Jenny',
 85 |     friends: ['Claire', 'Kate', 'Martha', 'Maggie', 'Jake'],
 86 |     country: {
 87 |       name: 'United States'
 88 |     },
 89 |     dateOfBirth: 843177600000 //'1996-09-20'
 90 |   },
 91 |   diff2: {
 92 |     obj: {
 93 |       _objectId: 'Contact/r1'
 94 |     },
 95 |     list: {
 96 |       add: {
 97 |         dateOfBirth: 843177600000 //'1996-09-20'
 98 |       },
 99 |       remove: {
100 |         gender: ''
101 |       },
102 |       insert: {
103 |         remove: {
104 |           country: {
105 |             code: 'US'
106 |           },
107 |           friends: [
108 |             {value: 'Sean'}
109 |           ]
110 |         }
111 |       }
112 |     }
113 |   }
114 | }


--------------------------------------------------------------------------------
/test/helpers.js:
--------------------------------------------------------------------------------
  1 | const hypercore = require('hypercore')
  2 | const { promisify } = require('util')
  3 | const ram = require('random-access-memory')
  4 | const isEqual = require('lodash/isEqual')
  5 | const MultiHyperbee = require('../')
  6 | 
  7 | const OPTIONS = {
  8 |         keyEncoding: 'utf-8',
  9 |         valueEncoding: 'json',
 10 |       }
 11 | 
 12 | const helpers = {
 13 |   async setup(count, storage) {
 14 |     let names = ['A', 'B', 'C', 'D', 'E', 'F', 'G']
 15 |     let multiHBs = []
 16 | 
 17 |     for (let i=0; i<count; i++) {
 18 |       let s = storage && `${storage}_${i}` || ram
 19 |       let mh = new MultiHyperbee(s, {...OPTIONS, name: names[i]})
 20 |       multiHBs.push(mh)
 21 |     }
 22 | 
 23 |     let hasPeers = await helpers.checkForPeers(multiHBs, storage)
 24 |     return {multiHBs, hasPeers}
 25 |   },
 26 |   async checkForPeers(multiHBs, storage) {
 27 |     if (!storage)
 28 |       return
 29 |     let peersMap = []
 30 |     let hasPeers
 31 |     for (let i=0; i<multiHBs.length; i++) {
 32 |       let multiHB = multiHBs[i]
 33 |       let peers = await multiHB.getPeers()
 34 | 
 35 |       if (peers  &&  peers.length)
 36 |         hasPeers = true
 37 |     }
 38 |     return hasPeers
 39 |   },
 40 |   async put(hyperbee, diff, value) {
 41 |     let key = `${value._objectId}`
 42 |     let val = cloneDeep(value)
 43 |     if (diff)
 44 |       val._diff = cloneDeep(diff)
 45 | 
 46 |     await hyperbee.put(key, val)
 47 |   },
 48 |   async delay (ms) {
 49 |     return new Promise(resolve => {
 50 |       setTimeout(() => {
 51 |         resolve()
 52 |       }, ms)
 53 |     })
 54 |   },
 55 | 
 56 |   async checkStoreAndDiff(t, multiHBs, storeArr, diffArr, print) {
 57 |     for (let i=0; i<multiHBs.length; i++) {
 58 |       let multiHB = multiHBs[i]
 59 |       let sec = multiHB.createHistoryStream()
 60 |       let counter = storeArr.length
 61 |       await new Promise((resolve, reject) => {
 62 |         sec.on('data', data => {
 63 |           if (print)
 64 |             console.log(multiHB.name + ' ' + JSON.stringify(data.value, null, 2))
 65 |           // Check that it's not peer list
 66 |           if (Array.isArray(data.value))
 67 |             return
 68 | 
 69 |           let { value } = data
 70 |           delete value._timestamp
 71 |           delete value._prevTimestamp
 72 |           delete value._prevSeq
 73 |           let v = storeArr.find(val => isEqual(val, value))
 74 |           t.same(value, v)
 75 |           counter--
 76 |         })
 77 |         sec.on('end', (data) => {
 78 |           if (counter)
 79 |             t.fail()
 80 |           resolve()
 81 |         })
 82 |       })
 83 |     }
 84 | 
 85 |     let diffs = await Promise.all(multiHBs.map(mh => mh.getDiff()))
 86 |     for (let i=0; i<diffs.length; i++) {
 87 |       let hstream = diffs[i].createHistoryStream()
 88 |       let multiHB = multiHBs[i]
 89 |       await new Promise((resolve, reject) => {
 90 |         hstream.on('data', ({value}) => {
 91 |           if (print)
 92 |             console.log(multiHB.name + 'Diff ' + JSON.stringify(value, null, 2))
 93 |           delete value._timestamp
 94 |           delete value.obj._prevTimestamp
 95 |           delete value._prevSeq
 96 |           t.same(value, diffArr[0])
 97 |           diffArr.shift()
 98 |         })
 99 |         hstream.on('end', (data) => {
100 |           resolve()
101 |         })
102 |       })
103 |     }
104 |   }
105 | }
106 | module.exports = helpers


--------------------------------------------------------------------------------
/test/hyperswarm.test.js:
--------------------------------------------------------------------------------
  1 | const test = require('tape')
  2 | const MultiHyperbee = require('../')
  3 | const hyperswarm = require('hyperswarm')
  4 | const crypto = require('crypto')
  5 | const pump = require('pump')
  6 | const rmdir = require('rimraf')
  7 | const log = require('why-is-node-running')
  8 | const { checkForPeers, checkStoreAndDiff, setup, delay } = require('./helpers')
  9 | const { object0, object1, object1_1, object2,
 10 |         diff0, diff1, diff1_1, diff2 } = require('./constants')
 11 | 
 12 | const topicHex = crypto.createHash('sha256')
 13 |   .update('my-multi-hyperbee')
 14 |   .digest()
 15 | 
 16 | var swarms = []
 17 | 
 18 | test('Multihyperbee - connected via hyperswarm', async t => {
 19 |   let storage = './test/mht/'
 20 |   let { multiHBs } = await setup(3, storage)
 21 |   let streams = []
 22 |   for (let i=0; i<multiHBs.length; i++) {
 23 |     let cur = i
 24 |     let j = 0
 25 |     let multiHB = multiHBs[i]
 26 |     let diffFeed = (await multiHB.getDiff()).feed
 27 |     for (; j<multiHBs.length; j++) {
 28 |       if (j === cur) continue
 29 |       await multiHBs[j].addPeer(diffFeed.key)
 30 |     }
 31 |   }
 32 |   const [ one, two, three ] = multiHBs
 33 |   let swarms = []
 34 |   for (let i=0; i<multiHBs.length; i++) {
 35 |     swarms.push(startSwarm(multiHBs[i], i))
 36 |   }
 37 | 
 38 |   let objects = [object0, object1, object2].map((obj, i) => {
 39 |     let _objectId = `Contact/r${i}`
 40 |     return {...obj, _objectId, someField: Math.random()}
 41 |   })
 42 |   for (let i=0; i<multiHBs.length; i++) {
 43 |     await multiHBs[i].put(objects[i]._objectId, objects[i])
 44 |   }
 45 | 
 46 |   await delay(2000)
 47 |   for (let i=0; i<multiHBs.length; i++) {
 48 |     let entries = await logFeed(multiHBs[i], i)
 49 |     let cmpObj = objects.slice()
 50 |     entries.forEach(entry => {
 51 |       const { key, value } = entry[`Peer ${i}`]
 52 |       if (key === '__peers')
 53 |         return
 54 |       delete value._prevSeq
 55 |       delete cmpObj[0]._prevSeq
 56 |       t.same(value, cmpObj[0])
 57 |       cmpObj.shift()
 58 |       // console.log(entry)
 59 |     })
 60 |   }
 61 |   // setTimeout(log, 1000)
 62 |   await delay(2000)
 63 |   // t.pass('done with test')
 64 |   t.end()
 65 | })
 66 | test('Multihyperbee - connected via hyperswarm - create object with the same key on 2 devices', async t => {
 67 |   let storage = './test/mht2/'
 68 |   let { multiHBs } = await setup(2, storage)
 69 |   let streams = []
 70 |   for (let i=0; i<multiHBs.length; i++) {
 71 |     let cur = i
 72 |     let j = 0
 73 |     let multiHB = multiHBs[i]
 74 |     let diffFeed = (await multiHB.getDiff()).feed
 75 |     for (; j<multiHBs.length; j++) {
 76 |       if (j === cur) continue
 77 |       await multiHBs[j].addPeer(diffFeed.key)
 78 |     }
 79 |   }
 80 |   const [ one, two ] = multiHBs
 81 |   let swarms = []
 82 |   for (let i=0; i<multiHBs.length; i++) {
 83 |     swarms.push(startSwarm(multiHBs[i], i))
 84 |   }
 85 | 
 86 |   for (let i=0; i<multiHBs.length; i++) {
 87 |     await multiHBs[i].put('sameKey', object0)
 88 |     await delay(1000)
 89 |   }
 90 |   delete object0._prevSeq
 91 |   await delay(2000)
 92 |   for (let i=0; i<multiHBs.length; i++) {
 93 |     let entries = await logFeed(multiHBs[i], i)
 94 |     entries.forEach(entry => {
 95 |       const { key, value } = entry[`Peer ${i}`]
 96 |       if (key === '__peers')
 97 |         return
 98 | 
 99 |       delete value._prevSeq
100 |       t.same(value, object0)
101 |     })
102 |   }
103 |   await delay(2000)
104 |   // t.pass('done with test 2')
105 |   t.end()
106 | })
107 | 
108 | test.onFinish(async () => {
109 |   await delay(2000)
110 |   swarms.forEach(swarm => swarm.destroy())
111 | })
112 | function startSwarm(db, i) {
113 |   var swarm = hyperswarm()
114 |   swarm.join(topicHex, {
115 |     lookup: true,
116 |     announce: true
117 |   })
118 |   swarms.push(swarm)
119 |   swarm.on('connection', async (socket, info) => {
120 |     // const key = db.feed.key.toString('hex')
121 |     // socket.write(key)
122 |     // socket.once('data', function (id) {
123 |     //   // debugger
124 |     //   info.deduplicate(Buffer.from(key), id)
125 |     // })
126 |     let stream = await db.replicate(info.client, {stream: socket, live: true})
127 |     pump(socket, stream, socket)
128 |   })
129 |   return swarm
130 | }
131 | async function logFeed(db, i) {
132 |   // console.log('watching', feed.key.toString('hex'), feed.length)
133 |   let stream = db.createReadStream({ live: true })
134 |   return await collect(stream, i)
135 | }
136 | async function collect(stream, i) {
137 |   return new Promise((resolve, reject) => {
138 |     const entries = []
139 |     stream.on('data', d => entries.push({[`Peer ${i}`]: d}))
140 |     stream.on('end', () => resolve(entries))
141 |     stream.on('error', err => reject(err))
142 |     stream.on('close', () => reject(new Error('Premature close')))
143 |   })
144 | }
145 | 
146 | 


--------------------------------------------------------------------------------
/test/multi-hyperbee.test.js:
--------------------------------------------------------------------------------
  1 | const test = require('tape')
  2 | const Hyperbee = require('hyperbee')
  3 | const ram = require('random-access-memory')
  4 | const cloneDeep = require('lodash/cloneDeep')
  5 | const pump = require('pump')
  6 | const fs = require('fs')
  7 | const path = require('path')
  8 | // const { promisify } = require('util')
  9 | const { checkForPeers, checkStoreAndDiff, setup, delay } = require('./helpers')
 10 | 
 11 | const MultiHyperbee = require('../')
 12 | var { object0, object1, object1_1, object2,
 13 |       diff0, diff1, diff1_1, diff2 } = require('./constants')
 14 | 
 15 | test('Multihyperbee - value should be JSON object', async t => {
 16 |   let { multiHBs } = await setupReplChannel(1)
 17 |   let mh = multiHBs[0]
 18 |   try {
 19 |     await mh.put('key', 'value')
 20 |   } catch(err) {
 21 |     t.ok(err, /value expected to be JSON object/)
 22 |   }
 23 |   t.end()
 24 | })
 25 | test('Multihyperbee - persistent storage, basic functionality', async t => {
 26 |   let storage = './test/mh/'
 27 |   // let { multiHBs, hasPeers, streams } = await setupReplChannel(2, storage)
 28 |   let { multiHBs } = await setupReplChannel(2, storage)
 29 | 
 30 |   let [primary, secondary] = multiHBs
 31 |   await put(primary, diff0, object0)
 32 |   await delay(100)
 33 |   await put(primary, diff1, object1)
 34 |   await delay(100)
 35 |   await put(secondary, diff1_1, object1_1)
 36 |   await delay(100)
 37 |   let storeArr = [object0, object1, object1_1]
 38 |   let diffArr = [diff0, diff1, diff1_1]
 39 |   await checkStoreAndDiff(t, multiHBs, storeArr, diffArr)
 40 |   t.end()
 41 | })
 42 | test('Multihyperbee - auto-generate diff', async t => {
 43 |   const { multiHBs } = await setupReplChannel(2)
 44 |   const [ primary, secondary ] = multiHBs
 45 | 
 46 |   // The delays are artificial. Without them the mesages get lost for some reason
 47 |   await put(primary, diff0, object0)
 48 | await delay(100)
 49 |   await put(primary, diff1, object1)
 50 | await delay(100)
 51 |   await put(secondary, diff1_1, object1_1)
 52 | await delay(100)
 53 |   await put(secondary, null, object2)
 54 | await delay(100)
 55 |   let storeArr = [object0, object1, object1_1, object2]
 56 |   let diffArr = [diff0, diff1, diff1_1, diff2]
 57 |   await checkStoreAndDiff(t, multiHBs, storeArr, diffArr)
 58 | 
 59 |   if (diffArr.length)
 60 |     t.fail()
 61 |   t.end()
 62 | })
 63 | 
 64 | async function setupReplChannel(count, storage) {
 65 |   let { hasPeers, multiHBs } = await setup(count, storage)
 66 | 
 67 |   let streams = []
 68 |   for (let i=0; i<multiHBs.length; i++) {
 69 |     let cur = i
 70 |     let j = 0
 71 |     let multiHB = multiHBs[i]
 72 |     let diffFeed = (await multiHB.getDiff()).feed
 73 |     for (; j<multiHBs.length; j++) {
 74 |       if (j === cur) continue
 75 |       await multiHBs[j].addPeer(diffFeed.key)
 76 |     }
 77 |   }
 78 | 
 79 |   for (let i=0; i<multiHBs.length; i++) {
 80 |     let cur = i
 81 |     let j = 0
 82 |     let multiHB = multiHBs[i]
 83 | 
 84 |     for (; j<multiHBs.length; j++) {
 85 |       if (j === cur) continue
 86 |       let cloneFeeds = await multiHBs[j].getPeers()
 87 |       for (let ii=0; ii<cloneFeeds.length; ii++) {
 88 |         let pstream = await multiHB.replicate(false, {live: true})
 89 |         streams.push(pstream)
 90 |         let cstream = cloneFeeds[ii].feed.replicate(true, {live: true})
 91 |         streams.push(cstream)
 92 |         pump(pstream, cstream, pstream)
 93 |       }
 94 |     }
 95 |   }
 96 |   return { multiHBs, hasPeers, streams }
 97 | }
 98 | async function put(hyperbee, diff, value) {
 99 |   // debugger
100 |   let key = `${value._objectId}`
101 |   let val = cloneDeep(value)
102 |   if (diff)
103 |     val._diff = cloneDeep(diff)
104 | 
105 |   await hyperbee.put(key, val)
106 | }
107 | 
108 | /*
109 |   // if (hasPeers) {
110 |   //   rmdir(storage, function(error) {
111 |   //     if (error)
112 |   //       console.log(`Error deleting directory ${storage}`, error)
113 |   //     else
114 |   //       console.log(`directory ${storage} was successfully deleted`)
115 |   //   })
116 |   // }
117 |   // for (let i=0; i<streams.length; i++) {
118 |   //   const stream = streams[i]
119 |   //   stream.end()
120 |   //   stream.destroy()
121 |   // }
122 |   // await delay(2000)
123 |   // debugger
124 |   // for (let i=0; i<multiHBs.length; i++) {
125 |   //   let peers = await multiHBs[i].getPeers()
126 |   //   for (let i=0; i<peers.length; i++) {
127 |   //     let peer = prPeers[i]
128 |   //     await promisify(peer.feed.close.bind(peer.feed))()
129 |   //   }
130 |   // }
131 |   // await promisify(primary.feed.close.bind(primary.feed))()
132 |   // await promisify(secondary.feed.close.bind(secondary.feed))()
133 |   // await promisify(primary.diffFeed.close.bind(primary.diffFeed))()
134 |   // await promisify(secondary.diffFeed.close.bind(secondary.diffFeed))()
135 |  */


--------------------------------------------------------------------------------
/test/peerRestoreAndCleanup.js:
--------------------------------------------------------------------------------
 1 | const test = require('tape')
 2 | const rmdir = require('rimraf')
 3 | const { checkStoreAndDiff, setup } = require('./helpers')
 4 | 
 5 | var { object0, object1, object1_1, object2,
 6 |       diff0, diff1, diff1_1, diff2 } = require('./constants')
 7 | 
 8 | test('Multihyperbee - restore peers, restore HLC clock, cleanup peers', async t => {
 9 |   let storage = './test/mh/'
10 |   let { multiHBs, hasPeers } = await setup(2, storage)
11 | 
12 |   if (hasPeers) {
13 |     let storeArr = [object0, object1, object1_1]
14 |     let diffArr = [diff0, diff1, diff1_1, diff2]
15 | 
16 |     await checkStoreAndDiff(t, multiHBs, storeArr, diffArr)
17 |     rmstorage()
18 |   }
19 |   else
20 |     t.pass('Nothing to check')
21 |   t.end()
22 | })
23 | 
24 | function rmstorage() {
25 |   let storages = ['./test/mh/', './test/mht/', './test/mht2']
26 |   storages.forEach(storage => {
27 |     rmdir(storage, error => {
28 |       if (error)
29 |         console.log(`Error deleting directory ${storage}`, error)
30 |       else
31 |         console.log(`directory ${storage} was successfully deleted`)
32 |     })
33 |   })
34 | }
35 | 


--------------------------------------------------------------------------------
/timestamp.js:
--------------------------------------------------------------------------------
  1 | module.exports = function timestamp() {
  2 |   var config = {
  3 |     // Maximum physical clock drift allowed, in ms
  4 |     maxDrift: 60000
  5 |   };
  6 | 
  7 |   class Timestamp {
  8 |     constructor(millis, counter, node) {
  9 |       this._state = {
 10 |         millis: millis,
 11 |         counter: counter,
 12 |         node: node
 13 |       };
 14 |     }
 15 | 
 16 |     valueOf() {
 17 |       return this.toString();
 18 |     }
 19 | 
 20 |     toString() {
 21 |       return [
 22 |         new Date(this.millis()).toISOString(),
 23 |         (
 24 |           '0000' +
 25 |           this.counter()
 26 |             .toString(16)
 27 |             .toUpperCase()
 28 |         ).slice(-4),
 29 |         ('0000000000000000' + this.node()).slice(-16)
 30 |       ].join('-');
 31 |     }
 32 | 
 33 |     millis() {
 34 |       return this._state.millis;
 35 |     }
 36 | 
 37 |     counter() {
 38 |       return this._state.counter;
 39 |     }
 40 | 
 41 |     node() {
 42 |       return this._state.node;
 43 |     }
 44 | 
 45 |     // hash() {
 46 |     //   return murmurhash.v3(this.toString());
 47 |     // }
 48 |   }
 49 | 
 50 |   class MutableTimestamp extends Timestamp {
 51 |     setMillis(n) {
 52 |       this._state.millis = n;
 53 |     }
 54 | 
 55 |     setCounter(n) {
 56 |       this._state.counter = n;
 57 |     }
 58 | 
 59 |     setNode(n) {
 60 |       this._state.node = n;
 61 |     }
 62 |   }
 63 | 
 64 |   MutableTimestamp.from = timestamp => {
 65 |     return new MutableTimestamp(
 66 |       timestamp.millis(),
 67 |       timestamp.counter(),
 68 |       timestamp.node()
 69 |     );
 70 |   };
 71 | 
 72 |   // Timestamp generator initialization
 73 |   // * sets the node ID to an arbitrary value
 74 |   // * useful for mocking/unit testing
 75 |   Timestamp.init = function(options = {}) {
 76 |     if (options.maxDrift) {
 77 |       config.maxDrift = options.maxDrift;
 78 |     }
 79 |   };
 80 | 
 81 |   /**
 82 |    * Timestamp send. Generates a unique, monotonic timestamp suitable
 83 |    * for transmission to another system in string format
 84 |    */
 85 |   Timestamp.send = function(clock) {
 86 |     // Retrieve the local wall time
 87 |     var phys = Date.now();
 88 | 
 89 |     // Unpack the clock.timestamp logical time and counter
 90 |     var lOld = clock.timestamp.millis();
 91 |     var cOld = clock.timestamp.counter();
 92 | 
 93 |     // Calculate the next logical time and counter
 94 |     // * ensure that the logical time never goes backward
 95 |     // * increment the counter if phys time does not advance
 96 |     var lNew = Math.max(lOld, phys);
 97 |     var cNew = lOld === lNew ? cOld + 1 : 0;
 98 | 
 99 |     // Check the result for drift and counter overflow
100 |     if (lNew - phys > config.maxDrift) {
101 |       throw new Timestamp.ClockDriftError(lNew, phys, config.maxDrift);
102 |     }
103 |     if (cNew > 65535) {
104 |       throw new Timestamp.OverflowError();
105 |     }
106 | 
107 |     // Repack the logical time/counter
108 |     clock.timestamp.setMillis(lNew);
109 |     clock.timestamp.setCounter(cNew);
110 | 
111 |     return new Timestamp(
112 |       clock.timestamp.millis(),
113 |       clock.timestamp.counter(),
114 |       clock.timestamp.node()
115 |     );
116 |   };
117 | 
118 |   // Timestamp receive. Parses and merges a timestamp from a remote
119 |   // system with the local timeglobal uniqueness and monotonicity are
120 |   // preserved
121 |   Timestamp.recv = function(clock, msg) {
122 |     var phys = Date.now();
123 | 
124 |     // Unpack the message wall time/counter
125 |     var lMsg = msg.millis();
126 |     var cMsg = msg.counter();
127 | 
128 |     // Assert the node id and remote clock drift
129 |     if (msg.node() === clock.timestamp.node()) {
130 |       throw new Timestamp.DuplicateNodeError(clock.timestamp.node());
131 |     }
132 |     if (lMsg - phys > config.maxDrift) {
133 |       throw new Timestamp.ClockDriftError();
134 |     }
135 | 
136 |     // Unpack the clock.timestamp logical time and counter
137 |     var lOld = clock.timestamp.millis();
138 |     var cOld = clock.timestamp.counter();
139 | 
140 |     // Calculate the next logical time and counter.
141 |     // Ensure that the logical time never goes backward;
142 |     // * if all logical clocks are equal, increment the max counter,
143 |     // * if max = old > message, increment local counter,
144 |     // * if max = messsage > old, increment message counter,
145 |     // * otherwise, clocks are monotonic, reset counter
146 |     var lNew = Math.max(Math.max(lOld, phys), lMsg);
147 |     var cNew =
148 |       lNew === lOld && lNew === lMsg
149 |         ? Math.max(cOld, cMsg) + 1
150 |         : lNew === lOld
151 |         ? cOld + 1
152 |         : lNew === lMsg
153 |         ? cMsg + 1
154 |         : 0;
155 | 
156 |     // Check the result for drift and counter overflow
157 |     if (lNew - phys > config.maxDrift) {
158 |       throw new Timestamp.ClockDriftError();
159 |     }
160 |     if (cNew > 65535) {
161 |       throw new Timestamp.OverflowError();
162 |     }
163 | 
164 |     // Repack the logical time/counter
165 |     clock.timestamp.setMillis(lNew);
166 |     clock.timestamp.setCounter(cNew);
167 | 
168 |     return new Timestamp(
169 |       clock.timestamp.millis(),
170 |       clock.timestamp.counter(),
171 |       clock.timestamp.node()
172 |     );
173 |   };
174 | 
175 |   /**
176 |    * Converts a fixed-length string timestamp to the structured value
177 |    */
178 |   Timestamp.parse = function(timestamp) {
179 |     if (typeof timestamp === 'string') {
180 |       var parts = timestamp.split('-');
181 |       if (parts && parts.length === 5) {
182 |         var millis = Date.parse(parts.slice(0, 3).join('-')).valueOf();
183 |         var counter = parseInt(parts[3], 16);
184 |         var node = parts[4];
185 |         if (!isNaN(millis) && !isNaN(counter))
186 |           return new Timestamp(millis, counter, node);
187 |       }
188 |     }
189 |     return null;
190 |   };
191 | 
192 |   Timestamp.since = isoString => {
193 |     return isoString + '-0000-0000000000000000';
194 |   };
195 | 
196 |   Timestamp.DuplicateNodeError = class extends Error {
197 |     constructor(node) {
198 |       super();
199 |       this.type = 'DuplicateNodeError';
200 |       this.message = 'duplicate node identifier ' + node;
201 |     }
202 |   };
203 | 
204 |   Timestamp.ClockDriftError = class extends Error {
205 |     constructor(...args) {
206 |       super();
207 |       this.type = 'ClockDriftError';
208 |       this.message = ['maximum clock drift exceeded'].concat(args).join(' ');
209 |     }
210 |   };
211 | 
212 |   Timestamp.OverflowError = class extends Error {
213 |     constructor() {
214 |       super();
215 |       this.type = 'OverflowError';
216 |       this.message = 'timestamp counter overflow';
217 |     }
218 |   };
219 | 
220 |   return { Timestamp, MutableTimestamp };
221 | }
222 | 


--------------------------------------------------------------------------------