├── .flowconfig ├── src ├── ipld-format.js ├── author.js ├── package │ ├── multicodec.js │ └── multicodec.js.flow ├── util.js ├── ipfs.js ├── crypto.js ├── format.js ├── data.js └── feed.js ├── package.json ├── .repl.js └── Readme.md /.flowconfig: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/ipld-format.js: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/author.js: -------------------------------------------------------------------------------- 1 | // @flow strict 2 | -------------------------------------------------------------------------------- /src/package/multicodec.js: -------------------------------------------------------------------------------- 1 | // @flow strict 2 | 3 | const { 4 | addPrefix, 5 | rmPrefix, 6 | getCodec, 7 | getCodeVarint, 8 | getVarint 9 | } = require("multicodec") 10 | 11 | export { addPrefix, rmPrefix, getCodec, getVarint } 12 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ipdf", 3 | "version": "0.0.1", 4 | "description": "Inter planetary Data Feed", 5 | "license": "MIT", 6 | "scripts": { 7 | "repl": "node -e \"require('esm')(module)('./.repl.js');\" -i" 8 | }, 9 | "dependencies": { 10 | "esm": "3.2.7", 11 | "ipfs": "0.35.0-pre.0" 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/util.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | /*:: 4 | import type { Callback } from "./data.js" 5 | */ 6 | 7 | export const passback = /*::*/ ( 8 | call /*:(Callback) => mixed*/ 9 | ) /*:Promise*/ => 10 | new Promise((resolve, reject) => { 11 | call((error, ok) => { 12 | if (error) { 13 | reject(error) 14 | } else { 15 | resolve(ok) 16 | } 17 | }) 18 | }) 19 | -------------------------------------------------------------------------------- /src/package/multicodec.js.flow: -------------------------------------------------------------------------------- 1 | // @flow strict 2 | 3 | export type Code = number 4 | export type Name = string 5 | 6 | declare export function addPrefix(prefix: Code | Name, data: Uint8Array): Buffer 7 | declare export function rmPrefix(prefixedData: Uint8Array): Buffer 8 | declare export function getCodec(prefixedData: Uint8Array): string 9 | 10 | declare export function getCodeVarint(name: Name): Buffer 11 | declare export function getVarint(code: Code): Buffer 12 | -------------------------------------------------------------------------------- /src/ipfs.js: -------------------------------------------------------------------------------- 1 | // @flow strict 2 | 3 | /*:: 4 | import type { IPFS, CID } from "./data.js" 5 | */ 6 | 7 | import { passback } from "./util.js" 8 | 9 | export class Dag { 10 | static async encode /*::*/( 11 | ipfs /*:IPFS*/, 12 | node /*:a*/, 13 | format /*:string*/ 14 | ) /*:Promise*/ { 15 | const codec = await passback(callback => 16 | ipfs._ipld._getFormat(format, callback) 17 | ) 18 | const buffer = await passback(callback => 19 | codec.util.serialize(node, callback) 20 | ) 21 | return buffer 22 | } 23 | static async decode /*::*/( 24 | ipfs /*:IPFS*/, 25 | buffer /*:Uint8Array*/, 26 | format /*:string*/ = "dag-cbor" 27 | ) /*:Promise*/ { 28 | const codec = await passback(callback => 29 | ipfs._ipld._getFormat(format, callback) 30 | ) 31 | const data = await passback(callback => 32 | codec.util.deserialize(buffer, callback) 33 | ) 34 | return data 35 | } 36 | static async get /*::*/(ipfs /*:IPFS*/, id /*:CID*/) /*:Promise*/ { 37 | const { value } = await ipfs.dag.get(id) 38 | return value 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /.repl.js: -------------------------------------------------------------------------------- 1 | // @flow strict 2 | 3 | import IPFS from "ipfs" 4 | import Crypto from "./src/crypto.js" 5 | import * as Feed from "./src/feed.js" 6 | import { Dag } from "./src/ipfs.js" 7 | 8 | export const ipfs = new IPFS({ 9 | EXPERIMENTAL: { pubsub: true, dht: true, ipnsPubsub: true } 10 | }) 11 | 12 | export const service = { ipfs, crypto: Crypto(ipfs.util.crypto) } 13 | 14 | export const wait = promise => { 15 | promise.then( 16 | value => { 17 | console.log((wait.result = promise.result = { ok: true, value })) 18 | }, 19 | error => { 20 | console.log((wait.result = promise.result = { ok: false, error })) 21 | } 22 | ) 23 | return promise 24 | } 25 | 26 | export const callback = (error, value) => { 27 | if (error) { 28 | console.log((callback.result = { ok: false, error })) 29 | } else { 30 | console.log((callback.result = { ok: true, value })) 31 | } 32 | } 33 | 34 | const main = async () => { 35 | Object.assign(global, { 36 | wait, 37 | callback, 38 | ipfs, 39 | service, 40 | IPFS, 41 | Crypto, 42 | Feed, 43 | Dag 44 | }) 45 | global.feed = await Feed.feed(service) 46 | global.agent = feed.agent 47 | console.log("Feed started", feed) 48 | } 49 | 50 | main() 51 | -------------------------------------------------------------------------------- /src/crypto.js: -------------------------------------------------------------------------------- 1 | // @flow strict 2 | 3 | /*:: 4 | import type { 5 | LibP2P$Crypto, 6 | Crypto, 7 | PrivateKey, 8 | PublicKey, 9 | SecretKey, 10 | KeyType, 11 | } from "./data.js" 12 | */ 13 | import { passback } from "./util.js" 14 | 15 | export default (libp2p$crypto /*:LibP2P$Crypto*/) /*:Crypto*/ => 16 | new Libp2pCrypto(libp2p$crypto) 17 | 18 | class Libp2pCrypto /*::implements Crypto*/ { 19 | /*:: 20 | provider:LibP2P$Crypto; 21 | supportedKeyType:{[string]:KeyType} 22 | */ 23 | constructor(provider) { 24 | this.provider = provider 25 | this.supportedKeyType = { 26 | ed25519: "ed25519", 27 | RSA: "RSA" 28 | } 29 | } 30 | randomBytes(size /*:number*/) { 31 | return this.provider.randomBytes(size) 32 | } 33 | generateKeyPair( 34 | type /*: KeyType*/, 35 | size /*: number*/ 36 | ) /*: Promise*/ { 37 | return passback(callback => 38 | this.provider.keys.generateKeyPair(type, size, callback) 39 | ) 40 | } 41 | generateKeyPairFromSeed( 42 | type /*: KeyType*/, 43 | seed /*: Uint8Array*/, 44 | size /*: number*/ 45 | ) /*: Promise*/ { 46 | return passback(callback => 47 | this.provider.keys.generateKeyPairFromSeed(type, seed, size, callback) 48 | ) 49 | } 50 | encodePrivateKey(key /*:PrivateKey*/) /*:Uint8Array*/ { 51 | return this.provider.keys.marshalPrivateKey(key) 52 | } 53 | encodePublicKey(key /*:PublicKey*/) /*:Uint8Array*/ { 54 | return this.provider.keys.marshalPublicKey(key) 55 | } 56 | decodePrivateKey(encodedKey /*:Uint8Array*/) /*:Promise*/ { 57 | return passback(out => 58 | this.provider.keys.unmarshalPrivateKey(encodedKey, out) 59 | ) 60 | } 61 | decodePublicKey(encodedKey /*:Uint8Array*/) /*:Promise*/ { 62 | return passback(callback => 63 | this.provider.keys.unmarshalPublicKey(encodedKey, callback) 64 | ) 65 | } 66 | verify( 67 | publicKey /*:PublicKey*/, 68 | data /*:Uint8Array*/, 69 | signature /*:Uint8Array*/ 70 | ) /*:Promise*/ { 71 | return passback(callback => publicKey.verify(data, signature, callback)) 72 | } 73 | sign( 74 | privateKey /*:PrivateKey*/, 75 | data /*:Uint8Array*/ 76 | ) /*:Promise*/ { 77 | return passback(callback => privateKey.sign(data, callback)) 78 | } 79 | async encrypt( 80 | data /*:Uint8Array*/, 81 | nonce /*:Uint8Array*/, 82 | secretKey /*:Uint8Array*/ 83 | ) /*:Promise*/ { 84 | const key = await passback($ => 85 | this.provider.aes.create(secretKey, nonce, $) 86 | ) 87 | return passback(callback => key.encrypt(data, callback)) 88 | } 89 | async decrypt( 90 | data /*:Uint8Array*/, 91 | nonce /*:Uint8Array*/, 92 | secretKey /*:Uint8Array*/ 93 | ) /*:Promise*/ { 94 | const key = await passback($ => 95 | this.provider.aes.create(secretKey, nonce, $) 96 | ) 97 | return passback(callback => key.decrypt(data, callback)) 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /src/format.js: -------------------------------------------------------------------------------- 1 | // @flow strict 2 | 3 | /*:: 4 | import type { CID } from "./data.js" 5 | 6 | type CryptoKey = Uint8Array 7 | type SecretPublicKey = Uint8Array 8 | type SecretPrivateKey = Uint8Array 9 | type BodyKey = Uint8Array 10 | type Signature = Uint8Array 11 | 12 | export opaque type Encoded:Uint8Array = Uint8Array 13 | 14 | export type ReplicatorKey = CryptoKey 15 | export type SubscriberKey = CryptoKey 16 | export type RecepientKey = CryptoKey 17 | export type AuthorPublicKey = CryptoKey 18 | export type AuthorPrivateKey = CryptoKey 19 | export type NamePublicKey = CryptoKey 20 | 21 | export type SecretKey = CryptoKey 22 | 23 | 24 | export type Head = { 25 | signature: Signature, ReplicatorKey>, AuthorPrivateKey>, 26 | block: Encoded, ReplicatorKey> 27 | } 28 | 29 | export type Root = { 30 | head:CID>, 31 | author:AuthorPublicKey, 32 | signature:Signature 33 | } 34 | 35 | export type Block = { 36 | links: CID>[], 37 | message: Encoded, SubscriberKey> 38 | } 39 | 40 | export type Message = { 41 | previous: CID>, 42 | size: number, 43 | content: a 44 | } 45 | 46 | type PrivateMessage = { 47 | type: "private", 48 | head: SecretPublicKey, 49 | // scalar multiplication is used to derive a shared secret for each recipient 50 | // which is then used as to encrypt a `BodyKey` for each recepient. 51 | // Each recepient will attempt to decode `BodyKey` by dervining shared secret 52 | // using `SercetPublicKey` (in head attribute) and own private key. If 53 | // successuful, recepient can decrypt content with it. 54 | // ----- 55 | // Unlike SSB this doesn't actually attempts to conceal number of recepients 56 | // which is not impossible just easier to do with raw buffers than with DAGs. 57 | secrets: Encoded>[], 58 | content: Encoded 59 | } 60 | 61 | type PublicMessage = { 62 | type: "public", 63 | body: a 64 | } 65 | 66 | type SSBLikeFeed = Head | PublicMessage> 67 | */ 68 | 69 | export const root = /*::*/ ( 70 | head /*:CID>*/, 71 | author /*:AuthorPublicKey*/, 72 | signature /*:Signature*/ 73 | ) /*:Root*/ => ({ head, author, signature }) 74 | 75 | export const head = /*::*/ ( 76 | block /*:Encoded, ReplicatorKey>*/, 77 | signature /*:Signature, ReplicatorKey>, AuthorPrivateKey>*/ 78 | ) /*:Head*/ => { 79 | return { block, signature } 80 | } 81 | 82 | export const block = /*::*/ ( 83 | message /*:Encoded, SubscriberKey>*/, 84 | head /*:?CID>*/ 85 | ) /*:Block*/ => { 86 | return { 87 | links: head ? [head] : [], 88 | message 89 | } 90 | } 91 | 92 | export const message = /*::*/ ( 93 | content /*:a*/, 94 | size /*:number*/, 95 | previous /*:CID>*/ 96 | ) /*:Message*/ => { 97 | return { 98 | previous, 99 | size, 100 | content 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | # IPDF - Content content-addressable data feed 2 | 3 | IPDF is the data model of content-addressable data feeds. It provides a way to 4 | distribute an immutable list of data posted & signed by a single identity. 5 | 6 | ### Status: Genesis 7 | 8 | At the moment project is in it's genesis, there are some code sketches, but in 9 | practice it's an outline of established requirements. 10 | 11 | If you care to provide feedabck [data format](https://github.com/gozala/ipdf/blob/master/src/format.js) is what you want to look at. 12 | 13 | ## Goals 14 | 15 | - Versioning for the data identifier (a.k.a mutable data). 16 | - Feed is signed and it's intergrity can be verified. (Each data block + link to the prior blocks is signed by an author allowing integrity checks) 17 | > There is a chance feed will be forked deliberetely or by an accident (e.g. bad migration to new device). Instead of treating such feed as corrupt we would allow user to choose between that or simply choosing a longer feed. 18 | - Feed id should be decoupled from the feed. That would allow an author to share both: 19 | - Live feed that others can subscribe to. 20 | - Specific version of the feed without permission to subscribe to updates. 21 | - Key rotation. It should be possible to swap keys. 22 | > Perhaps this is unecessary one could always just start a new feed instead, however that would require ID change. 23 | - Feed id should be bound to a specific feed. It should be possible to update it to reference new head of the feed but impossible to point to a different feed. 24 | - Multiple layers of privacy 25 | - Author should be able to publish ID of the feed without revealing anything at all. 26 | - Author should be able to share a tarversal key + feed ID with a _replicator_ that would allow it to replicate all the data in the feed without having access to actual data. 27 | - Author should be able to share a secret key + feed ID with with a _recepients_ allowing them to read all the data in the feed. 28 | > Perhaps case could be made to share access to just subset of the feed. Which would imply that different keys can be used for different data blocks. 29 | - Decouple feed from the data. Feed should be decoupled from storage, format, protocol. [IPLD][] provides a way to create links across protocol & format boundries which is a perfect fit. 30 | 31 | ## Prior Art 32 | 33 | ### [SSB Feeds][] 34 | 35 | A [Scuttlebutt feed][ssb feeds] is a primary source of inspiration. It is however tied to the [SSB Protocol][]. We would like to leverage [IPLD][], which provides content-addressable linked data model, and would allow us to work across multiple formats and protocols. 36 | 37 | ### [Hypercore][] 38 | 39 | [hypercore][] is a secure, distributed append-only log. It is decoupled from the transport but is coupled with a data storage model of [random-access-storage][] that uses byte rangens for addressing. On the upsite it is extremely fast, but on the downside it makes data deduplication more tricky. We do however want to take advantage of content-addressible model provided by [IPLD][] which addresses deduplication out of the box. Additionally unified read / write interface across multiple storage models provides additional flexibility e.g. one could implement [IPLD Resolver][] that reads / writes data into [random-access-storage][]. 40 | 41 | We also would like to have more granular access-rights in which you can have **replicator** participants that can make all the data available to the network but have no 0 knowledge of the data a hand. Another kind of participants **subscribers** could subscribe to the feed, verify it's integrity & access public data all without access to some private messages. Finally there can be specific **recepient** participants who can do all the above + access private messages addressed to them. 42 | 43 | ### [Textile Threads][] 44 | 45 | [Textile threads][] do leverage [IPLD][] and have very similar goals, however in the current iteration (v1) they additionally deal with managing messages from multiple parcticipants of the thread. Here we are modeling a feed with a single author, which reduces complexity & provides better scalability and defer multi-user participation to the next layer (e.g. [hypermerge][]). 46 | 47 | There are active discussions on [v2 threads](https://github.com/textileio/go-textile/issues/566) that could end up being very similar to approach here or we might even converge. Right now primary goal is to use this as an alternative backend for [hypermerge][] to enable collaborative use cases in [lunet][]. Starting with a blank page seems more effective way to pursue this goal that attempts to port textile threads from to JS while at the same time changing it's model. 48 | 49 | We do hope however that over time we'll be able to converge. 50 | 51 | ### [IPFS Log][] 52 | 53 | [IPFS Log](https://github.com/orbitdb/ipfs-log) from [Orbit DB][] also makes many of the same choices. However it also doen sot have granular access-rights (as described under [hypercore](#hypercore)). It also deals with joining forks as under the hood it's operation-based conflict-free replicated data structure ([CRDT][]) which is great but we prefer that to be done by the next layer - [hypermerge][] under the hood is also happens to be operation-based [CRDT][] and layering two would result doing more or less same kind of work twice. 54 | 55 | [ipld]: https://ipld.io/ 56 | [ipfs]: https://ipfs.io/ 57 | [ssb feeds]: https://ssbc.github.io/scuttlebutt-protocol-guide/#feeds 58 | [ssb protocol]: https://ssbc.github.io/scuttlebutt-protocol-guide/ 59 | [hypercore]: https://github.com/mafintosh/hypercore 60 | [dat]: http://datproject.org/ 61 | [dat protocol]: https://datprotocol.github.io/how-dat-works/#wire-protocol 62 | [textile threads]: https://medium.com/textileio/wip-textile-threads-whitepaper-just-kidding-6ce3a6624338 63 | [ipfs log]: https://github.com/orbitdb/ipfs-log 64 | [random-access-storage]: https://github.com/random-access-storage/ 65 | [ipld resolver]: https://github.com/ipld/interface-ipld-format 66 | [hypermerge]: https://github.com/automerge/hypermerge 67 | [lunet]: http://github.com/gozala/lunet 68 | [orbit db]: https://github.com/orbitdb/welcome 69 | [crdt]: https://en.wikipedia.org/wiki/Conflict-free_replicated_data_type 70 | -------------------------------------------------------------------------------- /src/data.js: -------------------------------------------------------------------------------- 1 | // @flow strict 2 | 3 | export interface Feed { 4 | publish(data: a): Promise>; 5 | nth(index: number): Promise; 6 | first(): Promise; 7 | slice(start: number, end: number): Promise; 8 | } 9 | 10 | export interface CID { 11 | constructor(string): void; 12 | toString(): string; 13 | toBaseEncodedString(base?: string): string; 14 | buffer: Uint8Array; 15 | equals(mixed): boolean; 16 | } 17 | 18 | export interface IPFS$DAG { 19 | put(content: a, options?: { onlyHash: boolean }): Promise>; 20 | get( 21 | CID, 22 | path: string 23 | ): Promise<{ cid: CID, remainderPath: string, value: b }>; 24 | get(CID): Promise<{ cid: CID, remainderPath: string, value: a }>; 25 | } 26 | 27 | export interface IPFSBlock { 28 | data: Buffer; 29 | cid: CID; 30 | } 31 | 32 | export interface IPLD { 33 | _getFormat(string, Callback>): void; 34 | } 35 | 36 | export interface IPLD$Codec { 37 | util: IPLD$Codec$Util; 38 | } 39 | 40 | export interface IPLD$Codec$Util { 41 | serialize(a, Callback): void; 42 | deserialize(Uint8Array, Callback): void; 43 | } 44 | 45 | type PeerIDJSON = { 46 | id: string, 47 | pubKey: ?string, 48 | privKey: ?string 49 | } 50 | 51 | export interface PeerID { 52 | toHexString(): string; 53 | toBytes(): Uint8Array; 54 | toB58String(): string; 55 | toJSON(): PeerIDJSON; 56 | toPrint(): string; 57 | isEqual(mixed): boolean; 58 | } 59 | 60 | export interface PeerIDAPI { 61 | create(options?: { bits: number }, Callback): void; 62 | createFromHexString(string): PeerID; 63 | createFromBytes(string): PeerID; 64 | createFromB58String(string): PeerID; 65 | createFromPubKey(Uint8Array, Callback): void; 66 | createFromJSON(PeerIDJSON): PeerID; 67 | } 68 | 69 | export type IPFS$Types = { 70 | CID: Class>, 71 | PeerId: PeerIDAPI 72 | } 73 | 74 | export type IPFS$Block$Put = { 75 | format?: "string", 76 | mhtype?: "string", 77 | mhlen?: number, 78 | version?: number 79 | } 80 | 81 | export type IPFS$Block$Stat = { 82 | key: string, 83 | size: number 84 | } 85 | 86 | export interface IPFS$Block { 87 | get( 88 | CID, 89 | option: ?{ cid: CID | Buffer | string } 90 | ): Promise; 91 | put( 92 | Buffer, 93 | options: { cid: CID | Buffer | string } & IPFS$Block$Put 94 | ): Promise; 95 | put(IPFS$Block, options?: IPFS$Block$Put): Promise; 96 | stat(cid: CID | Buffer | string): Promise; 97 | } 98 | 99 | export interface IPFS$Name { 100 | resolve( 101 | string, 102 | options?: { recursive?: boolean, nocache?: boolean } 103 | ): Promise<{ path: string }>; 104 | publish( 105 | string, 106 | options?: { 107 | resolve?: boolean, 108 | lifetime?: string, 109 | ttl?: string, 110 | key?: string 111 | } 112 | ): Promise<{ name: string, value: string }>; 113 | } 114 | 115 | export interface IPNS { 116 | publish( 117 | PrivateKey, 118 | string, 119 | lifetime: number, 120 | Callback 121 | ): void; 122 | } 123 | 124 | export interface IPFS$PubSub { 125 | publish(topic: string, data: Uint8Array): Promise; 126 | ls(): Promise; 127 | subscribe( 128 | topic: string, 129 | handler: IPFS$PubSub$Handler, 130 | options?: { discover?: boolean } 131 | ): Promise; 132 | unsubscribe(topic: string, handler: IPFS$PubSub$Handler): Promise; 133 | } 134 | 135 | export type IPFSPubSubMessage = { 136 | from: string, 137 | seqno: number, 138 | data: Buffer, 139 | topicIDs: string[] 140 | } 141 | type IPFS$PubSub$Handler = IPFSPubSubMessage => void 142 | export interface IPFS { 143 | types: IPFS$Types; 144 | dag: IPFS$DAG; 145 | block: IPFS$Block; 146 | name: IPFS$Name; 147 | pubsub: IPFS$PubSub; 148 | 149 | _ipld: IPLD; 150 | _ipns: IPNS; 151 | } 152 | 153 | export type FeedBlock = { 154 | +author: Uint8Array, 155 | +size: number, 156 | +content: CID, 157 | +previous: CID 158 | } 159 | 160 | export type SignedFeedBlock = FeedBlock & { +signature: Uint8Array } 161 | 162 | export interface Author { 163 | id(): Uint8Array; 164 | sign(bytes: Uint8Array): Promise; 165 | } 166 | 167 | export interface PublicKey { 168 | bytes: Uint8Array; 169 | verify( 170 | data: Uint8Array, 171 | signature: Uint8Array, 172 | Callback 173 | ): void; 174 | } 175 | 176 | export interface PrivateKey { 177 | bytes: Uint8Array; 178 | public: PublicKey; 179 | id(Callback): void; 180 | sign(data: Uint8Array, Callback): void; 181 | } 182 | 183 | export interface CryptoKey { 184 | bytes: Uint8Array; 185 | } 186 | 187 | export interface SecretKey extends CryptoKey { 188 | encrypt(data: Uint8Array, Callback): void; 189 | decrypt(data: Uint8Array, Callback): void; 190 | } 191 | 192 | export interface Crypto { 193 | sign(key: PrivateKey, message: Uint8Array): Promise; 194 | verify( 195 | key: PublicKey, 196 | data: Uint8Array, 197 | signature: Uint8Array 198 | ): Promise; 199 | decodePrivateKey(Uint8Array): Promise; 200 | decodePublicKey(Uint8Array): Promise; 201 | 202 | randomBytes(size: number): Uint8Array; 203 | generateKeyPair(type: KeyType, size: number): Promise; 204 | generateKeyPairFromSeed( 205 | type: KeyType, 206 | seed: Uint8Array, 207 | size: number 208 | ): Promise; 209 | 210 | encrypt( 211 | data: Uint8Array, 212 | nonce: Uint8Array, 213 | key: Uint8Array 214 | ): Promise; 215 | decrypt( 216 | data: Uint8Array, 217 | nonce: Uint8Array, 218 | key: Uint8Array 219 | ): Promise; 220 | } 221 | 222 | export type KeyType = "ed25519" | "RSA" 223 | 224 | interface LibP2P$Crypto$Keys { 225 | generateKeyPair( 226 | type: KeyType, 227 | size: number, 228 | Callback 229 | ): void; 230 | generateKeyPairFromSeed( 231 | type: KeyType, 232 | seed: Uint8Array, 233 | size: number, 234 | Callback 235 | ): void; 236 | unmarshalPrivateKey(Uint8Array, Callback): void; 237 | unmarshalPublicKey(Uint8Array, Callback): void; 238 | marshalPrivateKey(PrivateKey): Uint8Array; 239 | marshalPublicKey(PublicKey): Uint8Array; 240 | } 241 | 242 | interface LibP2P$Crypto$AES { 243 | create(key: Uint8Array, iv: Uint8Array, Callback): void; 244 | } 245 | 246 | export interface Callback { 247 | ($NonMaybeType, empty): void; 248 | (void, a): void; 249 | } 250 | 251 | export interface LibP2P$Crypto { 252 | keys: LibP2P$Crypto$Keys; 253 | aes: LibP2P$Crypto$AES; 254 | randomBytes(size: number): Uint8Array; 255 | } 256 | 257 | type AuthorKey = Uint8Array 258 | type MessageKey = Uint8Array 259 | type Message = { bytes: Uint8Array } 260 | type SecretBox = { key: MessageKey } 261 | type SharedKey = Uint8Array 262 | 263 | type Encoded = { 264 | decode(codec): data 265 | } 266 | 267 | type EncodedMessage = Encoded 268 | type ReceipentMessageKey = Encoded< 269 | messageKey, 270 | SharedKey 271 | > 272 | 273 | type Link = { 274 | cid: CID 275 | } 276 | 277 | type Block = { 278 | nonce: Link, 279 | publicKey: PublicKey, 280 | access: ReceipentMessageKey[], 281 | message: Encoded 282 | } 283 | 284 | type Plain = { 285 | type: "plain/text" 286 | } 287 | 288 | type Encrypted = { 289 | type: "encrypted", 290 | alorithm: string 291 | } 292 | 293 | type Entry = { 294 | format: Plain | Encrypted, 295 | data: Uint8Array, 296 | links: Link[] 297 | } 298 | -------------------------------------------------------------------------------- /src/feed.js: -------------------------------------------------------------------------------- 1 | // @flow strict 2 | 3 | import { Dag } from "./ipfs.js" 4 | import { addPrefix, rmPrefix, getCodec } from "./package/multicodec.js" 5 | import * as Format from "./format.js" 6 | import { passback } from "./util.js" 7 | 8 | /*:: 9 | import type { SecretKey, PrivateKey, PublicKey, IPFS, Crypto, CID, IPFSPubSubMessage } from "./data.js" 10 | import type { Root, Head, Block, Message, Encoded, ReplicatorKey, SubscriberKey } from "./format.js" 11 | 12 | interface FeedOptions { 13 | id:string; 14 | author:PrivateKey; 15 | subscriptionKey:Uint8Array; 16 | replicationKey:Uint8Array; 17 | size: number; 18 | head:CID>; 19 | ipfs:IPFS; 20 | crypto:Crypto; 21 | } 22 | */ 23 | 24 | /*:: 25 | 26 | interface FeedID { 27 | id:string; 28 | } 29 | interface Auditor { 30 | id:string; 31 | authorKey:PublicKey; 32 | } 33 | 34 | interface Publisher { 35 | id: string; 36 | feed: PrivateKey; 37 | author: PrivateKey; 38 | authorKey: PublicKey; 39 | subscriptionKey:Uint8Array; 40 | replicationKey:Uint8Array; 41 | } 42 | 43 | interface Subscriber { 44 | id:string; 45 | authorKey: PublicKey; 46 | subscriptionKey:Uint8Array; 47 | replicationKey:Uint8Array; 48 | } 49 | 50 | interface Replicator { 51 | id:string; 52 | replicationKey:Uint8Array; 53 | authorKey: PublicKey; 54 | } 55 | 56 | interface Service { 57 | ipfs:IPFS; 58 | crypto:Crypto; 59 | } 60 | */ 61 | 62 | const HEAD_FORMAT = "dag-cbor" 63 | const BLOCK_FORMAT = "dag-cbor" 64 | const MESSAGE_FORMAT = "dag-cbor" 65 | const PUBLISH_FORMAT = "dag-cbor" 66 | 67 | /*:: 68 | type Cursor = { 69 | size:number; 70 | head:CID> 71 | } 72 | */ 73 | 74 | class FeedReader /*::*/ { 75 | /*:: 76 | service:Service; 77 | +agent:Subscriber; 78 | cursor:Cursor; 79 | */ 80 | async last() /*:Promise*/ { 81 | const { service, agent, cursor } = this 82 | if (cursor.size === 0) { 83 | return null 84 | } else { 85 | const head = await decodeHead(service, agent, cursor.head) 86 | const block = await decodeBlock(service, agent, head.block) 87 | const message = await decodeMessage(service, agent, block.message) 88 | return message.content 89 | } 90 | } 91 | } 92 | 93 | class FeedPublisher /*::*/ extends FeedReader /*::*/ { 94 | /*:: 95 | service:Service; 96 | agent:Publisher; 97 | cursor:Cursor; 98 | */ 99 | constructor( 100 | cursor /*:Cursor*/, 101 | service /*:Service*/, 102 | agent /*:Publisher*/ 103 | ) { 104 | super() 105 | this.service = service 106 | this.agent = agent 107 | this.cursor = cursor 108 | } 109 | async publish(message /*:a*/) { 110 | const cursor = await publish(this, message) 111 | this.cursor = cursor 112 | return this 113 | } 114 | get size() { 115 | return this.cursor.size 116 | } 117 | toJSON() { 118 | const { 119 | cursor: { size, head }, 120 | agent: { id, feed, author, subscriptionKey, replicationKey } 121 | } = this 122 | return { 123 | agent: { 124 | id, 125 | feed: feed.bytes, 126 | author: author.bytes, 127 | subscriptionKey, 128 | replicationKey 129 | }, 130 | cursor: { 131 | head: head.toBaseEncodedString(), 132 | size 133 | } 134 | } 135 | } 136 | } 137 | 138 | class FeedSubscriber /*::*/ extends FeedReader /*::*/ { 139 | /*:: 140 | service:Service; 141 | subject:string; 142 | agent:Subscriber; 143 | cursor:Cursor; 144 | onpublish:(IPFSPubSubMessage) => void 145 | inbox:IPFSPubSubMessage[] 146 | notify:a[] => mixed 147 | isParked:boolean 148 | */ 149 | constructor( 150 | cursor /*:Cursor*/, 151 | service /*:Service*/, 152 | agent /*:Subscriber*/ 153 | ) { 154 | super() 155 | this.service = service 156 | this.agent = agent 157 | this.cursor = cursor 158 | this.onpublish = this.onpublish.bind(this) 159 | this.isParked = true 160 | this.inbox = [] 161 | } 162 | get subject() { 163 | return `/ipdf/${this.agent.id}/` 164 | } 165 | get size() { 166 | return this.cursor.size 167 | } 168 | onpublish(message /*:IPFSPubSubMessage*/) { 169 | this.inbox.push(message) 170 | this.unpark() 171 | } 172 | async unpark() { 173 | if (this.isParked) { 174 | this.isParked = false 175 | while (this.inbox.length) { 176 | await this.receive(this.inbox.shift()) 177 | } 178 | this.isParked = true 179 | } 180 | } 181 | async receive(payload) { 182 | const { service, agent, cursor } = this 183 | const id = await decodePublish(service, agent, payload.data) 184 | const root = await decodeRoot(service, agent, id) 185 | if (!cursor.head.equals(root.head)) { 186 | await this.reset(root.head) 187 | } 188 | } 189 | async reset(head /*:CID>*/) { 190 | const { service, agent, cursor } = this 191 | const messages = [] 192 | for await (const message of iterate(service, agent, head)) { 193 | messages.unshift(message.content) 194 | // Found current head 195 | if (message.previous.equals(cursor.head)) { 196 | this.cursor = { head, size: cursor.size + messages.length } 197 | break 198 | } 199 | // If we encounter message that is older than current head that implies 200 | // that feed was forked, in which case we panic. 201 | else if (cursor.size > message.size) { 202 | throw Error("Encountered fork in the message feed, panic!") 203 | } 204 | } 205 | return this.notify(messages) 206 | } 207 | subscribe(notify /*:a[] => mixed*/) { 208 | if (this.notify != null) { 209 | throw Error("Already subscribed") 210 | } else { 211 | this.notify = notify 212 | this.service.ipfs.pubsub.subscribe(this.subject, this.onpublish) 213 | } 214 | } 215 | unsubscribe(notify /*:a[] => mixed*/) { 216 | if (this.notify === notify) { 217 | this.service.ipfs.pubsub.unsubscribe(this.subject, this.onpublish) 218 | delete this.notify 219 | } else { 220 | throw Error("Invalid subscriber") 221 | } 222 | } 223 | } 224 | 225 | class FeedReplicator {} 226 | 227 | export const iterate = async function* iterate /*::*/( 228 | service /*:Service*/, 229 | agent /*:Subscriber*/, 230 | top /*:CID>*/ 231 | ) /*:AsyncIterable>*/ { 232 | let id = top 233 | while (id) { 234 | const head = await decodeHead(service, agent, id) 235 | const block = await decodeBlock(service, agent, head.block) 236 | const message = await decodeMessage(service, agent, block.message) 237 | yield message 238 | id = message.previous 239 | } 240 | } 241 | 242 | export const encode = async function encode /*::*/( 243 | { ipfs, crypto } /*:Service*/, 244 | data /*:a*/, 245 | nonce /*:Uint8Array*/, 246 | secretKey /*:Uint8Array*/, 247 | format /*:string*/ 248 | ) /*:Promise>*/ { 249 | const buffer = await Dag.encode(ipfs, data, format) 250 | const out /*:any*/ = await crypto.encrypt( 251 | addPrefix(format, buffer), 252 | nonce, 253 | secretKey 254 | ) 255 | return out 256 | } 257 | 258 | export const decode = async ( 259 | { ipfs, crypto } /*:Service*/, 260 | buffer /*:Uint8Array*/, 261 | nonce /*:Uint8Array*/, 262 | secretKey /*:Uint8Array*/ 263 | ) => { 264 | const data = await crypto.decrypt(buffer, nonce, secretKey) 265 | return await Dag.decode(ipfs, rmPrefix(data), getCodec(data)) 266 | } 267 | 268 | export const decodeBlock = /*::*/ ( 269 | service /*:Service*/, 270 | agent /*:Replicator*/, 271 | block /*:Encoded, ReplicatorKey>*/ 272 | ) /*:Promise>*/ => { 273 | return decode(service, block, REPLICATION_NONCE, agent.replicationKey) 274 | } 275 | 276 | export const encodeBlock = /*::*/ ( 277 | service /*:Service*/, 278 | agent /*:Publisher*/, 279 | block /*:Block*/ 280 | ) /*:Promise, ReplicatorKey>>*/ => 281 | encode(service, block, REPLICATION_NONCE, agent.replicationKey, BLOCK_FORMAT) 282 | 283 | export const encodeMessage = /*::*/ ( 284 | service /*:Service*/, 285 | agent /*:Publisher*/, 286 | message /*:Message*/ 287 | ) /*:Promise, SubscriberKey>>*/ => 288 | encode( 289 | service, 290 | message, 291 | SUBSCRIBTION_NONCE, 292 | agent.subscriptionKey, 293 | MESSAGE_FORMAT 294 | ) 295 | 296 | export const decodeMessage = /*::*/ ( 297 | service /*:Service*/, 298 | agent /*:Subscriber*/, 299 | message /*:Encoded, SubscriberKey>*/ 300 | ) /*:Promise>*/ => 301 | decode(service, message, SUBSCRIBTION_NONCE, agent.subscriptionKey) 302 | 303 | export const encodeHead = async function encodeHead /*::*/( 304 | service /*:Service*/, 305 | agent /*:Publisher*/, 306 | encodedBlock /*:Encoded, ReplicatorKey>*/ 307 | ) /*:Promise>>*/ { 308 | const signature = await service.crypto.sign(agent.author, encodedBlock) 309 | const head = Format.head(encodedBlock, signature) 310 | 311 | return await service.ipfs.dag.put(head) 312 | } 313 | 314 | export const decodeHead = async function decodeHead /*::*/( 315 | service /*:Service*/, 316 | agent /*:Auditor*/, 317 | id /*:CID>*/ 318 | ) /*:Promise>*/ { 319 | const head = await Dag.get(service.ipfs, id) 320 | const verified = await verifyHead(service, agent, head) 321 | if (verified) { 322 | return head 323 | } else { 324 | throw Error("Author of the feed has not signed this block") 325 | } 326 | } 327 | 328 | export const encodeRoot = async function encodeRoot /*::*/( 329 | service /*:Service*/, 330 | agent /*:Publisher*/, 331 | head /*:CID>*/ 332 | ) /*:Promise>>*/ { 333 | const id = Buffer.from(agent.id) 334 | const signature = await service.crypto.sign(agent.author, id) 335 | const root = Format.root(head, agent.authorKey.bytes, signature) 336 | return await service.ipfs.dag.put(root) 337 | } 338 | 339 | export const decodeRoot = async function decodeRoot /*::*/( 340 | service /*:Service*/, 341 | agent /*:Subscriber*/, 342 | id /*:CID>*/ 343 | ) /*:Promise>*/ { 344 | const root = await Dag.get(service.ipfs, id) 345 | const isAuthorized = await verifyRoot(service, agent, root) 346 | if (isAuthorized) { 347 | return root 348 | } else { 349 | throw Error( 350 | "Author of the feed has not authorized publishing under given name" 351 | ) 352 | } 353 | } 354 | 355 | export const encodePublish = async function encodePublish /*::*/( 356 | service /*:Service*/, 357 | agent /*:Publisher*/, 358 | id /*:CID>*/ 359 | ) /*:Promise*/ { 360 | const signature = await service.crypto.sign(agent.feed, id.buffer) 361 | return await Dag.encode(service.ipfs, { id, signature }, PUBLISH_FORMAT) 362 | } 363 | 364 | export const decodePublish = async function decodePublish /*::*/( 365 | service /*:Service*/, 366 | agent /*:Subscriber*/, 367 | data /*:Uint8Array*/ 368 | ) /*:Promise>>*/ { 369 | const { root, signature } = await Dag.decode( 370 | service.ipfs, 371 | data, 372 | PUBLISH_FORMAT 373 | ) 374 | // TODO: Verify signature of the feed 375 | return root 376 | } 377 | 378 | export const verifyRoot = /*::*/ ( 379 | service /*:Service*/, 380 | agent /*:Auditor*/, 381 | root /*:Root*/ 382 | ) /*:Promise*/ => 383 | service.crypto.verify(agent.authorKey, Buffer.from(agent.id), root.signature) 384 | 385 | export const verifyHead = /*::*/ ( 386 | service /*:Service*/, 387 | auditor /*:Auditor*/, 388 | head /*:Head*/ 389 | ) => service.crypto.verify(auditor.authorKey, head.block, head.signature) 390 | 391 | export const resolveHead = async function resolveHead /*::*/( 392 | service /*:Service*/, 393 | agent /*:Subscriber*/ 394 | ) /*:Promise>>*/ { 395 | const address = await service.ipfs.name.resolve(`/ipns/${agent.id}`, { 396 | recursive: !true 397 | }) 398 | const [_, protocol, key] = address.path.split("/") 399 | const root = await decodeRoot(service, agent, new service.ipfs.types.CID(key)) 400 | return root.head 401 | } 402 | 403 | const LIFETIME = 24 * 60 * 60 * 1000 // 24h 404 | const publishHead = async function publishHead /*::*/( 405 | service /*:Service*/, 406 | agent /*:Publisher*/, 407 | head /*:CID>*/ 408 | ) /*:Promise>>*/ { 409 | const root = await encodeRoot(service, agent, head) 410 | await passback(callback => 411 | service.ipfs._ipns.publish( 412 | agent.feed, 413 | `/ipdf/${root.toBaseEncodedString()}`, 414 | LIFETIME, 415 | callback 416 | ) 417 | ) 418 | const signature = await service.crypto.sign(agent.feed, root.buffer) 419 | const buffer = await Dag.encode( 420 | service.ipfs, 421 | { root, signature }, 422 | PUBLISH_FORMAT 423 | ) 424 | await service.ipfs.pubsub.publish(`/ipdf/${agent.id}/`, buffer) 425 | return root 426 | } 427 | 428 | // const create = /*::*/ (options /* :FeedOptions */) => new Feed(options) 429 | 430 | /*:: 431 | type PublisherOptions = { 432 | feed:PrivateKey; 433 | author:PrivateKey; 434 | authorKey:PublicKey; 435 | subscriptionKey:Uint8Array; 436 | replicationKey:Uint8Array; 437 | ipfs:IPFS; 438 | crypto:Crypto; 439 | } 440 | */ 441 | 442 | const feedID = async ( 443 | ipfs /*:IPFS*/, 444 | key /*:PublicKey*/ 445 | ) /*:Promise*/ => { 446 | const { PeerId } = ipfs.types 447 | const id = await passback($ => PeerId.createFromPubKey(key.bytes, $)) 448 | return id.toB58String() 449 | } 450 | 451 | export const publisher = async function publisher /*::*/( 452 | options /*:PublisherOptions*/ 453 | ) /*:Promise>*/ { 454 | const { 455 | ipfs, 456 | crypto, 457 | replicationKey, 458 | subscriptionKey, 459 | author, 460 | authorKey, 461 | feed 462 | } = options 463 | const service = { ipfs, crypto } 464 | 465 | const id = await feedID(ipfs, feed.public) 466 | const agent = { 467 | id, 468 | replicationKey, 469 | subscriptionKey, 470 | authorKey, 471 | author, 472 | feed 473 | } 474 | const headID = await resolveHead(service, agent) 475 | const head = await decodeHead(service, agent, headID) 476 | 477 | const block = await decodeBlock(service, agent, head.block) 478 | const message = await decodeMessage(service, agent, block.message) 479 | const cursor = { head: headID, size: message.size } 480 | return new FeedPublisher(cursor, service, agent) 481 | } 482 | 483 | const SUBSCRIBTION_NONCE = Buffer.from("The Subscribtion") 484 | const REPLICATION_NONCE = Buffer.from("Feed Replication") 485 | 486 | const END = "zdpuAxKCBsAKQpEw456S49oVDkWJ9PZa44KGRfVBWHiXN3UH8" 487 | 488 | export const feed = async function newFeed /*::*/( 489 | options /*:Service*/ 490 | ) /*:Promise>*/ { 491 | const { ipfs, crypto } = options 492 | const author = await crypto.generateKeyPair("ed25519", 256) 493 | const feed = await crypto.generateKeyPair("ed25519", 256) 494 | const authorKey = author.public 495 | const replicationKey = await crypto.randomBytes(32) 496 | const subscriptionKey = await crypto.randomBytes(32) 497 | const id = await feedID(ipfs, feed.public) 498 | const agent = { 499 | id, 500 | replicationKey, 501 | subscriptionKey, 502 | authorKey, 503 | author, 504 | feed 505 | } 506 | const cursor = { head: new ipfs.types.CID(END), size: 0 } 507 | return new FeedPublisher(cursor, options, agent) 508 | } 509 | 510 | /*:: 511 | type SubscriberOptions = { 512 | id:string; 513 | authorKey:PublicKey; 514 | subscriptionKey:Uint8Array; 515 | replicationKey:Uint8Array; 516 | 517 | ipfs:IPFS; 518 | crypto:Crypto; 519 | } 520 | */ 521 | export const subscriber = async function subscriber /*::*/( 522 | options /*:SubscriberOptions*/ 523 | ) /*:Promise>*/ { 524 | const { 525 | ipfs, 526 | crypto, 527 | replicationKey, 528 | subscriptionKey, 529 | authorKey, 530 | id 531 | } = options 532 | const service = { ipfs, crypto } 533 | const agent = { id, replicationKey, subscriptionKey, authorKey } 534 | const headID = await resolveHead(service, agent) 535 | const head = await decodeHead(service, agent, headID) 536 | const block = await decodeBlock(service, agent, head.block) 537 | const message = await decodeMessage(service, agent, block.message) 538 | const cursor = { head: headID, size: message.size } 539 | 540 | return new FeedSubscriber(cursor, service, agent) 541 | } 542 | 543 | export const publish = async function publish /*::*/( 544 | feed /*:FeedPublisher*/, 545 | content /*:a*/ 546 | ) /*:Promise>*/ { 547 | const { service, agent, cursor } = feed 548 | const previous = cursor.head 549 | const size = cursor.size + 1 550 | const message = await encodeMessage( 551 | service, 552 | agent, 553 | Format.message(content, size, previous) 554 | ) 555 | 556 | const block = await encodeBlock( 557 | service, 558 | agent, 559 | Format.block(message, previous) 560 | ) 561 | 562 | const head = await encodeHead(service, agent, block) 563 | void publishHead(service, agent, head) 564 | 565 | return { head, size } 566 | } 567 | --------------------------------------------------------------------------------