├── src ├── test │ ├── asserts.ts │ ├── test-deps.ts │ ├── misc │ │ ├── generate_share_address.test.ts │ │ ├── bytes.test.ts │ │ ├── characters.test.ts │ │ ├── buffers.test.ts │ │ ├── invite.test.ts │ │ └── base32.test.ts │ ├── benchmark │ │ ├── crypto.bench.ts │ │ └── replica.bench.ts │ ├── scenarios │ │ ├── utils.ts │ │ ├── types.ts │ │ └── scenarios.universal.ts │ ├── crypto │ │ └── crypto-driver-interop.test.ts │ ├── replica │ │ └── query_source.test.ts │ ├── server │ │ └── server.test.ts │ └── peer │ │ └── peer.test.ts ├── entries │ ├── npm.ts │ ├── browser.ts │ ├── node.ts │ ├── deno.ts │ └── universal.ts ├── crypto │ ├── default_driver.npm.ts │ ├── default_driver.ts │ ├── default_driver.web.ts │ ├── global-crypto-driver.ts │ ├── updatable_hash.ts │ ├── base32.ts │ ├── crypto-types.ts │ ├── crypto-driver-noble.ts │ ├── keypair.ts │ ├── crypto-driver-sodium.ts │ ├── crypto-driver-node.js │ └── crypto-driver-chloride.ts ├── sync-fs │ ├── constants.ts │ └── sync-fs-types.ts ├── node │ ├── chloride.ts │ └── chloride.d.ts ├── syncer │ ├── constants.ts │ ├── bumping_timeout.ts │ ├── multi_deferred.ts │ ├── doc_thumbnail_tree.ts │ ├── promise_enroller.ts │ ├── range_messenger.ts │ ├── plum_tree.ts │ ├── partner_web_client.ts │ ├── partner_web_server.ts │ └── transfer_queue.ts ├── util │ ├── bigint.ts │ ├── attachment_stream_info.ts │ ├── buffers.ts │ ├── streams.ts │ ├── misc.ts │ ├── bytes.ts │ ├── doc-types.ts │ ├── errors.ts │ ├── invite.ts │ └── log.ts ├── server │ ├── extensions │ │ ├── extension.ts │ │ ├── known_shares.ts │ │ ├── known_shares.node.ts │ │ ├── sync_web.ts │ │ └── sync_web.node.ts │ ├── server_core.ts │ ├── server.ts │ └── server.node.ts ├── tcp │ ├── types.ts │ └── tcp_provider.ts ├── replica │ ├── util-types.ts │ ├── driver_memory.ts │ ├── driver_web.ts │ ├── driver_fs.ts │ ├── driver_fs.node.ts │ ├── replica.ts │ ├── attachment_drivers │ │ └── memory.ts │ └── compare.ts ├── formats │ └── util.ts ├── peer │ └── peer-types.ts ├── discovery │ └── types.ts ├── query │ └── query-types.ts └── core-validators │ └── characters.ts ├── mod.browser.ts ├── .github ├── PULL_REQUEST_TEMPLATE.md ├── FUNDING.yml ├── ISSUE_TEMPLATE.md └── workflows │ ├── deno.js.yml │ └── node.js.yml ├── .nova ├── Configuration.json └── Tasks │ ├── Debug Syncer.json │ └── Debug LAN sync.json ├── .gitignore ├── examples └── servers │ ├── known_shares.json │ └── nimble_server.ts ├── debug ├── dropping_packets.ts ├── close_tcp_gracefully.ts ├── keys.ts ├── path_bytes.ts ├── lan.ts ├── partner_tcp.ts ├── idb.ts └── syncers.ts ├── deno.json ├── deps.ts ├── scripts ├── build_web_bundle.ts ├── release_beta.ts ├── release_alpha.ts └── build_npm.ts ├── Makefile └── mod.ts /src/test/asserts.ts: -------------------------------------------------------------------------------- 1 | export * from "https://deno.land/std@0.154.0/testing/asserts.ts"; 2 | -------------------------------------------------------------------------------- /src/test/test-deps.ts: -------------------------------------------------------------------------------- 1 | export { serve } from "https://deno.land/std@0.123.0/http/server.ts"; 2 | -------------------------------------------------------------------------------- /mod.browser.ts: -------------------------------------------------------------------------------- 1 | export * from "./src/entries/universal.ts"; 2 | export * from "./src/entries/browser.ts"; 3 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## What's the problem you solved? 2 | 3 | ## What solution are you recommending? 4 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | github: [sgwilym] 3 | open_collective: earthstar 4 | -------------------------------------------------------------------------------- /src/entries/npm.ts: -------------------------------------------------------------------------------- 1 | export * from "./universal.ts"; 2 | export * from "./node.ts"; 3 | export * from "./browser.ts"; 4 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## What's the problem you want solved? 2 | 3 | ## Is there a solution you'd like to recommend? 4 | -------------------------------------------------------------------------------- /.nova/Configuration.json: -------------------------------------------------------------------------------- 1 | { 2 | "co.gwil.deno.config.formatOnSave" : "null", 3 | "co.gwil.deno.config.tsconfig" : "deno.json" 4 | } 5 | -------------------------------------------------------------------------------- /src/crypto/default_driver.npm.ts: -------------------------------------------------------------------------------- 1 | import { CryptoDriverNoble } from "./crypto-driver-noble.ts"; 2 | 3 | export default CryptoDriverNoble; 4 | -------------------------------------------------------------------------------- /src/crypto/default_driver.ts: -------------------------------------------------------------------------------- 1 | import { CryptoDriverSodium } from "./crypto-driver-sodium.ts"; 2 | 3 | export default CryptoDriverSodium; 4 | -------------------------------------------------------------------------------- /src/crypto/default_driver.web.ts: -------------------------------------------------------------------------------- 1 | import { CryptoDriverNoble } from "./crypto-driver-noble.ts"; 2 | 3 | export default CryptoDriverNoble; 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | .DS_Store 3 | .nyc_output/ 4 | .vscode/ 5 | cov.lcov 6 | cov_profile/ 7 | cov_html/ 8 | npm/* 9 | test.db 10 | dist/* -------------------------------------------------------------------------------- /src/sync-fs/constants.ts: -------------------------------------------------------------------------------- 1 | export const MANIFEST_FILE_NAME = ".es-fs-manifest"; 2 | 3 | export const MIN_TIMESTAMP_MS = 10000000000; 4 | 5 | export const IGNORED_FILES = [ 6 | MANIFEST_FILE_NAME, 7 | ".DS_Store", 8 | ]; 9 | -------------------------------------------------------------------------------- /src/node/chloride.ts: -------------------------------------------------------------------------------- 1 | // This is a fake stub of chloride which will be swapped out for the real one in the npm package 2 | 3 | import * as chlorideTypes from "./chloride.d.ts"; 4 | 5 | export default undefined as unknown as typeof chlorideTypes; 6 | -------------------------------------------------------------------------------- /examples/servers/known_shares.json: -------------------------------------------------------------------------------- 1 | [ 2 | "+apples.btqswluholq6on2ci5mck66uzkmumb5uszgvqimtshff2f6zy5etq", 3 | "+bananas.bjhk5etxeg2g5ig655c55uig4f36zgbxwsly7pqjbztt3sldn4vxq", 4 | "+coconuts.bg26yyu6k6hkzj3re57u5sch7hn6orkufuqiqqybfud2xuw5nqpxa" 5 | ] 6 | -------------------------------------------------------------------------------- /src/syncer/constants.ts: -------------------------------------------------------------------------------- 1 | // Research indicates 10 seconds is the limit for holding someone's attention without any kind of feedback. 2 | /** The maximum number of milliseconds to wait for some kind of communication from the other peer. */ 3 | export const TIMEOUT_MS = 10000; 4 | -------------------------------------------------------------------------------- /src/util/bigint.ts: -------------------------------------------------------------------------------- 1 | export function bigIntToHex(number: bigint) { 2 | const base = 16; 3 | let hex = number.toString(base); 4 | if (hex.length % 2) { 5 | hex = "0" + hex; 6 | } 7 | return hex; 8 | } 9 | 10 | export function bigIntFromHex(hex: string) { 11 | return BigInt("0x" + hex); 12 | } 13 | -------------------------------------------------------------------------------- /src/entries/browser.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Earthstar APIs which run in browsers. 3 | * @module 4 | */ 5 | 6 | export { DocDriverLocalStorage } from "../replica/doc_drivers/localstorage.ts"; 7 | export { DocDriverIndexedDB } from "../replica/doc_drivers/indexeddb.ts"; 8 | export { AttachmentDriverIndexedDB } from "../replica/attachment_drivers/indexeddb.ts"; 9 | export { ReplicaDriverWeb } from "../replica/driver_web.ts"; 10 | -------------------------------------------------------------------------------- /.nova/Tasks/Debug Syncer.json: -------------------------------------------------------------------------------- 1 | { 2 | "extension" : { 3 | "identifier" : "panic.JavaScript", 4 | "name" : "JavaScript" 5 | }, 6 | "extensionTemplate" : "denoDebug", 7 | "extensionValues" : { 8 | "request" : "launch", 9 | "runtimeArgs" : [ 10 | "--unstable", 11 | "--allow-all" 12 | ], 13 | "scriptPath" : "debug\/syncers.ts", 14 | "stopOnEntry" : true 15 | }, 16 | "openLogOnRun" : "fail" 17 | } 18 | -------------------------------------------------------------------------------- /.nova/Tasks/Debug LAN sync.json: -------------------------------------------------------------------------------- 1 | { 2 | "extension" : { 3 | "identifier" : "panic.JavaScript", 4 | "name" : "JavaScript" 5 | }, 6 | "extensionTemplate" : "denoDebug", 7 | "extensionValues" : { 8 | "request" : "launch", 9 | "runtimeArgs" : [ 10 | "--unstable", 11 | "--allow-all" 12 | ], 13 | "scriptPath" : "debug\/partner_tcp.ts", 14 | "stopOnEntry" : true 15 | }, 16 | "openLogOnRun" : "fail" 17 | } 18 | -------------------------------------------------------------------------------- /.github/workflows/deno.js.yml: -------------------------------------------------------------------------------- 1 | name: Deno CI 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | pull_request: 7 | branches: [main] 8 | 9 | jobs: 10 | build: 11 | runs-on: ubuntu-latest 12 | 13 | strategy: 14 | matrix: 15 | deno-version: ["1.32.3"] 16 | 17 | steps: 18 | - uses: actions/checkout@v2 19 | 20 | - uses: denoland/setup-deno@v1 21 | with: 22 | deno-version: ${{ matrix.deno-version}} 23 | 24 | - run: deno task test 25 | -------------------------------------------------------------------------------- /src/server/extensions/extension.ts: -------------------------------------------------------------------------------- 1 | import { Peer } from "../../peer/peer.ts"; 2 | 3 | /** Implement this interface to create an Earthstar server extension. 4 | * 5 | * - `register` is called once by the server, and this is where you can get a reference to its underlying `Earthstar.Peer`. 6 | * - `handler` is called by the server when it is trying to fulfil an external request. If your extension does not interact with user requests you can return `Promise.resolve(null)`. 7 | */ 8 | export interface IServerExtension { 9 | register(peer: Peer): Promise; 10 | handler(req: Request): Promise; 11 | } 12 | -------------------------------------------------------------------------------- /src/tcp/types.ts: -------------------------------------------------------------------------------- 1 | export interface ITcpConn { 2 | read(bytes: Uint8Array): Promise; 3 | write(bytes: Uint8Array): Promise; 4 | readable: ReadableStream; 5 | writable: WritableStream; 6 | close(): void; 7 | remoteAddr: { 8 | hostname: string; 9 | port: number; 10 | }; 11 | } 12 | 13 | export interface ITcpListener extends AsyncIterable { 14 | close(): void; 15 | } 16 | 17 | export interface ITcpProvider { 18 | listen(opts: { port: number }): ITcpListener; 19 | connect(opts: { port: number; hostname: string }): Promise; 20 | } 21 | -------------------------------------------------------------------------------- /src/replica/util-types.ts: -------------------------------------------------------------------------------- 1 | //================================================================================ 2 | // BASIC UTILITY TYPES 3 | 4 | export type Thunk = () => void; 5 | export type Callback = (data: T) => void; 6 | export type AsyncCallback = (data: T) => Promise; 7 | export type SyncOrAsyncCallback = (data: T) => Promise | void; 8 | 9 | // The type of a class that implementes interface T. 10 | // let arr: ClassThatImplements = [Whatever1, Whatever2] 11 | export type ClassThatImplements = new (...args: any[]) => T; 12 | 13 | export enum Cmp { 14 | // this sorts ascendingly 15 | LT = -1, 16 | EQ = 0, 17 | GT = 1, 18 | } 19 | -------------------------------------------------------------------------------- /src/syncer/bumping_timeout.ts: -------------------------------------------------------------------------------- 1 | /** A timeout which can be 'bumped' manually, restarting the timer. */ 2 | export class BumpingTimeout { 3 | private timeout: number; 4 | private cb: () => void; 5 | private ms: number; 6 | private closed = false; 7 | 8 | constructor(cb: () => void, ms: number) { 9 | this.cb = cb; 10 | this.ms = ms; 11 | this.timeout = setTimeout(cb, ms); 12 | } 13 | 14 | bump() { 15 | if (this.closed) { 16 | return; 17 | } 18 | 19 | clearTimeout(this.timeout); 20 | this.timeout = setTimeout(this.cb, this.ms); 21 | } 22 | 23 | close() { 24 | this.closed = true; 25 | 26 | clearTimeout(this.timeout); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/crypto/global-crypto-driver.ts: -------------------------------------------------------------------------------- 1 | import { ICryptoDriver } from "./crypto-types.ts"; 2 | import DefaultCrypto from "./default_driver.ts"; 3 | 4 | //-------------------------------------------------- 5 | 6 | import { Logger } from "../util/log.ts"; 7 | let logger = new Logger("crypto", "cyan"); 8 | 9 | //================================================================================ 10 | 11 | export let GlobalCryptoDriver: ICryptoDriver = DefaultCrypto; 12 | 13 | /** Set the crypto driver used for all cryptographic operations. */ 14 | export function setGlobalCryptoDriver(driver: ICryptoDriver): void { 15 | logger.debug(`set global crypto driver: ${(driver as any).name}`); 16 | GlobalCryptoDriver = driver; 17 | } 18 | -------------------------------------------------------------------------------- /.github/workflows/node.js.yml: -------------------------------------------------------------------------------- 1 | name: NPM + Node CI 2 | 3 | on: 4 | push: 5 | branches: [main, squirrel] 6 | pull_request: 7 | branches: [main, squirrel] 8 | 9 | jobs: 10 | build: 11 | runs-on: ubuntu-latest 12 | 13 | strategy: 14 | matrix: 15 | node-version: ["16", "18"] 16 | deno-version: ["1.25.1"] 17 | 18 | steps: 19 | - uses: actions/checkout@v2 20 | 21 | - uses: denoland/setup-deno@v1 22 | with: 23 | deno-version: ${{ matrix.deno-version}} 24 | 25 | - name: Use Node.js version ${{ matrix.node-version }} 26 | uses: actions/setup-node@v2 27 | with: 28 | node-version: ${{ matrix.node-version }} 29 | 30 | - run: deno task npm 31 | -------------------------------------------------------------------------------- /src/replica/driver_memory.ts: -------------------------------------------------------------------------------- 1 | import { ShareAddress } from "../util/doc-types.ts"; 2 | import { AttachmentDriverMemory } from "./attachment_drivers/memory.ts"; 3 | import { DocDriverMemory } from "./doc_drivers/memory.ts"; 4 | import { 5 | IReplicaAttachmentDriver, 6 | IReplicaDocDriver, 7 | IReplicaDriver, 8 | } from "./replica-types.ts"; 9 | 10 | /** A replica driver which stores data in memory. All data is lost when the replica is closed. */ 11 | export class ReplicaDriverMemory implements IReplicaDriver { 12 | docDriver: IReplicaDocDriver; 13 | attachmentDriver: IReplicaAttachmentDriver; 14 | 15 | constructor(shareAddress: ShareAddress) { 16 | this.docDriver = new DocDriverMemory(shareAddress); 17 | this.attachmentDriver = new AttachmentDriverMemory(); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/test/misc/generate_share_address.test.ts: -------------------------------------------------------------------------------- 1 | import { assert, assertEquals } from "../asserts.ts"; 2 | import { generateShareAddress } from "../../util/misc.ts"; 3 | import { isErr } from "../../util/errors.ts"; 4 | import { checkShareIsValid } from "../../core-validators/addresses.ts"; 5 | 6 | Deno.test("generateShareAddress", () => { 7 | const address = generateShareAddress("testing"); 8 | assert(!isErr(address), "address is valid (according to itself)"); 9 | assert(checkShareIsValid(address as string), "address is valid"); 10 | assert( 11 | (address as string).startsWith("+testing."), 12 | "address contains the given name", 13 | ); 14 | 15 | const suffix = (address as string).split(".")[1]; 16 | assertEquals(suffix.length, 12, "suffix is 12 chars long"); 17 | }); 18 | -------------------------------------------------------------------------------- /src/replica/driver_web.ts: -------------------------------------------------------------------------------- 1 | import { ShareAddress } from "../util/doc-types.ts"; 2 | import { AttachmentDriverIndexedDB } from "./attachment_drivers/indexeddb.ts"; 3 | import { DocDriverIndexedDB } from "./doc_drivers/indexeddb.ts"; 4 | import { 5 | IReplicaAttachmentDriver, 6 | IReplicaDocDriver, 7 | IReplicaDriver, 8 | } from "./replica-types.ts"; 9 | 10 | /** A replica driver which persists data to IndexedDB. */ 11 | export class ReplicaDriverWeb implements IReplicaDriver { 12 | docDriver: IReplicaDocDriver; 13 | attachmentDriver: IReplicaAttachmentDriver; 14 | 15 | constructor(shareAddress: ShareAddress, namespace?: string) { 16 | this.docDriver = new DocDriverIndexedDB(shareAddress, namespace); 17 | this.attachmentDriver = new AttachmentDriverIndexedDB( 18 | shareAddress, 19 | namespace, 20 | ); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/crypto/updatable_hash.ts: -------------------------------------------------------------------------------- 1 | type UpdatableHashOpts = { 2 | hash: HashType; 3 | update: (hash: HashType, data: Uint8Array) => HashType; 4 | digest: (hash: HashType) => Uint8Array; 5 | }; 6 | 7 | export class UpdatableHash { 8 | private hash: HashType; 9 | private internalUpdate: UpdatableHashOpts["update"]; 10 | private internalDigest: UpdatableHashOpts["digest"]; 11 | 12 | constructor(opts: UpdatableHashOpts) { 13 | this.hash = opts.hash; 14 | this.internalUpdate = opts.update; 15 | this.internalDigest = opts.digest; 16 | } 17 | 18 | update(data: Uint8Array): HashType { 19 | this.hash = this.internalUpdate(this.hash, data); 20 | 21 | return this.hash; 22 | } 23 | 24 | /** Returns the digest of the hash. **The result is not encoded to base32**. */ 25 | digest(): Uint8Array { 26 | return this.internalDigest(this.hash); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /debug/dropping_packets.ts: -------------------------------------------------------------------------------- 1 | const SIZE = 800000; 2 | 3 | const listener = Deno.listen({ port: 17171 }); 4 | 5 | (async () => { 6 | for await (const conn of listener) { 7 | const id = Math.round(Math.random() * 100); 8 | 9 | let total = 0; 10 | 11 | conn.readable.pipeTo( 12 | new WritableStream({ 13 | write(chunk) { 14 | total += chunk.byteLength; 15 | 16 | console.log(id, "got", chunk.byteLength, "bytes..."); 17 | 18 | if (total >= SIZE) { 19 | console.log(id, "got everything!"); 20 | } 21 | }, 22 | }), 23 | ); 24 | } 25 | })(); 26 | 27 | async function run(id: number) { 28 | const conn = await Deno.connect({ port: 17171 }); 29 | 30 | const chunk = new Uint8Array(SIZE); 31 | 32 | // Send a big 33 | await conn.write(chunk); 34 | 35 | console.log(id, `sent`, chunk.byteLength, "bytes"); 36 | } 37 | 38 | for (const num of [1, 2, 3, 4, 5]) { 39 | run(num); 40 | } 41 | -------------------------------------------------------------------------------- /debug/close_tcp_gracefully.ts: -------------------------------------------------------------------------------- 1 | const SIZE = 800000; 2 | 3 | const listener = Deno.listen({ port: 17171 }); 4 | 5 | (async () => { 6 | for await (const conn of listener) { 7 | const id = Math.round(Math.random() * 100); 8 | 9 | let total = 0; 10 | 11 | conn.readable.pipeTo( 12 | new WritableStream({ 13 | write(chunk) { 14 | total += chunk.byteLength; 15 | 16 | console.log(id, "got", chunk.byteLength, "bytes..."); 17 | 18 | if (total >= SIZE) { 19 | console.log(id, "got everything!"); 20 | conn.close(); 21 | } 22 | }, 23 | }), 24 | ); 25 | } 26 | })(); 27 | 28 | async function run(id: number) { 29 | const conn = await Deno.connect({ port: 17171 }); 30 | 31 | const chunk = new Uint8Array(SIZE); 32 | 33 | // Send a big 34 | await conn.write(chunk); 35 | 36 | console.log(id, `sent`, chunk.byteLength, "bytes"); 37 | 38 | conn.close(); 39 | } 40 | 41 | for (const num of [1, 2, 3, 4, 5]) { 42 | run(num); 43 | } 44 | -------------------------------------------------------------------------------- /src/test/benchmark/crypto.bench.ts: -------------------------------------------------------------------------------- 1 | import { randomId } from "../../util/misc.ts"; 2 | import { cryptoScenarios } from "../scenarios/scenarios.ts"; 3 | 4 | for (const scenario of cryptoScenarios) { 5 | const keypairBytes = await scenario.item.generateKeypairBytes(); 6 | const message = "hello" + randomId() + randomId(); 7 | const sigBytes = await scenario.item.sign(keypairBytes, message); 8 | 9 | Deno.bench(`generateKeypairBytes (${scenario.name})`, { 10 | group: "generateKeypairBytes", 11 | }, async () => { 12 | await scenario.item.generateKeypairBytes(); 13 | }); 14 | 15 | Deno.bench(`sha256 (${scenario.name})`, { group: "sha256" }, async () => { 16 | await scenario.item.sha256(message); 17 | }); 18 | 19 | Deno.bench(`sign (${scenario.name})`, { group: "sign" }, async () => { 20 | await scenario.item.sign(keypairBytes, message); 21 | }); 22 | 23 | Deno.bench(`validate (${scenario.name})`, { group: "validate" }, async () => { 24 | await scenario.item.verify(keypairBytes.pubkey, sigBytes, message); 25 | }); 26 | } 27 | -------------------------------------------------------------------------------- /deno.json: -------------------------------------------------------------------------------- 1 | { 2 | "tasks": { 3 | "test": "deno test --allow-all --unstable src", 4 | "test-watch": "deno test --watch --allow-all --no-check --unstable src", 5 | "bench": "deno bench --allow-all --unstable --no-check src/test/benchmark", 6 | "bundle": "deno run --allow-all scripts/build_web_bundle.ts", 7 | "npm": "deno run --allow-all scripts/build_npm.ts", 8 | "example": "deno run examples/animal_story.ts", 9 | "coverage": "deno task test-coverage && deno task show-coverage", 10 | "test-coverage": "deno test --no-check --coverage=cov_profile src", 11 | "show-coverage": "deno coverage cov_profile --lcov > cov.lcov && genhtml -o cov_html cov.lcov", 12 | "clean": "rm -rf npm build coverage dist cov.lcov coverage_html cov_profile", 13 | "release-alpha": "deno run --allow-all scripts/release_alpha.ts", 14 | "release-beta": "deno run --allow-all scripts/release_beta.ts" 15 | }, 16 | "fmt": { 17 | "files": { 18 | "exclude": ["npm", ".git", "earthstar.bundle.js", ".nova"] 19 | } 20 | }, 21 | "lock": false 22 | } 23 | -------------------------------------------------------------------------------- /src/test/scenarios/utils.ts: -------------------------------------------------------------------------------- 1 | import { MultiplyScenarioOutput, Scenarios } from "./types.ts"; 2 | 3 | export function multiplyScenarios( 4 | ...scenarios: Scenarios[] 5 | ): MultiplyScenarioOutput { 6 | const output: MultiplyScenarioOutput = []; 7 | 8 | const [head, ...rest] = scenarios; 9 | 10 | if (!head) { 11 | return []; 12 | } 13 | 14 | for (const scenario of head.scenarios) { 15 | const restReses = multiplyScenarios(...rest); 16 | 17 | if (restReses.length === 0) { 18 | output.push({ 19 | name: scenario.name, 20 | subscenarios: { 21 | [head.description]: scenario.item, 22 | }, 23 | }); 24 | } 25 | 26 | for (const restRes of restReses) { 27 | const thing = { 28 | name: `${scenario.name} + ${restRes.name}`, 29 | subscenarios: { 30 | [head.description]: scenario.item, 31 | ...restRes.subscenarios, 32 | }, 33 | }; 34 | 35 | output.push(thing); 36 | } 37 | } 38 | 39 | return output; 40 | } 41 | -------------------------------------------------------------------------------- /src/syncer/multi_deferred.ts: -------------------------------------------------------------------------------- 1 | import { Deferred, deferred } from "../../deps.ts"; 2 | 3 | export class MultiDeferred { 4 | private deferreds = new Set>(); 5 | state: Deferred["state"] = "pending"; 6 | 7 | resolve(value?: ReturnType) { 8 | if (this.state !== "pending") { 9 | return; 10 | } 11 | 12 | this.state = "fulfilled"; 13 | 14 | for (const deferred of this.deferreds) { 15 | deferred.resolve(value); 16 | } 17 | } 18 | 19 | reject(reason?: any) { 20 | if (this.state !== "pending") { 21 | return; 22 | } 23 | 24 | this.state = "rejected"; 25 | 26 | for (const deferred of this.deferreds) { 27 | deferred.reject(reason); 28 | } 29 | } 30 | 31 | getPromise() { 32 | const promise = deferred(); 33 | 34 | if (this.state === "fulfilled") { 35 | promise.resolve(); 36 | } else if (this.state === "rejected") { 37 | promise.reject(); 38 | } else { 39 | this.deferreds.add(promise); 40 | } 41 | 42 | return promise; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/entries/node.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Earthstar APIs which run in the Node runtime. 3 | * @module 4 | */ 5 | 6 | export { CryptoDriverChloride } from "../crypto/crypto-driver-chloride.ts"; 7 | export { CryptoDriverNode } from "../crypto/crypto-driver-node.js"; 8 | export { DocDriverSqlite } from "../replica/doc_drivers/sqlite.node.ts"; 9 | export { PartnerWebServer } from "../syncer/partner_web_server.ts"; 10 | export { AttachmentDriverFilesystem } from "../replica/attachment_drivers/filesystem.node.ts"; 11 | export { ReplicaDriverFs } from "../replica/driver_fs.ts"; 12 | 13 | //export { syncReplicaAndFsDir } from "../sync-fs/sync-fs.ts"; 14 | 15 | // Servers 16 | export * from "../server/server_core.ts"; 17 | export * from "../server/server.node.ts"; 18 | export * from "../server/extensions/extension.ts"; 19 | export * from "../server/extensions/known_shares.node.ts"; 20 | export * from "../server/extensions/server_settings.ts"; 21 | export * from "../server/extensions/sync_web.node.ts"; 22 | export * from "../server/extensions/serve_content.ts"; 23 | 24 | // LAN discovery 25 | export * from "../discovery/discovery_lan.ts"; 26 | -------------------------------------------------------------------------------- /src/replica/driver_fs.ts: -------------------------------------------------------------------------------- 1 | import { join } from "https://deno.land/std@0.154.0/path/mod.ts"; 2 | import { ensureDirSync } from "https://deno.land/std@0.154.0/fs/ensure_dir.ts"; 3 | import { ShareAddress } from "../util/doc-types.ts"; 4 | import { AttachmentDriverFilesystem } from "./attachment_drivers/filesystem.ts"; 5 | import { DocDriverSqlite } from "./doc_drivers/sqlite.deno.ts"; 6 | import { 7 | IReplicaAttachmentDriver, 8 | IReplicaDocDriver, 9 | IReplicaDriver, 10 | } from "./replica-types.ts"; 11 | 12 | /** A replica driver which persists data to the filesystem. */ 13 | export class ReplicaDriverFs implements IReplicaDriver { 14 | docDriver: IReplicaDocDriver; 15 | attachmentDriver: IReplicaAttachmentDriver; 16 | 17 | constructor(shareAddress: ShareAddress, dirPath: string) { 18 | ensureDirSync(dirPath); 19 | 20 | this.docDriver = new DocDriverSqlite({ 21 | filename: join(dirPath, `${shareAddress}.sql`), 22 | mode: "create-or-open", 23 | share: shareAddress, 24 | }); 25 | this.attachmentDriver = new AttachmentDriverFilesystem( 26 | join(dirPath, "attachments"), 27 | ); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/util/attachment_stream_info.ts: -------------------------------------------------------------------------------- 1 | import { deferred } from "../../deps.ts"; 2 | import { base32BytesToString } from "../crypto/base32.ts"; 3 | import { Crypto } from "../crypto/crypto.ts"; 4 | 5 | export class AttachmentStreamInfo { 6 | private transformer: TransformStream; 7 | private updatableHash = Crypto.updatableSha256(); 8 | 9 | size = deferred(); 10 | hash = deferred(); 11 | 12 | constructor() { 13 | const { updatableHash, size, hash } = this; 14 | 15 | let currentSize = 0; 16 | 17 | this.transformer = new TransformStream({ 18 | transform(chunk, controller) { 19 | updatableHash.update(chunk); 20 | currentSize += chunk.byteLength; 21 | 22 | controller.enqueue(chunk); 23 | }, 24 | flush() { 25 | const digest = updatableHash.digest(); 26 | 27 | hash.resolve(base32BytesToString(digest)); 28 | size.resolve(currentSize); 29 | }, 30 | }); 31 | } 32 | 33 | get writable() { 34 | return this.transformer.writable; 35 | } 36 | 37 | get readable() { 38 | return this.transformer.readable; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/util/buffers.ts: -------------------------------------------------------------------------------- 1 | import { Buffer } from "https://deno.land/std@0.154.0/node/buffer.ts"; 2 | 3 | /** 4 | * This file provides common operations on Buffer. 5 | * Any util function that uses a Buffer should be here, not in bytes.ts. 6 | */ 7 | //-------------------------------------------------- 8 | 9 | import { isBuffer, isBytes } from "./bytes.ts"; 10 | 11 | export function bytesToBuffer(bytes: Uint8Array): Buffer { 12 | return Buffer.from(bytes); 13 | } 14 | 15 | export function bufferToBytes(buf: Buffer): Uint8Array { 16 | return new Uint8Array( 17 | buf.buffer, 18 | buf.byteOffset, 19 | buf.byteLength / Uint8Array.BYTES_PER_ELEMENT, 20 | ); 21 | } 22 | 23 | //-------------------------------------------------- 24 | 25 | export function stringToBuffer(str: string): Buffer { 26 | return Buffer.from(str, "utf-8"); 27 | } 28 | 29 | export function bufferToString(buf: Buffer): string { 30 | return buf.toString("utf-8"); 31 | } 32 | 33 | export function identifyBufOrBytes(bufOrBytes: Buffer | Uint8Array): string { 34 | if (isBytes(bufOrBytes)) return "bytes"; 35 | if (isBuffer(bufOrBytes)) return "buffer"; 36 | return "?"; 37 | } 38 | -------------------------------------------------------------------------------- /deps.ts: -------------------------------------------------------------------------------- 1 | export { 2 | shallowEqualArrays, 3 | shallowEqualObjects, 4 | } from "https://deno.land/x/shallow_equal@v0.1.3/mod.ts"; 5 | export { default as fast_json_stable_stringify } from "npm:fast-json-stable-stringify@2.1.0"; 6 | export * as rfc4648 from "https://esm.sh/rfc4648@1.5.0"; 7 | export * as sha256_uint8array from "https://esm.sh/sha256-uint8array@0.10.3"; 8 | export * as ed from "https://raw.githubusercontent.com/sgwilym/noble-ed25519/153f9e7e9952ad22885f5abb3f6abf777bef4a4c/mod.ts"; 9 | export { hash as xxhash64, XXH64 } from "./src/util/xxhash64.js"; 10 | export { 11 | FingerprintTree, 12 | RangeMessenger, 13 | } from "https://deno.land/x/range_reconcile@1.0.2/mod.ts"; 14 | export type { 15 | LiftingMonoid, 16 | RangeMessengerConfig, 17 | } from "https://deno.land/x/range_reconcile@1.0.2/mod.ts"; 18 | 19 | export { AsyncQueue } from "https://deno.land/x/for_awaitable_queue@1.0.0/mod.ts"; 20 | 21 | // Deno std lib 22 | 23 | export { 24 | type Deferred, 25 | deferred, 26 | } from "https://deno.land/std@0.167.0/async/deferred.ts"; 27 | export { equals as bytesEquals } from "https://deno.land/std@0.167.0/bytes/equals.ts"; 28 | export { concat } from "https://deno.land/std@0.167.0/bytes/concat.ts"; 29 | -------------------------------------------------------------------------------- /src/replica/driver_fs.node.ts: -------------------------------------------------------------------------------- 1 | import { join } from "https://deno.land/std@0.154.0/node/path.ts"; 2 | import { 3 | existsSync, 4 | mkdirSync, 5 | } from "https://deno.land/std@0.154.0/node/fs.ts"; 6 | import { ShareAddress } from "../util/doc-types.ts"; 7 | import { AttachmentDriverFilesystem } from "./attachment_drivers/filesystem.node.ts"; 8 | import { DocDriverSqlite } from "./doc_drivers/sqlite.node.ts"; 9 | import { 10 | IReplicaAttachmentDriver, 11 | IReplicaDocDriver, 12 | IReplicaDriver, 13 | } from "./replica-types.ts"; 14 | 15 | /** A replica driver which persists data to the filesystem. */ 16 | export class ReplicaDriverFs implements IReplicaDriver { 17 | docDriver: IReplicaDocDriver; 18 | attachmentDriver: IReplicaAttachmentDriver; 19 | 20 | constructor(shareAddress: ShareAddress, dirPath: string) { 21 | if (!existsSync(dirPath)) { 22 | mkdirSync(dirPath); 23 | } 24 | 25 | this.docDriver = new DocDriverSqlite({ 26 | filename: join(dirPath, `${shareAddress}.sql`), 27 | mode: "create-or-open", 28 | share: shareAddress, 29 | }); 30 | this.attachmentDriver = new AttachmentDriverFilesystem( 31 | join(dirPath, "attachments"), 32 | ); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/entries/deno.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Earthstar APIs which run in the Deno runtime. 3 | * @module 4 | */ 5 | 6 | export { ReplicaDriverFs } from "../replica/driver_fs.ts"; 7 | export { DocDriverLocalStorage } from "../replica/doc_drivers/localstorage.ts"; 8 | export { DocDriverSqlite } from "../replica/doc_drivers/sqlite.deno.ts"; 9 | 10 | // Uncomment when FFI APIs are stable 11 | // export { DocDriverSqliteFfi } from "../replica/doc_drivers/sqlite_ffi.ts"; 12 | export { AttachmentDriverFilesystem } from "../replica/attachment_drivers/filesystem.ts"; 13 | export { CryptoDriverSodium } from "../crypto/crypto-driver-sodium.ts"; 14 | export { PartnerWebServer } from "../syncer/partner_web_server.ts"; 15 | export { syncReplicaAndFsDir } from "../sync-fs/sync-fs.ts"; 16 | 17 | // Servers 18 | export * from "../server/server_core.ts"; 19 | export * from "../server/server.ts"; 20 | export * from "../server/extensions/extension.ts"; 21 | export * from "../server/extensions/known_shares.ts"; 22 | export * from "../server/extensions/server_settings.ts"; 23 | export * from "../server/extensions/sync_web.ts"; 24 | export * from "../server/extensions/serve_content.ts"; 25 | 26 | // LAN Discovery, uncomment when UDP APIs are stable. 27 | // export * from "../discovery/discovery_lan.ts"; 28 | -------------------------------------------------------------------------------- /src/sync-fs/sync-fs-types.ts: -------------------------------------------------------------------------------- 1 | import { AuthorKeypair } from "../crypto/crypto-types.ts"; 2 | import { Replica } from "../replica/replica.ts"; 3 | 4 | /** 5 | * Options for syncing a replica with a filesystem directory. 6 | * - `dirPath`: The filesystem path of the directory to be synced 7 | * - `replica`: The replica to be synced with the directory 8 | * - `keypair`: The keypair to be used to sign all writes derived from changes on the filesystem. 9 | * - `allowDirtyDirWithoutManifest`: Whether to allow syncing of a folder with pre-existing contents which has never been synced before. 10 | */ 11 | export type SyncFsOptions = { 12 | dirPath: string; 13 | replica: Replica; 14 | keypair: AuthorKeypair; 15 | allowDirtyDirWithoutManifest: boolean; 16 | overwriteFilesAtOwnedPaths?: boolean; 17 | }; 18 | 19 | export type SyncFsManifest = { 20 | share: string; 21 | entries: Record; 22 | }; 23 | 24 | export interface AbsenceEntry { 25 | path: string; 26 | fileLastSeenMs: number; 27 | } 28 | 29 | export interface FileInfoEntry { 30 | dirName: string; 31 | path: string; 32 | abspath: string; 33 | exposedContentSize: number; 34 | mtimeMs: number; // modified time (write) 35 | birthtimeMs: number; // created time 36 | exposedContentHash: string; 37 | } 38 | -------------------------------------------------------------------------------- /src/syncer/doc_thumbnail_tree.ts: -------------------------------------------------------------------------------- 1 | import { FingerprintTree, LiftingMonoid, xxhash64 } from "../../deps.ts"; 2 | import { DocThumbnail } from "./syncer_types.ts"; 3 | 4 | /** Derive an order from two document thumbnails. */ 5 | function compareThumbnails(a: DocThumbnail, b: DocThumbnail) { 6 | const [timestampA, hashA] = a.split(" "); 7 | const [timestampB, hashB] = b.split(" "); 8 | 9 | const timestampAInt = parseInt(timestampA); 10 | const timestampBInt = parseInt(timestampB); 11 | 12 | if (timestampAInt > timestampBInt) { 13 | return 1; 14 | } else if (timestampAInt < timestampBInt) { 15 | return -1; 16 | } 17 | 18 | if (hashA > hashB) { 19 | return 1; 20 | } else if (hashA < hashB) { 21 | return -1; 22 | } 23 | 24 | return 0; 25 | } 26 | 27 | /** A lifting monoid which hashes a document thumbnail, and combines them using addition. */ 28 | const docThumbnailMonoid: LiftingMonoid = { 29 | lift: (v: DocThumbnail) => { 30 | return xxhash64(v); 31 | }, 32 | combine: (a: bigint, b: bigint) => { 33 | return a + b; 34 | }, 35 | neutral: BigInt(0), 36 | }; 37 | 38 | /** A FingerprintTree preconfigured for the insertion of DocThumbnails. */ 39 | export class DocThumbnailTree extends FingerprintTree { 40 | constructor() { 41 | super(docThumbnailMonoid, compareThumbnails); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/replica/replica.ts: -------------------------------------------------------------------------------- 1 | import { DefaultFormats } from "../formats/format_types.ts"; 2 | import { IReplicaDriver, MultiFormatReplicaOpts } from "./replica-types.ts"; 3 | import { MultiformatReplica } from "./multiformat_replica.ts"; 4 | 5 | type ReplicaOpts = { 6 | /** The secret of the share this replica has been configured to use. 7 | * 8 | * If omitted the replica will be read only. 9 | */ 10 | shareSecret?: string; 11 | /** A replica driver which will be used to instruct the replica how to read and write data. 12 | */ 13 | driver: IReplicaDriver; 14 | }; 15 | 16 | /** 17 | * A replica holding a share's data, used to read, write, and synchronise data to. 18 | * 19 | * Should be closed using the `close` method when no longer being used. 20 | * 21 | * ```ts 22 | * const gardeningKeypair = await Crypto.generateShareKeypair("gardening"); 23 | * 24 | * const myReplica = new Replica({ 25 | * driver: new ReplicaDriverMemory(gardeningKeypair.shareAddress), 26 | * shareSecret: gardeningKeypair.secret 27 | * }); 28 | * ``` 29 | */ 30 | export class Replica extends MultiformatReplica { 31 | constructor(opts: ReplicaOpts) { 32 | super({ 33 | driver: opts.driver, 34 | config: { 35 | "es.5": { 36 | shareSecret: opts.shareSecret, 37 | }, 38 | }, 39 | } as MultiFormatReplicaOpts); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /scripts/build_web_bundle.ts: -------------------------------------------------------------------------------- 1 | import path from "https://deno.land/std@0.154.0/node/path.ts"; 2 | import * as esbuild from "https://deno.land/x/esbuild@v0.15.14/mod.js"; 3 | import { denoPlugin } from "https://deno.land/x/esbuild_deno_loader@0.6.0/mod.ts"; 4 | 5 | const version = Deno.args[0]; 6 | 7 | const defaultWebCryptoDriverAbsPath = path.resolve( 8 | "./src/crypto/default_driver.web.ts", 9 | ); 10 | 11 | const replaceDefaultDriverPlugin: esbuild.Plugin = { 12 | name: "replaceDefaultDriver", 13 | setup(build) { 14 | build.onResolve({ filter: /default\_driver\.ts$/ }, () => { 15 | return { path: defaultWebCryptoDriverAbsPath }; 16 | }); 17 | }, 18 | }; 19 | 20 | const result = await esbuild.build({ 21 | plugins: [ 22 | replaceDefaultDriverPlugin, 23 | denoPlugin(), 24 | ], 25 | entryPoints: ["./mod.browser.ts"], 26 | outfile: `./dist/earthstar${version ? `-${version}` : ""}.web.js`, 27 | bundle: true, 28 | format: "esm", 29 | platform: "browser", 30 | sourcemap: "linked", 31 | minify: true, 32 | metafile: true, 33 | banner: { 34 | js: `/** 35 | * Earthstar ${version || ""} 36 | * https://earthstar-project.org 37 | * 38 | * This source code is licensed under the LGPL-3.0 license. 39 | */ 40 | 41 | `, 42 | }, 43 | }); 44 | 45 | if (result.metafile) { 46 | await Deno.writeTextFile( 47 | "./dist/metafile.json", 48 | JSON.stringify(result.metafile), 49 | ); 50 | } 51 | 52 | Deno.exit(0); 53 | -------------------------------------------------------------------------------- /src/util/streams.ts: -------------------------------------------------------------------------------- 1 | export function bytesToStream(bytes: Uint8Array): ReadableStream { 2 | return new ReadableStream({ 3 | start(controller) { 4 | for (let i = 0; i <= bytes.length; i += 8) { 5 | controller.enqueue(bytes.slice(i, i + 8)); 6 | } 7 | 8 | controller.close(); 9 | }, 10 | }); 11 | } 12 | 13 | export async function streamToBytes( 14 | stream: ReadableStream, 15 | ): Promise { 16 | let bytes = new Uint8Array(); 17 | 18 | await stream.pipeTo( 19 | new WritableStream({ 20 | write(chunk) { 21 | const nextBytes = new Uint8Array(bytes.length + chunk.length); 22 | nextBytes.set(bytes); 23 | nextBytes.set(chunk, bytes.length); 24 | bytes = nextBytes; 25 | }, 26 | }), 27 | ); 28 | 29 | return bytes; 30 | } 31 | 32 | export async function getStreamSize(stream: ReadableStream) { 33 | let size = 0; 34 | 35 | const sink = new WritableStream({ 36 | write(chunk) { 37 | size += chunk.byteLength; 38 | }, 39 | }); 40 | 41 | await stream.pipeTo(sink); 42 | 43 | return size; 44 | } 45 | 46 | export async function readStream( 47 | stream: ReadableStream, 48 | ): Promise { 49 | const arr: ChunkType[] = []; 50 | 51 | const writable = new WritableStream({ 52 | write(entry) { 53 | arr.push(entry); 54 | }, 55 | }); 56 | 57 | await stream.pipeTo(writable); 58 | 59 | return arr; 60 | } 61 | -------------------------------------------------------------------------------- /src/tcp/tcp_provider.ts: -------------------------------------------------------------------------------- 1 | import { ITcpConn, ITcpListener, ITcpProvider } from "./types.ts"; 2 | 3 | export class TcpProvider implements ITcpProvider { 4 | listen(opts: { port: number }): TcpListener { 5 | return new TcpListener(Deno.listen(opts)); 6 | } 7 | async connect(opts: { port: number; hostname: string }): Promise { 8 | const conn = await Deno.connect(opts); 9 | 10 | return new TcpConn(conn); 11 | } 12 | } 13 | 14 | export class TcpListener implements ITcpListener { 15 | listener: Deno.Listener; 16 | 17 | constructor(listener: Deno.Listener) { 18 | this.listener = listener; 19 | } 20 | 21 | close(): void { 22 | this.listener.close(); 23 | } 24 | 25 | async *[Symbol.asyncIterator]() { 26 | for await (const conn of this.listener) { 27 | yield new TcpConn(conn); 28 | } 29 | } 30 | } 31 | 32 | export class TcpConn implements ITcpConn { 33 | private conn: Deno.Conn; 34 | 35 | constructor(conn: Deno.Conn) { 36 | this.conn = conn; 37 | } 38 | 39 | read(bytes: Uint8Array): Promise { 40 | return this.conn.read(bytes); 41 | } 42 | 43 | write(bytes: Uint8Array): Promise { 44 | return this.conn.write(bytes); 45 | } 46 | 47 | close() { 48 | return this.conn.close(); 49 | } 50 | 51 | get readable() { 52 | return this.conn.readable; 53 | } 54 | 55 | get writable() { 56 | return this.conn.writable; 57 | } 58 | 59 | get remoteAddr() { 60 | return this.conn.remoteAddr as Deno.NetAddr; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/server/extensions/known_shares.ts: -------------------------------------------------------------------------------- 1 | import { Peer } from "../../peer/peer.ts"; 2 | import { Replica } from "../../replica/replica.ts"; 3 | import { IServerExtension } from "./extension.ts"; 4 | 5 | interface ExtensionKnownSharesOpts { 6 | /** The path where the known shares (a JSON array of share addresses) is located. */ 7 | knownSharesPath: string; 8 | /** A callback used to create the replicas in the known shares list. Mostly useful for choosing how you'd like your shares to be persisted, e.g. probably by Sqlite. */ 9 | onCreateReplica: (shareAddress: string) => Replica; 10 | } 11 | 12 | /** A server extension for populating a server with known shares. Use this to specify which shares you'd like your server to sync with others. 13 | * 14 | * You most likely want to pass this as the first extension to your server. 15 | */ 16 | export class ExtensionKnownShares implements IServerExtension { 17 | private peer: Peer | null = null; 18 | private knownSharesPath: string; 19 | private onCreateReplica: (shareAddress: string) => Replica; 20 | 21 | constructor(opts: ExtensionKnownSharesOpts) { 22 | this.knownSharesPath = opts.knownSharesPath; 23 | this.onCreateReplica = opts.onCreateReplica; 24 | } 25 | 26 | async register(peer: Peer) { 27 | this.peer = peer; 28 | 29 | const knownSharesRaw = await Deno.readTextFile(this.knownSharesPath); 30 | 31 | const knownShares = JSON.parse(knownSharesRaw) as string[]; 32 | 33 | for (const shareAddress of knownShares) { 34 | const replica = this.onCreateReplica(shareAddress); 35 | 36 | await this.peer.addReplica(replica); 37 | } 38 | } 39 | 40 | handler() { 41 | return Promise.resolve(null); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: test test-watch npm fmt clean bundle 2 | 3 | clean: 4 | rm -rf npm build .nyc_output coverage earthstar.bundle.js cov.lcov coverage_html cov_profile node_modules 5 | 6 | example: 7 | deno run ./example-app.ts 8 | 9 | test: 10 | deno test --allow-all src 11 | 12 | test-watch: 13 | deno test --watch --allow-all src 14 | 15 | test-coverage: 16 | deno test --no-check --coverage=cov_profile src 17 | 18 | # to get "genhtml", run "sudo apt-get install lcov" (on linux) or "brew install lcov" (on mac) 19 | show-coverage: 20 | deno coverage cov_profile --lcov > cov.lcov && genhtml -o cov_html cov.lcov 21 | 22 | coverage: test-coverage show-coverage 23 | 24 | npm: 25 | deno run --allow-all scripts/build_npm.ts $(VERSION) 26 | 27 | bundle: 28 | deno bundle --no-check=remote ./mod.browser.ts ./earthstar.bundle.js 29 | 30 | run-bundle: 31 | deno run --allow-all ./earthstar.bundle.js --help 32 | 33 | depchart-no-types: 34 | mkdir -p depchart && npx depchart `find src | grep .ts` --exclude deps.ts src/print-platform-support.ts src/decls.d.ts src/index.ts src/index.browser.ts src/shims/*.ts src/entries/*.ts `find src | grep '/test/'` `find src | grep '/util/'` `find src | grep '/experimental/'` `find src | grep types.ts` --rankdir LR -o depchart/depchart-no-types --node_modules omit 35 | 36 | depchart-deps: 37 | mkdir -p depchart && npx depchart deps.ts `find src | grep .ts` --exclude src/print-platform-support.ts src/decls.d.ts src/index.ts src/index.browser.ts src/shims/*.ts src/entries/*.ts `find src | grep '/test/'` `find src | grep '/util/'` `find src | grep '/experimental/'` --rankdir LR -o depchart/depchart-deps --node_modules separated 38 | 39 | depchart: depchart-no-types depchart-deps 40 | -------------------------------------------------------------------------------- /src/server/server_core.ts: -------------------------------------------------------------------------------- 1 | import { Peer } from "../peer/peer.ts"; 2 | import { IServerExtension } from "./extensions/extension.ts"; 3 | 4 | /** The core server logic. Combine this with a HTTP framework to create a fully-fledged server. */ 5 | export class ServerCore { 6 | private extensions: IServerExtension[]; 7 | private peer: Peer; 8 | private isReady: Promise; 9 | 10 | /** 11 | * Create a new server with an array of extensions. 12 | * @param extensions - The extensions used by the server. Extensions will be registered in the order you provide them in, as one extension may depend on the actions of another. For example, the `ExtensionServeContent` may rely on a replica created by `ExtensionKnownShares`. 13 | */ 14 | constructor(extensions: IServerExtension[]) { 15 | this.peer = new Peer(); 16 | 17 | this.extensions = extensions; 18 | 19 | this.isReady = this.registerExtensions(); 20 | 21 | console.log("Your server is running."); 22 | } 23 | 24 | private async registerExtensions() { 25 | // Extensions must be registered sequentially, one-by-one, 26 | // As later extensions may depend on the actions of previous ones 27 | // e.g. The serve content extension using a share added by the known shares extension 28 | for (const extension of this.extensions) { 29 | await extension.register(this.peer); 30 | } 31 | } 32 | 33 | async handler(req: Request): Promise { 34 | await this.isReady; 35 | 36 | for (const extension of this.extensions) { 37 | const response = await extension.handler(req); 38 | 39 | if (response) { 40 | return response; 41 | } 42 | } 43 | 44 | return new Response("Not found", { status: 404 }); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /debug/keys.ts: -------------------------------------------------------------------------------- 1 | const keyPair = await crypto.subtle.generateKey( 2 | { 3 | name: "ECDH", 4 | namedCurve: "P-256", 5 | }, 6 | true, 7 | ["deriveKey", "deriveBits"], 8 | ); 9 | 10 | const publicKeyExported = await crypto.subtle.exportKey( 11 | "raw", 12 | keyPair.publicKey, 13 | ); 14 | 15 | console.log(publicKeyExported.byteLength); 16 | 17 | const publicKeyImported = await crypto.subtle.importKey( 18 | "raw", 19 | publicKeyExported, 20 | { 21 | name: "ECDH", 22 | namedCurve: "P-256", 23 | }, 24 | true, 25 | [], 26 | ); 27 | 28 | // Derive a new thing 29 | 30 | const keyPair2 = await crypto.subtle.generateKey( 31 | { 32 | name: "ECDH", 33 | namedCurve: "P-256", 34 | }, 35 | true, 36 | ["deriveKey", "deriveBits"], 37 | ); 38 | 39 | const derivedKey = await crypto.subtle.deriveKey( 40 | { name: "ECDH", public: publicKeyImported }, 41 | keyPair2.privateKey, 42 | { name: "AES-GCM", length: 256 }, 43 | true, 44 | ["encrypt", "decrypt"], 45 | ); 46 | 47 | // Encrypt... 48 | 49 | const message = "Hello, there! This is my secret message."; 50 | 51 | const iv = crypto.getRandomValues(new Uint8Array(12)); 52 | 53 | const encrypted = await crypto.subtle.encrypt( 54 | { 55 | name: "AES-GCM", 56 | iv, 57 | }, 58 | derivedKey, 59 | new TextEncoder().encode(message), 60 | ); 61 | 62 | // Decrypt 63 | const decrypted = await crypto.subtle.decrypt( 64 | { 65 | name: "AES-GCM", 66 | iv, 67 | }, 68 | derivedKey, 69 | encrypted, 70 | ); 71 | 72 | console.log(new TextDecoder().decode(encrypted)); 73 | console.log(new TextDecoder().decode(decrypted)); 74 | 75 | const t = crypto.getRandomValues(new Uint8Array(4)); 76 | 77 | const dv = new DataView(t.buffer); 78 | 79 | console.log(dv.getUint32(0)); 80 | -------------------------------------------------------------------------------- /examples/servers/nimble_server.ts: -------------------------------------------------------------------------------- 1 | import { parse } from "https://deno.land/std@0.119.0/flags/mod.ts"; 2 | import { 3 | AttachmentDriverFilesystem, 4 | DocDriverSqlite, 5 | ExtensionServerSettings, 6 | ExtensionSyncWeb, 7 | } from "../../src/entries/deno.ts"; 8 | import { Replica } from "../../src/replica/replica.ts"; 9 | import { Server, ServerOpts } from "../../src/server/server.ts"; 10 | 11 | const flags = parse(Deno.args, { 12 | string: ["port", "hostname"], 13 | default: { 14 | port: 8080, 15 | hostname: "0.0.0.0", 16 | }, 17 | }); 18 | 19 | export class NimbleServer { 20 | private server: Server; 21 | 22 | constructor(opts: ServerOpts) { 23 | this.server = new Server([ 24 | new ExtensionServerSettings({ 25 | configurationShare: 26 | "+apples.btqswluholq6on2ci5mck66uzkmumb5uszgvqimtshff2f6zy5etq", 27 | onCreateReplica: (shareAddress) => { 28 | return new Replica( 29 | { 30 | driver: { 31 | docDriver: new DocDriverSqlite({ 32 | share: shareAddress, 33 | filename: `./data/${shareAddress}.sql`, 34 | mode: "create-or-open", 35 | }), 36 | attachmentDriver: new AttachmentDriverFilesystem( 37 | `./data/${shareAddress}_attachments`, 38 | ), 39 | }, 40 | }, 41 | ); 42 | }, 43 | }), 44 | new ExtensionSyncWeb({ path: "/sync" }), 45 | ], opts); 46 | } 47 | 48 | close() { 49 | return this.server.close(); 50 | } 51 | } 52 | 53 | console.log(`Started Nimble server on ${flags.hostname}:${flags.port}`); 54 | 55 | const server = new NimbleServer({ 56 | hostname: flags.hostname, 57 | port: flags.port, 58 | }); 59 | -------------------------------------------------------------------------------- /src/formats/util.ts: -------------------------------------------------------------------------------- 1 | import { FormatEs5 } from "./format_es5.ts"; 2 | import { 3 | DefaultFormat, 4 | DefaultFormats, 5 | FormatArg, 6 | FormatsArg, 7 | } from "./format_types.ts"; 8 | 9 | export const DEFAULT_FORMAT = FormatEs5; 10 | export const DEFAULT_FORMATS = [DEFAULT_FORMAT]; 11 | 12 | /** Returns the default format if no formats are given. */ 13 | export function getFormatWithFallback( 14 | format?: FormatArg, 15 | ): FormatArg { 16 | return format || DEFAULT_FORMAT as unknown as FormatArg; 17 | } 18 | 19 | /** Returns the default formats if no formats are given. */ 20 | export function getFormatsWithFallback( 21 | formats?: FormatsArg, 22 | ): FormatsArg { 23 | return formats || DEFAULT_FORMATS as unknown as FormatsArg; 24 | } 25 | 26 | /** Given an array of format names, and an array of `IFormat`, returns an array of `IFormat` restricted to those with matching names. */ 27 | export function getFormatIntersection( 28 | formatNames: string[], 29 | formats: FormatsArg, 30 | ): FormatsArg { 31 | const intersection = []; 32 | 33 | for (const f of formats) { 34 | if (formatNames.includes(f.id)) { 35 | intersection.push(f); 36 | } 37 | } 38 | 39 | return intersection as FormatsArg; 40 | } 41 | 42 | /** Returns an object with format names as keys, and corresponding `IFormat` as values. */ 43 | export function getFormatLookup( 44 | formats?: FormatsArg, 45 | ): Record> { 46 | const f = formats || DEFAULT_FORMATS; 47 | 48 | const formatLookup: Record> = {}; 49 | 50 | for (const format of f) { 51 | formatLookup[format.id] = format as typeof formatLookup[string]; 52 | } 53 | 54 | return formatLookup; 55 | } 56 | -------------------------------------------------------------------------------- /src/server/extensions/known_shares.node.ts: -------------------------------------------------------------------------------- 1 | import { Peer } from "../../peer/peer.ts"; 2 | import { Replica } from "../../replica/replica.ts"; 3 | import { IServerExtension } from "./extension.ts"; 4 | import * as fs from "https://deno.land/std@0.154.0/node/fs/promises.ts"; 5 | 6 | interface ExtensionKnownSharesOpts { 7 | /** The path where the known shares (a JSON array of share addresses) is located. */ 8 | knownSharesPath: string; 9 | /** A callback used to create the replicas in the known shares list. Mostly useful for choosing how you'd like your shares to be persisted, e.g. probably by Sqlite. */ 10 | onCreateReplica: (shareAddress: string) => Replica; 11 | } 12 | 13 | /** A server extension for populating a server with known shares. Use this to specify which shares you'd like your server to sync with others. 14 | * 15 | * You most likely want to pass this as the first extension to your server. 16 | */ 17 | export class ExtensionKnownShares implements IServerExtension { 18 | private peer: Peer | null = null; 19 | private knownSharesPath: string; 20 | private onCreateReplica: (shareAddress: string) => Replica; 21 | 22 | constructor(opts: ExtensionKnownSharesOpts) { 23 | this.knownSharesPath = opts.knownSharesPath; 24 | this.onCreateReplica = opts.onCreateReplica; 25 | } 26 | 27 | async register(peer: Peer) { 28 | this.peer = peer; 29 | 30 | const knownSharesRaw = await fs.readFile(this.knownSharesPath, "utf-8"); 31 | 32 | const knownShares = JSON.parse(knownSharesRaw) as string[]; 33 | 34 | for (const shareAddress of knownShares) { 35 | const replica = this.onCreateReplica(shareAddress); 36 | 37 | await this.peer.addReplica(replica); 38 | } 39 | } 40 | 41 | handler() { 42 | return Promise.resolve(null); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/syncer/promise_enroller.ts: -------------------------------------------------------------------------------- 1 | import { EarthstarError } from "../util/errors.ts"; 2 | import { MultiDeferred } from "./multi_deferred.ts"; 3 | 4 | export class PromiseEnroller { 5 | private promises = new Set>(); 6 | private sealed = false; 7 | private multiDeferred = new MultiDeferred(); 8 | private allowRejectedPromises: boolean; 9 | 10 | constructor(allowRejectedPromises: boolean = false) { 11 | this.allowRejectedPromises = allowRejectedPromises; 12 | } 13 | 14 | enrol(promise: Promise) { 15 | if (this.sealed) { 16 | throw new EarthstarError( 17 | "Tried to enrol a promise when enrolment was already sealed.", 18 | ); 19 | } 20 | 21 | this.promises.add(promise); 22 | 23 | promise.then(() => { 24 | this.checkAllDone(); 25 | }).catch((err) => { 26 | if (this.allowRejectedPromises) { 27 | // Swallow the error. 28 | return; 29 | } 30 | 31 | throw err; 32 | }); 33 | } 34 | 35 | checkAllDone() { 36 | if (!this.sealed) { 37 | return; 38 | } 39 | 40 | if (this.allowRejectedPromises) { 41 | Promise.allSettled(this.promises).then(() => { 42 | this.multiDeferred.resolve(); 43 | }); 44 | } else { 45 | Promise.all(this.promises).then(() => { 46 | this.multiDeferred.resolve(); 47 | }).catch(() => { 48 | this.multiDeferred.reject(); 49 | }); 50 | } 51 | } 52 | 53 | seal() { 54 | if (this.sealed) { 55 | return; 56 | } 57 | 58 | this.sealed = true; 59 | 60 | this.checkAllDone(); 61 | } 62 | 63 | isSealed() { 64 | return this.sealed; 65 | } 66 | 67 | isDone() { 68 | const promise = this.multiDeferred.getPromise(); 69 | 70 | return promise; 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /debug/path_bytes.ts: -------------------------------------------------------------------------------- 1 | import { AuthorKeypair, Crypto, ShareKeypair } from "../mod.ts"; 2 | 3 | const shareAddr = await Crypto.generateShareKeypair( 4 | "gardening", 5 | ) as ShareKeypair; 6 | const keypair = await Crypto.generateAuthorKeypair("suzy") as AuthorKeypair; 7 | 8 | const path = 9 | "/where-are-my-socks/i/left/them/here/somewhere/has-anyone-seen-them"; 10 | const format = "es.5"; 11 | 12 | const encoder = new TextEncoder(); 13 | 14 | const shareAddressBytes = encoder.encode(shareAddr.shareAddress); 15 | const authorBytes = encoder.encode(keypair.address); 16 | 17 | const pathBytes = encoder.encode(path); 18 | const formatBytes = encoder.encode(format); 19 | 20 | // share address len can be Uint8 21 | 22 | // author len is 59 23 | 24 | // path len can be 512, so Uint16. 25 | 26 | // format length can uint8 27 | 28 | // download upload is 1 or 0 29 | 30 | const transferDescBytes = new Uint8Array( 31 | 59 + 32 | 2 + shareAddressBytes.byteLength + 33 | 1 + formatBytes.byteLength + 34 | 2 + pathBytes.byteLength, 35 | ); 36 | 37 | const transferView = new DataView(transferDescBytes.buffer); 38 | 39 | let position = 0; 40 | 41 | transferDescBytes.set(authorBytes, position); 42 | 43 | position += authorBytes.byteLength; 44 | 45 | transferView.setUint8(position, shareAddressBytes.byteLength); 46 | 47 | position += 1; 48 | 49 | transferDescBytes.set(shareAddressBytes, position); 50 | 51 | position += shareAddressBytes.byteLength; 52 | 53 | transferView.setUint8(position, formatBytes.byteLength); 54 | 55 | position += 1; 56 | 57 | transferDescBytes.set(formatBytes, position); 58 | 59 | position += formatBytes.byteLength; 60 | 61 | transferView.setUint16(position, pathBytes.byteLength); 62 | 63 | position += 2; 64 | 65 | transferDescBytes.set(pathBytes, position); 66 | 67 | console.log(transferDescBytes.subarray(0, 4)); 68 | -------------------------------------------------------------------------------- /src/peer/peer-types.ts: -------------------------------------------------------------------------------- 1 | import { ShareAddress } from "../util/doc-types.ts"; 2 | import { Replica } from "../replica/replica.ts"; 3 | import { Syncer } from "../syncer/syncer.ts"; 4 | import { FormatsArg } from "../formats/format_types.ts"; 5 | import { ISyncPartner } from "../syncer/syncer_types.ts"; 6 | 7 | //================================================================================ 8 | // PEER 9 | 10 | export type PeerId = string; 11 | 12 | /** Holds many shares' replicas and manages their synchronisation with other peers. Recommended as the point of contact between your application and Earthstar shares. */ 13 | export interface IPeer { 14 | // getters 15 | hasShare(share: ShareAddress): boolean; 16 | shares(): ShareAddress[]; 17 | replicas(): Replica[]; 18 | size(): number; 19 | getReplica( 20 | share: ShareAddress, 21 | ): Replica | undefined; 22 | 23 | // setters 24 | addReplica(replica: Replica): Promise; 25 | removeReplicaByShare(share: ShareAddress): Promise; 26 | removeReplica(replica: Replica): Promise; 27 | 28 | sync( 29 | target: IPeer | string, 30 | continuous?: boolean, 31 | formats?: FormatsArg, 32 | ): Syncer; 33 | 34 | addSyncPartner( 35 | partner: ISyncPartner, 36 | description: string, 37 | formats?: FormatsArg, 38 | ): Syncer; 39 | 40 | getSyncers(): Map< 41 | string, 42 | { description: string; syncer: Syncer } 43 | >; 44 | 45 | onReplicasChange( 46 | callback: (map: Map) => void | Promise, 47 | ): () => void; 48 | 49 | onSyncersChange( 50 | callback: ( 51 | map: Map< 52 | string, 53 | { description: string; syncer: Syncer } 54 | >, 55 | ) => void | Promise, 56 | ): () => void; 57 | } 58 | -------------------------------------------------------------------------------- /src/server/server.ts: -------------------------------------------------------------------------------- 1 | import { serve } from "https://deno.land/std@0.167.0/http/server.ts"; 2 | import { IServerExtension } from "./extensions/extension.ts"; 3 | import { ServerCore } from "./server_core.ts"; 4 | 5 | export type ServerOpts = { 6 | port?: number; 7 | hostname?: string; 8 | }; 9 | 10 | /** 11 | * An extensible Earthstar server able to synchronise with other peers. 12 | * 13 | * A server's functionality can be extended using extensions of type `IServerExtension`. 14 | * 15 | * ```ts 16 | * const server = new Server([ 17 | * new ExtensionKnownShares({ 18 | * knownSharesPath: "./known_shares.json", 19 | * onCreateReplica: (shareAddress) => { 20 | * return new Earthstar.Replica({ 21 | * driver: new ReplicaDriverFs(shareAddress, "./share_data"), 22 | * }); 23 | * }, 24 | * }), 25 | * new ExtensionSyncWeb(), 26 | * ]); 27 | * ``` 28 | */ 29 | export class Server { 30 | private core: ServerCore; 31 | private abortController: AbortController; 32 | private server: Promise; 33 | 34 | /** 35 | * Create a new server with an array of extensions. 36 | * @param extensions - The extensions used by the server. Extensions will be registered in the order you provide them in, as one extension may depend on the actions of another. For example, the `ExtensionServeContent` may rely on a replica created by `ExtensionKnownShares`. 37 | */ 38 | constructor(extensions: IServerExtension[], opts?: ServerOpts) { 39 | this.core = new ServerCore(extensions); 40 | 41 | this.abortController = new AbortController(); 42 | 43 | this.server = serve(this.core.handler.bind(this.core), { 44 | port: opts?.port, 45 | hostname: opts?.hostname, 46 | signal: this.abortController.signal, 47 | }); 48 | } 49 | 50 | async close() { 51 | this.abortController.abort(); 52 | 53 | await this.server; 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /mod.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * [Earthstar](https://earthstar-project.org) is a small and resilient distributed storage protocol designed with a strong focus on simplicity and versatility, with the social realities of peer-to-peer computing kept in mind. 3 | * 4 | * This is a reference implementation written in Typescript. You can use it to add Earthstar functionality to applications running on servers, browsers, the command line, or anywhere else JavaScript can be run. 5 | * 6 | * ### Example usage 7 | * 8 | * ```ts 9 | * import { Replica, ReplicaDriverMemory, Crypto, Peer } from "earthstar"; 10 | * 11 | * const shareKeypair = await Crypto.generateShareKeypair("gardening"); 12 | * 13 | * const replica = new Replica({ 14 | * driver: ReplicaDriverMemory(shareKeypair.shareAddress), 15 | * shareSecret: shareKeypair.secret, 16 | * }); 17 | * 18 | * const authorKeypair = await Crypto.generateAuthorKeypair("suzy"); 19 | * 20 | * await replica.set(authorKeypair, { 21 | * path: "/my-note", 22 | * text: "Saw seven magpies today", 23 | * }); 24 | * 25 | * const allDocs = await replica.getAllDocs(); 26 | * 27 | * const peer = new Peer(); 28 | * 29 | * peer.addReplica(replica); 30 | * 31 | * peer.sync("https://my.server") 32 | * ``` 33 | * 34 | * This module also exposes server APIs for for building always-online peers. The below example reads some on-disk JSON to initiate some share replicas, and stores their data using the filesystem. 35 | * 36 | * ```ts 37 | * import { 38 | * ExtensionKnownShares, 39 | * ExtensionSyncWeb, 40 | * Server, 41 | * } from "https://deno.land/x/earthstar/mod.ts"; 42 | * 43 | * const server = new Server([ 44 | * new ExtensionKnownShares({ 45 | * knownSharesPath: "./known_shares.json", 46 | * onCreateReplica: (shareAddress) => { 47 | * return new Earthstar.Replica({ 48 | * driver: new ReplicaDriverFs(shareAddress, "./share_data"), 49 | * }); 50 | * }, 51 | * }), 52 | * new ExtensionSyncWebsocket(), 53 | * ]); 54 | * 55 | * @module 56 | */ 57 | 58 | export * from "./src/entries/universal.ts"; 59 | export * from "./src/entries/deno.ts"; 60 | -------------------------------------------------------------------------------- /src/test/scenarios/types.ts: -------------------------------------------------------------------------------- 1 | import { FormatsArg, IFormat } from "../../formats/format_types.ts"; 2 | import { IPeer } from "../../peer/peer-types.ts"; 3 | import { 4 | IReplicaAttachmentDriver, 5 | IReplicaDocDriver, 6 | } from "../../replica/replica-types.ts"; 7 | import { IServerExtension } from "../../server/extensions/extension.ts"; 8 | import { Syncer } from "../../syncer/syncer.ts"; 9 | import { SyncAppetite } from "../../syncer/syncer_types.ts"; 10 | import { 11 | DocBase, 12 | DocInputBase, 13 | FormatName, 14 | ShareAddress, 15 | } from "../../util/doc-types.ts"; 16 | 17 | export type Scenario = { 18 | name: string; 19 | item: T; 20 | }; 21 | 22 | export type ScenarioItem = T extends Scenario[] ? ItemType 23 | : never; 24 | 25 | export type Scenarios = { 26 | description: DescType; 27 | scenarios: Scenario[]; 28 | }; 29 | 30 | export type MultiplyScenarioOutput> = 31 | { 32 | name: string; 33 | subscenarios: RecordType; 34 | }[]; 35 | 36 | export interface SyncPartnerScenario { 37 | formats: FormatsArg; 38 | appetite: SyncAppetite; 39 | setup( 40 | peerA: IPeer, 41 | peerB: IPeer, 42 | ): Promise<[Syncer, Syncer]>; 43 | teardown(): Promise; 44 | syncContinuousWait: number; 45 | } 46 | 47 | export interface ServerScenario { 48 | start(testExtension: IServerExtension): Promise; 49 | close(): Promise; 50 | } 51 | 52 | export type DocDriverScenario = { 53 | makeDriver: (share: ShareAddress, variant?: string) => IReplicaDocDriver; 54 | persistent: boolean; 55 | builtInConfigKeys: string[]; 56 | }; 57 | 58 | export type AttachmentDriverScenario = { 59 | makeDriver: (shareAddr: string, variant?: string) => IReplicaAttachmentDriver; 60 | persistent: boolean; 61 | }; 62 | 63 | export type FormatScenario< 64 | N extends FormatName, 65 | I extends DocInputBase, 66 | O extends DocBase, 67 | C extends Record, 68 | F extends IFormat, 69 | > = { 70 | format: F; 71 | makeInputDoc: () => I; 72 | }; 73 | -------------------------------------------------------------------------------- /src/crypto/base32.ts: -------------------------------------------------------------------------------- 1 | import { rfc4648 } from "../../deps.ts"; 2 | import { Base32String } from "../util/doc-types.ts"; 3 | import { ValidationError } from "../util/errors.ts"; 4 | 5 | const { codec } = rfc4648; 6 | 7 | // For base32 encoding we use rfc4648, no padding, lowercase, prefixed with "b". 8 | // 9 | // Base32 character set: `abcdefghijklmnopqrstuvwxyz234567` 10 | // 11 | // The Multibase format adds a "b" prefix to specify this particular encoding. 12 | // We leave the "b" prefix there because we don't want the encoded string 13 | // to start with a number (so we can use it as a URL location). 14 | // 15 | // When decoding, we require it to start with a "b" -- 16 | // no other multibase formats are allowed. 17 | // 18 | // The decoding must be strict (it doesn't allow a 1 in place of an i, etc). 19 | 20 | const myEncoding = { 21 | // this should match b32chars from characters.ts 22 | chars: "abcdefghijklmnopqrstuvwxyz234567", 23 | bits: 5, 24 | }; 25 | 26 | /** Encode uint8array bytes to base32 string */ 27 | export function base32BytesToString(bytes: Uint8Array): Base32String { 28 | return "b" + codec.stringify(bytes, myEncoding, { pad: false }); 29 | } 30 | 31 | /** Decode base32 string to a uint8array of bytes. Throw a ValidationError if the string is bad. */ 32 | export function base32StringToBytes(str: Base32String): Uint8Array { 33 | if (!str.startsWith("b")) { 34 | throw new ValidationError( 35 | "can't decode base32 string - it should start with a 'b'. " + str, 36 | ); 37 | } 38 | 39 | // this library combines padding and looseness settings into a single "loose" option, so 40 | // we have to set "loose: true" in order to handle unpadded inputs. 41 | // with a custom codec, loose mode: 42 | // -- allows padding or no padding -- we have to check for this 43 | // -- does not allow uppercase -- good 44 | // -- does not allow 1/i substitution -- good 45 | 46 | // make sure no padding characters are on the end 47 | if (str[str.length - 1] === "=") { 48 | throw new ValidationError( 49 | "can't decode base32 string - it contains padding characters ('=')", 50 | ); 51 | } 52 | return codec.parse(str.slice(1), myEncoding, { loose: true }); 53 | } 54 | -------------------------------------------------------------------------------- /src/crypto/crypto-types.ts: -------------------------------------------------------------------------------- 1 | import { 2 | AuthorAddress, 3 | Base32String, 4 | ShareAddress, 5 | } from "../util/doc-types.ts"; 6 | import { ValidationError } from "../util/errors.ts"; 7 | import { UpdatableHash } from "./updatable_hash.ts"; 8 | 9 | export interface KeypairBytes { 10 | pubkey: Uint8Array; 11 | secret: Uint8Array; 12 | } 13 | 14 | /** A keypair used by individual entities to sign documents. */ 15 | export interface AuthorKeypair { 16 | address: AuthorAddress; 17 | secret: string; 18 | } 19 | 20 | /** A keypair used to write to a specific share */ 21 | export type ShareKeypair = { 22 | shareAddress: ShareAddress; 23 | secret: string; 24 | }; 25 | 26 | /** Higher-level crypto functions. Not used directly for the most part, but useful for generating new keypairs. */ 27 | // These all handle base32-encoded strings. 28 | export interface ICrypto { 29 | sha256base32( 30 | input: string | Uint8Array, 31 | ): Promise; 32 | updatableSha256(): UpdatableHash; 33 | generateAuthorKeypair( 34 | name: string, 35 | ): Promise; 36 | generateShareKeypair( 37 | name: string, 38 | ): Promise; 39 | sign( 40 | keypair: AuthorKeypair | ShareKeypair, 41 | msg: string | Uint8Array, 42 | ): Promise; 43 | verify( 44 | address: AuthorAddress | ShareAddress, 45 | sig: Base32String, 46 | msg: string | Uint8Array, 47 | ): Promise; 48 | checkKeypairIsValid( 49 | keypair: AuthorKeypair | ShareKeypair, 50 | ): Promise; 51 | } 52 | 53 | /** A crypto driver provides low-level access to an implementation providing ed25519 cryptography, e.g. Chloride, noble/ed25519, Node crypto. */ 54 | // These all handle Uint8Arrays (bytes) 55 | export interface ICryptoDriver { 56 | sha256( 57 | input: string | Uint8Array, 58 | ): Promise; 59 | updatableSha256(): UpdatableHash; 60 | generateKeypairBytes(): Promise; 61 | sign( 62 | keypairBytes: KeypairBytes, 63 | msg: string | Uint8Array, 64 | ): Promise; 65 | verify( 66 | publicKey: Uint8Array, 67 | sig: Uint8Array, 68 | msg: string | Uint8Array, 69 | ): Promise; 70 | } 71 | -------------------------------------------------------------------------------- /src/util/misc.ts: -------------------------------------------------------------------------------- 1 | import { checkShareIsValid } from "../core-validators/addresses.ts"; 2 | import { 3 | alphaLower, 4 | workspaceKeyChars, 5 | } from "../core-validators/characters.ts"; 6 | import { isErr, ValidationError } from "./errors.ts"; 7 | 8 | //================================================================================ 9 | // TIME 10 | 11 | export function microsecondNow() { 12 | return Date.now() * 1000; 13 | } 14 | 15 | /** Returns a promise which is fulfilled after a given number of milliseconds. */ 16 | export function sleep(ms: number): Promise { 17 | return new Promise((res) => { 18 | setTimeout(res, ms); 19 | }); 20 | } 21 | 22 | // TODO: better randomness here 23 | export function randomId(): string { 24 | return "" + Math.floor(Math.random() * 1000) + 25 | Math.floor(Math.random() * 1000); 26 | } 27 | 28 | // replace all occurrences of substring "from" with "to" 29 | export function replaceAll(str: string, from: string, to: string): string { 30 | return str.split(from).join(to); 31 | } 32 | 33 | // how many times does the character occur in the string? 34 | 35 | export function countChars(str: string, char: string) { 36 | if (char.length != 1) { 37 | throw new Error("char must have length 1 but is " + JSON.stringify(char)); 38 | } 39 | return str.split(char).length - 1; 40 | } 41 | 42 | export function isObjectEmpty(obj: Object): Boolean { 43 | return Object.keys(obj).length === 0; 44 | } 45 | 46 | //================================================================================ 47 | // Share 48 | 49 | /** Returns a valid share address generated using a given name. 50 | * @returns A share address or a validation error resulting from the name given. 51 | * @deprecated This function only generates valid es.4 addresses. Use Crypto.generateShareKeypair to generate es.5 share addresses. 52 | */ 53 | export function generateShareAddress(name: string): string | ValidationError { 54 | const randomFromString = (str: string) => { 55 | return str[Math.floor(Math.random() * str.length)]; 56 | }; 57 | 58 | const firstLetter = randomFromString(alphaLower); 59 | const rest = Array.from(Array(11), () => randomFromString(workspaceKeyChars)) 60 | .join(""); 61 | 62 | const suffix = `${firstLetter}${rest}`; 63 | const address = `+${name}.${suffix}`; 64 | 65 | return address; 66 | } 67 | -------------------------------------------------------------------------------- /src/crypto/crypto-driver-noble.ts: -------------------------------------------------------------------------------- 1 | import { ICryptoDriver, KeypairBytes } from "./crypto-types.ts"; 2 | import { stringToBytes } from "../util/bytes.ts"; 3 | import { ed, sha256_uint8array } from "../../deps.ts"; 4 | 5 | const { createHash } = sha256_uint8array; 6 | 7 | //-------------------------------------------------- 8 | 9 | import { Logger } from "../util/log.ts"; 10 | import { UpdatableHash } from "./updatable_hash.ts"; 11 | const logger = new Logger("crypto-driver-noble", "cyan"); 12 | 13 | //================================================================================ 14 | /** 15 | * A version of the ICryptoDriver interface backed by noble/ed25519. 16 | * The slowest crypto driver available, but works everywhere. 17 | */ 18 | export const CryptoDriverNoble: ICryptoDriver = class { 19 | static sha256( 20 | input: string | Uint8Array, 21 | ): Promise { 22 | if (typeof input === "string") { 23 | return Promise.resolve( 24 | createHash("sha256").update(input, "utf-8").digest(), 25 | ); 26 | } else { 27 | return Promise.resolve(createHash("sha256").update(input).digest()); 28 | } 29 | } 30 | 31 | static updatableSha256() { 32 | return new UpdatableHash({ 33 | hash: createHash("sha256"), 34 | update: (hash, data) => hash.update(data), 35 | digest: (hash) => hash.digest(), 36 | }); 37 | } 38 | 39 | static async generateKeypairBytes(): Promise { 40 | logger.debug("generateKeypairBytes"); 41 | const secret = ed.utils.randomPrivateKey(); 42 | const pubkey = await ed.getPublicKey(secret); 43 | 44 | return { 45 | pubkey, 46 | secret, 47 | }; 48 | } 49 | static sign( 50 | keypairBytes: KeypairBytes, 51 | msg: string | Uint8Array, 52 | ): Promise { 53 | logger.debug("sign"); 54 | if (typeof msg === "string") msg = stringToBytes(msg); 55 | return ed.sign(msg, keypairBytes.secret); 56 | } 57 | static async verify( 58 | publicKey: Uint8Array, 59 | sig: Uint8Array, 60 | msg: string | Uint8Array, 61 | ): Promise { 62 | logger.debug("verify"); 63 | try { 64 | if (typeof msg === "string") msg = stringToBytes(msg); 65 | const result = await ed.verify(sig, msg, publicKey); 66 | return result; 67 | } catch { 68 | return false; 69 | } 70 | } 71 | }; 72 | -------------------------------------------------------------------------------- /src/util/bytes.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * This file provides common operations on Uint8Arrays. 3 | * It should not use any Buffers to do so. 4 | * Any function that uses Buffer should be in buffers.ts. 5 | * This helps us avoid bringing in the heavy polyfill for Buffer 6 | * when bundling for the browser. 7 | */ 8 | 9 | import { rfc4648 } from "../../deps.ts"; 10 | 11 | const decoder: TextDecoder = new TextDecoder(); 12 | const encoder: TextEncoder = new TextEncoder(); 13 | 14 | //-------------------------------------------------- 15 | 16 | export function bytesToString(bytes: Uint8Array): string { 17 | return decoder.decode(bytes); 18 | } 19 | 20 | export function stringToBytes(str: string): Uint8Array { 21 | return encoder.encode(str); 22 | } 23 | 24 | //-------------------------------------------------- 25 | 26 | export function stringLengthInBytes(str: string): number { 27 | // TODO: is there a more efficient way to do this? 28 | // If we had a Buffer we could just do Buffer.byteLength(str, 'utf-8'); 29 | return stringToBytes(str).length; 30 | } 31 | 32 | export function concatBytes(a: Uint8Array, b: Uint8Array): Uint8Array { 33 | // Checks for truthy values or empty arrays on each argument 34 | // to avoid the unnecessary construction of a new array and 35 | // the type comparison 36 | if (!b || b.length === 0) return a; 37 | if (!a || a.length === 0) return b; 38 | 39 | var c = new Uint8Array(a.length + b.length); 40 | c.set(a); 41 | c.set(b, a.length); 42 | 43 | return c; 44 | } 45 | 46 | //-------------------------------------------------- 47 | 48 | export function b64StringToBytes(b64string: string): Uint8Array { 49 | return rfc4648.base64.parse(b64string); 50 | } 51 | 52 | //-------------------------------------------------- 53 | 54 | export function isBytes(bytes: any): bytes is Uint8Array { 55 | return bytes?.constructor?.name === "Uint8Array"; 56 | //return bytes.writeUInt8 === undefined && bytes instanceof Uint8Array; 57 | } 58 | 59 | export function isBuffer(buf: any): boolean { 60 | // do this without any official reference to Buffer 61 | // to avoid bringing in the Buffer polyfill 62 | return buf?.constructor?.name === "Buffer"; 63 | //buf instanceof Buffer; 64 | } 65 | 66 | export function identifyBufOrBytes(bufOrBytes: any | Uint8Array): string { 67 | if (isBytes(bufOrBytes)) return "bytes"; 68 | if (isBuffer(bufOrBytes)) return "buffer"; 69 | return "?"; 70 | } 71 | -------------------------------------------------------------------------------- /src/node/chloride.d.ts: -------------------------------------------------------------------------------- 1 | import { type Buffer } from "https://deno.land/std@0.154.0/node/buffer.ts"; 2 | 3 | export interface KeyPair { 4 | publicKey: Buffer; 5 | secretKey: Buffer; 6 | } 7 | // *** hash *** 8 | // sha512 9 | export function crypto_hash(plainText: Buffer): Buffer; 10 | // sha256 11 | export function crypto_hash_sha256(plainText: Buffer): Buffer; 12 | 13 | /** Signatures */ 14 | export function crypto_sign_keypair(): KeyPair; 15 | // seed's length is 24 bytes 16 | export function crypto_sign_seed_keypair(seed: Buffer): KeyPair; 17 | // return concat( {the signature of the message}, {message} ), 18 | // if just only need signature, use crypto_sign_detached instead 19 | export function crypto_sign(message: Buffer, secretKey: Buffer): Buffer; 20 | export function crypto_sign_open(signed: Buffer, publicKey: Buffer): Buffer; 21 | export function crypto_sign_detached( 22 | message: Buffer, 23 | secretKey: Buffer, 24 | ): Buffer; 25 | export function crypto_sign_verify_detached( 26 | signed: Buffer, 27 | message: Buffer, 28 | publicKey: Buffer, 29 | ): Buffer; 30 | 31 | // *** Box *** 32 | export function crypto_box_keypair(): KeyPair; 33 | export function crypto_box_seed_keypair(seed: Buffer): KeyPair; 34 | export function crypto_box_easy( 35 | data: Buffer, 36 | nonce: Buffer, 37 | publicKey: Buffer, 38 | secretKey: Buffer, 39 | ): Buffer; 40 | export function crypto_box_open_easy( 41 | boxed: Buffer, 42 | nonce: Buffer, 43 | publicKey: Buffer, 44 | secretKey: Buffer, 45 | ): Buffer; 46 | 47 | // *** SecretBox *** 48 | export function crypto_secretbox_easy( 49 | plainText: Buffer, 50 | nonce: Buffer, 51 | key: Buffer, 52 | ): Buffer; 53 | export function crypto_secretbox_open_easy( 54 | encrypted: Buffer, 55 | nonce: Buffer, 56 | key: Buffer, 57 | ): Buffer; 58 | 59 | // *** random bytes *** 60 | export function randombytes(buf: Buffer): void; 61 | 62 | // *** auth *** 63 | export function crypto_auth(data: Buffer, key: Buffer): Buffer; 64 | export function crypto_auth_verify( 65 | signed: Buffer, 66 | data: Buffer, 67 | key: Buffer, 68 | ): boolean; 69 | 70 | // *** scalar multiplication *** 71 | export function crypto_scalarmult(secretKey: Buffer, publicKey: Buffer): Buffer; 72 | 73 | // *** conversions *** 74 | export function crypto_sign_ed25519_sk_to_curve25519(secretKey: Buffer): Buffer; 75 | export function crypto_sign_ed25519_pk_to_curve25519(publicKey: Buffer): Buffer; 76 | -------------------------------------------------------------------------------- /src/test/scenarios/scenarios.universal.ts: -------------------------------------------------------------------------------- 1 | import { CryptoDriverNoble } from "../../crypto/crypto-driver-noble.ts"; 2 | import { ICryptoDriver } from "../../crypto/crypto-types.ts"; 3 | import { FormatsArg } from "../../formats/format_types.ts"; 4 | import { IPeer } from "../../peer/peer-types.ts"; 5 | import { AttachmentDriverMemory } from "../../replica/attachment_drivers/memory.ts"; 6 | import { DocDriverMemory } from "../../replica/doc_drivers/memory.ts"; 7 | import { PartnerLocal } from "../../syncer/partner_local.ts"; 8 | import { Syncer } from "../../syncer/syncer.ts"; 9 | 10 | import { SyncAppetite } from "../../syncer/syncer_types.ts"; 11 | import { 12 | AttachmentDriverScenario, 13 | DocDriverScenario, 14 | Scenario, 15 | SyncPartnerScenario, 16 | } from "./types.ts"; 17 | 18 | export const universalCryptoDrivers: Scenario[] = [{ 19 | name: "Noble", 20 | item: CryptoDriverNoble, 21 | }]; 22 | 23 | export const universalReplicaDocDrivers: Scenario[] = [ 24 | { 25 | name: "Memory", 26 | item: { 27 | persistent: false, 28 | builtInConfigKeys: [], 29 | makeDriver: (addr) => new DocDriverMemory(addr), 30 | }, 31 | }, 32 | ]; 33 | 34 | export const universalReplicaAttachmentDrivers: Scenario< 35 | AttachmentDriverScenario 36 | >[] = [ 37 | { 38 | name: "Memory", 39 | item: { makeDriver: () => new AttachmentDriverMemory(), persistent: false }, 40 | }, 41 | ]; 42 | 43 | export class SyncScenarioLocal implements SyncPartnerScenario { 44 | formats: FormatsArg; 45 | appetite: SyncAppetite; 46 | syncContinuousWait = 800; 47 | 48 | constructor(formats: FormatsArg, appetite: SyncAppetite) { 49 | this.formats = formats; 50 | this.appetite = appetite; 51 | } 52 | 53 | setup(peerA: IPeer, peerB: IPeer) { 54 | const partner = new PartnerLocal(peerB, peerA, this.appetite, this.formats); 55 | 56 | const syncerA = peerA.addSyncPartner(partner, "Test local"); 57 | 58 | return Promise.resolve( 59 | [syncerA, partner.partnerSyncer] as [ 60 | Syncer, 61 | Syncer, 62 | ], 63 | ); 64 | } 65 | 66 | teardown() { 67 | return Promise.resolve(); 68 | } 69 | } 70 | 71 | export const universalPartners: Scenario< 72 | (formats: FormatsArg, appetite: SyncAppetite) => SyncPartnerScenario 73 | >[] = [{ 74 | name: "Local", 75 | item: (formats, appetite) => new SyncScenarioLocal(formats, appetite), 76 | }]; 77 | -------------------------------------------------------------------------------- /debug/lan.ts: -------------------------------------------------------------------------------- 1 | import { 2 | AttachmentDriverMemory, 3 | AuthorKeypair, 4 | Crypto, 5 | CryptoDriverSodium, 6 | DocDriverMemory, 7 | Peer, 8 | Replica, 9 | setGlobalCryptoDriver, 10 | Syncer, 11 | } from "../mod.ts"; 12 | import { DiscoveryLAN } from "../src/lan/discovery_lan.ts"; 13 | import { writeRandomDocs } from "../src/test/test-utils.ts"; 14 | 15 | const keypair = await Crypto.generateAuthorKeypair("test") as AuthorKeypair; 16 | 17 | setGlobalCryptoDriver(CryptoDriverSodium); 18 | 19 | const shareKeypair = { 20 | shareAddress: 21 | "+landisco.blvfxy5iv4mpkiq37pag5ol65hzwx3vwnz7ksedwepb66zff6lqjq", 22 | secret: "bev6ybcxxjwdcn337ctj4rv5nwqi54xsavqoeo3zbxay7whe3r64q", 23 | }; 24 | 25 | const replicaA = new Replica({ 26 | driver: { 27 | docDriver: new DocDriverMemory(shareKeypair.shareAddress), 28 | attachmentDriver: new AttachmentDriverMemory(), 29 | }, 30 | shareSecret: shareKeypair.secret, 31 | }); 32 | 33 | await writeRandomDocs(keypair, replicaA, 100); 34 | 35 | const peer = new Peer(); 36 | 37 | peer.addReplica(replicaA); 38 | 39 | const disco = new DiscoveryLAN({ 40 | name: "LAN Peer", 41 | }); 42 | 43 | console.log("Discovering..."); 44 | 45 | for await (const event of peer.discover(disco)) { 46 | switch (event.kind) { 47 | case "PEER_DISCOVERED": { 48 | try { 49 | const syncer = await event.sync(); 50 | 51 | console.log(`Discovered ${event.description}`); 52 | 53 | logSyncer(syncer, event.description); 54 | } catch { 55 | // already had a session active 56 | } 57 | 58 | break; 59 | } 60 | case "PEER_INITIATED_SYNC": { 61 | console.log(`Was discovered by ${event.description}`); 62 | 63 | logSyncer(event.syncer, event.description); 64 | } 65 | } 66 | } 67 | 68 | function logSyncer(syncer: Syncer, description: string) { 69 | console.log("Started syncing with", description); 70 | 71 | syncer.isDone().then(() => { 72 | console.log("Finished syncing with", description); 73 | 74 | const status = syncer.getStatus(); 75 | 76 | for (const share in status) { 77 | const report = status[share]; 78 | 79 | console.log( 80 | `Got ${report.docs.receivedCount} / ${report.docs.requestedCount} | Sent ${report.docs.sentCount} | Transfers: ${ 81 | report.attachments.filter((transfer) => 82 | transfer.status === "complete" 83 | ).length 84 | } / ${report.attachments.length}`, 85 | ); 86 | } 87 | }); 88 | } 89 | -------------------------------------------------------------------------------- /src/discovery/types.ts: -------------------------------------------------------------------------------- 1 | import { IPeer } from "../peer/peer-types.ts"; 2 | import { Syncer } from "../syncer/syncer.ts"; 3 | import { SyncAppetite } from "../syncer/syncer_types.ts"; 4 | 5 | /** A service which discovers other remote or local Earthstar peers. */ 6 | export interface DiscoveryService { 7 | events: AsyncIterable; 8 | /** Stop the discovery service from finding other peers and advertising itself. */ 9 | stop(): void; 10 | } 11 | 12 | export type DiscoveryServiceEvent = 13 | | { 14 | kind: "PEER_DISCOVERED"; 15 | /** The description of the peer */ 16 | description: string; 17 | /** A callback which starts a new sync session and returns the resulting `Syncer`. */ 18 | begin: ( 19 | peer: IPeer, 20 | appetite: SyncAppetite, 21 | ) => Promise>; 22 | } 23 | | { 24 | kind: "PEER_INITIATED_SYNC"; 25 | /** The description of the peer */ 26 | description: string; 27 | /** A callback which begins the incoming sync session and returns the resulting `Syncer`. */ 28 | begin: ( 29 | peer: IPeer, 30 | ) => Promise>; 31 | } 32 | | { 33 | kind: "PEER_EXITED"; 34 | /** The description of the peer */ 35 | description: string; 36 | } 37 | /** For when the service has halted. Used to break the async iterable. */ 38 | | { 39 | kind: "SERVICE_STOPPED"; 40 | }; 41 | 42 | /** A discovery service event, indicating: 43 | * 44 | * - That a new peer has been discovered 45 | * - That another peer discovered us and initiated sync 46 | * - That a peer which had been previously discovered has exited. 47 | */ 48 | export type DiscoveryEvent = 49 | /** An event indicating a peer has been discovered by the service. */ 50 | | { 51 | kind: "PEER_DISCOVERED"; 52 | /** The description of the peer */ 53 | description: string; 54 | /** A callback to initiate sync with this peer. */ 55 | sync: ( 56 | opts?: { syncContinuously: boolean }, 57 | ) => Promise>; 58 | } 59 | /** An event indicating another peer has discovered us and initiated sync. */ 60 | | { 61 | kind: "PEER_INITIATED_SYNC"; 62 | /** The description of the peer */ 63 | description: string; 64 | /** The `Syncer` created for this sync session. */ 65 | syncer: Syncer; 66 | } 67 | /** A peer which had previously been discovered has exited. */ 68 | | { 69 | kind: "PEER_EXITED"; 70 | /** The description of the peer */ 71 | description: string; 72 | }; 73 | -------------------------------------------------------------------------------- /src/query/query-types.ts: -------------------------------------------------------------------------------- 1 | import { 2 | AuthorAddress, 3 | FormatName, 4 | Path, 5 | Timestamp, 6 | } from "../util/doc-types.ts"; 7 | 8 | //================================================================================ 9 | 10 | /** Filters a query by document attributes. */ 11 | export interface QueryFilter { 12 | /** Match an exact path. */ 13 | path?: Path; 14 | pathStartsWith?: string; 15 | pathEndsWith?: string; 16 | /** Match documents with a given author. */ 17 | author?: AuthorAddress; 18 | timestamp?: Timestamp; 19 | /** Match documents newer than the given timestamp. */ 20 | timestampGt?: Timestamp; 21 | /** Match documents older than the given timestamp. */ 22 | timestampLt?: Timestamp; 23 | } 24 | 25 | /** Represents fetching all historical versions of a document by authors, or just the latest versions. */ 26 | export type HistoryMode = "latest" | "all"; 27 | 28 | /** Describes a query for fetching documents from a replica. */ 29 | export interface Query { 30 | // for each property, the first option is the default if it's omitted 31 | 32 | // this is in the order that processing happens: 33 | 34 | // first, limit to latest docs or all doc. 35 | 36 | /** Whether to fetch all historical versions of a document or just the latest versions. */ 37 | historyMode?: HistoryMode; 38 | 39 | // then iterate in this order 40 | // "path ASC" is actually "path ASC then break ties with timestamp DESC" 41 | // "path DESC" is the reverse of that 42 | 43 | /** The order to return docs in. Defaults to `path ASC`. */ 44 | orderBy?: "path ASC" | "path DESC" | "localIndex ASC" | "localIndex DESC"; 45 | 46 | // start iterating immediately after this item (e.g. get items which are > startAfter) 47 | 48 | /** Only fetch documents which come after a certain point. */ 49 | startAfter?: { 50 | /** Only documents after this localIndex. Only works when ordering by localIndex. */ 51 | localIndex?: number; 52 | 53 | /** Only documents after this path. Only works when ordering by path. */ 54 | path?: string; 55 | }; 56 | 57 | // then apply filters, if any 58 | filter?: QueryFilter; 59 | 60 | // stop iterating after this number of docs 61 | /** The maximum number of documents to return. */ 62 | limit?: number; 63 | 64 | formats?: FormatsType; 65 | } 66 | 67 | export const DEFAULT_QUERY: Query<["es.5"]> = { 68 | historyMode: "latest", 69 | orderBy: "path ASC", 70 | startAfter: undefined, 71 | limit: undefined, 72 | filter: undefined, 73 | formats: ["es.5"], 74 | }; 75 | -------------------------------------------------------------------------------- /src/server/server.node.ts: -------------------------------------------------------------------------------- 1 | import { createServer } from "https://deno.land/std@0.167.0/node/http.ts"; 2 | import { IServerExtension } from "./extensions/extension.ts"; 3 | import { Buffer } from "https://deno.land/std@0.167.0/node/buffer.ts"; 4 | import { ServerCore } from "./server_core.ts"; 5 | 6 | export type ServerOpts = { 7 | port?: number; 8 | server: ReturnType; 9 | }; 10 | 11 | /** 12 | * An extensible server able to synchronise with other peers. 13 | * 14 | * A server's functionality can be extended using extensions of type `IServerExtension`. 15 | */ 16 | export class Server { 17 | private core: ServerCore; 18 | private server: ReturnType; 19 | 20 | /** 21 | * Create a new server with an array of extensions. 22 | * @param extensions - The extensions used by the server. Extensions will be registered in the order you provide them in, as one extension may depend on the actions of another. For example, the `ExtensionServeContent` may rely on a replica created by `ExtensionKnownShares`. 23 | */ 24 | constructor(extensions: IServerExtension[], opts: ServerOpts) { 25 | this.core = new ServerCore(extensions); 26 | 27 | this.server = opts.server; 28 | 29 | this.server.on("request", async (req, res) => { 30 | let data = undefined; 31 | 32 | if (req.method === "POST") { 33 | const buffers = []; 34 | 35 | for await (const chunk of req) { 36 | buffers.push(chunk); 37 | } 38 | 39 | data = Buffer.concat(buffers).toString(); 40 | } 41 | 42 | const headers = new Headers(); 43 | 44 | for (const key in req.headers) { 45 | if (req.headers[key]) headers.append(key, req.headers[key] as string); 46 | } 47 | 48 | // Need the hostname here so the URL plays nice with Node's URL class. 49 | const url = `http://0.0.0.0${req.url}`; 50 | 51 | const request = new Request(url, { 52 | method: req.method, 53 | headers, 54 | body: data, 55 | }); 56 | 57 | const response = await this.core.handler(request); 58 | 59 | // Headers 60 | for (const [key, value] of response.headers) { 61 | res.setHeader(key, value); 62 | } 63 | 64 | // Status 65 | res.statusCode = response.status; 66 | 67 | // Body 68 | // TODO: Handle streaming responses. 69 | if (response.body) { 70 | res.end(response.body); 71 | } 72 | 73 | res.end(); 74 | }); 75 | 76 | this.server.listen(opts?.port); 77 | } 78 | 79 | close() { 80 | this.server.close(); 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/test/crypto/crypto-driver-interop.test.ts: -------------------------------------------------------------------------------- 1 | import { assert, assertEquals } from "../asserts.ts"; 2 | 3 | import { ICryptoDriver, KeypairBytes } from "../../crypto/crypto-types.ts"; 4 | import { identifyBufOrBytes } from "../../util/bytes.ts"; 5 | import { cryptoScenarios } from "../scenarios/scenarios.ts"; 6 | import { Scenario } from "../scenarios/types.ts"; 7 | 8 | //================================================================================ 9 | 10 | export function runCryptoDriverInteropTests( 11 | scenario: Scenario[], 12 | ) { 13 | const TEST_NAME = "crypto-driver-interop shared tests"; 14 | const SUBTEST_NAME = scenario.map((scenario) => scenario.name).join( 15 | " + ", 16 | ); 17 | const drivers = scenario.map(({ item }) => item); 18 | 19 | Deno.test( 20 | SUBTEST_NAME + ": compare sigs from each driver", 21 | async () => { 22 | const msg = "hello"; 23 | const keypairBytes: KeypairBytes = await drivers[0] 24 | .generateKeypairBytes(); 25 | const keypairName = (drivers[0] as any).name; 26 | const sigs: { name: string; sig: Uint8Array }[] = []; 27 | for (const signer of drivers) { 28 | const sig = await signer.sign(keypairBytes, msg); 29 | assertEquals( 30 | identifyBufOrBytes(sig), 31 | "bytes", 32 | "signature is bytes, not buffer", 33 | ); 34 | sigs.push({ name: (signer as any).name, sig }); 35 | } 36 | for (let ii = 0; ii < sigs.length - 1; ii++) { 37 | const sigs0 = sigs[ii]; 38 | const sigs1 = sigs[ii + 1]; 39 | assertEquals( 40 | sigs0.sig, 41 | sigs1.sig, 42 | `keypair by ${keypairName}; signature by ${sigs0.name} matches signature by ${sigs1.name}`, 43 | ); 44 | } 45 | }, 46 | ); 47 | 48 | Deno.test( 49 | SUBTEST_NAME + ": sign with one driver, verify with another", 50 | async () => { 51 | const msg = "hello"; 52 | for (const signer of drivers) { 53 | const keypairBytes: KeypairBytes = await drivers[0] 54 | .generateKeypairBytes(); 55 | const sig = await signer.sign(keypairBytes, msg); 56 | const signerName = (signer as any).name; 57 | for (const verifier of drivers) { 58 | const verifierName = (verifier as any).name; 59 | assert( 60 | verifier.verify(keypairBytes.pubkey, sig, msg), 61 | `keypair and signature by ${signerName} was verified by ${verifierName}`, 62 | ); 63 | } 64 | } 65 | }, 66 | ); 67 | } 68 | 69 | runCryptoDriverInteropTests( 70 | cryptoScenarios, 71 | ); 72 | -------------------------------------------------------------------------------- /src/util/doc-types.ts: -------------------------------------------------------------------------------- 1 | //================================================================================ 2 | // PRIMITIVE DATA TYPES SPECIFIC TO OUR CODE 3 | 4 | import { ValidationError } from "./errors.ts"; 5 | 6 | /** An identity's public address. */ 7 | export type AuthorAddress = string; 8 | /** The human-identifiable portion of an identity's public address, e.g. `suzy`. */ 9 | export type AuthorShortname = string; 10 | /** A share's public address. */ 11 | export type ShareAddress = string; 12 | /** The human-identifiable portion of a share's address, e.g. `gardening`. */ 13 | export type ShareName = string; 14 | /** The path of a document, e.g. `/images/teapot.png`. */ 15 | export type Path = string; 16 | export type Signature = string; 17 | /** A UNIX timestamp in microseconds. */ 18 | export type Timestamp = number; 19 | export type LocalIndex = number; 20 | export type Base32String = string; 21 | export type FormatName = string; 22 | 23 | export type ParsedAddress = { 24 | address: AuthorAddress; 25 | name: AuthorShortname; 26 | pubkey: Base32String; 27 | }; 28 | 29 | //================================================================================ 30 | // DOCUMENTS 31 | 32 | /** The core properties all documents must implement, regardless of format. */ 33 | export interface DocBase { 34 | format: FormatType; 35 | path: string; 36 | author: AuthorAddress; 37 | timestamp: Timestamp; 38 | signature: Signature; 39 | deleteAfter?: number | null; 40 | _localIndex?: number; 41 | } 42 | 43 | export interface DocInputBase { 44 | format: FormatType; 45 | path: string; 46 | timestamp?: Timestamp; 47 | } 48 | 49 | export type DocWithFormat< 50 | FormatType extends string, 51 | DocType extends DocBase, 52 | > = Extract; 53 | 54 | export type DocInputWithFormat< 55 | FormatType extends string, 56 | DocInputType extends DocInputBase, 57 | > = Extract; 58 | 59 | /** An attachment associated with a document. */ 60 | export type DocAttachment = { 61 | /** Returns a stream to use the attachment's bytes chunk by chunk. Useful if the attachment is very big. */ 62 | stream: () => Promise>; 63 | /** Returns all of the attachments bytes in one go. Handier if you know the attachment is small. */ 64 | bytes: () => Promise; 65 | }; 66 | 67 | /** A document with it's attachment merged onto a new `attachment` property. */ 68 | export type DocWithAttachment> = D & { 69 | attachment: DocAttachment | undefined | ValidationError; 70 | }; 71 | -------------------------------------------------------------------------------- /src/crypto/keypair.ts: -------------------------------------------------------------------------------- 1 | import { AuthorShortname } from "../util/doc-types.ts"; 2 | import { isErr, ValidationError } from "../util/errors.ts"; 3 | import { base32BytesToString, base32StringToBytes } from "./base32.ts"; 4 | import { AuthorKeypair, KeypairBytes, ShareKeypair } from "./crypto-types.ts"; 5 | import { 6 | assembleAuthorAddress, 7 | assembleShareAddress, 8 | parseAuthorOrShareAddress, 9 | } from "../core-validators/addresses.ts"; 10 | 11 | //================================================================================ 12 | 13 | /** Combine a shortname with a raw KeypairBytes to make an AuthorKeypair */ 14 | export function encodeAuthorKeypairToStrings( 15 | shortname: AuthorShortname, 16 | pair: KeypairBytes, 17 | ): AuthorKeypair { 18 | return ({ 19 | address: assembleAuthorAddress(shortname, base32BytesToString(pair.pubkey)), 20 | secret: base32BytesToString(pair.secret), 21 | }); 22 | } 23 | 24 | /** Combine a name with a raw KeypairBytes to make an ShareKeypair */ 25 | export function encodeShareKeypairToStrings( 26 | name: string, 27 | pair: KeypairBytes, 28 | ) { 29 | return ({ 30 | address: assembleShareAddress(name, base32BytesToString(pair.pubkey)), 31 | secret: base32BytesToString(pair.secret), 32 | }); 33 | } 34 | 35 | /** Convert a keypair (author / share) back into a raw KeypairBytes for use in crypto operations. */ 36 | export function decodeKeypairToBytes( 37 | pair: AuthorKeypair | ShareKeypair, 38 | ): KeypairBytes | ValidationError { 39 | try { 40 | const address = isAuthorKeypair(pair) ? pair.address : pair.shareAddress; 41 | 42 | const parsed = parseAuthorOrShareAddress(address); 43 | if (isErr(parsed)) return parsed; 44 | const bytes = { 45 | pubkey: base32StringToBytes(parsed.pubkey), 46 | secret: base32StringToBytes(pair.secret), 47 | }; 48 | if (bytes.pubkey.length !== 32) { 49 | // this is already checked by parseAuthorAddress so we can't test it here 50 | // but we'll test it again just to make sure. 51 | return new ValidationError( 52 | `pubkey bytes should be 32 bytes long, not ${bytes.pubkey.length} after base32 decoding. ${address}`, 53 | ); 54 | } 55 | if (bytes.secret.length !== 32) { 56 | return new ValidationError( 57 | `secret bytes should be 32 bytes long, not ${bytes.secret.length} after base32 decoding. ${pair.secret}`, 58 | ); 59 | } 60 | return bytes; 61 | } catch (err) { 62 | return new ValidationError( 63 | "crash while decoding author keypair: " + err.message, 64 | ); 65 | } 66 | } 67 | 68 | export function isAuthorKeypair( 69 | keypair: AuthorKeypair | ShareKeypair, 70 | ): keypair is AuthorKeypair { 71 | return "address" in keypair; 72 | } 73 | -------------------------------------------------------------------------------- /src/util/errors.ts: -------------------------------------------------------------------------------- 1 | /** Generic top-level error class that other Earthstar errors inherit from. */ 2 | export class EarthstarError extends Error { 3 | constructor(message?: string) { 4 | super(message || ""); 5 | this.name = "EarthstarError"; 6 | } 7 | } 8 | 9 | /** Validation failed on a document, share address, author address, etc. */ 10 | export class ValidationError extends EarthstarError { 11 | constructor(message?: string) { 12 | super(message || "Validation error"); 13 | this.name = "ValidationError"; 14 | } 15 | } 16 | 17 | /** An IReplica or IReplicaDriver was used after close() was called on it. */ 18 | export class ReplicaIsClosedError extends EarthstarError { 19 | constructor(message?: string) { 20 | super( 21 | message || "a Replica or ReplicaDriver was used after being closed", 22 | ); 23 | this.name = "ReplicaIsClosedError"; 24 | } 25 | } 26 | 27 | /** An IReplica or IReplicaDriver was used after close() was called on it. */ 28 | export class ReplicaCacheIsClosedError extends EarthstarError { 29 | constructor(message?: string) { 30 | super( 31 | message || "a ReplicaCache was used after being closed", 32 | ); 33 | this.name = "ReplicaCacheIsClosedError"; 34 | } 35 | } 36 | 37 | /** A server URL is bad or the network is down */ 38 | export class NetworkError extends EarthstarError { 39 | constructor(message?: string) { 40 | super(message || "network error"); 41 | this.name = "NetworkError"; 42 | } 43 | } 44 | 45 | export class TimeoutError extends EarthstarError { 46 | constructor(message?: string) { 47 | super(message || "timeout error"); 48 | this.name = "TimeoutError"; 49 | } 50 | } 51 | 52 | /** A server won't accept writes */ 53 | export class ConnectionRefusedError extends EarthstarError { 54 | constructor(message?: string) { 55 | super(message || "connection refused"); 56 | this.name = "ConnectionRefused"; 57 | } 58 | } 59 | 60 | export class NotImplementedError extends EarthstarError { 61 | constructor(message?: string) { 62 | super(message || "not implemented yet"); 63 | this.name = "NotImplementedError"; 64 | } 65 | } 66 | 67 | export class NotSupportedError extends EarthstarError { 68 | constructor(message?: string) { 69 | super(message || "Not supported"); 70 | this.name = "NotSupportedError"; 71 | } 72 | } 73 | 74 | /** Check if any value is a subclass of EarthstarError (return true) or not (return false) */ 75 | export function isErr(x: T | Error): x is EarthstarError { 76 | return x instanceof EarthstarError; 77 | } 78 | 79 | /** Check if any value is a subclass of EarthstarError (return false) or not (return true) */ 80 | export function notErr(x: T | Error): x is T { 81 | return !(x instanceof EarthstarError); 82 | } 83 | -------------------------------------------------------------------------------- /scripts/release_beta.ts: -------------------------------------------------------------------------------- 1 | // Make sure we're on squirrel 2 | console.group("Checking branch..."); 3 | 4 | const currentBranch = await run([ 5 | "git", 6 | "rev-parse", 7 | "--abbrev-ref", 8 | "HEAD", 9 | ]); 10 | 11 | if (currentBranch !== "squirrel") { 12 | console.error("Trying to cut a release from a branch other than squirrel."); 13 | Deno.exit(1); 14 | } else { 15 | console.log("squirrel ✔"); 16 | } 17 | 18 | console.groupEnd(); 19 | 20 | // Get the last tag and bump it 21 | console.group("Bumping tag..."); 22 | 23 | const lastTag = await run([ 24 | "git", 25 | "describe", 26 | "--tags", 27 | "--abbrev=0", 28 | ]); 29 | 30 | const regex = /v10.0.0-beta.(\d+)/; 31 | 32 | const match = lastTag.match(regex); 33 | 34 | let nextTag: string; 35 | 36 | if (match) { 37 | const nextNum = parseInt(match[1]) + 1; 38 | nextTag = `v10.0.0-beta.${nextNum}`; 39 | } else { 40 | nextTag = `v10.0.0-beta.1`; 41 | } 42 | 43 | console.log(`Next tag: ${nextTag}`); 44 | 45 | console.groupEnd(); 46 | 47 | console.group("Creating web bundle..."); 48 | await run(["deno", "task", "bundle", nextTag]); 49 | console.log("... done."); 50 | console.groupEnd(); 51 | 52 | // Call NPM with new version 53 | console.group("Creating NPM distribution..."); 54 | await run(["deno", "task", "npm", nextTag]); 55 | 56 | console.log("... done."); 57 | console.groupEnd(); 58 | 59 | console.group("Creating web bundle..."); 60 | await run(["deno", "task", "bundle", nextTag]); 61 | console.log("... done."); 62 | console.groupEnd(); 63 | 64 | const proceed = confirm(`Publish ${nextTag} to git and NPM?`); 65 | 66 | if (proceed) { 67 | console.group("Creating release..."); 68 | 69 | // Create tag and push to remote 70 | console.log("Creating git tag..."); 71 | await run(["git", "tag", nextTag]); 72 | 73 | // Push to origin 74 | console.log("Pushing git tag to origin..."); 75 | await run(["git", "push", "origin", nextTag]); 76 | 77 | console.log("Publishing to NPM..."); 78 | await run(["npm", "publish", "./npm", "--tag", "beta"]); 79 | 80 | console.groupEnd(); 81 | 82 | console.log(`Released ${nextTag}`); 83 | } else { 84 | console.log("Aborted release."); 85 | Deno.exit(0); 86 | } 87 | 88 | async function run(cmd: string[], cwd?: string): Promise { 89 | const process = Deno.run({ 90 | cmd, 91 | cwd, 92 | stdout: "piped", 93 | stderr: "piped", 94 | }); 95 | 96 | const { code } = await process.status(); 97 | 98 | if (code === 0) { 99 | return new TextDecoder().decode(await process.output()).trim(); 100 | } else { 101 | console.error(new TextDecoder().decode(await process.stderrOutput())); 102 | Deno.exit(1); 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /scripts/release_alpha.ts: -------------------------------------------------------------------------------- 1 | // Make sure we're on squirrel 2 | console.group("Checking branch..."); 3 | 4 | const currentBranch = await run([ 5 | "git", 6 | "rev-parse", 7 | "--abbrev-ref", 8 | "HEAD", 9 | ]); 10 | 11 | if (currentBranch !== "squirrel") { 12 | console.error("Trying to cut a release from a branch other than squirrel."); 13 | Deno.exit(1); 14 | } else { 15 | console.log("squirrel ✔"); 16 | } 17 | 18 | console.groupEnd(); 19 | 20 | // Get the last tag and bump it 21 | console.group("Bumping tag..."); 22 | 23 | const lastTag = await run([ 24 | "git", 25 | "describe", 26 | "--tags", 27 | "--abbrev=0", 28 | ]); 29 | 30 | const regex = /v10.0.0-alpha.(\d+)/; 31 | 32 | const match = lastTag.match(regex); 33 | 34 | let nextTag: string; 35 | 36 | if (match) { 37 | const nextNum = parseInt(match[1]) + 1; 38 | nextTag = `v10.0.0-alpha.${nextNum}`; 39 | } else { 40 | nextTag = `v10.0.0-alpha.1`; 41 | } 42 | 43 | console.log(`Next tag: ${nextTag}`); 44 | 45 | console.groupEnd(); 46 | 47 | console.group("Creating web bundle..."); 48 | await run(["deno", "task", "bundle", nextTag]); 49 | console.log("... done."); 50 | console.groupEnd(); 51 | 52 | // Call NPM with new version 53 | console.group("Creating NPM distribution..."); 54 | await run(["deno", "task", "npm", nextTag]); 55 | 56 | console.log("... done."); 57 | console.groupEnd(); 58 | 59 | console.group("Creating web bundle..."); 60 | await run(["deno", "task", "bundle", nextTag]); 61 | console.log("... done."); 62 | console.groupEnd(); 63 | 64 | const proceed = confirm(`Publish ${nextTag} to git and NPM?`); 65 | 66 | if (proceed) { 67 | console.group("Creating release..."); 68 | 69 | // Create tag and push to remote 70 | console.log("Creating git tag..."); 71 | await run(["git", "tag", nextTag]); 72 | 73 | // Push to origin 74 | console.log("Pushing git tag to origin..."); 75 | await run(["git", "push", "origin", nextTag]); 76 | 77 | console.log("Publishing to NPM..."); 78 | await run(["npm", "publish", "./npm", "--tag", "alpha"]); 79 | 80 | console.groupEnd(); 81 | 82 | console.log(`Released ${nextTag}`); 83 | } else { 84 | console.log("Aborted release."); 85 | Deno.exit(0); 86 | } 87 | 88 | async function run(cmd: string[], cwd?: string): Promise { 89 | const process = Deno.run({ 90 | cmd, 91 | cwd, 92 | stdout: "piped", 93 | stderr: "piped", 94 | }); 95 | 96 | const { code } = await process.status(); 97 | 98 | if (code === 0) { 99 | return new TextDecoder().decode(await process.output()).trim(); 100 | } else { 101 | console.error(new TextDecoder().decode(await process.stderrOutput())); 102 | Deno.exit(1); 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /src/test/misc/bytes.test.ts: -------------------------------------------------------------------------------- 1 | import { assert, assertEquals } from "../asserts.ts"; 2 | import { snowmanBytes, snowmanString } from "../test-utils.ts"; 3 | //t.runOnly = true; 4 | 5 | let TEST_NAME = "bytes"; 6 | 7 | import { 8 | bytesToString, 9 | concatBytes, 10 | isBuffer, 11 | isBytes, 12 | stringLengthInBytes, 13 | stringToBytes, 14 | } from "../../util/bytes.ts"; 15 | 16 | //================================================================================ 17 | 18 | let simpleString = "aa"; 19 | let simpleBytes = Uint8Array.from([97, 97]); 20 | 21 | Deno.test("bytesToString", () => { 22 | assertEquals( 23 | bytesToString(simpleBytes), 24 | simpleString, 25 | "simple bytes to string", 26 | ); 27 | assertEquals( 28 | bytesToString(snowmanBytes), 29 | snowmanString, 30 | "snowman bytes to string", 31 | ); 32 | assert(typeof bytesToString(snowmanBytes) === "string", "returns a string"); 33 | }); 34 | 35 | Deno.test("stringToBytes", () => { 36 | assertEquals( 37 | stringToBytes(simpleString), 38 | simpleBytes, 39 | "simple string to bytes", 40 | ); 41 | assertEquals( 42 | stringToBytes(snowmanString), 43 | snowmanBytes, 44 | "snowman string to bytes", 45 | ); 46 | }); 47 | 48 | //-------------------------------------------------- 49 | 50 | Deno.test("stringLengthInBytes", () => { 51 | assertEquals(stringLengthInBytes(simpleString), 2, "simple string"); 52 | assertEquals(stringLengthInBytes(snowmanString), 3, "snowman string"); 53 | }); 54 | 55 | Deno.test("concatBytes", () => { 56 | let coldSnowmanString = "cold" + snowmanString; 57 | let coldSnowmanBytes = stringToBytes(coldSnowmanString); 58 | let concatted = concatBytes(stringToBytes("cold"), snowmanBytes); 59 | assertEquals(concatted, coldSnowmanBytes, "concat bytes"); 60 | 61 | assertEquals( 62 | concatBytes(Uint8Array.from([]), Uint8Array.from([1, 2, 3])), 63 | Uint8Array.from([1, 2, 3]), 64 | "optimization when a is empty", 65 | ); 66 | assertEquals( 67 | concatBytes(Uint8Array.from([1, 2, 3]), Uint8Array.from([])), 68 | Uint8Array.from([1, 2, 3]), 69 | "optimization when b is empty", 70 | ); 71 | }); 72 | 73 | //-------------------------------------------------- 74 | 75 | // TODO: b64stringtobytes 76 | 77 | // TODO: hexstringtobytes 78 | 79 | //-------------------------------------------------- 80 | 81 | Deno.test("bytes: identifyBufOrBytes, isBuffer, isBytes", () => { 82 | let bytes = Uint8Array.from([1]); 83 | let other = [1, 2, 3]; 84 | 85 | assertEquals(isBuffer(bytes), false, "isBuffer false"); 86 | assertEquals(isBytes(bytes), true, "isBytes true"); 87 | 88 | assertEquals(isBuffer(other), false, "isBuffer false on other"); 89 | assertEquals(isBytes(other), false, "isBytes false on other"); 90 | }); 91 | -------------------------------------------------------------------------------- /debug/partner_tcp.ts: -------------------------------------------------------------------------------- 1 | import { deferred } from "../deps.ts"; 2 | import { 3 | AttachmentDriverMemory, 4 | AuthorKeypair, 5 | Crypto, 6 | CryptoDriverSodium, 7 | DocDriverMemory, 8 | Peer, 9 | Replica, 10 | setGlobalCryptoDriver, 11 | ShareKeypair, 12 | } from "../mod.ts"; 13 | import { LANSession } from "../src/discovery/discovery_lan.ts"; 14 | import { TcpProvider } from "../src/tcp/tcp_provider.ts"; 15 | import { writeRandomDocs } from "../src/test/test-utils.ts"; 16 | 17 | const keypair = await Crypto.generateAuthorKeypair("test") as AuthorKeypair; 18 | 19 | setGlobalCryptoDriver(CryptoDriverSodium); 20 | 21 | const shareKeypair = await Crypto.generateShareKeypair( 22 | "apples", 23 | ) as ShareKeypair; 24 | 25 | const replicaA = new Replica({ 26 | driver: { 27 | docDriver: new DocDriverMemory(shareKeypair.shareAddress), 28 | attachmentDriver: new AttachmentDriverMemory(), 29 | }, 30 | shareSecret: shareKeypair.secret, 31 | }); 32 | 33 | const replicaB = new Replica({ 34 | driver: { 35 | docDriver: new DocDriverMemory(shareKeypair.shareAddress), 36 | attachmentDriver: new AttachmentDriverMemory(), 37 | }, 38 | shareSecret: shareKeypair.secret, 39 | }); 40 | 41 | await writeRandomDocs(keypair, replicaA, 1000); 42 | await writeRandomDocs(keypair, replicaB, 1000); 43 | 44 | const peerA = new Peer(); 45 | const peerB = new Peer(); 46 | 47 | peerA.addReplica(replicaA); 48 | peerB.addReplica(replicaB); 49 | 50 | const lanSessionA = deferred(); 51 | const lanSessionB = deferred(); 52 | 53 | const tcpProvider = new TcpProvider(); 54 | 55 | // Set up listeners... 56 | const listenerA = tcpProvider.listen({ port: 17171 }); 57 | const listenerB = tcpProvider.listen({ port: 17172 }); 58 | 59 | (async () => { 60 | for await (const conn of listenerA) { 61 | const session = await lanSessionA; 62 | 63 | await session.addConn(conn); 64 | } 65 | })(); 66 | 67 | (async () => { 68 | for await (const conn of listenerB) { 69 | const session = await lanSessionB; 70 | 71 | await session.addConn(conn); 72 | } 73 | })(); 74 | 75 | console.log("Started syncing..."); 76 | 77 | lanSessionA.resolve( 78 | new LANSession(false, peerA, "once", { 79 | hostname: "127.0.0.1", 80 | port: 17172, 81 | name: "Peer B", 82 | }), 83 | ); 84 | 85 | lanSessionB.resolve( 86 | new LANSession(true, peerB, "once", { 87 | hostname: "127.0.0.1", 88 | port: 17171, 89 | name: "Peer A", 90 | }), 91 | ); 92 | 93 | const syncerA = await (await lanSessionA).syncer; 94 | const syncerB = await (await lanSessionB).syncer; 95 | 96 | await Promise.all([syncerA.isDone(), syncerB.isDone()]); 97 | 98 | console.log("Synced"); 99 | 100 | listenerA.close(); 101 | listenerB.close(); 102 | 103 | replicaA.close(false); 104 | replicaB.close(false); 105 | -------------------------------------------------------------------------------- /src/crypto/crypto-driver-sodium.ts: -------------------------------------------------------------------------------- 1 | import { ICryptoDriver, KeypairBytes } from "./crypto-types.ts"; 2 | import { concatBytes, stringToBytes } from "../util/bytes.ts"; 3 | import sodium from "https://deno.land/x/sodium@0.2.0/basic.ts"; 4 | 5 | await sodium.ready; 6 | const { createHash } = sha256_uint8array; 7 | 8 | //-------------------------------------------------- 9 | 10 | import { Logger } from "../util/log.ts"; 11 | import { sha256_uint8array } from "../../deps.ts"; 12 | import { UpdatableHash } from "./updatable_hash.ts"; 13 | const logger = new Logger("crypto-driver-noble", "cyan"); 14 | 15 | //================================================================================ 16 | /** 17 | * A version of the ICryptoDriver interface backed a WASM wrapper of libsodium. 18 | * Faster than noble by several magnitudes. 19 | * Works in Deno. 20 | */ 21 | export const CryptoDriverSodium: ICryptoDriver = class { 22 | static async sha256( 23 | input: string | Uint8Array, 24 | ): Promise { 25 | if (typeof input === "string") { 26 | const encoded = new TextEncoder().encode(input); 27 | const result = await crypto.subtle.digest("SHA-256", encoded); 28 | return Promise.resolve(new Uint8Array(result)); 29 | } else { 30 | const result = await crypto.subtle.digest("SHA-256", input); 31 | return Promise.resolve(new Uint8Array(result)); 32 | } 33 | } 34 | 35 | static updatableSha256() { 36 | return new UpdatableHash({ 37 | hash: createHash("sha256"), 38 | update: (hash, data) => hash.update(data), 39 | digest: (hash) => hash.digest(), 40 | }); 41 | } 42 | 43 | static generateKeypairBytes(): Promise { 44 | logger.debug("generateKeypairBytes"); 45 | 46 | const seed = sodium.randombytes_buf(32); 47 | const keys = sodium.crypto_sign_seed_keypair(seed); 48 | 49 | return Promise.resolve({ 50 | pubkey: keys.publicKey, 51 | secret: keys.privateKey.slice(0, 32), 52 | }); 53 | } 54 | static sign( 55 | keypairBytes: KeypairBytes, 56 | msg: string | Uint8Array, 57 | ): Promise { 58 | logger.debug("sign"); 59 | if (typeof msg === "string") msg = stringToBytes(msg); 60 | 61 | const identity = concatBytes(keypairBytes.secret, keypairBytes.pubkey); 62 | 63 | return Promise.resolve(sodium.crypto_sign_detached(msg, identity)); 64 | } 65 | static verify( 66 | publicKey: Uint8Array, 67 | sig: Uint8Array, 68 | msg: string | Uint8Array, 69 | ): Promise { 70 | logger.debug("verify"); 71 | try { 72 | if (typeof msg === "string") msg = stringToBytes(msg); 73 | 74 | const verified = sodium.crypto_sign_verify_detached( 75 | sig, 76 | msg, 77 | publicKey, 78 | ); 79 | 80 | return Promise.resolve(verified); 81 | } catch { 82 | return Promise.resolve(false); 83 | } 84 | } 85 | }; 86 | -------------------------------------------------------------------------------- /src/test/misc/characters.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "../asserts.ts"; 2 | import { snowmanString } from "../test-utils.ts"; 3 | //t.runOnly = true; 4 | 5 | let TEST_NAME = "characters"; 6 | 7 | import { 8 | alphaLower, 9 | isDigit, 10 | isOnlyPrintableAscii, 11 | onlyHasChars, 12 | } from "../../core-validators/characters.ts"; 13 | import { bytesToString } from "../../util/bytes.ts"; 14 | 15 | //================================================================================ 16 | 17 | Deno.test("onlyHasCharacters", () => { 18 | type Vector = [str: string, allowedChars: string, result: boolean]; 19 | let vectors: Vector[] = [ 20 | ["a", "a", true], 21 | ["abc", "a", false], 22 | ["a", "abc", true], 23 | ["", "abc", true], 24 | ["abc", "", false], 25 | ["helloworld", alphaLower, true], 26 | ["helloWorld", alphaLower, false], 27 | [snowmanString, snowmanString, true], 28 | [snowmanString, "a", false], 29 | ["a", snowmanString, false], 30 | [snowmanString, "abc" + snowmanString + "def", true], 31 | ["a" + snowmanString + "a", "abc" + snowmanString + "def", true], 32 | ]; 33 | for (let [str, allowedChars, expectedResult] of vectors) { 34 | assertEquals( 35 | onlyHasChars(str, allowedChars), 36 | expectedResult, 37 | `onlyHasChars("${str}", "${allowedChars}") should === ${expectedResult}`, 38 | ); 39 | } 40 | }); 41 | 42 | Deno.test("isOnlyPrintableAscii", () => { 43 | type Vector = [ch: string, result: boolean]; 44 | let vectors: Vector[] = [ 45 | ["hello", true], 46 | [" ", true], 47 | ["", true], 48 | ["\n", false], 49 | ["\t", false], 50 | ["\x00", false], 51 | [snowmanString, false], 52 | [bytesToString(Uint8Array.from([200])), false], 53 | [bytesToString(Uint8Array.from([127])), false], 54 | [bytesToString(Uint8Array.from([126])), true], 55 | [bytesToString(Uint8Array.from([55, 127])), false], 56 | [bytesToString(Uint8Array.from([55, 126])), true], 57 | [bytesToString(Uint8Array.from([32])), true], 58 | [bytesToString(Uint8Array.from([31])), false], 59 | [bytesToString(Uint8Array.from([0])), false], 60 | ]; 61 | for (let [str, expectedResult] of vectors) { 62 | assertEquals( 63 | isOnlyPrintableAscii(str), 64 | expectedResult, 65 | `isOnlyPrintableAscii("${str}") should === ${expectedResult}`, 66 | ); 67 | } 68 | }); 69 | 70 | Deno.test("isDigit", () => { 71 | type Vector = [ch: string, result: boolean]; 72 | let vectors: Vector[] = [ 73 | ["0", true], 74 | ["1", true], 75 | ["2", true], 76 | ["3", true], 77 | ["4", true], 78 | ["5", true], 79 | ["6", true], 80 | ["7", true], 81 | ["8", true], 82 | ["9", true], 83 | 84 | ["a", false], 85 | [" ", false], 86 | ["", false], 87 | ["00", false], // only one digit at a time 88 | ["0.0", false], 89 | ]; 90 | for (let [ch, expectedResult] of vectors) { 91 | assertEquals( 92 | isDigit(ch), 93 | expectedResult, 94 | `isDigit("${ch}") should === ${expectedResult}`, 95 | ); 96 | } 97 | }); 98 | -------------------------------------------------------------------------------- /src/syncer/range_messenger.ts: -------------------------------------------------------------------------------- 1 | import { RangeMessenger, RangeMessengerConfig } from "../../deps.ts"; 2 | import { bigIntFromHex, bigIntToHex } from "../util/bigint.ts"; 3 | import { DocThumbnailTree } from "./doc_thumbnail_tree.ts"; 4 | import { DocThumbnail, RangeMessage } from "./syncer_types.ts"; 5 | 6 | const TYPE_MAPPINGS = { 7 | "EMPTY_SET": "emptySet" as const, 8 | "LOWER_BOUND": "lowerBound" as const, 9 | "PAYLOAD": "payload" as const, 10 | "EMPTY_PAYLOAD": "emptyPayload" as const, 11 | "DONE": "done" as const, 12 | "FINGERPRINT": "fingerprint" as const, 13 | "TERMINAL": "terminal" as const, 14 | }; 15 | 16 | const encoding: RangeMessengerConfig< 17 | RangeMessage, 18 | DocThumbnail, 19 | bigint 20 | > = { 21 | encode: { 22 | emptySet: (canRespond) => ({ 23 | type: "EMPTY_SET", 24 | canRespond, 25 | }), 26 | lowerBound: (x) => ({ 27 | type: "LOWER_BOUND", 28 | value: x, 29 | }), 30 | payload: (v, end) => ({ 31 | type: "PAYLOAD", 32 | payload: v, 33 | ...(end ? { end } : {}), 34 | }), 35 | emptyPayload: (upperBound) => ({ 36 | type: "EMPTY_PAYLOAD", 37 | upperBound, 38 | }), 39 | done: (y) => ({ 40 | type: "DONE", 41 | upperBound: y, 42 | }), 43 | fingerprint: (fp, y) => ({ 44 | type: "FINGERPRINT", 45 | fingerprint: bigIntToHex(fp), 46 | upperBound: y, 47 | }), 48 | terminal: () => ({ 49 | type: "TERMINAL", 50 | }), 51 | }, 52 | decode: { 53 | getType: (obj) => { 54 | return TYPE_MAPPINGS[obj.type]; 55 | }, 56 | emptySet: (obj) => { 57 | if (obj.type === "EMPTY_SET") { 58 | return obj.canRespond; 59 | } 60 | throw "Can't decode"; 61 | }, 62 | lowerBound: (obj) => { 63 | if (obj.type === "LOWER_BOUND") { 64 | return obj.value; 65 | } 66 | throw "Can't decode"; 67 | }, 68 | payload: (obj) => { 69 | if (obj.type === "PAYLOAD") { 70 | return { 71 | value: obj.payload, 72 | ...(obj.end ? { end: obj.end } : {}), 73 | }; 74 | } 75 | throw "Can't decode"; 76 | }, 77 | emptyPayload: (obj) => { 78 | if (obj.type === "EMPTY_PAYLOAD") { 79 | return obj.upperBound; 80 | } 81 | throw "Can't decode"; 82 | }, 83 | done: (obj) => { 84 | if (obj.type === "DONE") { 85 | return obj.upperBound; 86 | } 87 | throw "Can't decode"; 88 | }, 89 | fingerprint: (obj) => { 90 | if (obj.type === "FINGERPRINT") { 91 | return { 92 | fingerprint: bigIntFromHex(obj.fingerprint), 93 | upperBound: obj.upperBound, 94 | }; 95 | } 96 | throw "Can't decode"; 97 | }, 98 | terminal: (obj) => { 99 | if (obj.type === "TERMINAL") { 100 | return true; 101 | } 102 | throw "Can't decode"; 103 | }, 104 | }, 105 | }; 106 | 107 | export class EarthstarRangeMessenger 108 | extends RangeMessenger { 109 | constructor( 110 | tree: DocThumbnailTree, 111 | payloadThreshold: number, 112 | rangeDivision: number, 113 | ) { 114 | super({ 115 | tree, 116 | encoding, 117 | fingerprintEquals: (a, b) => a === b, 118 | payloadThreshold, 119 | rangeDivision, 120 | }); 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /src/syncer/plum_tree.ts: -------------------------------------------------------------------------------- 1 | import { 2 | DocThumbnail, 3 | SyncAgentDocEvent, 4 | SyncAgentHaveEvent, 5 | } from "./syncer_types.ts"; 6 | 7 | /** A push-lazy-push-multicast tree, or 'PlumTree'. Organises a network of interconnected peers into a spanning tree where messaging resiliency, latency, and bandwidth are optimised. 8 | * 9 | * When Earthstar sync agents finish their initial reconciliation phase they are switched to a mode where they are managed by a plumtree. 10 | */ 11 | export class PlumTree { 12 | private messagingModes = new Map< 13 | string, 14 | "EAGER" | "LAZY" 15 | >(); 16 | 17 | getMode(id: string): "EAGER" | "LAZY" { 18 | const maybeMode = this.messagingModes.get(id); 19 | 20 | if (maybeMode) { 21 | return maybeMode; 22 | } 23 | 24 | const initialMode = this.messagingModes.size === 0 ? "EAGER" : "LAZY"; 25 | 26 | this.messagingModes.set(id, initialMode); 27 | 28 | return "EAGER"; 29 | } 30 | 31 | /** A list of previously received message IDs, used to check incoming messages for duplicates. 32 | */ 33 | private eagerMessageThumbnails = new Set(); 34 | 35 | /** A map of DocThumbnails to timers. */ 36 | private haveTimeouts = new Map(); 37 | 38 | /** Triggered when the other peer sends a DOC message. Returns a boolean indicating whether to send a PRUNE event to the peer we got this message from. */ 39 | onEagerMessage(id: string, event: SyncAgentDocEvent): boolean { 40 | // Check the list of have timers to see if we're waiting for this event. 41 | // If it's there, clear the timer. 42 | const maybeTimeout = this.haveTimeouts.get(event.thumbnail); 43 | 44 | if (maybeTimeout) { 45 | clearTimeout(maybeTimeout); 46 | // TODO: stop here? 47 | return false; 48 | } 49 | 50 | // Check the list of previously received eager messages for this ID. 51 | if (this.eagerMessageThumbnails.has(event.thumbnail)) { 52 | // If already present, switch to lazy messaging this peer. 53 | this.messagingModes.set(id, "LAZY"); 54 | 55 | // ask syncagent to tell partner to make become lazy (PRUNE). 56 | return true; 57 | } else { 58 | // If not present, add to the list. 59 | this.eagerMessageThumbnails.add(event.thumbnail); 60 | return false; 61 | } 62 | } 63 | 64 | /** Triggered when the other peer sends a HAVE message */ 65 | onLazyMessage( 66 | event: SyncAgentHaveEvent, 67 | dispatchGraft: (thumbnail: DocThumbnail) => void, 68 | ): void { 69 | // Set a timer for the this message to arrive. 70 | // (if a timer already exists, do nothing) 71 | if (!this.haveTimeouts.has(event.thumbnail)) { 72 | const timeout = setTimeout(() => { 73 | // When the timer expires, get the syncagent to send a WANT for this thumbnail. 74 | dispatchGraft(event.thumbnail); 75 | }, 5); 76 | // TODO: What should the timeout be? 77 | 78 | this.haveTimeouts.set(event.thumbnail, timeout); 79 | } 80 | } 81 | 82 | /** Triggered when the other peer sends a graft, i.e. WANT message. */ 83 | onGraftMessage(id: string): void { 84 | // Move the peer who requested this to our eager peers. 85 | this.messagingModes.set(id, "EAGER"); 86 | } 87 | 88 | /** Triggered when the other peer sends a PRUNE message. */ 89 | onPrune(id: string): void { 90 | this.messagingModes.set(id, "LAZY"); 91 | } 92 | } 93 | 94 | // We have to store a bit of plumtree info the messaging mode - in the sync agent. right? 95 | -------------------------------------------------------------------------------- /src/test/misc/buffers.test.ts: -------------------------------------------------------------------------------- 1 | import { Buffer } from "https://deno.land/std@0.154.0/node/buffer.ts"; 2 | import { assert, assertEquals } from "../asserts.ts"; 3 | import { snowmanBytes, snowmanString } from "../test-utils.ts"; 4 | //t.runOnly = true; 5 | 6 | let TEST_NAME = "buffers"; 7 | 8 | // Boilerplate to help browser-run know when this test is completed. 9 | // When run in the browser we'll be running tape, not tap, so we have to use tape's onFinish function. 10 | /* istanbul ignore next */ 11 | 12 | import { 13 | bufferToBytes, 14 | bufferToString, 15 | bytesToBuffer, 16 | stringToBuffer, 17 | } from "../../util/buffers.ts"; 18 | 19 | import { identifyBufOrBytes, isBuffer, isBytes } from "../../util/bytes.ts"; 20 | 21 | //================================================================================ 22 | 23 | let snowmanBuffer = Buffer.from([0xe2, 0x98, 0x83]); 24 | 25 | let simpleString = "aa"; 26 | let simpleBuffer = Buffer.from([97, 97]); 27 | 28 | //================================================================================ 29 | 30 | Deno.test("bytesToBuffer", () => { 31 | assertEquals( 32 | bytesToBuffer(snowmanBytes), 33 | snowmanBuffer, 34 | "snowman bytes to buffer", 35 | ); 36 | assertEquals( 37 | identifyBufOrBytes(bytesToBuffer(snowmanBytes)), 38 | "buffer", 39 | "returns buffer", 40 | ); 41 | }); 42 | 43 | Deno.test("bufferToBytes", () => { 44 | assertEquals( 45 | bufferToBytes(snowmanBuffer), 46 | snowmanBytes, 47 | "snowman buffer to bytes", 48 | ); 49 | assertEquals( 50 | identifyBufOrBytes(bufferToBytes(snowmanBuffer)), 51 | "bytes", 52 | "returns bytes", 53 | ); 54 | }); 55 | 56 | //-------------------------------------------------- 57 | 58 | Deno.test("bufferToString", () => { 59 | assertEquals( 60 | bufferToString(simpleBuffer), 61 | simpleString, 62 | "simple buffer to string", 63 | ); 64 | assertEquals( 65 | bufferToString(snowmanBuffer), 66 | snowmanString, 67 | "snowman buffer to string", 68 | ); 69 | assert( 70 | typeof bufferToString(snowmanBuffer) === "string", 71 | "returns a string", 72 | ); 73 | }); 74 | 75 | Deno.test("stringToBuffer", () => { 76 | assertEquals( 77 | stringToBuffer(simpleString), 78 | simpleBuffer, 79 | "simple string to buffer", 80 | ); 81 | assertEquals( 82 | stringToBuffer(snowmanString), 83 | snowmanBuffer, 84 | "snowman string to buffer", 85 | ); 86 | assertEquals( 87 | identifyBufOrBytes(stringToBuffer(snowmanString)), 88 | "buffer", 89 | "returns buffer", 90 | ); 91 | }); 92 | 93 | Deno.test("buffer: identifyBufOrBytes, isBuffer, isBytes", () => { 94 | let buf = Buffer.from([1]); 95 | let bytes = Uint8Array.from([1]); 96 | let other = [1, 2, 3]; 97 | 98 | assertEquals(identifyBufOrBytes(buf), "buffer", "can identify Buffer"); 99 | assertEquals(isBuffer(buf), true, "isBuffer true"); 100 | assertEquals(isBytes(buf), false, "isBytes false"); 101 | 102 | assertEquals(identifyBufOrBytes(bytes), "bytes", "can identify bytes"); 103 | assertEquals(isBuffer(bytes), false, "isBuffer false"); 104 | assertEquals(isBytes(bytes), true, "isBytes true"); 105 | 106 | assertEquals( 107 | identifyBufOrBytes(other as any), 108 | "?", 109 | "is not tricked by other kinds of object", 110 | ); 111 | assertEquals(isBuffer(other), false, "isBuffer false on other"); 112 | assertEquals(isBytes(other), false, "isBytes false on other"); 113 | }); 114 | -------------------------------------------------------------------------------- /src/test/misc/invite.test.ts: -------------------------------------------------------------------------------- 1 | import { Crypto } from "../../crypto/crypto.ts"; 2 | import { isErr, notErr } from "../../util/errors.ts"; 3 | import { createInvitationURL, parseInvitationURL } from "../../util/invite.ts"; 4 | import { assert, assertEquals } from "../asserts.ts"; 5 | 6 | Deno.test("encodeInvitationURL", async () => { 7 | const shareKeypair = await Crypto.generateShareKeypair("test"); 8 | 9 | assert(notErr(shareKeypair)); 10 | 11 | // Catches bad share addresses 12 | const badShareRes = await createInvitationURL( 13 | "+BAD.addr", 14 | [], 15 | ); 16 | 17 | assert(isErr(badShareRes)); 18 | 19 | // Catches bad URLs 20 | const badUrlRes = await createInvitationURL( 21 | shareKeypair.shareAddress, 22 | ["https://server.com", "NOT_A_URL"], 23 | ); 24 | 25 | assert(isErr(badUrlRes)); 26 | 27 | // Catches bad secrets 28 | const badSecretRes = await createInvitationURL( 29 | shareKeypair.shareAddress, 30 | ["https://server.com"], 31 | "NOT_THE_SECRET", 32 | ); 33 | 34 | assert(isErr(badSecretRes)); 35 | 36 | // Puts out a good code too. 37 | const goodRes = await createInvitationURL( 38 | shareKeypair.shareAddress, 39 | ["https://server.com", "https://server2.com"], 40 | shareKeypair.secret, 41 | ); 42 | 43 | assert(notErr(goodRes)); 44 | }); 45 | 46 | Deno.test("parseInvitationURL", async () => { 47 | const shareKeypair = await Crypto.generateShareKeypair("test"); 48 | 49 | assert(notErr(shareKeypair)); 50 | 51 | // Catches non-URLs 52 | const notURL = "Hello there."; 53 | const notUrlRes = await parseInvitationURL(notURL); 54 | assert(isErr(notUrlRes)); 55 | 56 | // Catches bad share address 57 | const badAddress = "earthstar://notashare/?invite"; 58 | const badAddressRes = await parseInvitationURL(badAddress); 59 | assert(isErr(badAddressRes)); 60 | 61 | // Catches non-invite 62 | const notInvite = `earthstar://${shareKeypair.shareAddress}/some/path`; 63 | const notInviteRes = await parseInvitationURL(notInvite); 64 | assert(isErr(notInviteRes)); 65 | 66 | // Catches missing version 67 | const missingVersion = `earthstar://${shareKeypair.shareAddress}/?invite`; 68 | const missingVersionRes = await parseInvitationURL(missingVersion); 69 | assert(isErr(missingVersionRes)); 70 | 71 | // Catches wrong version 72 | const wrongVersion = `earthstar://${shareKeypair.shareAddress}/?invite&v=1`; 73 | const wrongVersionRes = await parseInvitationURL(wrongVersion); 74 | assert(isErr(wrongVersionRes)); 75 | 76 | // Catches bad URLs 77 | const badServerUrl = 78 | `earthstar://${shareKeypair.shareAddress}/?invite&server=NOT_URL&v=2`; 79 | const badServerUrlRes = await parseInvitationURL(badServerUrl); 80 | assert(isErr(badServerUrlRes)); 81 | 82 | // Catches bad secret 83 | const badSecret = 84 | `earthstar://${shareKeypair.shareAddress}/?invite&server=https://server.com&secret=NOT_REAL_SECRET&v=2`; 85 | const badSecretRes = await parseInvitationURL(badSecret); 86 | assert(isErr(badSecretRes)); 87 | 88 | const goodUrl = 89 | `earthstar://${shareKeypair.shareAddress}/?invite&server=https://server.com&server=https://server2.com&secret=${shareKeypair.secret}&v=2`; 90 | const goodResult = await parseInvitationURL(goodUrl); 91 | assert(notErr(goodResult)); 92 | 93 | assertEquals(goodResult.shareAddress, shareKeypair.shareAddress); 94 | assertEquals(goodResult.servers, [ 95 | "https://server.com", 96 | "https://server2.com", 97 | ]); 98 | assertEquals(goodResult.secret, shareKeypair.secret); 99 | }); 100 | -------------------------------------------------------------------------------- /src/crypto/crypto-driver-node.js: -------------------------------------------------------------------------------- 1 | // This file has no type annotations because we can't get the typings for Node's crypto library here. 2 | 3 | import crypto from "https://deno.land/std@0.119.0/node/crypto.ts"; 4 | 5 | import { b64StringToBytes, concatBytes } from "../util/bytes.ts"; 6 | import { 7 | bufferToBytes, 8 | bytesToBuffer, 9 | stringToBuffer, 10 | } from "../util/buffers.ts"; 11 | 12 | //-------------------------------------------------- 13 | 14 | import { Logger } from "../util/log.ts"; 15 | let logger = new Logger("crypto-driver-node", "cyan"); 16 | 17 | //================================================================================ 18 | 19 | const _generateKeypairDerBytes = () => { 20 | // Generate a keypair in "der" format, which we will have to process 21 | // to remove some prefixes. 22 | // 23 | // Typescript has outdated definitions, doesn't know about ed25519. 24 | // So fight it with "as any". 25 | let pair = crypto.generateKeyPairSync( 26 | "ed25519", 27 | { 28 | publicKeyEncoding: { 29 | format: "der", 30 | type: "spki", 31 | }, 32 | privateKeyEncoding: { 33 | format: "der", 34 | type: "pkcs8", 35 | }, 36 | }, 37 | ); 38 | // Typescript thinks these are strings, but they're Buffers... 39 | // and we need to convert them to bytes (uint8arrays) 40 | return { 41 | pubkey: bufferToBytes(pair.publicKey), 42 | secret: bufferToBytes(pair.privateKey), 43 | }; 44 | }; 45 | 46 | function _shortenDer(k) { 47 | return ({ 48 | pubkey: k.pubkey.slice(-32), 49 | secret: k.secret.slice(-32), 50 | }); 51 | } 52 | let _derPrefixPublic = b64StringToBytes("MCowBQYDK2VwAyEA"); 53 | let _derPrefixSecret = b64StringToBytes("MC4CAQAwBQYDK2VwBCIEIA=="); 54 | function _lengthenDerPublic(b) { 55 | return concatBytes(_derPrefixPublic, b); 56 | } 57 | function _lengthenDerSecret(b) { 58 | return concatBytes(_derPrefixSecret, b); 59 | } 60 | 61 | /** 62 | * A verison of the ICrptoDriver interface backed by native Node crypto functions. 63 | * Requires a recent version of Node, perhaps 12+? 64 | * Does not work in the browser. 65 | */ 66 | export const CryptoDriverNode = class { 67 | static async sha256(input) { 68 | return bufferToBytes( 69 | crypto.createHash("sha256").update(input).digest(), 70 | ); 71 | } 72 | static updatableSha256() { 73 | return new UpdatableHash({ 74 | hash: crypto.createHash("sha256"), 75 | update: (hash, data) => hash.update(data), 76 | digest: (hash) => { 77 | const digest = hash.digest(); 78 | 79 | if (typeof digest === "string") { 80 | return stringToBytes(digest); 81 | } 82 | 83 | return bufferToBytes(digest); 84 | }, 85 | }); 86 | } 87 | static async generateKeypairBytes() { 88 | logger.debug("generateKeypairBytes"); 89 | return _shortenDer(_generateKeypairDerBytes()); 90 | } 91 | static async sign(keypairBytes, msg) { 92 | logger.debug("sign"); 93 | if (typeof msg === "string") msg = stringToBuffer(msg); 94 | return bufferToBytes(crypto.sign( 95 | null, 96 | msg, 97 | { 98 | key: bytesToBuffer(_lengthenDerSecret(keypairBytes.secret)), 99 | format: "der", 100 | type: "pkcs8", 101 | }, 102 | )); 103 | } 104 | static async verify(publicKey, sig, msg) { 105 | logger.debug("verif"); 106 | if (typeof msg === "string") msg = stringToBuffer(msg); 107 | try { 108 | return crypto.verify( 109 | null, 110 | msg, 111 | { 112 | key: _lengthenDerPublic(publicKey), 113 | format: "der", 114 | type: "spki", 115 | }, 116 | sig, 117 | ); 118 | } catch { 119 | return false; 120 | } 121 | } 122 | }; 123 | -------------------------------------------------------------------------------- /src/entries/universal.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * [Earthstar](https://earthstar-project.org) is a small and resilient distributed storage protocol designed with a strong focus on simplicity and versatility, with the social realities of peer-to-peer computing kept in mind. 3 | * 4 | * This is a reference implementation written in Typescript. You can use it to add Earthstar functionality to applications running on servers, browsers, the command line, or anywhere else JavaScript can be run. 5 | * 6 | * ### Example usage 7 | * 8 | * ```ts 9 | * import { Replica, ReplicaDriverMemory, Crypto, Peer } from "earthstar"; 10 | * 11 | * const shareKeypair = await Crypto.generateShareKeypair("gardening"); 12 | * 13 | * const replica = new Replica({ 14 | * driver: ReplicaDriverMemory(shareKeypair.shareAddress), 15 | * shareSecret: shareKeypair.secret, 16 | * }); 17 | * 18 | * const authorKeypair = await Crypto.generateAuthorKeypair("suzy"); 19 | * 20 | * await replica.set(authorKeypair, { 21 | * path: "/my-note", 22 | * text: "Saw seven magpies today", 23 | * }); 24 | * 25 | * const allDocs = await replica.getAllDocs(); 26 | * 27 | * const peer = new Peer(); 28 | * 29 | * peer.addReplica(replica); 30 | * 31 | * peer.sync("https://my.server") 32 | * ``` 33 | * 34 | * This module also exposes server APIs for for building always-online peers. The below example reads some on-disk JSON to initiate some share replicas, and stores their data using the filesystem. 35 | * 36 | * ```ts 37 | * import { 38 | * ExtensionKnownShares, 39 | * ExtensionSyncWeb, 40 | * Server, 41 | * } from "https://deno.land/x/earthstar/mod.ts"; 42 | * 43 | * const server = new Server([ 44 | * new ExtensionKnownShares({ 45 | * knownSharesPath: "./known_shares.json", 46 | * onCreateReplica: (shareAddress) => { 47 | * return new Earthstar.Replica({ 48 | * driver: new ReplicaDriverFs(shareAddress, "./share_data"), 49 | * }); 50 | * }, 51 | * }), 52 | * new ExtensionSyncWebsocket(), 53 | * ]); 54 | * 55 | * @module 56 | */ 57 | 58 | export * from "../core-validators/addresses.ts"; 59 | export * from "../core-validators/characters.ts"; 60 | export * from "../core-validators/checkers.ts"; 61 | 62 | export * from "../crypto/base32.ts"; 63 | export * from "../crypto/crypto-driver-noble.ts"; 64 | export * from "../crypto/crypto-types.ts"; 65 | export * from "../crypto/crypto.ts"; 66 | export * from "../crypto/global-crypto-driver.ts"; 67 | export * from "../crypto/keypair.ts"; 68 | 69 | export * from "../formats/format_es4.ts"; 70 | export * from "../formats/format_es5.ts"; 71 | export * from "../formats/util.ts"; 72 | export * from "../formats/format_types.ts"; 73 | 74 | export * from "../syncer/syncer.ts"; 75 | export * from "../syncer/partner_local.ts"; 76 | export * from "../syncer/partner_web_client.ts"; 77 | export * from "../syncer/syncer_types.ts"; 78 | 79 | export * from "../peer/peer-types.ts"; 80 | export * from "../peer/peer.ts"; 81 | 82 | export * from "../query/query-types.ts"; 83 | export * from "../query/query.ts"; 84 | export * from "../query/query-helpers.ts"; 85 | 86 | export * from "../replica/compare.ts"; 87 | export * from "../replica/replica.ts"; 88 | export * from "../replica/replica_cache.ts"; 89 | export * from "../replica/replica-types.ts"; 90 | export * from "../replica/multiformat_replica.ts"; 91 | export * from "../replica/util-types.ts"; 92 | export * from "../replica/doc_drivers/memory.ts"; 93 | export * from "../replica/attachment_drivers/memory.ts"; 94 | export * from "../replica/driver_memory.ts"; 95 | 96 | export * from "../util/bytes.ts"; 97 | export * from "../util/doc-types.ts"; 98 | export * from "../util/errors.ts"; 99 | export * from "../util/invite.ts"; 100 | export * from "../util/log.ts"; 101 | export * from "../util/misc.ts"; 102 | export * from "../util/shared_settings.ts"; 103 | -------------------------------------------------------------------------------- /src/core-validators/characters.ts: -------------------------------------------------------------------------------- 1 | import { stringToBytes } from "../util/bytes.ts"; 2 | 3 | /** Check that a string only contains character from a string of allowed characters. */ 4 | export function onlyHasChars(str: string, allowedChars: string): boolean { 5 | for (let s of str) { 6 | if (allowedChars.indexOf(s) === -1) return false; 7 | } 8 | return true; 9 | } 10 | 11 | /** Check that a string contains only printable ASCII */ 12 | export function isOnlyPrintableAscii(s: string): boolean { 13 | let bytes = stringToBytes(s); 14 | for (let byte of bytes) { 15 | // char must be between ' ' (space) and '~' inclusive 16 | if (byte < 32 || byte > 126) return false; 17 | } 18 | return true; 19 | } 20 | 21 | /* Check that a string is exactly one digit. */ 22 | export function isDigit(ch: string): boolean { 23 | if (ch === "") return false; 24 | return digits.indexOf(ch) !== -1; 25 | } 26 | 27 | /** Lowercase alphabetical characters. */ 28 | export const alphaLower = "abcdefghijklmnopqrstuvwxyz"; 29 | /** Uppercase alphabetical characters. */ 30 | export const alphaUpper = alphaLower.toUpperCase(); 31 | /** All digits. */ 32 | export const digits = "0123456789"; 33 | /** All characters allowed in base32. */ 34 | export const b32chars = alphaLower + "234567"; 35 | 36 | /** All characters allowed in an identity's short name. */ 37 | export const authorNameChars = alphaLower + digits; 38 | /** All characters allowed in an identity's pub key. */ 39 | export const authorKeyChars = b32chars; 40 | /** All characters allowed in an identity's public address. */ 41 | export const authorAddressChars = authorNameChars + b32chars + "@."; 42 | 43 | /** All characters allowed in a share's name. */ 44 | export const workspaceNameChars = alphaLower + digits; 45 | /** All characters allowed in a share's key. */ 46 | export const workspaceKeyChars = b32chars; 47 | /** All characters allowed in a share's address. */ 48 | export const workspaceAddressChars = workspaceNameChars + b32chars + "+."; 49 | 50 | // Characters allowed in Earthstar paths 51 | //--------------------------------------------- 52 | // These allowed path characters should be usable in regular web URLs 53 | // without percent-encoding them or interfering with the rest of the URL. 54 | // 55 | // Allowed Earthstar Meaning 56 | // a-zA-Z0-9 no meaning 57 | // '()-_$&,:= no meaning 58 | // / starts paths; path segment separator 59 | // ! ephemeral docs must contain at least one '!' 60 | // ~ denotes path ownership (write permissions) 61 | // +@. used by workspace and author names but allowed elsewhere too 62 | // % used for percent-encoding 63 | // 64 | // Disallowed Reason 65 | // space not allowed in URLs 66 | // <>"[\]^`{|} not allowed in URLs (though some browsers allow some of them) 67 | // ? to avoid confusion with URL query parameters 68 | // # to avoid confusion with URL anchors 69 | // ; no reason 70 | // * no reason; maybe useful for querying in the future 71 | // non-ASCII chars to avoid trouble with Unicode normalization 72 | // ASCII whitespace 73 | // ASCII control characters 74 | // 75 | // (Regular URL character rules are in RFC3986, RFC1738, and https://url.spec.whatwg.org/#url-code-points ) 76 | // 77 | // To use other characters in a path, percent-encode them using encodeURI. 78 | // For example 79 | // desiredPath = '/food/🍆/nutrition' 80 | // earthstarPath = encodeURI(desiredPath); // --> '/food/%F0%9F%8D%86/nutrition' 81 | // store it as earthstarPath 82 | // for display to users again, run decodeURI(earthstarPath) 83 | // 84 | 85 | /** All special characters permitted in a document's path. */ 86 | export const pathPunctuation = "/'()-._~!$&+,:=@%"; // note double quotes are not included 87 | /** All characters permitted in a document's path. */ 88 | export const pathChars = alphaLower + alphaUpper + digits + pathPunctuation; 89 | -------------------------------------------------------------------------------- /src/crypto/crypto-driver-chloride.ts: -------------------------------------------------------------------------------- 1 | import { default as chloride } from "../node/chloride.ts"; 2 | import { Buffer } from "https://deno.land/std@0.154.0/node/buffer.ts"; 3 | import crypto from "https://deno.land/std@0.154.0/node/crypto.ts"; 4 | import { ICryptoDriver, KeypairBytes } from "./crypto-types.ts"; 5 | import { 6 | concatBytes, 7 | identifyBufOrBytes, 8 | stringToBytes, 9 | } from "../util/bytes.ts"; 10 | import { 11 | bufferToBytes, 12 | bytesToBuffer, 13 | stringToBuffer, 14 | } from "../util/buffers.ts"; 15 | 16 | //-------------------------------------------------- 17 | 18 | import { Logger, LogLevel, setLogLevel } from "../util/log.ts"; 19 | import { UpdatableHash } from "./updatable_hash.ts"; 20 | let logger = new Logger("crypto-driver-chloride", "cyan"); 21 | 22 | setLogLevel("crypto-driver-chloride", LogLevel.Info); 23 | 24 | //================================================================================ 25 | 26 | /** 27 | * A version of the ICryptoDriver interface backed by Chloride. 28 | * Faster than noble. 29 | * Works in Node and the browser (with some polyfilling on your part). 30 | */ 31 | export const CryptoDriverChloride: ICryptoDriver = class { 32 | static sha256(input: string | Buffer): Promise { 33 | if (typeof input === "string") input = stringToBuffer(input); 34 | if (identifyBufOrBytes(input) === "bytes") input = bytesToBuffer(input); 35 | const resultBuf = chloride.crypto_hash_sha256(input); 36 | return Promise.resolve(bufferToBytes(resultBuf)); 37 | } 38 | static updatableSha256() { 39 | return new UpdatableHash({ 40 | hash: crypto.createHash("sha256"), 41 | update: (hash, data) => hash.update(data), 42 | digest: (hash) => { 43 | const digest = hash.digest(); 44 | 45 | if (typeof digest === "string") { 46 | return stringToBytes(digest); 47 | } 48 | 49 | return bufferToBytes(digest); 50 | }, 51 | }); 52 | } 53 | static generateKeypairBytes( 54 | seed?: Uint8Array, 55 | ): Promise { 56 | // If provided, the seed is used as the secret key. 57 | // If omitted, a random secret key is generated. 58 | logger.debug("generateKeypairBytes"); 59 | let seedBuf = seed === undefined ? undefined : bytesToBuffer(seed); 60 | if (!seedBuf) { 61 | seedBuf = Buffer.alloc(32); 62 | chloride.randombytes(seedBuf); 63 | } 64 | const keys = chloride.crypto_sign_seed_keypair(seedBuf); 65 | return Promise.resolve({ 66 | //curve: 'ed25519', 67 | pubkey: bufferToBytes(keys.publicKey), 68 | // so that this works with either sodium or libsodium-wrappers (in browser): 69 | secret: bufferToBytes((keys.secretKey).slice(0, 32)), 70 | }); 71 | } 72 | static sign( 73 | keypairBytes: KeypairBytes, 74 | msg: string | Buffer, 75 | ): Promise { 76 | logger.debug("sign"); 77 | const secretBuf = bytesToBuffer( 78 | concatBytes(keypairBytes.secret, keypairBytes.pubkey), 79 | ); 80 | if (typeof msg === "string") msg = stringToBuffer(msg); 81 | if (msg instanceof Uint8Array) msg = bytesToBuffer(msg); 82 | return Promise.resolve(bufferToBytes( 83 | // this returns a Buffer 84 | chloride.crypto_sign_detached(msg, secretBuf), 85 | )); 86 | } 87 | static verify( 88 | publicKey: Buffer, 89 | sig: Uint8Array, 90 | msg: string | Buffer, 91 | ): Promise { 92 | logger.debug("verify"); 93 | try { 94 | if (typeof msg === "string") msg = stringToBuffer(msg); 95 | if (msg instanceof Uint8Array) msg = bytesToBuffer(msg); 96 | return Promise.resolve(chloride.crypto_sign_verify_detached( 97 | bytesToBuffer(sig), 98 | msg, 99 | publicKey, 100 | ) as unknown as boolean); 101 | } catch { 102 | /* istanbul ignore next */ 103 | return Promise.resolve(false); 104 | } 105 | } 106 | }; 107 | -------------------------------------------------------------------------------- /debug/idb.ts: -------------------------------------------------------------------------------- 1 | import { IDBDatabase } from "https://deno.land/x/indexeddb@v1.1.0/lib/indexeddb.ts"; 2 | import { 3 | IDBKeyRange, 4 | indexedDB, 5 | } from "https://deno.land/x/indexeddb@v1.1.0/ponyfill_memory.ts"; 6 | import { deferred } from "../deps.ts"; 7 | 8 | // setup 9 | 10 | const req = indexedDB.open("test_db"); 11 | 12 | const dbPromise = deferred(); 13 | 14 | req.onupgradeneeded = () => { 15 | req.result.createObjectStore("test_store"); 16 | const store = req.result.createObjectStore("doc_store", { 17 | keyPath: "localIndex", 18 | }); 19 | store.createIndex("pathAndTimestamp", ["path", "timestamp"], { 20 | //multiEntry: true, 21 | }); 22 | 23 | store.createIndex("localIndex", "localIndex", { 24 | // multiEntry: true, 25 | }); 26 | }; 27 | 28 | req.onsuccess = () => { 29 | dbPromise.resolve(req.result); 30 | }; 31 | 32 | const db = await dbPromise; 33 | 34 | // set 35 | 36 | const bytes = new TextEncoder().encode("Hello there"); 37 | 38 | const putReq = db.transaction(["test_store"], "readwrite").objectStore( 39 | "test_store", 40 | ).put( 41 | bytes, 42 | "test_key", 43 | ); 44 | 45 | const putPromise = deferred(); 46 | 47 | putReq.onsuccess = () => { 48 | putPromise.resolve(); 49 | }; 50 | 51 | await putPromise; 52 | 53 | // get 54 | 55 | const getReq = db.transaction(["test_store"], "readwrite").objectStore( 56 | "test_store", 57 | ).get("test_key"); 58 | 59 | const getPromise = deferred(); 60 | 61 | getReq.onsuccess = () => { 62 | getPromise.resolve(getReq.result); 63 | }; 64 | 65 | const res = await getPromise; 66 | 67 | const blob = new Blob([res]); 68 | 69 | const url = URL.createObjectURL(blob); 70 | 71 | console.log({ blob, url }); 72 | 73 | const putReq2 = db.transaction(["doc_store"], "readwrite").objectStore( 74 | "doc_store", 75 | ).put({ 76 | path: "/hey", 77 | timestamp: 400, 78 | localIndex: 3, 79 | }); 80 | 81 | const putReq3 = db.transaction(["doc_store"], "readwrite").objectStore( 82 | "doc_store", 83 | ).put({ 84 | path: "/hey", 85 | timestamp: 300, 86 | localIndex: 1, 87 | }); 88 | 89 | const putReq4 = db.transaction(["doc_store"], "readwrite").objectStore( 90 | "doc_store", 91 | ).put({ 92 | path: "/bo", 93 | timestamp: 200, 94 | localIndex: 2, 95 | }); 96 | 97 | const putDeferred2 = deferred(); 98 | const putDeferred3 = deferred(); 99 | const putDeferred4 = deferred(); 100 | 101 | putReq2.onsuccess = () => putDeferred2.resolve(); 102 | putReq3.onsuccess = () => putDeferred3.resolve(); 103 | putReq4.onsuccess = () => putDeferred4.resolve(); 104 | 105 | await putDeferred2; 106 | await putDeferred3; 107 | await putDeferred4; 108 | 109 | const index = db.transaction(["doc_store"], "readwrite").objectStore( 110 | "doc_store", 111 | ).index("pathAndTimestamp"); 112 | 113 | const indexGetAll = index.getAll( 114 | IDBKeyRange.bound([" ", 0], ["/hey", Number.MAX_SAFE_INTEGER]), 115 | ); 116 | 117 | indexGetAll.onsuccess = () => { 118 | console.log("hmmm"); 119 | console.log(indexGetAll.result); 120 | console.log("..."); 121 | }; 122 | 123 | const indexGet = index.get( 124 | IDBKeyRange.bound(["/hey"], ["/hey", Number.MAX_SAFE_INTEGER]), 125 | ); 126 | 127 | indexGet.onsuccess = () => { 128 | console.log("yo"); 129 | console.log(indexGet.result); 130 | console.log("..."); 131 | }; 132 | 133 | const cursorGet0 = index.openCursor( 134 | IDBKeyRange.bound(["/hey"], ["/hey", Number.MAX_SAFE_INTEGER]), 135 | "prev", 136 | ); 137 | 138 | cursorGet0.onsuccess = () => { 139 | console.log("ehh"); 140 | console.log(cursorGet0.result?.value); 141 | console.log("..."); 142 | }; 143 | 144 | cursorGet0.onerror = () => { 145 | console.log("whaa"); 146 | }; 147 | 148 | const localIndex = db.transaction(["doc_store"], "readwrite").objectStore( 149 | "doc_store", 150 | ).index("localIndex"); 151 | 152 | const cursorGet = localIndex.openCursor(null, "prev"); 153 | 154 | cursorGet.onsuccess = () => { 155 | console.log(cursorGet.result?.value); 156 | }; 157 | -------------------------------------------------------------------------------- /src/server/extensions/sync_web.ts: -------------------------------------------------------------------------------- 1 | import { deferred } from "https://deno.land/std@0.138.0/async/deferred.ts"; 2 | import { FormatsArg } from "../../formats/format_types.ts"; 3 | import { Peer } from "../../peer/peer.ts"; 4 | import { PartnerWebClient } from "../../syncer/partner_web_client.ts"; 5 | import { Syncer } from "../../syncer/syncer.ts"; 6 | import { randomId } from "../../util/misc.ts"; 7 | import { IServerExtension } from "./extension.ts"; 8 | 9 | interface ExtensionSyncOpts { 10 | /** The path to accept HTTP sync requests from, e.g. `/earthstar-api/v2`. Make sure to set this if you're using other extensions which handle requests, as by default this will match any request to /. */ 11 | path?: string; 12 | formats?: FormatsArg; 13 | } 14 | 15 | /** An extension which enables synchronisation over the web via HTTP. */ 16 | export class ExtensionSyncWeb implements IServerExtension { 17 | private path = ""; 18 | private syncers = new Map>(); 19 | private peer = deferred(); 20 | private formats: FormatsArg | undefined; 21 | 22 | constructor(opts?: ExtensionSyncOpts) { 23 | if (opts?.path) { 24 | this.path = opts.path; 25 | } 26 | 27 | if (opts?.formats) { 28 | this.formats = opts.formats; 29 | } 30 | } 31 | 32 | register(peer: Peer) { 33 | this.peer.resolve(peer); 34 | 35 | return Promise.resolve(); 36 | } 37 | 38 | async handler(req: Request): Promise { 39 | const transferPattern = new URLPattern({ 40 | pathname: 41 | `${this.path}/:syncerId/:kind/:shareAddress/:formatName/:author/:path*`, 42 | }); 43 | 44 | const initiatePattern = new URLPattern({ 45 | pathname: `${this.path}/:mode`, 46 | }); 47 | 48 | const transferMatch = transferPattern.exec(req.url); 49 | 50 | if (transferMatch) { 51 | const { path, kind } = transferMatch.pathname.groups; 52 | 53 | const syncer = this.syncers.get( 54 | transferMatch.pathname.groups["syncerId"]!, 55 | ); 56 | 57 | if (!syncer) { 58 | return new Response("Not found", { 59 | status: 404, 60 | }); 61 | } 62 | 63 | const { socket, response } = Deno.upgradeWebSocket(req); 64 | 65 | // We don't await this, as we need to return the response. 66 | syncer.handleTransferRequest({ 67 | shareAddress: transferMatch.pathname.groups["shareAddress"]!, 68 | formatName: transferMatch.pathname.groups["formatName"]!, 69 | path: `/${path}`, 70 | author: transferMatch.pathname.groups["author"]!, 71 | kind: kind as "download" | "upload", 72 | source: socket, 73 | }); 74 | 75 | return response; 76 | } 77 | 78 | const initiateMatch = initiatePattern.exec(req.url); 79 | 80 | if (initiateMatch) { 81 | const { mode } = initiateMatch.pathname.groups; 82 | 83 | if (mode !== "once" && mode !== "continuous") { 84 | return Promise.resolve(null); 85 | } 86 | 87 | const { socket, response } = Deno.upgradeWebSocket(req, {}); 88 | 89 | const peer = await this.peer; 90 | 91 | const partner = new PartnerWebClient({ 92 | socket, 93 | appetite: mode === "once" ? "once" : "continuous", 94 | }); 95 | 96 | const description = `Client ${randomId()}`; 97 | 98 | const newSyncer = peer.addSyncPartner(partner, description, this.formats); 99 | 100 | console.log(`${description}: started sync`); 101 | 102 | newSyncer.isDone().then(() => { 103 | console.log(`${description}: completed sync`); 104 | }).catch((err) => { 105 | console.error(`Syncer ${newSyncer.id}: cancelled`, err); 106 | }).finally(() => { 107 | console.log(`${description}: removed`); 108 | this.syncers.delete(newSyncer.id); 109 | }); 110 | 111 | this.syncers.set(newSyncer.id, newSyncer); 112 | 113 | return response; 114 | } 115 | 116 | return Promise.resolve(null); 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /src/util/invite.ts: -------------------------------------------------------------------------------- 1 | import { parseShareAddress } from "../core-validators/addresses.ts"; 2 | import { Crypto } from "../crypto/crypto.ts"; 3 | import { ShareAddress } from "./doc-types.ts"; 4 | import { isErr, ValidationError } from "./errors.ts"; 5 | 6 | /** Creates an invitation URL. Validates the share address, that servers are valid URLs, and the secret against the share address if given. */ 7 | export async function createInvitationURL( 8 | shareAddress: ShareAddress, 9 | servers: string[], 10 | secret?: string, 11 | ): Promise { 12 | const parsedShareAddress = parseShareAddress(shareAddress); 13 | 14 | if (isErr(parsedShareAddress)) { 15 | return new ValidationError(`Invalid share address.`); 16 | } 17 | 18 | for (const server of servers) { 19 | try { 20 | new URL(server); 21 | } catch { 22 | return new ValidationError(`Could not parse ${server} as a URL.`); 23 | } 24 | } 25 | 26 | const serverParams = servers.map((url) => `&server=${url}`).join("&"); 27 | 28 | let secretParam = ""; 29 | 30 | if (secret) { 31 | const isValid = await Crypto.checkKeypairIsValid({ 32 | shareAddress, 33 | secret, 34 | }); 35 | 36 | if (isErr(isValid)) { 37 | return new ValidationError( 38 | `Supplied the wrong secret for ${shareAddress}`, 39 | ); 40 | } 41 | 42 | secretParam = `&secret=${secret}`; 43 | } 44 | 45 | const invitationURL = 46 | `earthstar://${shareAddress}/?invite${serverParams}${secretParam}&v=2`; 47 | 48 | return invitationURL; 49 | } 50 | 51 | type ParsedInvitation = { 52 | shareAddress: ShareAddress; 53 | secret?: string; 54 | servers: string[]; 55 | }; 56 | 57 | /** Parses an invitation URL. Validates the share address, secret (if given), and any server URLs. */ 58 | export async function parseInvitationURL( 59 | invitationURL: string, 60 | ): Promise { 61 | try { 62 | const url = new URL(invitationURL); 63 | 64 | // Firefox and Chrome do not parse the share component as a path name and put it in the pathname instead. Bummer. 65 | const shareAddress = url.hostname.length > 0 66 | ? url.hostname 67 | : url.pathname.replaceAll("/", ""); 68 | 69 | const isValidShareAddress = parseShareAddress(shareAddress); 70 | 71 | if (isErr(isValidShareAddress)) { 72 | return new ValidationError( 73 | "Invitation did not include a valid share address.", 74 | ); 75 | } 76 | 77 | const params = new URLSearchParams(url.search); 78 | 79 | const isInvitation = params.get("invite"); 80 | const version = params.get("v"); 81 | 82 | if (isInvitation === null) { 83 | return new ValidationError("Not an invitation URL"); 84 | } 85 | 86 | if (version === null) { 87 | return new ValidationError("Invitation version not specified."); 88 | } 89 | 90 | if (version !== "2") { 91 | return new ValidationError( 92 | `Invitation version is ${version}, expected version 2.`, 93 | ); 94 | } 95 | 96 | const servers = params.getAll("server"); 97 | 98 | for (const server of servers) { 99 | try { 100 | new URL(server); 101 | } catch { 102 | return new ValidationError( 103 | `Invitation's servers included a malformed URL: ${server}`, 104 | ); 105 | } 106 | } 107 | 108 | const secret = params.get("secret"); 109 | 110 | if (secret) { 111 | const isValid = await Crypto.checkKeypairIsValid({ 112 | shareAddress: shareAddress, 113 | secret: secret, 114 | }); 115 | 116 | if (isErr(isValid)) { 117 | return new ValidationError( 118 | `Invitation contains the wrong secret for share ${shareAddress}.`, 119 | ); 120 | } 121 | } 122 | 123 | return { 124 | shareAddress: shareAddress, 125 | secret: secret || undefined, 126 | servers: servers, 127 | }; 128 | } catch { 129 | return new ValidationError("Could not parse the invitation URL."); 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /src/test/benchmark/replica.bench.ts: -------------------------------------------------------------------------------- 1 | import { Crypto } from "../../crypto/crypto.ts"; 2 | import { setGlobalCryptoDriver } from "../../crypto/global-crypto-driver.ts"; 3 | import { Replica } from "../../replica/replica.ts"; 4 | import { randomId } from "../../util/misc.ts"; 5 | import { writeRandomDocs } from "../test-utils.ts"; 6 | import { cryptoScenarios, docDriverScenarios } from "../scenarios/scenarios.ts"; 7 | import { MultiplyScenarioOutput, ScenarioItem } from "../scenarios/types.ts"; 8 | import { multiplyScenarios } from "../scenarios/utils.ts"; 9 | import { AttachmentDriverMemory } from "../../replica/attachment_drivers/memory.ts"; 10 | import { AuthorKeypair } from "../../crypto/crypto-types.ts"; 11 | import { notErr } from "../../util/errors.ts"; 12 | import { assert } from "../asserts.ts"; 13 | 14 | const scenarios: MultiplyScenarioOutput<{ 15 | "docDriver": ScenarioItem; 16 | "crypto": ScenarioItem; 17 | }> = multiplyScenarios({ 18 | description: "docDriver", 19 | scenarios: docDriverScenarios, 20 | }, { 21 | description: "crypto", 22 | scenarios: cryptoScenarios, 23 | }); 24 | 25 | for (const scenario of scenarios) { 26 | const replicaDriver = scenario.subscenarios.docDriver; 27 | const crypto = scenario.subscenarios.crypto; 28 | 29 | const shareKeypair = await Crypto.generateShareKeypair("test"); 30 | 31 | assert(notErr(shareKeypair)); 32 | 33 | const driverToClose = replicaDriver.makeDriver( 34 | shareKeypair.shareAddress, 35 | scenario.name, 36 | ); 37 | 38 | const keypair = await Crypto.generateAuthorKeypair("test") as AuthorKeypair; 39 | const keypairB = await Crypto.generateAuthorKeypair("nest") as AuthorKeypair; 40 | 41 | const replicaToClose = new Replica({ 42 | driver: { 43 | docDriver: driverToClose, 44 | attachmentDriver: new AttachmentDriverMemory(), 45 | }, 46 | }); 47 | 48 | await replicaToClose.close(true); 49 | const driver = replicaDriver.makeDriver( 50 | shareKeypair.shareAddress, 51 | scenario.name, 52 | ); 53 | const replica = new Replica({ 54 | driver: { 55 | docDriver: driver, 56 | attachmentDriver: new AttachmentDriverMemory(), 57 | }, 58 | }); 59 | 60 | await writeRandomDocs(keypair, replica, 100); 61 | 62 | await replica.set(keypair, { 63 | text: "hello", 64 | path: `/stable`, 65 | }); 66 | 67 | await replica.set(keypairB, { 68 | text: "howdy", 69 | path: `/stable`, 70 | }); 71 | 72 | Deno.bench(`Replica.set (${scenario.name})`, { group: "set" }, async () => { 73 | setGlobalCryptoDriver(crypto); 74 | await replica.set(keypair, { 75 | text: "hi", 76 | path: `/test/${randomId()}`, 77 | }); 78 | }); 79 | 80 | Deno.bench( 81 | `Replica.queryDocs (${scenario.name})`, 82 | { group: "queryDocs" }, 83 | async () => { 84 | setGlobalCryptoDriver(crypto); 85 | await replica.queryDocs({}); 86 | }, 87 | ); 88 | 89 | Deno.bench( 90 | `Replica.queryDocs (path ASC) (${scenario.name})`, 91 | { group: "queryDocs.pathAsc" }, 92 | async () => { 93 | setGlobalCryptoDriver(crypto); 94 | await replica.queryDocs({ 95 | orderBy: "path ASC", 96 | }); 97 | }, 98 | ); 99 | 100 | Deno.bench( 101 | `Replica.queryDocs (localIndex ASC) (${scenario.name})`, 102 | { group: "queryDocs.localIndexAsc" }, 103 | async () => { 104 | setGlobalCryptoDriver(crypto); 105 | await replica.queryDocs({ 106 | orderBy: "localIndex ASC", 107 | }); 108 | }, 109 | ); 110 | 111 | Deno.bench( 112 | `Replica.getLatestDocAtPath (${scenario.name})`, 113 | { group: "getLatestDocAtPath" }, 114 | async () => { 115 | setGlobalCryptoDriver(crypto); 116 | await replica.getLatestDocAtPath("/stable.txt"); 117 | }, 118 | ); 119 | 120 | Deno.bench( 121 | `Replica.getAllDocsAtPath (${scenario.name})`, 122 | { group: "getAllDocsAtPath" }, 123 | async () => { 124 | setGlobalCryptoDriver(crypto); 125 | await replica.getAllDocsAtPath("/stable.txt"); 126 | }, 127 | ); 128 | } 129 | -------------------------------------------------------------------------------- /src/test/replica/query_source.test.ts: -------------------------------------------------------------------------------- 1 | import { AuthorKeypair, ShareKeypair } from "../../crypto/crypto-types.ts"; 2 | import { Crypto } from "../../crypto/crypto.ts"; 3 | import { DocEs5 } from "../../formats/format_es5.ts"; 4 | import { AttachmentDriverMemory } from "../../replica/attachment_drivers/memory.ts"; 5 | import { DocDriverMemory } from "../../replica/doc_drivers/memory.ts"; 6 | import { QuerySourceEvent } from "../../replica/replica-types.ts"; 7 | import { Replica } from "../../replica/replica.ts"; 8 | import { CallbackSink } from "../../streams/stream_utils.ts"; 9 | import { sleep } from "../../util/misc.ts"; 10 | import { readStream } from "../../util/streams.ts"; 11 | import { assertEquals } from "../asserts.ts"; 12 | 13 | Deno.test("QuerySource", async () => { 14 | const shareKeypair = await Crypto.generateShareKeypair( 15 | "test", 16 | ) as ShareKeypair; 17 | 18 | const SHARE_ADDR = shareKeypair.shareAddress; 19 | 20 | const keypairA = await Crypto.generateAuthorKeypair("test") as AuthorKeypair; 21 | const keypairB = await Crypto.generateAuthorKeypair( 22 | "suzy", 23 | ) as AuthorKeypair; 24 | 25 | const replica = new Replica( 26 | { 27 | driver: { 28 | docDriver: new DocDriverMemory(SHARE_ADDR), 29 | attachmentDriver: new AttachmentDriverMemory(), 30 | }, 31 | shareSecret: shareKeypair.secret, 32 | }, 33 | ); 34 | 35 | await replica.set(keypairA, { 36 | text: "a", 37 | path: "/wanted/1", 38 | }); 39 | 40 | await replica.set(keypairA, { 41 | text: "b", 42 | path: "/wanted/2", 43 | }); 44 | 45 | await replica.set(keypairB, { 46 | text: "c", 47 | path: "/wanted/1", 48 | }); 49 | 50 | await replica.set(keypairA, { 51 | text: "🐸", 52 | path: "/unwanted/1", 53 | }); 54 | 55 | const existingStream = replica.getQueryStream( 56 | { 57 | historyMode: "all", 58 | orderBy: "localIndex ASC", 59 | filter: { 60 | pathStartsWith: "/wanted", 61 | }, 62 | }, 63 | "existing", 64 | ); 65 | 66 | const results = await readStream(existingStream); 67 | const existingWantedContent = results.map((event) => { 68 | if (event.kind === "processed_all_existing") { 69 | return "STOP"; 70 | } 71 | 72 | return event.doc.text; 73 | }); 74 | 75 | assertEquals( 76 | existingWantedContent, 77 | ["a", "b", "c", "STOP"], 78 | "QueryStream returned existing content which matched filter", 79 | ); 80 | 81 | const everythingStream = replica.getQueryStream( 82 | { 83 | historyMode: "all", 84 | orderBy: "localIndex ASC", 85 | filter: { 86 | pathStartsWith: "/wanted", 87 | }, 88 | }, 89 | "everything", 90 | ); 91 | 92 | const onlyNewStream = replica.getQueryStream( 93 | { 94 | historyMode: "all", 95 | orderBy: "localIndex ASC", 96 | filter: { 97 | pathStartsWith: "/wanted", 98 | }, 99 | }, 100 | "new", 101 | ); 102 | 103 | await replica.set(keypairA, { 104 | text: "d", 105 | path: "/wanted/3", 106 | }); 107 | 108 | const everythingWantedContent: string[] = []; 109 | const newWantedContent: string[] = []; 110 | 111 | const everythingCallbackSink = new CallbackSink>(); 112 | 113 | everythingCallbackSink.onWrite((event) => { 114 | if (event.kind === "processed_all_existing") { 115 | return; 116 | } 117 | everythingWantedContent.push(event.doc.text); 118 | }); 119 | 120 | const newCallbackSink = new CallbackSink>(); 121 | 122 | newCallbackSink.onWrite((event) => { 123 | if (event.kind === "processed_all_existing") { 124 | return; 125 | } 126 | newWantedContent.push(event.doc.text); 127 | }); 128 | 129 | everythingStream.pipeTo(new WritableStream(everythingCallbackSink)); 130 | onlyNewStream.pipeTo(new WritableStream(newCallbackSink)); 131 | 132 | await sleep(10); 133 | 134 | assertEquals( 135 | everythingWantedContent, 136 | ["a", "b", "c", "d"], 137 | ); 138 | 139 | assertEquals( 140 | newWantedContent, 141 | ["d"], 142 | ); 143 | 144 | await replica.close(true); 145 | }); 146 | -------------------------------------------------------------------------------- /src/test/server/server.test.ts: -------------------------------------------------------------------------------- 1 | import { deferred } from "../../../deps.ts"; 2 | import { Crypto } from "../../crypto/crypto.ts"; 3 | import { AuthorKeypair, ShareKeypair } from "../../crypto/crypto-types.ts"; 4 | import { Peer } from "../../peer/peer.ts"; 5 | import { Replica } from "../../replica/replica.ts"; 6 | import { IServerExtension } from "../../server/extensions/extension.ts"; 7 | import { WebServerScenario } from "../scenarios/scenarios.ts"; 8 | import { 9 | makeOverlappingReplicaTuple, 10 | replicaAttachmentsAreSynced, 11 | replicaDocsAreSynced, 12 | } from "../test-utils.ts"; 13 | import { assert } from "../asserts.ts"; 14 | 15 | class ExtensionTest implements IServerExtension { 16 | private peer = deferred(); 17 | private replicas: Replica[]; 18 | 19 | constructor(replicas: Replica[]) { 20 | this.replicas = replicas; 21 | } 22 | 23 | register(peer: Peer): Promise { 24 | for (const replica of this.replicas) { 25 | peer.addReplica(replica); 26 | } 27 | 28 | this.peer.resolve(peer); 29 | 30 | return Promise.resolve(); 31 | } 32 | 33 | handler(): Promise { 34 | return Promise.resolve(null); 35 | } 36 | 37 | async getPeer() { 38 | const peer = await this.peer; 39 | 40 | return peer; 41 | } 42 | } 43 | 44 | Deno.test("Server", async (test) => { 45 | const authorKeypair = await Crypto.generateAuthorKeypair( 46 | "test", 47 | ) as AuthorKeypair; 48 | 49 | // Create three shares 50 | const shareKeypairA = await Crypto.generateShareKeypair( 51 | "apples", 52 | ) as ShareKeypair; 53 | const shareKeypairB = await Crypto.generateShareKeypair( 54 | "bananas", 55 | ) as ShareKeypair; 56 | const shareKeypairC = await Crypto.generateShareKeypair( 57 | "coconuts", 58 | ) as ShareKeypair; 59 | 60 | const [replicaClientA, serverA] = await makeOverlappingReplicaTuple( 61 | authorKeypair, 62 | shareKeypairA, 63 | 50, 64 | 2, 65 | 100, 66 | ); 67 | 68 | const [replicaClientB, serverB] = await makeOverlappingReplicaTuple( 69 | authorKeypair, 70 | shareKeypairB, 71 | 50, 72 | 2, 73 | 100, 74 | ); 75 | 76 | const [replicaClientC, serverC] = await makeOverlappingReplicaTuple( 77 | authorKeypair, 78 | shareKeypairC, 79 | 50, 80 | 2, 81 | 100, 82 | ); 83 | 84 | const peer = new Peer(); 85 | 86 | peer.addReplica(replicaClientA); 87 | peer.addReplica(replicaClientB); 88 | peer.addReplica(replicaClientC); 89 | 90 | const testExtension = new ExtensionTest([ 91 | serverA, 92 | serverB, 93 | serverC, 94 | ]); 95 | 96 | const serverScenario = new WebServerScenario(8087); 97 | 98 | await serverScenario.start(testExtension); 99 | 100 | await test.step({ 101 | name: "Syncs", 102 | fn: async () => { 103 | const syncer = peer.sync("http://localhost:8087"); 104 | 105 | await syncer.isDone(); 106 | 107 | assert( 108 | await replicaDocsAreSynced([replicaClientA, serverA]), 109 | `+a docs are in sync`, 110 | ); 111 | assert( 112 | await replicaDocsAreSynced([replicaClientB, serverB]), 113 | `+b docs are in sync`, 114 | ); 115 | assert( 116 | await replicaDocsAreSynced([replicaClientC, serverC]), 117 | `+c docs are in sync`, 118 | ); 119 | 120 | assert( 121 | await replicaAttachmentsAreSynced([replicaClientA, serverA]), 122 | `+a attachments are in sync`, 123 | ); 124 | 125 | assert( 126 | await replicaAttachmentsAreSynced([replicaClientB, serverB]), 127 | `+b attachments are in sync`, 128 | ); 129 | assert( 130 | await replicaAttachmentsAreSynced([replicaClientC, serverC]), 131 | `+c attachments are in sync`, 132 | ); 133 | }, 134 | sanitizeOps: false, 135 | sanitizeResources: false, 136 | }); 137 | 138 | await serverScenario.close(); 139 | 140 | await replicaClientA.close(true); 141 | await replicaClientB.close(true); 142 | await replicaClientC.close(true); 143 | 144 | await serverA.close(true); 145 | await serverB.close(true); 146 | await serverC.close(true); 147 | }); 148 | -------------------------------------------------------------------------------- /src/replica/attachment_drivers/memory.ts: -------------------------------------------------------------------------------- 1 | import { Crypto } from "../../crypto/crypto.ts"; 2 | import { DocAttachment } from "../../util/doc-types.ts"; 3 | import { ReplicaIsClosedError, ValidationError } from "../../util/errors.ts"; 4 | import { streamToBytes } from "../../util/streams.ts"; 5 | import { IReplicaAttachmentDriver } from "../replica-types.ts"; 6 | 7 | /** An attachment driver which persists attachments in memory. 8 | * Works everywhere. 9 | */ 10 | export class AttachmentDriverMemory implements IReplicaAttachmentDriver { 11 | private stagingMap = new Map(); 12 | private attachmentMap = new Map(); 13 | private closed = false; 14 | 15 | private getKey(formatName: string, attachmentHash: string) { 16 | return `${formatName}___${attachmentHash}`; 17 | } 18 | 19 | getAttachment( 20 | formatName: string, 21 | attachmentHash: string, 22 | ): Promise { 23 | if (this.closed) throw new ReplicaIsClosedError(); 24 | const key = this.getKey(formatName, attachmentHash); 25 | const attachment = this.attachmentMap.get(key); 26 | 27 | if (!attachment) { 28 | return Promise.resolve(undefined); 29 | } 30 | 31 | return Promise.resolve({ 32 | bytes: async () => new Uint8Array(await attachment.arrayBuffer()), 33 | stream: () => 34 | Promise.resolve( 35 | // Need to do this for Node's sake. 36 | attachment.stream() as unknown as ReadableStream, 37 | ), 38 | }); 39 | } 40 | 41 | async stage( 42 | formatName: string, 43 | attachment: ReadableStream | Uint8Array, 44 | ) { 45 | if (this.closed) throw new ReplicaIsClosedError(); 46 | const bytes = attachment instanceof Uint8Array 47 | ? attachment 48 | : await streamToBytes(attachment); 49 | 50 | const hash = await Crypto.sha256base32(bytes); 51 | 52 | const newAttachment = new Blob([bytes]); 53 | 54 | const key = this.getKey(formatName, hash); 55 | 56 | this.stagingMap.set(key, newAttachment); 57 | 58 | return Promise.resolve({ 59 | hash, 60 | size: bytes.byteLength, 61 | commit: () => { 62 | this.attachmentMap.set(key, newAttachment); 63 | this.stagingMap.delete(key); 64 | 65 | return Promise.resolve(); 66 | }, 67 | reject: () => { 68 | this.stagingMap.delete(key); 69 | 70 | return Promise.resolve(); 71 | }, 72 | }); 73 | } 74 | 75 | erase(formatName: string, attachmentHash: string) { 76 | if (this.closed) throw new ReplicaIsClosedError(); 77 | const key = this.getKey(formatName, attachmentHash); 78 | if (this.attachmentMap.has(key)) { 79 | this.attachmentMap.delete(key); 80 | return Promise.resolve(true as true); 81 | } 82 | 83 | return Promise.resolve( 84 | new ValidationError("No attachment with that signature found."), 85 | ); 86 | } 87 | 88 | wipe() { 89 | if (this.closed) throw new ReplicaIsClosedError(); 90 | this.attachmentMap.clear(); 91 | return Promise.resolve(); 92 | } 93 | 94 | async filter( 95 | hashes: Record>, 96 | ): Promise<{ format: string; hash: string }[]> { 97 | if (this.closed) throw new ReplicaIsClosedError(); 98 | const erasedAttachments = []; 99 | 100 | for (const key of this.attachmentMap.keys()) { 101 | const [format, hash] = key.split("___"); 102 | 103 | const hashesToKeep = hashes[format]; 104 | 105 | if (hashesToKeep && !hashesToKeep.has(hash)) { 106 | const result = await this.erase(format, hash); 107 | 108 | if (result) { 109 | erasedAttachments.push({ format, hash }); 110 | } 111 | } 112 | } 113 | 114 | return erasedAttachments; 115 | } 116 | 117 | clearStaging() { 118 | if (this.closed) throw new ReplicaIsClosedError(); 119 | this.stagingMap.clear(); 120 | return Promise.resolve(); 121 | } 122 | 123 | isClosed(): boolean { 124 | return this.closed; 125 | } 126 | 127 | async close(erase: boolean) { 128 | if (this.closed) throw new ReplicaIsClosedError(); 129 | 130 | if (erase) { 131 | await this.wipe(); 132 | } 133 | 134 | this.closed = true; 135 | 136 | return; 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /src/replica/compare.ts: -------------------------------------------------------------------------------- 1 | import { shallowEqualArrays, shallowEqualObjects } from "../../deps.ts"; 2 | import { Cmp } from "./util-types.ts"; 3 | 4 | //================================================================================ 5 | 6 | export type SortOrder = "ASC" | "DESC"; 7 | 8 | export function sortedInPlace(array: T[]): T[] { 9 | array.sort(); 10 | return array; 11 | } 12 | 13 | // myStrings.sort(baseCompare) 14 | export function compareBasic(a: T, b: T, order: SortOrder = "ASC"): Cmp { 15 | if (Array.isArray(a) && shallowEqualArrays(a, b)) { 16 | return Cmp.EQ; 17 | } 18 | 19 | if (typeof a === "object" && shallowEqualObjects(a, b)) { 20 | return Cmp.EQ; 21 | } 22 | 23 | if (a === b) return Cmp.EQ; 24 | if (order === "ASC" || order === undefined) { 25 | return (a < b) ? Cmp.LT : Cmp.GT; 26 | } else if (order === "DESC") { 27 | return (a > b) ? Cmp.LT : Cmp.GT; 28 | } else { 29 | throw new Error( 30 | "unexpected sort order to compareBasic: " + JSON.stringify(order), 31 | ); 32 | } 33 | } 34 | 35 | /** 36 | * example usage: myArrayOfArrays.sort(arrayCompare) 37 | * 38 | * Compare arrays element by element, stopping and returning the first non-EQ comparison. 39 | * Earlier array items are more important. 40 | * When arrays are different lengths and share the same prefix, the shorter one 41 | * is less than the longer one. In other words, the undefined you would get by 42 | * reading of the end of the array counts as lower than any other value. 43 | * 44 | * For example, this list of arrays is sorted: 45 | * - [1], 46 | * - [1, 1], 47 | * - [1, 1, 99], 48 | * - [1, 2], 49 | * - [1, 2], 50 | * - [2], 51 | * - [2, 99], 52 | * - [2, 99, 1], 53 | * 54 | * sortOrders is an array of 'ASC' | 'DESC' strings. Imagine it's applied 55 | * to the columns of a spreadsheet. 56 | * 57 | * For example, to sort DESC by the first item, and ASC by the second item: 58 | * compareArrays(['hello', 123], ['goodbye', 456], ['DESC', 'ASC']). 59 | * 60 | * Sort order defaults to 'ASC' when the sortOrders array is not provided. 61 | * If the sortOrders array is shorter than the arrays to be sorted, it acts 62 | * as if it was filled out with additional 'ASC' entries as needed. 63 | * A sort order of 'DESC' in the appropriate column can make longer arrays 64 | * come before shorter arrays. 65 | * 66 | * sortOrders ['ASC', 'DESC'] sorts in this order: 67 | * - [1, 99], 68 | * - [1, 2], 69 | * - [1], // shorter array comes last, because of DESC in this column 70 | * - [2], // but first element is still sorted ASC 71 | */ 72 | export function compareArrays( 73 | a: T[], 74 | b: T[], 75 | sortOrders?: SortOrder[], 76 | ): Cmp { 77 | let minLen = Math.min(a.length, b.length); 78 | for (let ii = 0; ii < minLen; ii++) { 79 | let sortOrder = sortOrders?.[ii] ?? "ASC"; // default to ASC if sortOrders is undefined or too short 80 | let elemCmp = compareBasic(a[ii], b[ii], sortOrder); 81 | if (elemCmp !== Cmp.EQ) return elemCmp; 82 | } 83 | // arrays are the same length, and all elements are the same 84 | if (a.length === b.length) return Cmp.EQ; 85 | 86 | // arrays are not the same length. 87 | // use the sort order for one past the end of the shorter array, 88 | // and apply it to the lengths of the array (so that DESC makes the 89 | // shorter one come first). 90 | let ii = Math.min(a.length, b.length); 91 | let sortOrder = sortOrders?.[ii] ?? "ASC"; // default to ASC if sortOrders is undefined or too short 92 | return compareBasic(a.length, b.length, sortOrder); 93 | } 94 | 95 | // myArray.sort(compareByObjKey('signature', 'ASC')); 96 | export function compareByObjKey(key: string, sortOrder: SortOrder = "ASC") { 97 | return (a: Record, b: Record): Cmp => 98 | compareBasic(a[key], b[key], sortOrder); 99 | } 100 | 101 | // myArray.sort(compareByFn((x) => x.signature + x.path)); 102 | export function compareByFn>(fn: (x: any) => T) { 103 | return (a: R, b: R): Cmp => compareBasic(fn(a), fn(b)); 104 | } 105 | 106 | // myArray.sort(compareByObjArrayFn((x) => [x.signature, x.path])); 107 | export function compareByObjArrayFn(fn: (x: any) => any[]) { 108 | return (a: Record, b: Record): Cmp => 109 | compareArrays(fn(a), fn(b)); 110 | } 111 | -------------------------------------------------------------------------------- /src/syncer/partner_web_client.ts: -------------------------------------------------------------------------------- 1 | import { AsyncQueue, deferred } from "../../deps.ts"; 2 | import { 3 | websocketReadable, 4 | websocketWritable, 5 | } from "../streams/stream_utils.ts"; 6 | import { EarthstarError, NotSupportedError } from "../util/errors.ts"; 7 | import { 8 | GetTransferOpts, 9 | ISyncPartner, 10 | SyncAppetite, 11 | SyncerEvent, 12 | } from "./syncer_types.ts"; 13 | 14 | type SyncerDriverWebServerOpts = { 15 | /** A websocket created from the initial sync request. */ 16 | socket: WebSocket; 17 | appetite: SyncAppetite; 18 | }; 19 | 20 | /** A syncing partner created from an inbound HTTP connection (i.e. a web client). 21 | * 22 | * Works everywhere, but is really meant for servers running on Deno and Node. 23 | */ 24 | export class PartnerWebClient< 25 | IncomingTransferSourceType extends WebSocket, 26 | > implements ISyncPartner { 27 | concurrentTransfers = 24; 28 | payloadThreshold = 8; 29 | rangeDivision = 8; 30 | syncAppetite: SyncAppetite; 31 | 32 | private socket: WebSocket; 33 | private incomingQueue = new AsyncQueue(); 34 | private socketIsReady = deferred(); 35 | 36 | constructor({ socket, appetite }: SyncerDriverWebServerOpts) { 37 | this.syncAppetite = appetite; 38 | 39 | if (socket.readyState === socket.OPEN) { 40 | this.socketIsReady.resolve(); 41 | } 42 | 43 | socket.onopen = () => { 44 | this.socketIsReady.resolve(); 45 | }; 46 | 47 | this.socket = socket; 48 | 49 | this.socket.binaryType = "arraybuffer"; 50 | 51 | this.socket.onmessage = (event) => { 52 | // Casting as string for Node's incorrect WebSocket types. 53 | this.incomingQueue.push(JSON.parse(event.data as string)); 54 | }; 55 | 56 | this.socket.onclose = () => { 57 | this.incomingQueue.close(); 58 | }; 59 | 60 | this.socket.onerror = (event) => { 61 | if ("error" in event) { 62 | this.incomingQueue.close({ 63 | withError: event.error, 64 | }); 65 | 66 | return; 67 | } 68 | 69 | this.incomingQueue.close({ 70 | withError: new EarthstarError("Websocket error."), 71 | }); 72 | }; 73 | } 74 | 75 | async sendEvent(event: SyncerEvent): Promise { 76 | await this.socketIsReady; 77 | 78 | if (this.socket.readyState !== this.socket.OPEN) { 79 | return; 80 | } 81 | 82 | return this.socket.send(JSON.stringify(event)); 83 | } 84 | 85 | getEvents(): AsyncIterable { 86 | return this.incomingQueue; 87 | } 88 | 89 | closeConnection(): Promise { 90 | this.socket.close(); 91 | 92 | return Promise.resolve(); 93 | } 94 | 95 | getDownload( 96 | _opts: GetTransferOpts, 97 | ): Promise | NotSupportedError> { 98 | // Server can't initiate a request with a client. 99 | return Promise.resolve( 100 | new NotSupportedError( 101 | "SyncDriverWebServer does not support download requests.", 102 | ), 103 | ); 104 | } 105 | 106 | handleUploadRequest( 107 | _opts: GetTransferOpts, 108 | ): Promise | NotSupportedError> { 109 | // Server won't get in-band BLOB_REQ messages 110 | return Promise.resolve( 111 | new NotSupportedError( 112 | "SyncDriverWebServer does not support upload requests.", 113 | ), 114 | ); 115 | } 116 | 117 | handleTransferRequest( 118 | socket: IncomingTransferSourceType, 119 | kind: "upload" | "download", 120 | ): Promise< 121 | | ReadableStream 122 | | WritableStream 123 | | undefined 124 | > { 125 | // Return a stream which writes to the socket. nice. 126 | // They want to download data from us 127 | if (kind === "download") { 128 | const writable = websocketWritable( 129 | socket, 130 | (outgoing: Uint8Array) => outgoing, 131 | ); 132 | 133 | return Promise.resolve(writable); 134 | } else { 135 | // they want to upload data to us. 136 | const readable = websocketReadable(socket, (event) => { 137 | if (event.data instanceof ArrayBuffer) { 138 | const bytes = new Uint8Array(event.data); 139 | return bytes; 140 | } 141 | 142 | return null as never; 143 | }); 144 | 145 | return Promise.resolve(readable); 146 | } 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /src/server/extensions/sync_web.node.ts: -------------------------------------------------------------------------------- 1 | import { deferred } from "https://deno.land/std@0.138.0/async/deferred.ts"; 2 | import { FormatsArg } from "../../formats/format_types.ts"; 3 | import { Peer } from "../../peer/peer.ts"; 4 | import { PartnerWebClient } from "../../syncer/partner_web_client.ts"; 5 | import { Syncer } from "../../syncer/syncer.ts"; 6 | import { randomId } from "../../util/misc.ts"; 7 | import { IServerExtension } from "./extension.ts"; 8 | import { createServer } from "https://deno.land/std@0.167.0/node/http.ts"; 9 | import { WebSocketServer } from "https://esm.sh/ws@8.8.1"; 10 | 11 | interface ExtensionSyncOpts { 12 | /** The path to accept HTTP sync requests from, e.g. `/earthstar-api/v2`. Make sure to set this if you're using other extensions which handle requests, as by default this will match any request to /. */ 13 | path?: string; 14 | formats?: FormatsArg; 15 | server: ReturnType; 16 | } 17 | 18 | /** An extension which enables synchronisation over the web via HTTP. */ 19 | export class ExtensionSyncWeb implements IServerExtension { 20 | private path = ""; 21 | private syncers = new Map>(); 22 | private peer = deferred(); 23 | private formats: FormatsArg | undefined; 24 | private wss: WebSocketServer; 25 | private server: ReturnType; 26 | 27 | constructor(opts: ExtensionSyncOpts) { 28 | if (opts?.path) { 29 | this.path = opts.path; 30 | } 31 | 32 | if (opts?.formats) { 33 | this.formats = opts.formats; 34 | } 35 | 36 | this.server = opts.server; 37 | this.wss = new WebSocketServer({ noServer: true }); 38 | } 39 | 40 | register(peer: Peer) { 41 | this.peer.resolve(peer); 42 | 43 | const transferPattern = new URLPattern({ 44 | pathname: 45 | `${this.path}/:syncerId/:kind/:shareAddress/:formatName/:author/:path*`, 46 | }); 47 | 48 | const initiatePattern = new URLPattern({ 49 | pathname: `${this.path}/:mode`, 50 | }); 51 | 52 | this.server.on("upgrade", (req, socket, head) => { 53 | const reqUrl = `http://0.0.0.0${req.url}`; 54 | 55 | if (transferPattern.test(reqUrl) || initiatePattern.test(reqUrl)) { 56 | this.wss.handleUpgrade(req, socket, head, (ws) => { 57 | // @ts-ignore 58 | this.wss.emit("connection", ws, req); 59 | }); 60 | } 61 | }); 62 | 63 | this.wss.on("connection", async (socket: WebSocket, req: any) => { 64 | const reqUrl = `http://0.0.0.0${req.url}`; 65 | 66 | const transferMatch = transferPattern.exec(reqUrl); 67 | 68 | if (transferMatch) { 69 | const { syncerId, shareAddress, formatName, path, author, kind } = 70 | transferMatch.pathname.groups; 71 | 72 | const syncer = this.syncers.get(syncerId); 73 | 74 | if (!syncer) { 75 | return; 76 | } 77 | 78 | await syncer.handleTransferRequest({ 79 | shareAddress, 80 | formatName, 81 | path: `/${path}`, 82 | author, 83 | kind: kind as "download" | "upload", 84 | source: socket, 85 | }); 86 | 87 | return; 88 | } 89 | 90 | const initiateMatch = initiatePattern.exec(reqUrl); 91 | 92 | if (initiateMatch) { 93 | const { mode } = initiateMatch.pathname.groups; 94 | 95 | if (mode !== "once" && mode !== "continuous") { 96 | return; 97 | } 98 | 99 | const peer = await this.peer; 100 | 101 | const partner = new PartnerWebClient({ 102 | socket, 103 | appetite: mode === "once" ? "once" : "continuous", 104 | }); 105 | 106 | const description = `Client ${randomId()}`; 107 | 108 | const newSyncer = peer.addSyncPartner( 109 | partner, 110 | description, 111 | this.formats, 112 | ); 113 | 114 | console.log(`${description}: started sync`); 115 | 116 | newSyncer.isDone().then(() => { 117 | console.log(`${description}: completed sync`); 118 | }).catch((err) => { 119 | console.error(`Syncer ${newSyncer.id}: cancelled`, err); 120 | }).finally(() => { 121 | console.log(`${description}: removed`); 122 | this.syncers.delete(newSyncer.id); 123 | }); 124 | 125 | this.syncers.set(newSyncer.id, newSyncer); 126 | } 127 | }); 128 | 129 | return Promise.resolve(); 130 | } 131 | 132 | async handler(req: Request): Promise { 133 | return Promise.resolve(null); 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /debug/syncers.ts: -------------------------------------------------------------------------------- 1 | import { 2 | AuthorKeypair, 3 | Crypto, 4 | CryptoDriverSodium, 5 | Peer, 6 | Replica, 7 | setGlobalCryptoDriver, 8 | ShareKeypair, 9 | } from "../mod.ts"; 10 | import { AttachmentDriverMemory } from "../src/replica/attachment_drivers/memory.ts"; 11 | import { DocDriverMemory } from "../src/replica/doc_drivers/memory.ts"; 12 | import { 13 | docAttachmentsAreEquivalent, 14 | docsAreEquivalent, 15 | writeRandomDocs, 16 | } from "../src/test/test-utils.ts"; 17 | 18 | const keypair = await Crypto.generateAuthorKeypair("test") as AuthorKeypair; 19 | 20 | setGlobalCryptoDriver(CryptoDriverSodium); 21 | 22 | // create three replicas x 2. 23 | const shareKeypairA = await Crypto.generateShareKeypair( 24 | "apples", 25 | ) as ShareKeypair; 26 | const shareKeypairB = await Crypto.generateShareKeypair( 27 | "bananas", 28 | ) as ShareKeypair; 29 | const shareKeypairC = await Crypto.generateShareKeypair( 30 | "coconuts", 31 | ) as ShareKeypair; 32 | 33 | const ADDRESS_A = shareKeypairA.shareAddress; 34 | const ADDRESS_B = shareKeypairB.shareAddress; 35 | const ADDRESS_C = shareKeypairC.shareAddress; 36 | 37 | const makeReplicaTrio = (addr: string, shareSecret: string) => { 38 | return [ 39 | new Replica({ 40 | driver: { 41 | docDriver: new DocDriverMemory(addr), 42 | attachmentDriver: new AttachmentDriverMemory(), 43 | }, 44 | shareSecret, 45 | }), 46 | new Replica({ 47 | driver: { 48 | docDriver: new DocDriverMemory(addr), 49 | attachmentDriver: new AttachmentDriverMemory(), 50 | }, 51 | shareSecret, 52 | }), 53 | new Replica({ 54 | driver: { 55 | docDriver: new DocDriverMemory(addr), 56 | attachmentDriver: new AttachmentDriverMemory(), 57 | }, 58 | shareSecret, 59 | }), 60 | ] as [Replica, Replica, Replica]; 61 | }; 62 | 63 | const [a1, a2, a3] = makeReplicaTrio(ADDRESS_A, shareKeypairA.secret); 64 | const [b1, b2, b3] = makeReplicaTrio(ADDRESS_B, shareKeypairB.secret); 65 | const [c1, c2, c3] = makeReplicaTrio(ADDRESS_C, shareKeypairC.secret); 66 | 67 | const docCount = 1000; 68 | 69 | console.log("writing docs"); 70 | 71 | await Promise.all([ 72 | writeRandomDocs(keypair, a1, docCount), 73 | writeRandomDocs(keypair, a2, docCount), 74 | writeRandomDocs(keypair, a3, docCount), 75 | 76 | writeRandomDocs(keypair, b1, docCount), 77 | writeRandomDocs(keypair, b2, docCount), 78 | writeRandomDocs(keypair, b3, docCount), 79 | 80 | writeRandomDocs(keypair, c1, docCount), 81 | writeRandomDocs(keypair, c2, docCount), 82 | writeRandomDocs(keypair, c3, docCount), 83 | ]); 84 | 85 | console.log("wrote docs"); 86 | 87 | const peer1 = new Peer(); 88 | const peer2 = new Peer(); 89 | const peer3 = new Peer(); 90 | 91 | peer1.addReplica(a1); 92 | peer1.addReplica(b1); 93 | peer1.addReplica(c1); 94 | 95 | peer2.addReplica(a2); 96 | peer2.addReplica(b2); 97 | peer2.addReplica(c2); 98 | 99 | peer3.addReplica(a3); 100 | peer3.addReplica(b3); 101 | peer3.addReplica(c3); 102 | 103 | const syncer = peer1.sync(peer2, false); 104 | await syncer.isDone(); 105 | 106 | console.log("Sync 1 <> 2 done"); 107 | 108 | const syncer2 = peer2.sync(peer3, false); 109 | await syncer2.isDone(); 110 | console.log("Sync 2 <> 3 done"); 111 | 112 | const syncer3 = peer3.sync(peer1, false); 113 | await syncer3.isDone(); 114 | console.log("Sync 3 <> 1 done"); 115 | 116 | const trios = [[a1, a2, a3], [b1, b2, b3], [c1, c2, c3]]; 117 | 118 | for (const [x, y, z] of trios) { 119 | // get all docs. 120 | const fstDocs = await x.getAllDocs(); 121 | const sndDocs = await y.getAllDocs(); 122 | const thdDocs = await z.getAllDocs(); 123 | 124 | const fstWithAttachments = await x.addAttachments(fstDocs); 125 | const sndWithAttachments = await y.addAttachments(sndDocs); 126 | const thdWithAttachments = await z.addAttachments(sndDocs); 127 | 128 | console.log(fstDocs.length, sndDocs.length, thdDocs.length); 129 | 130 | console.group(x.share); 131 | 132 | const docsSynced = docsAreEquivalent(fstDocs, sndDocs); 133 | const docsSynced2 = docsAreEquivalent(sndDocs, thdDocs); 134 | 135 | if (docsSynced && docsSynced2) { 136 | console.log(`Docs synced!`); 137 | } else { 138 | console.log(`%c Docs did not sync...`, "color: red"); 139 | } 140 | 141 | const res = await docAttachmentsAreEquivalent( 142 | fstWithAttachments, 143 | sndWithAttachments, 144 | ); 145 | 146 | const res2 = await docAttachmentsAreEquivalent( 147 | sndWithAttachments, 148 | thdWithAttachments, 149 | ); 150 | 151 | if (res && res2) { 152 | console.log(`Attachments synced!`); 153 | } else { 154 | console.log(`%c Attachments did not sync...`, "color: red"); 155 | } 156 | console.groupEnd(); 157 | } 158 | 159 | Deno.exit(0); 160 | -------------------------------------------------------------------------------- /src/util/log.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Logs are assigned a priority number. 3 | * Higher numbers are less important information. 4 | * Set your desired log level higher to get more info. 5 | * 6 | * -1 nothing 7 | * 0 only errors 8 | * 1 also warnings 9 | * 2 also logs 10 | * 3 also debugs 11 | * 12 | * Logs also come from different "sources" which can 13 | * have different log level settings. 14 | * 15 | * Two ways to modify the log level settings: 16 | * 17 | * 1. Set an environment variable in the shell. 18 | * This applies to all "sources" (can't be set individually) 19 | * EARTHSTAR_LOG_LEVEL=2 npm run test 20 | * 21 | * 2. Use setLogLevels() to globally modify the levels: 22 | * setLogLevels({ sync: 2 }); 23 | * 24 | * The environment variable wins over the numbers set by setLogLevels. 25 | */ 26 | 27 | //================================================================================ 28 | // TYPES 29 | 30 | type LogSource = string; 31 | 32 | export enum LogLevel { 33 | None = -1, 34 | Error = 0, // default 35 | Warn = 1, 36 | Log = 2, 37 | Info = 3, 38 | Debug = 4, // most verbose 39 | } 40 | export const DEFAULT_LOG_LEVEL = LogLevel.Error; 41 | 42 | type LogLevels = Record; 43 | 44 | //================================================================================ 45 | // ENV VAR 46 | 47 | /* 48 | // get the single log level number from the environment, or undefined if not set 49 | const readEnvLogLevel = (): LogLevel | undefined => { 50 | if (process?.env?.EARTHSTAR_LOG_LEVEL) { 51 | const parsed = parseInt(process.env.EARTHSTAR_LOG_LEVEL); 52 | if (isNaN(parsed)) { return undefined } 53 | if (parsed !== Math.floor(parsed)) { return undefined; } 54 | return parsed; 55 | } 56 | return undefined; 57 | } 58 | 59 | // apply env var setting 60 | const ENV_LOG_LEVEL = readEnvLogLevel(); 61 | */ 62 | 63 | //================================================================================ 64 | // GLOBAL SETTINGS 65 | 66 | // make global singleton to hold log levels 67 | let globalLogLevels: LogLevels = { 68 | // result is the min of (_env) and anything else 69 | _default: DEFAULT_LOG_LEVEL, 70 | }; 71 | 72 | export function updateLogLevels(newLogLevels: LogLevels): void { 73 | globalLogLevels = { 74 | ...globalLogLevels, 75 | ...newLogLevels, 76 | }; 77 | } 78 | 79 | export function setLogLevel(source: LogSource, level: LogLevel) { 80 | globalLogLevels[source] = level; 81 | } 82 | 83 | export function setDefaultLogLevel(level: LogLevel) { 84 | globalLogLevels._default = level; 85 | } 86 | 87 | export function getLogLevel(source: LogSource): LogLevel { 88 | if (source in globalLogLevels) { 89 | return globalLogLevels[source]; 90 | } else { 91 | return globalLogLevels._default; 92 | } 93 | } 94 | 95 | export function getLogLevels(): LogLevels { 96 | return globalLogLevels; 97 | } 98 | 99 | //================================================================================ 100 | // Logger class 101 | 102 | type LogColor = 103 | | "blue" 104 | | "aqua" 105 | | "darkcyan" 106 | | "cyan" 107 | | "dimgray" 108 | | "slategray" 109 | | "green" 110 | | "springgreen" 111 | | "grey" 112 | | "darkMagenta" 113 | | "magenta" 114 | | "red" 115 | | "orangeRed" 116 | | "salmon" 117 | | "lightsalmon" 118 | | "gold" 119 | | "yellow"; 120 | 121 | export class Logger { 122 | source: LogSource; 123 | color: LogColor | undefined = undefined; 124 | 125 | constructor(source: LogSource, color?: LogColor) { 126 | this.source = source; 127 | this.color = color || "aqua"; 128 | } 129 | 130 | _print(level: LogLevel, showTag: boolean, indent: string, ...args: any[]) { 131 | if (level <= getLogLevel(this.source)) { 132 | if (showTag) { 133 | const tag = `[${this.source}]`; 134 | 135 | if (this.color !== undefined) { 136 | const tagArgs = [`%c ${tag}`, `color: ${this.color}`]; 137 | console.log(indent, ...tagArgs, ...args); 138 | } else { 139 | console.log(indent, tag, ...args); 140 | } 141 | } else { 142 | console.log(indent, ...args); 143 | } 144 | } 145 | } 146 | 147 | error(...args: any[]) { 148 | this._print(LogLevel.Error, true, "!!", ...args); 149 | } 150 | warn(...args: any[]) { 151 | this._print(LogLevel.Warn, true, "! ", ...args); 152 | } 153 | log(...args: any[]) { 154 | this._print(LogLevel.Log, true, " ", ...args); 155 | } 156 | info(...args: any[]) { 157 | this._print(LogLevel.Info, true, " ", ...args); 158 | } 159 | debug(...args: any[]) { 160 | this._print(LogLevel.Debug, true, " ", ...args); 161 | } 162 | 163 | blank() { 164 | this._print(LogLevel.Info, false, ""); 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /src/syncer/partner_web_server.ts: -------------------------------------------------------------------------------- 1 | import { AsyncQueue, deferred } from "../../deps.ts"; 2 | import { 3 | websocketReadable, 4 | websocketWritable, 5 | } from "../streams/stream_utils.ts"; 6 | import { EarthstarError, NotSupportedError } from "../util/errors.ts"; 7 | import { 8 | GetTransferOpts, 9 | ISyncPartner, 10 | SyncAppetite, 11 | SyncerEvent, 12 | } from "./syncer_types.ts"; 13 | 14 | type SyncerDriverWebClientOpts = { 15 | /** The URL of the server to sync with. */ 16 | url: string; 17 | appetite: SyncAppetite; 18 | }; 19 | 20 | /** A syncing partner to be used with servers reachable via the internet. 21 | * Works everywhere. 22 | */ 23 | export class PartnerWebServer< 24 | IncomingTransferSourceType extends undefined, 25 | > implements ISyncPartner { 26 | syncAppetite: SyncAppetite; 27 | concurrentTransfers = 24; 28 | payloadThreshold = 8; 29 | rangeDivision = 8; 30 | 31 | private isSecure: boolean; 32 | private wsUrl: string; 33 | 34 | private socket: WebSocket; 35 | private incomingQueue = new AsyncQueue(); 36 | private socketIsReady = deferred(); 37 | 38 | constructor(opts: SyncerDriverWebClientOpts) { 39 | this.syncAppetite = opts.appetite; 40 | 41 | // Check if it's a URL of some kind. 42 | const url = new URL(opts.url); 43 | 44 | // Check if it's a web syncer 45 | const hostAndPath = `${url.host}${ 46 | url.pathname === "/" ? "" : url.pathname 47 | }`; 48 | 49 | this.isSecure = url.protocol === "https:" || 50 | url.protocol === "wss:"; 51 | 52 | this.wsUrl = `${this.isSecure ? "wss://" : "ws://"}${hostAndPath}/'`; 53 | 54 | const urlWithMode = new URL(opts.appetite, this.wsUrl); 55 | 56 | this.socket = new WebSocket( 57 | urlWithMode.toString(), 58 | ); 59 | 60 | this.socket.onopen = () => { 61 | this.socketIsReady.resolve(); 62 | }; 63 | 64 | this.socket.binaryType = "arraybuffer"; 65 | 66 | this.socket.onmessage = (event) => { 67 | // Casting to string due to Node's incorrect websocket types 68 | this.incomingQueue.push(JSON.parse(event.data as string)); 69 | }; 70 | 71 | this.socket.onclose = () => { 72 | this.incomingQueue.close(); 73 | }; 74 | 75 | this.socket.onerror = (event) => { 76 | if ("error" in event) { 77 | this.incomingQueue.close({ 78 | withError: event.error, 79 | }); 80 | 81 | return; 82 | } 83 | 84 | this.incomingQueue.close({ 85 | withError: new EarthstarError("Websocket error."), 86 | }); 87 | }; 88 | } 89 | 90 | async sendEvent(event: SyncerEvent): Promise { 91 | await this.socketIsReady; 92 | 93 | if ( 94 | this.socket.readyState === this.socket.CLOSED || 95 | this.socket.readyState === this.socket.CLOSING 96 | ) { 97 | return; 98 | } 99 | 100 | return this.socket.send(JSON.stringify(event)); 101 | } 102 | 103 | getEvents(): AsyncIterable { 104 | return this.incomingQueue; 105 | } 106 | 107 | closeConnection(): Promise { 108 | this.socket.close(); 109 | 110 | return Promise.resolve(); 111 | } 112 | 113 | getDownload( 114 | opts: GetTransferOpts, 115 | ): Promise | undefined> { 116 | // create a new url with the share, path, and syncer ID embedded 117 | 118 | const url = new URL( 119 | `${opts.syncerId}/download/${opts.shareAddress}/${opts.doc.format}/${opts.doc.author}${opts.doc.path}`, 120 | this.wsUrl, 121 | ); 122 | 123 | const readable = websocketReadable(url.toString(), (event) => { 124 | if (event.data instanceof ArrayBuffer) { 125 | const bytes = new Uint8Array(event.data); 126 | return bytes; 127 | } 128 | 129 | return null as never; 130 | }); 131 | 132 | return Promise.resolve(readable); 133 | } 134 | 135 | handleUploadRequest( 136 | opts: GetTransferOpts, 137 | ): Promise | NotSupportedError> { 138 | const url = new URL( 139 | `${opts.syncerId}/upload/${opts.shareAddress}/${opts.doc.format}/${opts.doc.author}${opts.doc.path}`, 140 | this.wsUrl, 141 | ); 142 | 143 | const writable = websocketWritable( 144 | url.toString(), 145 | (outgoing: Uint8Array) => outgoing, 146 | ); 147 | 148 | return Promise.resolve(writable); 149 | } 150 | 151 | handleTransferRequest( 152 | _source: IncomingTransferSourceType, 153 | _kind: "upload" | "download", 154 | ): Promise< 155 | | ReadableStream 156 | | WritableStream 157 | | undefined 158 | | NotSupportedError 159 | > { 160 | // We don't expect any external requests. 161 | return Promise.resolve( 162 | new NotSupportedError( 163 | "SyncDriverWebClient does not support external transfer requests.", 164 | ), 165 | ); 166 | } 167 | } 168 | -------------------------------------------------------------------------------- /src/syncer/transfer_queue.ts: -------------------------------------------------------------------------------- 1 | import { BlockingBus } from "../streams/stream_utils.ts"; 2 | import { AttachmentTransfer } from "./attachment_transfer.ts"; 3 | import { PromiseEnroller } from "./promise_enroller.ts"; 4 | import { AttachmentTransferReport } from "./syncer_types.ts"; 5 | 6 | export class TransferQueue { 7 | private waiting: AttachmentTransfer[] = []; 8 | private active = new Set>(); 9 | private failed = new Set>(); 10 | private completed = new Set>(); 11 | 12 | private activeLimit: number; 13 | 14 | private transfersRequestedByUsEnroller = new PromiseEnroller(true); 15 | 16 | private closed = false; 17 | 18 | // This status is going to be modified a LOT so it's better to mutate than recreate from scratch. 19 | private reports: Record> = 20 | {}; 21 | 22 | private reportBus = new BlockingBus< 23 | Record 24 | >(); 25 | 26 | constructor(activeLimit: number) { 27 | this.activeLimit = activeLimit; 28 | } 29 | 30 | private async activate(transfer: AttachmentTransfer) { 31 | this.active.add(transfer); 32 | 33 | transfer.isDone().then(() => { 34 | this.completed.add(transfer); 35 | }).catch(() => { 36 | this.failed.add(transfer); 37 | }).finally(() => { 38 | this.active.delete(transfer); 39 | this.admitNext(); 40 | }); 41 | 42 | await transfer.start(); 43 | } 44 | 45 | private queue(transfer: AttachmentTransfer) { 46 | this.waiting.push(transfer); 47 | } 48 | 49 | private admitNext() { 50 | if (this.waiting.length === 0) { 51 | return; 52 | } 53 | 54 | if (this.active.size >= this.activeLimit) { 55 | return; 56 | } 57 | 58 | const first = this.waiting.shift(); 59 | 60 | if (first) { 61 | this.activate(first); 62 | } 63 | } 64 | 65 | async addTransfer(transfer: AttachmentTransfer) { 66 | if (this.closed) { 67 | transfer.abort(); 68 | return; 69 | } 70 | 71 | transfer.onProgress(() => { 72 | this.updateTransferStatus(transfer); 73 | }); 74 | 75 | if (transfer.requester === "us") { 76 | this.transfersRequestedByUsEnroller.enrol(transfer.isDone()); 77 | } 78 | 79 | if (this.active.size < this.activeLimit) { 80 | await this.activate(transfer); 81 | } else { 82 | this.queue(transfer); 83 | } 84 | } 85 | 86 | gotAllTransfersRequestedByUs() { 87 | this.transfersRequestedByUsEnroller.seal(); 88 | } 89 | 90 | cancel() { 91 | this.closed = true; 92 | 93 | this.transfersRequestedByUsEnroller.seal(); 94 | 95 | for (const transfer of this.active) { 96 | transfer.abort(); 97 | } 98 | } 99 | 100 | private updateTransferStatus(transfer: AttachmentTransfer) { 101 | const shareReports = this.reports[transfer.share]; 102 | 103 | if (!shareReports) { 104 | this.reports[transfer.share] = {}; 105 | } 106 | 107 | this.reports[transfer.share][transfer.hash + transfer.kind] = { 108 | author: transfer.doc.author, 109 | path: transfer.doc.path, 110 | format: transfer.doc.format, 111 | hash: transfer.hash, 112 | kind: transfer.kind, 113 | status: transfer.status, 114 | bytesLoaded: transfer.loaded, 115 | totalBytes: transfer.expectedSize, 116 | }; 117 | 118 | this.reportBus.send(this.getReport()); 119 | } 120 | 121 | getReport(): Record { 122 | const report: Record = {}; 123 | 124 | for (const shareKey in this.reports) { 125 | const transferReports = []; 126 | 127 | const shareReport = this.reports[shareKey]; 128 | 129 | for (const key in shareReport) { 130 | const transferReport = shareReport[key]; 131 | transferReports.push(transferReport); 132 | } 133 | 134 | report[shareKey] = transferReports; 135 | } 136 | 137 | return report; 138 | } 139 | 140 | hasQueuedTransfer(hash: string, kind: "upload" | "download") { 141 | for (const waiting of this.waiting) { 142 | if (waiting.hash === hash && waiting.kind === kind) { 143 | return true; 144 | } 145 | } 146 | 147 | for (const active of this.active) { 148 | if (active.hash === hash && active.kind === kind) { 149 | return true; 150 | } 151 | } 152 | 153 | for (const complete of this.completed) { 154 | if (complete.hash === hash && complete.kind === kind) { 155 | return true; 156 | } 157 | } 158 | 159 | return false; 160 | } 161 | 162 | onReportUpdate( 163 | cb: (report: Record) => void, 164 | ) { 165 | return this.reportBus.on(cb); 166 | } 167 | 168 | transfersRequestedByUsFinished() { 169 | return this.transfersRequestedByUsEnroller.isDone(); 170 | } 171 | } 172 | -------------------------------------------------------------------------------- /src/test/peer/peer.test.ts: -------------------------------------------------------------------------------- 1 | import { assert, assertEquals } from "../asserts.ts"; 2 | 3 | import { ShareAddress } from "../../util/doc-types.ts"; 4 | 5 | import { 6 | GlobalCryptoDriver, 7 | setGlobalCryptoDriver, 8 | } from "../../crypto/global-crypto-driver.ts"; 9 | import { compareByFn, sortedInPlace } from "../../replica/compare.ts"; 10 | import { Replica } from "../../replica/replica.ts"; 11 | import { Peer } from "../../peer/peer.ts"; 12 | 13 | //================================================================================ 14 | 15 | import { Logger } from "../../util/log.ts"; 16 | import { MultiplyScenarioOutput, ScenarioItem } from "../scenarios/types.ts"; 17 | import { cryptoScenarios, docDriverScenarios } from "../scenarios/scenarios.ts"; 18 | import { multiplyScenarios } from "../scenarios/utils.ts"; 19 | import { AttachmentDriverMemory } from "../../replica/attachment_drivers/memory.ts"; 20 | import { Crypto } from "../../crypto/crypto.ts"; 21 | import { ShareKeypair } from "../../crypto/crypto-types.ts"; 22 | 23 | const loggerTest = new Logger("test", "lightsalmon"); 24 | const loggerTestCb = new Logger("test cb", "salmon"); 25 | const J = JSON.stringify; 26 | 27 | //setDefaultLogLevel(LogLevel.None); 28 | //setLogLevel('peer', LogLevel.Debug); 29 | 30 | //================================================================================ 31 | 32 | const scenarios: MultiplyScenarioOutput<{ 33 | "replicaDriver": ScenarioItem; 34 | "cryptoDriver": ScenarioItem; 35 | }> = multiplyScenarios({ 36 | description: "replicaDriver", 37 | scenarios: docDriverScenarios, 38 | }, { 39 | description: "cryptoDriver", 40 | scenarios: cryptoScenarios, 41 | }); 42 | 43 | function runPeerTests( 44 | scenario: typeof scenarios[number], 45 | ) { 46 | const SUBTEST_NAME = scenario.name; 47 | 48 | setGlobalCryptoDriver(scenario.subscenarios.cryptoDriver); 49 | 50 | function makeStorage(share: ShareAddress, shareSecret: string): Replica { 51 | const storage = new Replica({ 52 | driver: { 53 | docDriver: scenario.subscenarios.replicaDriver.makeDriver(share), 54 | attachmentDriver: new AttachmentDriverMemory(), 55 | }, 56 | shareSecret, 57 | }); 58 | return storage; 59 | } 60 | 61 | Deno.test(SUBTEST_NAME + ": peer basics", async () => { 62 | const initialCryptoDriver = GlobalCryptoDriver; 63 | 64 | const shares = [ 65 | await Crypto.generateShareKeypair("one"), 66 | await Crypto.generateShareKeypair("two"), 67 | await Crypto.generateShareKeypair("three"), 68 | ] as ShareKeypair[]; 69 | const storages = shares.map((ws) => makeStorage(ws.shareAddress, "")); 70 | 71 | const sortedShares = sortedInPlace([ 72 | ...shares.map((kepair) => kepair.shareAddress), 73 | ]); 74 | const sortedStorages = [...storages]; 75 | sortedStorages.sort(compareByFn((storage) => storage.share)); 76 | 77 | const peer = new Peer(); 78 | 79 | assertEquals( 80 | peer.hasShare(shares[1].shareAddress), 81 | false, 82 | "does not yet have +two", 83 | ); 84 | assertEquals(peer.shares(), [], "has no shares"); 85 | assertEquals(peer.replicas(), [], "has no replicas"); 86 | assertEquals(peer.size(), 0, "size is zero"); 87 | 88 | for (const storage of storages) { 89 | await peer.addReplica(storage); 90 | } 91 | 92 | assertEquals( 93 | peer.hasShare("nope"), 94 | false, 95 | "does not have invalid share address", 96 | ); 97 | assertEquals( 98 | peer.hasShare("+nope.ws"), 99 | false, 100 | "does not have +nope.ws share", 101 | ); 102 | assertEquals( 103 | peer.hasShare(shares[1].shareAddress), 104 | true, 105 | "now it does have +two", 106 | ); 107 | 108 | assertEquals( 109 | peer.shares(), 110 | sortedShares, 111 | "has all 3 shares, sorted", 112 | ); 113 | assertEquals( 114 | peer.replicas(), 115 | sortedStorages, 116 | "has all 3 storages sorted by share", 117 | ); 118 | assertEquals(peer.size(), 3, "size is 3"); 119 | 120 | await peer.removeReplicaByShare(shares[0].shareAddress); 121 | assertEquals( 122 | peer.shares(), 123 | [shares[2].shareAddress, shares[1].shareAddress], 124 | "removed by share address", 125 | ); 126 | assertEquals(peer.size(), 2, "size is 2"); 127 | 128 | await peer.removeReplica(storages[1]); // that's two.ws 129 | assertEquals( 130 | peer.shares(), 131 | [shares[2].shareAddress], 132 | "removed storage instance", 133 | ); 134 | assertEquals(peer.size(), 1, "size is 1"); 135 | 136 | assertEquals( 137 | initialCryptoDriver, 138 | GlobalCryptoDriver, 139 | `GlobalCryptoDriver has not changed unexpectedly. started as ${ 140 | (initialCryptoDriver as any).name 141 | }, ended as ${(GlobalCryptoDriver as any).name}`, 142 | ); 143 | 144 | for (const storage of storages) { 145 | await storage.close(true); 146 | } 147 | 148 | // TODO: eventually test peer events when we have them 149 | }); 150 | } 151 | 152 | for (const scenario of scenarios) { 153 | runPeerTests(scenario); 154 | } 155 | -------------------------------------------------------------------------------- /src/test/misc/base32.test.ts: -------------------------------------------------------------------------------- 1 | import { assert, assertEquals, assertThrows } from "../asserts.ts"; 2 | 3 | let TEST_NAME = "base32"; 4 | 5 | import { 6 | base32BytesToString, 7 | base32StringToBytes, 8 | } from "../../crypto/base32.ts"; 9 | import { ValidationError } from "../../util/errors.ts"; 10 | 11 | //================================================================================ 12 | 13 | Deno.test("base32 encoding", () => { 14 | let bytes = Uint8Array.from([1, 2, 3, 4, 5]); 15 | let str = base32BytesToString(bytes); 16 | let bytes2 = base32StringToBytes(str); 17 | let str2 = base32BytesToString(bytes2); 18 | 19 | assert(bytes2 instanceof Uint8Array, "decoding creates a Uint8Array"); 20 | 21 | assertEquals(bytes, bytes2, "bytes roundtrip to base32"); 22 | assertEquals(str, str2, "base32 roundtrip to bytes"); 23 | assert(str.startsWith("b"), "base32 startswith b"); 24 | 25 | assertEquals( 26 | base32BytesToString(Uint8Array.from([])), 27 | "b", 28 | 'base32 can encode Uint8Array([]) to "b"', 29 | ); 30 | assertEquals( 31 | base32BytesToString(Uint8Array.from([0])), 32 | "baa", 33 | 'base32 can encode Uint8Array([0]) to "baa"', 34 | ); 35 | 36 | assertEquals( 37 | base32StringToBytes("b"), 38 | Uint8Array.from([]), 39 | 'base32 can decode just the string "b" to an empty Uint8Array', 40 | ); 41 | assertEquals( 42 | base32StringToBytes("baa"), 43 | Uint8Array.from([0]), 44 | 'base32 can decode the string "baa" to Uint8Array([0])', 45 | ); 46 | 47 | assertThrows( 48 | () => base32StringToBytes(""), 49 | ValidationError, 50 | undefined, 51 | "decoding base32 throws an exception if string is empty", 52 | ); 53 | assertThrows( 54 | () => base32StringToBytes("abc"), 55 | ValidationError, 56 | undefined, 57 | 'decoding base32 throws an exception when it does not start with "b"', 58 | ); 59 | assertThrows( 60 | () => base32StringToBytes("b123"), 61 | SyntaxError, 62 | undefined, 63 | "decoding base32 throws when encountering invalid base32 character", 64 | ); 65 | assertThrows( 66 | () => base32StringToBytes("babc?xyz"), 67 | SyntaxError, 68 | undefined, 69 | "decoding base32 throws when encountering invalid base32 character", 70 | ); 71 | assertThrows( 72 | () => base32StringToBytes("babc xyz"), 73 | SyntaxError, 74 | undefined, 75 | "decoding base32 throws when encountering invalid base32 character", 76 | ); 77 | assertThrows( 78 | () => base32StringToBytes("b abcxyz"), 79 | SyntaxError, 80 | undefined, 81 | "decoding base32 throws when encountering invalid base32 character", 82 | ); 83 | assertThrows( 84 | () => base32StringToBytes("babcxyz "), 85 | SyntaxError, 86 | undefined, 87 | "decoding base32 throws when encountering invalid base32 character", 88 | ); 89 | assertThrows( 90 | () => base32StringToBytes("babcxyz\n"), 91 | SyntaxError, 92 | undefined, 93 | "decoding base32 throws when encountering invalid base32 character", 94 | ); 95 | assertThrows( 96 | () => base32StringToBytes("BABC"), 97 | ValidationError, 98 | undefined, 99 | "decoding base32 throws when encountering a different multibase encoding", 100 | ); 101 | assertThrows( 102 | () => base32StringToBytes("b???"), 103 | SyntaxError, 104 | undefined, 105 | 'decoding base32 throws on "b???"', 106 | ); 107 | assertThrows( 108 | () => base32StringToBytes("b11"), 109 | SyntaxError, 110 | undefined, 111 | 'decoding base32 throws on "b11"', 112 | ); 113 | 114 | // make sure we have a multibase version that fixed this bug: 115 | // https://github.com/multiformats/js-multibase/issues/17 116 | let exampleBytes = base32StringToBytes( 117 | "bciqbed3k6ya5i3qqwljochwxdrk5exzqilbckapedujenz5b5hj5r3a", 118 | ); 119 | let exampleString = 120 | "bciqbed3k6ya5i3qqwljochwxdrk5exzqilbckapedujenz5b5hj5r3a"; 121 | assertEquals( 122 | base32BytesToString(exampleBytes), 123 | exampleString, 124 | "edge case works", 125 | ); 126 | 127 | let bytes_11 = Uint8Array.from([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]); 128 | let str_11_correct = "baebagbafaydqqcikbm"; 129 | let str_11_loose1 = "baebagbafaydqqc1kbm"; // i to 1 130 | let str_11_loose2 = "baebagbafaydqqcikbM"; // uppercase M at end 131 | assertEquals( 132 | base32BytesToString(bytes_11), 133 | str_11_correct, 134 | "encodes bytes to correct string", 135 | ); 136 | assertEquals( 137 | base32StringToBytes(str_11_correct), 138 | bytes_11, 139 | "decodes string to correct bytes", 140 | ); 141 | assertThrows( 142 | () => base32StringToBytes(str_11_loose1), 143 | SyntaxError, 144 | undefined, 145 | "throws on loose string (i vs 1)", 146 | ); 147 | assertThrows( 148 | () => base32StringToBytes(str_11_loose2), 149 | SyntaxError, 150 | undefined, 151 | "throws on loose string (case change)", 152 | ); 153 | 154 | let padded_b32 = "baa======"; 155 | let unpadded_b32 = "baa"; 156 | let matching_buf = Uint8Array.from([0]); 157 | 158 | assertEquals( 159 | base32StringToBytes(unpadded_b32), 160 | matching_buf, 161 | "unpadded base32 is handled ok", 162 | ); 163 | assertThrows( 164 | () => base32StringToBytes(padded_b32), 165 | ValidationError, 166 | undefined, 167 | "padded base32 is not allowed", 168 | ); 169 | }); 170 | -------------------------------------------------------------------------------- /scripts/build_npm.ts: -------------------------------------------------------------------------------- 1 | import { build } from "https://deno.land/x/dnt@0.34.0/mod.ts"; 2 | 3 | await Deno.remove("npm", { recursive: true }).catch((_) => {}); 4 | 5 | await build({ 6 | entryPoints: [ 7 | { name: ".", path: "./src/entries/universal.ts" }, 8 | { name: "./node", path: "./src/entries/node.ts" }, 9 | { name: "./browser", path: "./src/entries/browser.ts" }, 10 | ], 11 | testPattern: "**/!(sync_fs)/*.test.{ts,tsx,js,mjs,jsx}", 12 | outDir: "./npm", 13 | compilerOptions: { 14 | lib: ["dom", "es2021"], 15 | }, 16 | shims: { 17 | deno: "dev", 18 | timers: true, 19 | weakRef: true, 20 | crypto: true, 21 | custom: [ 22 | { 23 | package: { 24 | name: "isomorphic-blob", 25 | version: "1.0.1", 26 | }, 27 | 28 | globalNames: ["Blob"], 29 | }, 30 | { 31 | package: { 32 | name: "isomorphic-undici-ponyfill", 33 | version: "1.0.0", 34 | }, 35 | globalNames: [ 36 | "Request", 37 | "Response", 38 | "Headers", 39 | ], 40 | }, 41 | { 42 | package: { 43 | name: "@sgwilym/isomorphic-streams", 44 | version: "1.0.4", 45 | }, 46 | globalNames: [ 47 | "WritableStream", 48 | "TransformStream", 49 | "ReadableStream", 50 | "TransformStreamDefaultController", 51 | { name: "UnderlyingSink", typeOnly: true }, 52 | "WritableStreamDefaultWriter", 53 | ], 54 | }, 55 | { 56 | package: { 57 | name: "isomorphic-ws", 58 | version: "5.0.0", 59 | }, 60 | globalNames: [{ name: "WebSocket", exportName: "default" }], 61 | }, 62 | { 63 | package: { 64 | name: "textencoder-ponyfill", 65 | version: "1.0.2", 66 | }, 67 | globalNames: ["TextEncoder", "TextDecoder"], 68 | }, 69 | { 70 | package: { 71 | name: "@sgwilym/urlpattern-polyfill", 72 | version: "1.0.0-rc8", 73 | }, 74 | globalNames: [{ 75 | name: "URLPattern", 76 | exportName: "URLPattern", 77 | }], 78 | }, 79 | ], 80 | }, 81 | 82 | mappings: { 83 | "./src/test/scenarios/scenarios.ts": 84 | "./src/test/scenarios/scenarios.node.ts", 85 | 86 | "./src/tcp/tcp_provider.ts": "./src/tcp/tcp_provider.node.ts", 87 | 88 | "./src/node/chloride.ts": { 89 | name: "chloride", 90 | version: "2.4.1", 91 | }, 92 | 93 | "./src/crypto/default_driver.ts": "./src/crypto/default_driver.npm.ts", 94 | 95 | "./src/replica/driver_fs.ts": "./src/replica/driver_fs.node.ts", 96 | "https://esm.sh/better-sqlite3?dts": { 97 | name: "better-sqlite3", 98 | version: "8.5.0", 99 | }, 100 | "https://raw.githubusercontent.com/sgwilym/noble-ed25519/153f9e7e9952ad22885f5abb3f6abf777bef4a4c/mod.ts": 101 | { 102 | name: "@noble/ed25519", 103 | version: "1.6.0", 104 | }, 105 | "https://esm.sh/path-to-regexp@6.2.1": { 106 | name: "path-to-regexp", 107 | version: "6.2.1", 108 | }, 109 | "https://deno.land/std@0.154.0/node/fs/promises.ts": { 110 | name: "node:fs/promises", 111 | }, 112 | "https://deno.land/std@0.154.0/node/path.ts": { 113 | name: "node:path", 114 | }, 115 | "https://esm.sh/@nodelib/fs.walk@1.2.8": { 116 | name: "@nodelib/fs.walk", 117 | version: "1.2.8", 118 | }, 119 | "https://esm.sh/ws@8.8.1": { 120 | name: "ws", 121 | version: "8.8.1", 122 | }, 123 | "https://deno.land/x/dns_sd@2.0.0/mod.ts": { 124 | name: "ya-dns-sd", 125 | version: "2.0.0", 126 | }, 127 | 128 | "https://deno.land/std@0.167.0/node/http.ts": "node:http", 129 | "https://deno.land/std@0.167.0/node/buffer.ts": "node:buffer", 130 | }, 131 | package: { 132 | // package.json properties 133 | name: "earthstar", 134 | version: Deno.args[0], 135 | engines: { 136 | node: ">=16.0.0", 137 | }, 138 | description: 139 | "Earthstar is a tool for private, undiscoverable, offline-first networks.", 140 | license: "LGPL-3.0-only", 141 | homepage: "https://earthstar-project.org", 142 | funding: { 143 | type: "opencollective", 144 | url: "https://opencollective.com/earthstar", 145 | }, 146 | repository: { 147 | type: "git", 148 | url: "git+https://github.com/earthstar-project/earthstar.git", 149 | }, 150 | bugs: { 151 | url: "https://github.com/earthstar-project/earthstar/issues", 152 | }, 153 | devDependencies: { 154 | "@types/better-sqlite3": "7.4.2", 155 | "@types/chloride": "2.4.0", 156 | "@types/ws": "8.5.3", 157 | "@types/node": "20.1.1", 158 | }, 159 | }, 160 | }); 161 | 162 | // post build steps 163 | Deno.copyFileSync("LICENSE", "npm/LICENSE"); 164 | Deno.copyFileSync("README.md", "npm/README.md"); 165 | 166 | // A truly filthy hack to compensate for Typescript's lack of support for the exports field 167 | Deno.writeTextFileSync( 168 | "npm/browser.js", 169 | `export * from "./esm/src/entries/browser";`, 170 | ); 171 | 172 | Deno.writeTextFileSync( 173 | "npm/browser.d.ts", 174 | `export * from './types/src/entries/browser';`, 175 | ); 176 | 177 | Deno.writeTextFileSync( 178 | "npm/node.js", 179 | `export * from "./esm/src/entries/node";`, 180 | ); 181 | 182 | Deno.writeTextFileSync( 183 | "npm/node.d.ts", 184 | `export * from './types/src/entries/node';`, 185 | ); 186 | --------------------------------------------------------------------------------