├── .dockerignore ├── .gitignore ├── npm ├── napi │ ├── .prettierignore │ ├── npm │ │ ├── darwin-x64 │ │ │ ├── README.md │ │ │ └── package.json │ │ ├── darwin-arm64 │ │ │ ├── README.md │ │ │ └── package.json │ │ ├── linux-x64-gnu │ │ │ ├── README.md │ │ │ └── package.json │ │ └── win32-x64-msvc │ │ │ ├── README.md │ │ │ └── package.json │ ├── .yarnrc.yml │ ├── build.rs │ ├── .taplo.toml │ ├── tsconfig.json │ ├── .editorconfig │ ├── .gitattributes │ ├── Cargo.toml │ ├── __test__ │ │ └── index.spec.ts │ ├── index.d.ts │ ├── .gitignore │ ├── package.json │ └── .eslintrc.yml ├── src │ ├── proto │ │ ├── messages │ │ │ ├── index.ts │ │ │ └── com │ │ │ │ ├── deno │ │ │ │ ├── index.ts │ │ │ │ └── kv │ │ │ │ │ ├── index.ts │ │ │ │ │ ├── backup │ │ │ │ │ ├── index.ts │ │ │ │ │ ├── BackupKvMutationKind.ts │ │ │ │ │ ├── BackupKvPair.ts │ │ │ │ │ ├── BackupSnapshotRange.ts │ │ │ │ │ └── BackupMutationRange.ts │ │ │ │ │ └── datapath │ │ │ │ │ ├── SnapshotReadStatus.ts │ │ │ │ │ ├── ValueEncoding.ts │ │ │ │ │ ├── AtomicWriteStatus.ts │ │ │ │ │ ├── MutationType.ts │ │ │ │ │ ├── index.ts │ │ │ │ │ ├── WatchKey.ts │ │ │ │ │ ├── Watch.ts │ │ │ │ │ ├── SnapshotRead.ts │ │ │ │ │ ├── ReadRangeOutput.ts │ │ │ │ │ ├── Check.ts │ │ │ │ │ ├── KvValue.ts │ │ │ │ │ ├── WatchKeyOutput.ts │ │ │ │ │ ├── WatchOutput.ts │ │ │ │ │ ├── AtomicWriteOutput.ts │ │ │ │ │ ├── ReadRange.ts │ │ │ │ │ ├── AtomicWrite.ts │ │ │ │ │ ├── Enqueue.ts │ │ │ │ │ ├── KvEntry.ts │ │ │ │ │ ├── Mutation.ts │ │ │ │ │ └── SnapshotReadOutput.ts │ │ │ │ └── index.ts │ │ ├── index.ts │ │ └── runtime │ │ │ ├── async │ │ │ ├── wait.ts │ │ │ ├── async-generator.ts │ │ │ ├── event-emitter.ts │ │ │ ├── event-buffer.ts │ │ │ └── observer.ts │ │ │ ├── scalar.ts │ │ │ ├── array.ts │ │ │ ├── wire │ │ │ ├── zigzag.ts │ │ │ ├── index.ts │ │ │ ├── varint.ts │ │ │ ├── serialize.ts │ │ │ └── deserialize.ts │ │ │ ├── base64.ts │ │ │ ├── json │ │ │ └── scalar.ts │ │ │ ├── Long.ts │ │ │ └── rpc.ts │ ├── native.ts │ ├── v8_test.ts │ ├── sleep.ts │ ├── kv_u64.ts │ ├── kv_u64_test.ts │ ├── scripts │ │ ├── process.ts │ │ ├── generate_napi_index.ts │ │ └── build_npm.ts │ ├── unraw_watch_stream.ts │ ├── bytes.ts │ ├── kv_key_test.ts │ ├── check.ts │ ├── e2e_test.ts │ ├── v8.ts │ ├── kv_key.ts │ └── npm.ts ├── deno.jsonc └── LICENSE ├── .rustfmt.toml ├── rust-toolchain.toml ├── .github ├── diagram-dark.png ├── diagram-light.png ├── npm_publish.sh └── workflows │ ├── docker.yml │ └── ci.yml ├── timemachine ├── src │ ├── lib.rs │ ├── key_metadata.rs │ └── backup.rs └── Cargo.toml ├── proto ├── protobuf.rs ├── README.md ├── lib.rs ├── Cargo.toml ├── time.rs ├── limits.rs ├── schema │ ├── kv-metadata-exchange-request.json │ ├── backup.proto │ ├── kv-metadata-exchange-response.v1.json │ └── kv-metadata-exchange-response.v2.json ├── build.rs └── protobuf │ └── com.deno.kv.backup.rs ├── Dockerfile ├── remote ├── time.rs └── Cargo.toml ├── sqlite ├── time.rs ├── Cargo.toml └── sum_operand.rs ├── LICENSE ├── denokv ├── Cargo.toml └── config.rs └── Cargo.toml /.dockerignore: -------------------------------------------------------------------------------- 1 | target/ -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | /*.sqlite* -------------------------------------------------------------------------------- /npm/napi/.prettierignore: -------------------------------------------------------------------------------- 1 | target 2 | .yarn -------------------------------------------------------------------------------- /.rustfmt.toml: -------------------------------------------------------------------------------- 1 | max_width = 80 2 | tab_spaces = 2 3 | edition = "2021" 4 | -------------------------------------------------------------------------------- /rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | [toolchain] 2 | channel = "1.88.0" 3 | components = ["rustfmt", "clippy"] 4 | -------------------------------------------------------------------------------- /.github/diagram-dark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/denoland/denokv/main/.github/diagram-dark.png -------------------------------------------------------------------------------- /.github/diagram-light.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/denoland/denokv/main/.github/diagram-light.png -------------------------------------------------------------------------------- /timemachine/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod backup; 2 | pub mod backup_source_s3; 3 | pub mod key_metadata; 4 | pub mod time_travel; 5 | -------------------------------------------------------------------------------- /npm/napi/npm/darwin-x64/README.md: -------------------------------------------------------------------------------- 1 | # `@deno/kv-darwin-x64` 2 | 3 | This is the **x86_64-apple-darwin** binary for `@deno/kv` 4 | -------------------------------------------------------------------------------- /npm/napi/npm/darwin-arm64/README.md: -------------------------------------------------------------------------------- 1 | # `@deno/kv-darwin-arm64` 2 | 3 | This is the **aarch64-apple-darwin** binary for `@deno/kv` 4 | -------------------------------------------------------------------------------- /npm/src/proto/messages/index.ts: -------------------------------------------------------------------------------- 1 | // @ts-nocheck 2 | import * as com from "./com/index.ts"; 3 | 4 | export type { 5 | com, 6 | }; 7 | -------------------------------------------------------------------------------- /npm/napi/npm/linux-x64-gnu/README.md: -------------------------------------------------------------------------------- 1 | # `@deno/kv-linux-x64-gnu` 2 | 3 | This is the **x86_64-unknown-linux-gnu** binary for `@deno/kv` 4 | -------------------------------------------------------------------------------- /npm/napi/npm/win32-x64-msvc/README.md: -------------------------------------------------------------------------------- 1 | # `@deno/kv-win32-x64-msvc` 2 | 3 | This is the **x86_64-pc-windows-msvc** binary for `@deno/kv` 4 | -------------------------------------------------------------------------------- /npm/src/proto/index.ts: -------------------------------------------------------------------------------- 1 | // @ts-nocheck 2 | import * as messages from "./messages/index.ts"; 3 | 4 | export type { 5 | messages, 6 | }; 7 | -------------------------------------------------------------------------------- /npm/src/proto/messages/com/deno/index.ts: -------------------------------------------------------------------------------- 1 | // @ts-nocheck 2 | import * as kv from "./kv/index.ts"; 3 | 4 | export type { 5 | kv, 6 | }; 7 | -------------------------------------------------------------------------------- /npm/src/proto/messages/com/index.ts: -------------------------------------------------------------------------------- 1 | // @ts-nocheck 2 | import * as deno from "./deno/index.ts"; 3 | 4 | export type { 5 | deno, 6 | }; 7 | -------------------------------------------------------------------------------- /npm/napi/.yarnrc.yml: -------------------------------------------------------------------------------- 1 | nodeLinker: node-modules 2 | 3 | npmAuditRegistry: "https://registry.npmjs.org" 4 | 5 | yarnPath: .yarn/releases/yarn-4.0.1.cjs 6 | -------------------------------------------------------------------------------- /npm/src/proto/runtime/async/wait.ts: -------------------------------------------------------------------------------- 1 | export default function wait(ms: number): Promise { 2 | return new Promise((resolve) => setTimeout(resolve, ms)); 3 | } 4 | -------------------------------------------------------------------------------- /npm/napi/build.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 the Deno authors. All rights reserved. MIT license. 2 | 3 | extern crate napi_build; 4 | 5 | fn main() { 6 | napi_build::setup(); 7 | } 8 | -------------------------------------------------------------------------------- /npm/napi/.taplo.toml: -------------------------------------------------------------------------------- 1 | exclude = ["node_modules/**/*.toml"] 2 | 3 | # https://taplo.tamasfe.dev/configuration/formatter-options.html 4 | [formatting] 5 | align_entries = true 6 | indent_tables = true 7 | reorder_keys = true 8 | -------------------------------------------------------------------------------- /npm/src/proto/messages/com/deno/kv/index.ts: -------------------------------------------------------------------------------- 1 | // @ts-nocheck 2 | import * as backup from "./backup/index.ts"; 3 | import * as datapath from "./datapath/index.ts"; 4 | 5 | export type { 6 | backup, 7 | datapath, 8 | }; 9 | -------------------------------------------------------------------------------- /proto/protobuf.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. 2 | 3 | // Generated code, disable lints 4 | #[allow(clippy::all, non_snake_case)] 5 | #[path = "protobuf/com.deno.kv.datapath.rs"] 6 | pub mod datapath; 7 | 8 | #[allow(clippy::all, non_snake_case)] 9 | #[path = "protobuf/com.deno.kv.backup.rs"] 10 | pub mod backup; 11 | -------------------------------------------------------------------------------- /proto/README.md: -------------------------------------------------------------------------------- 1 | # denokv_proto 2 | 3 | The `denokv_proto` crate provides foundational types and structures related to 4 | Deno KV protocol. 5 | 6 | It contains the protobuf definitions for the KV Connect protocol, as well as the 7 | [KV Connect specification](./kv-connect.md). It also contains a `Database` trait 8 | that can be implemented to provide a Deno KV compatible database. 9 | -------------------------------------------------------------------------------- /npm/napi/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2018", 4 | "strict": true, 5 | "moduleResolution": "node", 6 | "module": "CommonJS", 7 | "noUnusedLocals": true, 8 | "noUnusedParameters": true, 9 | "esModuleInterop": true, 10 | "allowSyntheticDefaultImports": true 11 | }, 12 | "include": ["."], 13 | "exclude": ["node_modules"] 14 | } 15 | -------------------------------------------------------------------------------- /npm/src/proto/runtime/async/async-generator.ts: -------------------------------------------------------------------------------- 1 | export async function* fromSingle(value: T): AsyncGenerator { 2 | yield value; 3 | } 4 | 5 | export async function first( 6 | generator: AsyncGenerator, 7 | ): Promise { 8 | const { done, value } = await generator.next(); 9 | if (done) throw Error("The generator should yield at least one value."); 10 | return value; 11 | } 12 | -------------------------------------------------------------------------------- /npm/napi/.editorconfig: -------------------------------------------------------------------------------- 1 | # EditorConfig helps developers define and maintain consistent 2 | # coding styles between different editors or IDEs 3 | # http://editorconfig.org 4 | root = true 5 | 6 | [*] 7 | indent_style = space 8 | indent_size = 2 9 | end_of_line = lf 10 | charset = utf-8 11 | trim_trailing_whitespace = true 12 | insert_final_newline = true 13 | 14 | [*.md] 15 | trim_trailing_whitespace = false 16 | -------------------------------------------------------------------------------- /proto/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 the Deno authors. All rights reserved. MIT license. 2 | 3 | mod codec; 4 | mod convert; 5 | mod interface; 6 | mod limits; 7 | mod protobuf; 8 | pub mod time; 9 | pub use crate::codec::decode_key; 10 | pub use crate::codec::encode_key; 11 | pub use crate::convert::ConvertError; 12 | pub use crate::interface::*; 13 | pub use crate::protobuf::backup; 14 | pub use crate::protobuf::datapath; 15 | -------------------------------------------------------------------------------- /npm/napi/.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | 4 | 5 | *.ts text eol=lf merge=union 6 | *.tsx text eol=lf merge=union 7 | *.rs text eol=lf merge=union 8 | *.js text eol=lf merge=union 9 | *.json text eol=lf merge=union 10 | *.debug text eol=lf merge=union 11 | 12 | # Generated codes 13 | index.js linguist-detectable=false 14 | index.d.ts linguist-detectable=false -------------------------------------------------------------------------------- /npm/src/proto/messages/com/deno/kv/backup/index.ts: -------------------------------------------------------------------------------- 1 | // @ts-nocheck 2 | export type { Type as BackupSnapshotRange } from "./BackupSnapshotRange.ts"; 3 | export type { Type as BackupKvPair } from "./BackupKvPair.ts"; 4 | export type { Type as BackupMutationRange } from "./BackupMutationRange.ts"; 5 | export type { Type as BackupReplicationLogEntry } from "./BackupReplicationLogEntry.ts"; 6 | export type { Type as BackupKvMutationKind } from "./BackupKvMutationKind.ts"; 7 | -------------------------------------------------------------------------------- /timemachine/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "denokv_timemachine" 3 | description = "Point-in-time recovery tool for Deno KV" 4 | version = "0.13.0" 5 | edition.workspace = true 6 | license.workspace = true 7 | repository.workspace = true 8 | authors.workspace = true 9 | 10 | [dependencies] 11 | anyhow.workspace = true 12 | async-trait.workspace = true 13 | aws-sdk-s3.workspace = true 14 | chrono.workspace = true 15 | denokv_proto.workspace = true 16 | futures.workspace = true 17 | hex.workspace = true 18 | prost.workspace = true 19 | rusqlite.workspace = true 20 | tokio.workspace = true 21 | -------------------------------------------------------------------------------- /npm/src/proto/messages/com/deno/kv/datapath/SnapshotReadStatus.ts: -------------------------------------------------------------------------------- 1 | // @ts-nocheck 2 | export declare namespace $.com.deno.kv.datapath { 3 | export type SnapshotReadStatus = 4 | | "SR_UNSPECIFIED" 5 | | "SR_SUCCESS" 6 | | "SR_READ_DISABLED"; 7 | } 8 | 9 | export type Type = $.com.deno.kv.datapath.SnapshotReadStatus; 10 | 11 | export const num2name = { 12 | 0: "SR_UNSPECIFIED", 13 | 1: "SR_SUCCESS", 14 | 2: "SR_READ_DISABLED", 15 | } as const; 16 | 17 | export const name2num = { 18 | SR_UNSPECIFIED: 0, 19 | SR_SUCCESS: 1, 20 | SR_READ_DISABLED: 2, 21 | } as const; 22 | -------------------------------------------------------------------------------- /npm/src/proto/messages/com/deno/kv/datapath/ValueEncoding.ts: -------------------------------------------------------------------------------- 1 | // @ts-nocheck 2 | export declare namespace $.com.deno.kv.datapath { 3 | export type ValueEncoding = 4 | | "VE_UNSPECIFIED" 5 | | "VE_V8" 6 | | "VE_LE64" 7 | | "VE_BYTES"; 8 | } 9 | 10 | export type Type = $.com.deno.kv.datapath.ValueEncoding; 11 | 12 | export const num2name = { 13 | 0: "VE_UNSPECIFIED", 14 | 1: "VE_V8", 15 | 2: "VE_LE64", 16 | 3: "VE_BYTES", 17 | } as const; 18 | 19 | export const name2num = { 20 | VE_UNSPECIFIED: 0, 21 | VE_V8: 1, 22 | VE_LE64: 2, 23 | VE_BYTES: 3, 24 | } as const; 25 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM rust:1.83-bookworm as builder 2 | 3 | RUN apt-get update && apt-get install -y protobuf-compiler 4 | 5 | WORKDIR /usr/src/denokv 6 | COPY . . 7 | 8 | RUN cargo build --release 9 | 10 | FROM gcr.io/distroless/cc-debian12:debug 11 | 12 | LABEL org.opencontainers.image.source=https://github.com/denoland/denokv 13 | LABEL org.opencontainers.image.description="A self-hosted backend for Deno KV" 14 | LABEL org.opencontainers.image.licenses=MIT 15 | 16 | COPY --from=builder /usr/src/denokv/target/release/denokv /usr/local/bin/ 17 | 18 | ENTRYPOINT ["/usr/local/bin/denokv"] 19 | CMD ["serve"] -------------------------------------------------------------------------------- /npm/src/proto/messages/com/deno/kv/datapath/AtomicWriteStatus.ts: -------------------------------------------------------------------------------- 1 | // @ts-nocheck 2 | export declare namespace $.com.deno.kv.datapath { 3 | export type AtomicWriteStatus = 4 | | "AW_UNSPECIFIED" 5 | | "AW_SUCCESS" 6 | | "AW_CHECK_FAILURE" 7 | | "AW_WRITE_DISABLED"; 8 | } 9 | 10 | export type Type = $.com.deno.kv.datapath.AtomicWriteStatus; 11 | 12 | export const num2name = { 13 | 0: "AW_UNSPECIFIED", 14 | 1: "AW_SUCCESS", 15 | 2: "AW_CHECK_FAILURE", 16 | 5: "AW_WRITE_DISABLED", 17 | } as const; 18 | 19 | export const name2num = { 20 | AW_UNSPECIFIED: 0, 21 | AW_SUCCESS: 1, 22 | AW_CHECK_FAILURE: 2, 23 | AW_WRITE_DISABLED: 5, 24 | } as const; 25 | -------------------------------------------------------------------------------- /npm/src/proto/messages/com/deno/kv/datapath/MutationType.ts: -------------------------------------------------------------------------------- 1 | // @ts-nocheck 2 | export declare namespace $.com.deno.kv.datapath { 3 | export type MutationType = 4 | | "M_UNSPECIFIED" 5 | | "M_SET" 6 | | "M_DELETE" 7 | | "M_SUM" 8 | | "M_MAX" 9 | | "M_MIN"; 10 | } 11 | 12 | export type Type = $.com.deno.kv.datapath.MutationType; 13 | 14 | export const num2name = { 15 | 0: "M_UNSPECIFIED", 16 | 1: "M_SET", 17 | 2: "M_DELETE", 18 | 3: "M_SUM", 19 | 4: "M_MAX", 20 | 5: "M_MIN", 21 | } as const; 22 | 23 | export const name2num = { 24 | M_UNSPECIFIED: 0, 25 | M_SET: 1, 26 | M_DELETE: 2, 27 | M_SUM: 3, 28 | M_MAX: 4, 29 | M_MIN: 5, 30 | } as const; 31 | -------------------------------------------------------------------------------- /proto/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "denokv_proto" 3 | description = "Fundamental types, traits, and protobuf models for denokv" 4 | version = "0.13.0" 5 | edition.workspace = true 6 | license.workspace = true 7 | repository.workspace = true 8 | authors.workspace = true 9 | 10 | [lib] 11 | path = "lib.rs" 12 | 13 | [features] 14 | build_protos = ["prost-build"] 15 | 16 | [dependencies] 17 | async-trait.workspace = true 18 | chrono.workspace = true 19 | futures.workspace = true 20 | num-bigint.workspace = true 21 | prost.workspace = true 22 | serde.workspace = true 23 | uuid.workspace = true 24 | deno_error.workspace = true 25 | 26 | [build-dependencies] 27 | prost-build = { workspace = true, optional = true } 28 | -------------------------------------------------------------------------------- /npm/src/proto/messages/com/deno/kv/backup/BackupKvMutationKind.ts: -------------------------------------------------------------------------------- 1 | // @ts-nocheck 2 | export declare namespace $.com.deno.kv.backup { 3 | export type BackupKvMutationKind = 4 | | "MK_UNSPECIFIED" 5 | | "MK_SET" 6 | | "MK_CLEAR" 7 | | "MK_SUM" 8 | | "MK_MAX" 9 | | "MK_MIN"; 10 | } 11 | 12 | export type Type = $.com.deno.kv.backup.BackupKvMutationKind; 13 | 14 | export const num2name = { 15 | 0: "MK_UNSPECIFIED", 16 | 1: "MK_SET", 17 | 2: "MK_CLEAR", 18 | 3: "MK_SUM", 19 | 4: "MK_MAX", 20 | 5: "MK_MIN", 21 | } as const; 22 | 23 | export const name2num = { 24 | MK_UNSPECIFIED: 0, 25 | MK_SET: 1, 26 | MK_CLEAR: 2, 27 | MK_SUM: 3, 28 | MK_MAX: 4, 29 | MK_MIN: 5, 30 | } as const; 31 | -------------------------------------------------------------------------------- /npm/src/proto/runtime/scalar.ts: -------------------------------------------------------------------------------- 1 | export type ScalarValueTypePath = `.${ScalarValueType}`; 2 | export type ScalarValueTypes = Writable; 3 | export type ScalarValueType = ScalarValueTypes[number]; 4 | export const _scalarValueTypes = [ 5 | "double", 6 | "float", 7 | "int32", 8 | "int64", 9 | "uint32", 10 | "uint64", 11 | "sint32", 12 | "sint64", 13 | "fixed32", 14 | "fixed64", 15 | "sfixed32", 16 | "sfixed64", 17 | "bool", 18 | "string", 19 | "bytes", 20 | ] as const; 21 | export const scalarValueTypes: ScalarValueTypes = 22 | _scalarValueTypes as ScalarValueTypes; 23 | 24 | type Writable = { 25 | -readonly [K in keyof T]: T[K]; 26 | }; 27 | -------------------------------------------------------------------------------- /npm/deno.jsonc: -------------------------------------------------------------------------------- 1 | { 2 | "lint": { 3 | "exclude": [ 4 | "src/proto", // generated 5 | ], 6 | "include": [ 7 | "src" 8 | ], 9 | "rules": { 10 | "tags": [ 11 | "recommended" 12 | ], 13 | "include": [ 14 | "camelcase", 15 | "guard-for-in" 16 | ], 17 | "exclude": [ 18 | "no-invalid-triple-slash-reference" 19 | ] 20 | } 21 | }, 22 | "fmt": { 23 | "exclude": [ 24 | "src/proto", // generated 25 | ], 26 | "include": [ 27 | "src" 28 | ], 29 | }, 30 | "lock": false, 31 | } 32 | -------------------------------------------------------------------------------- /.github/npm_publish.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # abort on any non-zero exit code 4 | set -e 5 | 6 | # ensure required environment variables are defined 7 | if [ -z "$VERSION" ]; then 8 | echo "\$VERSION is required" 9 | exit 1 10 | fi 11 | 12 | # install deno 13 | DENO_VERSION="v1.38.3" 14 | curl -fsSL https://deno.land/x/install/install.sh | DENO_INSTALL=./deno-$DENO_VERSION sh -s $DENO_VERSION 15 | 16 | # run unit tests as a sanity check 17 | NO_COLOR=1 ./deno-$DENO_VERSION/bin/deno test --allow-read --allow-write --unstable 18 | 19 | # build root package, then publish it (and native subpackages) 20 | NO_COLOR=1 ./deno-$DENO_VERSION/bin/deno run --unstable --allow-all ./src/scripts/build_npm.ts $VERSION --napi=$VERSION --publish=$(which npm) ${DRY_RUN:+--dry-run} 21 | -------------------------------------------------------------------------------- /npm/napi/Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | 3 | [package] 4 | authors = [] 5 | edition = "2021" 6 | name = "deno-kv-napi" 7 | version = "0.0.0" # never published 8 | 9 | [lib] 10 | crate-type = ["cdylib"] 11 | 12 | [dependencies] 13 | futures = "0.3.28" 14 | napi = { version = "2", features = ["async", "anyhow"] } 15 | napi-derive = "2" 16 | denokv_sqlite = { path = "../../sqlite" } 17 | denokv_proto = { path = "../../proto" } 18 | rand = "0.8.5" 19 | rand_distr = "0.3.0" 20 | tokio = { version = "1.33.0", features = ["full"] } 21 | anyhow = "1" 22 | prost = "0.13" 23 | once_cell = "1.18.0" 24 | rusqlite = { version = "0.37.0", features = ["bundled"] } 25 | deno_error = "0.7.0" 26 | 27 | [build-dependencies] 28 | napi-build = "2" 29 | 30 | [profile.release] 31 | lto = true 32 | -------------------------------------------------------------------------------- /proto/time.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 the Deno authors. All rights reserved. MIT license. 2 | 3 | /// Identical to chrono::Utc::now() but without the system "clock" 4 | /// feature flag. 5 | /// 6 | /// The "clock" feature flag pulls in the "iana-time-zone" crate 7 | /// which links to macOS's "CoreFoundation" framework which increases 8 | /// startup time for the CLI. 9 | pub fn utc_now() -> chrono::DateTime { 10 | let now = std::time::SystemTime::now() 11 | .duration_since(std::time::UNIX_EPOCH) 12 | .expect("system time before Unix epoch"); 13 | let naive = chrono::NaiveDateTime::from_timestamp_opt( 14 | now.as_secs() as i64, 15 | now.subsec_nanos(), 16 | ) 17 | .unwrap(); 18 | chrono::DateTime::from_naive_utc_and_offset(naive, chrono::Utc) 19 | } 20 | -------------------------------------------------------------------------------- /remote/time.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 the Deno authors. All rights reserved. MIT license. 2 | 3 | /// Identical to chrono::Utc::now() but without the system "clock" 4 | /// feature flag. 5 | /// 6 | /// The "clock" feature flag pulls in the "iana-time-zone" crate 7 | /// which links to macOS's "CoreFoundation" framework which increases 8 | /// startup time for the CLI. 9 | pub fn utc_now() -> chrono::DateTime { 10 | let now = std::time::SystemTime::now() 11 | .duration_since(std::time::UNIX_EPOCH) 12 | .expect("system time before Unix epoch"); 13 | let naive = chrono::NaiveDateTime::from_timestamp_opt( 14 | now.as_secs() as i64, 15 | now.subsec_nanos(), 16 | ) 17 | .unwrap(); 18 | chrono::DateTime::from_naive_utc_and_offset(naive, chrono::Utc) 19 | } 20 | -------------------------------------------------------------------------------- /sqlite/time.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 the Deno authors. All rights reserved. MIT license. 2 | 3 | /// Identical to chrono::Utc::now() but without the system "clock" 4 | /// feature flag. 5 | /// 6 | /// The "clock" feature flag pulls in the "iana-time-zone" crate 7 | /// which links to macOS's "CoreFoundation" framework which increases 8 | /// startup time for the CLI. 9 | pub fn utc_now() -> chrono::DateTime { 10 | let now = std::time::SystemTime::now() 11 | .duration_since(std::time::UNIX_EPOCH) 12 | .expect("system time before Unix epoch"); 13 | let naive = chrono::NaiveDateTime::from_timestamp_opt( 14 | now.as_secs() as i64, 15 | now.subsec_nanos(), 16 | ) 17 | .unwrap(); 18 | chrono::DateTime::from_naive_utc_and_offset(naive, chrono::Utc) 19 | } 20 | -------------------------------------------------------------------------------- /proto/limits.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 the Deno authors. All rights reserved. MIT license. 2 | 3 | pub const MAX_WRITE_KEY_SIZE_BYTES: usize = 2048; 4 | pub const MAX_READ_KEY_SIZE_BYTES: usize = MAX_WRITE_KEY_SIZE_BYTES + 1; 5 | pub const MAX_VALUE_SIZE_BYTES: usize = 65536; 6 | pub const MAX_READ_RANGES: usize = 10; 7 | pub const MAX_READ_ENTRIES: usize = 1000; 8 | pub const MAX_CHECKS: usize = 10; 9 | pub const MAX_MUTATIONS: usize = 1000; 10 | pub const MAX_TOTAL_MUTATION_SIZE_BYTES: usize = 819200; 11 | pub const MAX_QUEUE_DELAY_MS: u64 = 30 * 24 * 60 * 60 * 1000; // 30 days 12 | pub const MAX_QUEUE_UNDELIVERED_KEYS: usize = 10; 13 | pub const MAX_QUEUE_BACKOFF_INTERVALS: usize = 10; 14 | pub const MAX_QUEUE_BACKOFF_MS: u32 = 3600000; // 1 hour 15 | pub const MAX_WATCHED_KEYS: usize = 10; 16 | -------------------------------------------------------------------------------- /npm/napi/__test__/index.spec.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2023 the Deno authors. All rights reserved. MIT license. 2 | 3 | import test from 'ava' 4 | 5 | import { open, close, atomicWrite, dequeueNextMessage, finishMessage, snapshotRead } from '../index' 6 | 7 | test('native function exports', (t) => { 8 | t.is(typeof open, 'function'); 9 | t.is(typeof close, 'function'); 10 | t.is(typeof atomicWrite, 'function'); 11 | t.is(typeof dequeueNextMessage, 'function'); 12 | t.is(typeof finishMessage, 'function'); 13 | t.is(typeof snapshotRead, 'function'); 14 | }) 15 | 16 | test('memory db open/close', (t) => { 17 | t.notThrows(() => { 18 | const debug = false; 19 | const inMemory = undefined; 20 | const dbId = open(':memory:', inMemory, debug); 21 | close(dbId, debug); 22 | }) 23 | }) 24 | -------------------------------------------------------------------------------- /npm/src/proto/runtime/array.ts: -------------------------------------------------------------------------------- 1 | // @ts-ignore 2 | export type PojoSet = { [key in T]: T }; 3 | export function toPojoSet( 4 | arr: readonly T[], 5 | ): PojoSet { 6 | const result: any = {}; 7 | for (const item of arr) result[item] = item; 8 | return result; 9 | } 10 | 11 | export function removeItem(arr: T[], item: T): T[] { 12 | const index = arr.indexOf(item); 13 | arr.splice(index, 1); 14 | return arr; 15 | } 16 | 17 | export function groupBy(arr: T[], by: U): Map { 18 | const result = new Map(); 19 | for (const item of arr) { 20 | const key = item[by]; 21 | if (result.has(key)) result.get(key)!.push(item); 22 | else result.set(key, [item]); 23 | } 24 | return result; 25 | } 26 | -------------------------------------------------------------------------------- /npm/src/native.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2023 the Deno authors. All rights reserved. MIT license. 2 | 3 | import { KvService } from "./kv_types.ts"; 4 | 5 | /** 6 | * Creates a new KvService instance that can be used to access Deno's native implementation (only works in the Deno runtime!) 7 | * 8 | * Requires the --unstable flag to `deno run` and any applicable --allow-read/allow-write/allow-net flags 9 | */ 10 | export function makeNativeService(): KvService { 11 | if ("Deno" in globalThis) { 12 | // deno-lint-ignore no-explicit-any 13 | const { openKv } = (globalThis as any).Deno; 14 | if (typeof openKv === "function") { 15 | return { 16 | // deno-lint-ignore no-explicit-any 17 | openKv: openKv as any, 18 | }; 19 | } 20 | } 21 | throw new Error(`Global 'Deno.openKv' not found`); 22 | } 23 | -------------------------------------------------------------------------------- /sqlite/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "denokv_sqlite" 3 | description = "SQLite storage backend for Deno KV" 4 | version = "0.13.0" 5 | edition.workspace = true 6 | license.workspace = true 7 | repository.workspace = true 8 | authors.workspace = true 9 | 10 | [lib] 11 | path = "lib.rs" 12 | 13 | [dependencies] 14 | async-stream.workspace = true 15 | async-trait.workspace = true 16 | chrono.workspace = true 17 | denokv_proto.workspace = true 18 | futures.workspace = true 19 | hex.workspace = true 20 | log.workspace = true 21 | num-bigint.workspace = true 22 | rand.workspace = true 23 | rusqlite.workspace = true 24 | serde_json.workspace = true 25 | thiserror.workspace = true 26 | tokio.workspace = true 27 | uuid.workspace = true 28 | v8_valueserializer.workspace = true 29 | tokio-stream.workspace = true 30 | deno_error.workspace = true 31 | -------------------------------------------------------------------------------- /proto/schema/kv-metadata-exchange-request.json: -------------------------------------------------------------------------------- 1 | { 2 | "$id": "https://raw.githubusercontent.com/denoland/denokv/main/proto/schemas/kv-metadata-exchange-request.json", 3 | "$schema": "http://json-schema.org/draft-07/schema#", 4 | "definitions": { 5 | "ClientMetadata": { 6 | "type": "object", 7 | "properties": { 8 | "supportedVersions": { 9 | "type": "array", 10 | "items": { 11 | "type": "number", 12 | "anyOf": [ 13 | { 14 | "const": 1 15 | }, 16 | { 17 | "const": 2 18 | } 19 | ] 20 | } 21 | } 22 | }, 23 | "required": ["supportedVersions"], 24 | "additionalProperties": false 25 | } 26 | }, 27 | "$ref": "#/definitions/ClientMetadata" 28 | } 29 | -------------------------------------------------------------------------------- /npm/napi/npm/darwin-x64/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@deno/kv-darwin-x64", 3 | "version": "0.0.0-local", 4 | "os": [ 5 | "darwin" 6 | ], 7 | "cpu": [ 8 | "x64" 9 | ], 10 | "main": "deno-kv-napi.darwin-x64.node", 11 | "files": [ 12 | "deno-kv-napi.darwin-x64.node" 13 | ], 14 | "description": "A Deno KV client library optimized for Node.js", 15 | "keywords": [ 16 | "deno", 17 | "kv", 18 | "napi-rs", 19 | "NAPI", 20 | "N-API", 21 | "Rust", 22 | "node-addon", 23 | "node-addon-api" 24 | ], 25 | "license": "MIT", 26 | "engines": { 27 | "node": ">= 18" 28 | }, 29 | "publishConfig": { 30 | "registry": "https://registry.npmjs.org/", 31 | "access": "public" 32 | }, 33 | "repository": { 34 | "type": "git", 35 | "url": "https://github.com/denoland/denokv.git", 36 | "directory": "npm/napi" 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /npm/napi/npm/darwin-arm64/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@deno/kv-darwin-arm64", 3 | "version": "0.0.0-local", 4 | "os": [ 5 | "darwin" 6 | ], 7 | "cpu": [ 8 | "arm64" 9 | ], 10 | "main": "deno-kv-napi.darwin-arm64.node", 11 | "files": [ 12 | "deno-kv-napi.darwin-arm64.node" 13 | ], 14 | "description": "A Deno KV client library optimized for Node.js", 15 | "keywords": [ 16 | "deno", 17 | "kv", 18 | "napi-rs", 19 | "NAPI", 20 | "N-API", 21 | "Rust", 22 | "node-addon", 23 | "node-addon-api" 24 | ], 25 | "license": "MIT", 26 | "engines": { 27 | "node": ">= 18" 28 | }, 29 | "publishConfig": { 30 | "registry": "https://registry.npmjs.org/", 31 | "access": "public" 32 | }, 33 | "repository": { 34 | "type": "git", 35 | "url": "https://github.com/denoland/denokv.git", 36 | "directory": "npm/napi" 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /remote/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "denokv_remote" 3 | description = "Remote (KV Connect) backend for Deno KV" 4 | version = "0.13.0" 5 | edition.workspace = true 6 | license.workspace = true 7 | repository.workspace = true 8 | authors.workspace = true 9 | 10 | [lib] 11 | path = "lib.rs" 12 | 13 | [dependencies] 14 | async-stream.workspace = true 15 | async-trait.workspace = true 16 | bytes.workspace = true 17 | chrono.workspace = true 18 | denokv_proto.workspace = true 19 | futures.workspace = true 20 | http.workspace = true 21 | log.workspace = true 22 | prost.workspace = true 23 | rand.workspace = true 24 | serde_json.workspace = true 25 | serde.workspace = true 26 | tokio.workspace = true 27 | tokio-util.workspace = true 28 | url.workspace = true 29 | uuid.workspace = true 30 | thiserror.workspace = true 31 | deno_error.workspace = true 32 | 33 | [dev-dependencies] 34 | reqwest.workspace = true 35 | -------------------------------------------------------------------------------- /npm/napi/npm/win32-x64-msvc/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@deno/kv-win32-x64-msvc", 3 | "version": "0.0.0-local", 4 | "os": [ 5 | "win32" 6 | ], 7 | "cpu": [ 8 | "x64" 9 | ], 10 | "main": "deno-kv-napi.win32-x64-msvc.node", 11 | "files": [ 12 | "deno-kv-napi.win32-x64-msvc.node" 13 | ], 14 | "description": "A Deno KV client library optimized for Node.js", 15 | "keywords": [ 16 | "deno", 17 | "kv", 18 | "napi-rs", 19 | "NAPI", 20 | "N-API", 21 | "Rust", 22 | "node-addon", 23 | "node-addon-api" 24 | ], 25 | "license": "MIT", 26 | "engines": { 27 | "node": ">= 18" 28 | }, 29 | "publishConfig": { 30 | "registry": "https://registry.npmjs.org/", 31 | "access": "public" 32 | }, 33 | "repository": { 34 | "type": "git", 35 | "url": "https://github.com/denoland/denokv.git", 36 | "directory": "npm/napi" 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /proto/schema/backup.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. 2 | 3 | syntax = "proto3"; 4 | 5 | package com.deno.kv.backup; 6 | 7 | message BackupSnapshotRange { 8 | repeated BackupKvPair data_list = 1; 9 | repeated BackupKvPair metadata_list = 2; 10 | } 11 | 12 | message BackupKvPair { 13 | bytes key = 1; 14 | bytes value = 2; 15 | } 16 | 17 | message BackupMutationRange { 18 | repeated BackupReplicationLogEntry entries = 1; 19 | uint64 timestamp_ms = 2; 20 | } 21 | 22 | message BackupReplicationLogEntry { 23 | bytes versionstamp = 1; 24 | BackupKvMutationKind kind = 2; 25 | bytes key = 3; 26 | bytes value = 4; 27 | int32 value_encoding = 5; 28 | uint64 expire_at_ms = 6; 29 | } 30 | 31 | enum BackupKvMutationKind { 32 | MK_UNSPECIFIED = 0; 33 | MK_SET = 1; 34 | MK_CLEAR = 2; 35 | MK_SUM = 3; 36 | MK_MAX = 4; 37 | MK_MIN = 5; 38 | } -------------------------------------------------------------------------------- /npm/src/proto/runtime/wire/zigzag.ts: -------------------------------------------------------------------------------- 1 | import Long from "../Long.ts"; 2 | 3 | export function encode(value: T): T { 4 | if (value instanceof Long) { 5 | const l = new Long( 6 | value[0] << 1, 7 | (value[1] << 1) | (value[0] >>> 31), 8 | ); 9 | const r = value[1] >>> 31 ? new Long(0xFFFFFFFF, 0xFFFFFFFF) : new Long(); 10 | return new Long(l[0] ^ r[0], l[1] ^ r[1]) as T; 11 | } 12 | return (((value as number) * 2) ^ ((value as number) >> 31)) >>> 0 as T; 13 | } 14 | 15 | export function decode(value: T): T { 16 | if (value instanceof Long) { 17 | const l = new Long((value[0] >>> 1) | (value[1] << 31), (value[1]) >>> 1); 18 | const r = value[0] & 1 ? new Long(0xFFFFFFFF, 0xFFFFFFFF) : new Long(); 19 | return new Long(l[0] ^ r[0], l[1] ^ r[1]) as T; 20 | } 21 | return (((value as number) >>> 1) ^ -((value as number) & 1)) as T; 22 | } 23 | -------------------------------------------------------------------------------- /npm/napi/npm/linux-x64-gnu/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@deno/kv-linux-x64-gnu", 3 | "version": "0.0.0-local", 4 | "os": [ 5 | "linux" 6 | ], 7 | "cpu": [ 8 | "x64" 9 | ], 10 | "main": "deno-kv-napi.linux-x64-gnu.node", 11 | "files": [ 12 | "deno-kv-napi.linux-x64-gnu.node" 13 | ], 14 | "description": "A Deno KV client library optimized for Node.js", 15 | "keywords": [ 16 | "deno", 17 | "kv", 18 | "napi-rs", 19 | "NAPI", 20 | "N-API", 21 | "Rust", 22 | "node-addon", 23 | "node-addon-api" 24 | ], 25 | "license": "MIT", 26 | "engines": { 27 | "node": ">= 18" 28 | }, 29 | "publishConfig": { 30 | "registry": "https://registry.npmjs.org/", 31 | "access": "public" 32 | }, 33 | "repository": { 34 | "type": "git", 35 | "url": "https://github.com/denoland/denokv.git", 36 | "directory": "npm/napi" 37 | }, 38 | "libc": [ 39 | "glibc" 40 | ] 41 | } 42 | -------------------------------------------------------------------------------- /npm/src/v8_test.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2023 the Deno authors. All rights reserved. MIT license. 2 | 3 | import { decodeV8, encodeV8 } from "./v8.ts"; 4 | import { assertEquals } from "https://deno.land/std@0.208.0/assert/assert_equals.ts"; 5 | 6 | Deno.test({ 7 | name: "encodeV8/decodeV8", 8 | fn: () => { 9 | const tests: [unknown, number[]][] = [ 10 | ["bar", [255, 15, 34, 3, 98, 97, 114]], 11 | [null, [255, 15, 48]], 12 | [undefined, [255, 15, 95]], 13 | [true, [255, 15, 84]], 14 | [false, [255, 15, 70]], 15 | ["a😮b", [255, 15, 99, 8, 97, 0, 61, 216, 46, 222, 98, 0]], 16 | [0n, [255, 15, 90, 0]], 17 | // [ 1n, [ 255, 15, 90, 16, 1, 0, 0, 0, 0, 0, 0, 0 ] ], 18 | ]; 19 | for (const [value, expected] of tests) { 20 | const encoded = encodeV8(value); 21 | assertEquals(encoded, new Uint8Array(expected), `${value}`); 22 | assertEquals(decodeV8(encoded), value); 23 | } 24 | }, 25 | }); 26 | -------------------------------------------------------------------------------- /timemachine/src/key_metadata.rs: -------------------------------------------------------------------------------- 1 | #[derive(Clone, Debug)] 2 | pub struct KeyMetadata { 3 | pub versionstamp: [u8; 10], 4 | pub value_encoding: i32, 5 | pub expire_at_ms: i64, 6 | } 7 | 8 | impl KeyMetadata { 9 | pub fn decode(raw: &[u8]) -> Option { 10 | if raw.len() < 11 { 11 | return None; 12 | } 13 | 14 | let mut versionstamp = [0; 10]; 15 | versionstamp.copy_from_slice(&raw[0..10]); 16 | let value_encoding = raw[10] as i32; 17 | 18 | let expire_at_ms = if raw.len() >= 19 { 19 | i64::from_le_bytes(raw[11..19].try_into().unwrap()) 20 | } else { 21 | -1 22 | }; 23 | 24 | Some(Self { 25 | versionstamp, 26 | value_encoding, 27 | expire_at_ms, 28 | }) 29 | } 30 | 31 | pub fn encode(&self) -> Vec { 32 | let mut buf = Vec::with_capacity(10 + 1 + 8); 33 | buf.extend_from_slice(&self.versionstamp); 34 | buf.push(self.value_encoding as u8); 35 | buf.extend_from_slice(&self.expire_at_ms.to_le_bytes()); 36 | buf 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /npm/src/proto/runtime/wire/index.ts: -------------------------------------------------------------------------------- 1 | import Long from "../Long.ts"; 2 | 3 | export type WireMessage = [FieldNumber, Field][]; 4 | export type FieldNumber = number; 5 | export type Field = 6 | | Varint 7 | | Fixed64 8 | | LengthDelimited 9 | | StartGroup 10 | | EndGroup 11 | | Fixed32; 12 | 13 | export enum WireType { 14 | Varint = 0, 15 | Fixed64 = 1, 16 | LengthDelimited = 2, 17 | StartGroup = 3, 18 | EndGroup = 4, 19 | Fixed32 = 5, 20 | } 21 | 22 | interface FieldBase { 23 | type: T; 24 | } 25 | export interface Varint extends FieldBase { 26 | value: Long; 27 | } 28 | export interface Fixed64 extends FieldBase { 29 | value: Long; 30 | } 31 | export interface LengthDelimited extends FieldBase { 32 | value: Uint8Array; 33 | } 34 | export interface StartGroup extends FieldBase {} 35 | export interface EndGroup extends FieldBase {} 36 | export interface Fixed32 extends FieldBase { 37 | value: number; 38 | } 39 | -------------------------------------------------------------------------------- /npm/napi/index.d.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | 4 | /* auto-generated by NAPI-RS */ 5 | 6 | export function open(path: string, inMemory: boolean | undefined | null, debug: boolean): number 7 | export function close(dbId: number, debug: boolean): void 8 | export function snapshotRead(dbId: number, snapshotReadBytes: Buffer, debug: boolean): Promise 9 | export function atomicWrite(dbId: number, atomicWriteBytes: Buffer, debug: boolean): Promise 10 | export interface QueueMessage { 11 | bytes: Buffer 12 | messageId: number 13 | } 14 | export function dequeueNextMessage(dbId: number, debug: boolean): Promise 15 | export function finishMessage(dbId: number, messageId: number, success: boolean, debug: boolean): Promise 16 | export function startWatch(dbId: number, watchBytes: Buffer, debug: boolean): Promise 17 | export function dequeueNextWatchMessage(dbId: number, watchId: number, debug: boolean): Promise 18 | export function endWatch(dbId: number, watchId: number, debug: boolean): void 19 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 the Deno authors 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /npm/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 the Deno authors 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /npm/src/sleep.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2023 the Deno authors. All rights reserved. MIT license. 2 | 3 | export function sleep(ms: number) { 4 | return new Promise((resolve) => setTimeout(resolve, ms)); 5 | } 6 | 7 | export async function executeWithRetries( 8 | tag: string, 9 | fn: () => Promise, 10 | opts: { maxRetries?: number; isRetryable?: (e: Error) => boolean } = {}, 11 | ): Promise { 12 | const { maxRetries = 10, isRetryable = (e) => e instanceof RetryableError } = 13 | opts; 14 | let retries = 0; 15 | while (true) { 16 | try { 17 | if (retries > 0) { 18 | const waitMillis = retries * 1000; 19 | await sleep(waitMillis); 20 | } 21 | return await fn(); 22 | } catch (e) { 23 | if (isRetryable(e)) { 24 | if (retries >= maxRetries) { 25 | throw new Error( 26 | `${tag}: Out of retries (max=${maxRetries}): ${e.stack || e}`, 27 | ); 28 | } 29 | retries++; 30 | } else { 31 | throw e; 32 | } 33 | } 34 | } 35 | } 36 | 37 | export class RetryableError extends Error { 38 | constructor(message: string) { 39 | super(message); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /npm/src/proto/runtime/async/event-emitter.ts: -------------------------------------------------------------------------------- 1 | export interface EventEmitter> { 2 | emit(type: Type, event: Events[Type]): void; 3 | on( 4 | type: Type | "*", 5 | listener: Listener, 6 | ): Off; 7 | off(type: keyof Events): void; 8 | } 9 | export type Listener = (event: Event, type: string) => void; 10 | export type Off = () => void; 11 | 12 | export function createEventEmitter< 13 | Events extends Record, 14 | >(): EventEmitter { 15 | const listeners = {} as Record>>; 16 | const eventEmitter: EventEmitter = { 17 | emit(type, event) { 18 | listeners[type]?.forEach( 19 | (listener) => listener(event, type as string), 20 | ); 21 | (type !== "*") && listeners["*"]?.forEach( 22 | (listener) => listener(event, type as string), 23 | ); 24 | }, 25 | on(type, listener) { 26 | (listeners[type] ||= new Set()).add(listener); 27 | return () => listeners[type]?.delete(listener); 28 | }, 29 | off(type) { 30 | delete listeners[type]; 31 | }, 32 | }; 33 | return eventEmitter; 34 | } 35 | -------------------------------------------------------------------------------- /npm/src/proto/messages/com/deno/kv/datapath/index.ts: -------------------------------------------------------------------------------- 1 | // @ts-nocheck 2 | export type { Type as SnapshotRead } from "./SnapshotRead.ts"; 3 | export type { Type as SnapshotReadOutput } from "./SnapshotReadOutput.ts"; 4 | export type { Type as SnapshotReadStatus } from "./SnapshotReadStatus.ts"; 5 | export type { Type as ReadRange } from "./ReadRange.ts"; 6 | export type { Type as ReadRangeOutput } from "./ReadRangeOutput.ts"; 7 | export type { Type as AtomicWrite } from "./AtomicWrite.ts"; 8 | export type { Type as AtomicWriteOutput } from "./AtomicWriteOutput.ts"; 9 | export type { Type as Check } from "./Check.ts"; 10 | export type { Type as Mutation } from "./Mutation.ts"; 11 | export type { Type as KvValue } from "./KvValue.ts"; 12 | export type { Type as KvEntry } from "./KvEntry.ts"; 13 | export type { Type as MutationType } from "./MutationType.ts"; 14 | export type { Type as ValueEncoding } from "./ValueEncoding.ts"; 15 | export type { Type as AtomicWriteStatus } from "./AtomicWriteStatus.ts"; 16 | export type { Type as Enqueue } from "./Enqueue.ts"; 17 | export type { Type as Watch } from "./Watch.ts"; 18 | export type { Type as WatchOutput } from "./WatchOutput.ts"; 19 | export type { Type as WatchKey } from "./WatchKey.ts"; 20 | export type { Type as WatchKeyOutput } from "./WatchKeyOutput.ts"; 21 | -------------------------------------------------------------------------------- /npm/src/kv_u64.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2023 the Deno authors. All rights reserved. MIT license. 2 | 3 | const max = (1n << 64n) - 1n; 4 | 5 | export class _KvU64 { 6 | readonly value: bigint; 7 | 8 | constructor(value: bigint) { 9 | if (typeof value !== "bigint") { 10 | throw new TypeError("value must be a bigint"); 11 | } 12 | if (value < 0n) throw new Error("value must be a positive bigint"); 13 | if (value > max) { 14 | throw new Error("value must fit in a 64-bit unsigned integer"); 15 | } 16 | this.value = value; 17 | } 18 | 19 | sum(other: { readonly value: bigint }): _KvU64 { 20 | checkValueHolder(other); 21 | return new _KvU64((this.value + other.value) % (1n << 64n)); 22 | } 23 | 24 | min(other: { readonly value: bigint }): _KvU64 { 25 | checkValueHolder(other); 26 | return other.value < this.value ? new _KvU64(other.value) : this; 27 | } 28 | 29 | max(other: { readonly value: bigint }): _KvU64 { 30 | checkValueHolder(other); 31 | return other.value > this.value ? new _KvU64(other.value) : this; 32 | } 33 | } 34 | 35 | // 36 | 37 | function checkValueHolder(obj: unknown) { 38 | const valid = typeof obj === "object" && obj !== null && 39 | !Array.isArray(obj) && "value" in obj && typeof obj.value === "bigint"; 40 | if (!valid) throw new Error(`Expected bigint holder, found: ${obj}`); 41 | } 42 | -------------------------------------------------------------------------------- /npm/src/kv_u64_test.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2023 the Deno authors. All rights reserved. MIT license. 2 | 3 | import { assertEquals } from "https://deno.land/std@0.208.0/assert/assert_equals.ts"; 4 | import { assertThrows } from "https://deno.land/std@0.208.0/assert/assert_throws.ts"; 5 | import { _KvU64 } from "./kv_u64.ts"; 6 | 7 | Deno.test({ 8 | name: "_KvU64", 9 | fn: () => { 10 | // deno-lint-ignore no-explicit-any 11 | [[], {}, 1.2, null, undefined, 0, 123, "01", -1n, 1n << 64n].forEach((v) => 12 | assertThrows(() => new _KvU64(v as any)) 13 | ); 14 | [0n, 123n, (1n << 64n) - 1n].forEach((v) => 15 | assertEquals(new _KvU64(v).value, v) 16 | ); 17 | [[0n, 0n, 0n], [1n, 2n, 3n], [(1n << 64n) - 1n, 1n, 0n]].forEach((v) => 18 | assertEquals( 19 | new _KvU64(v[0]).sum(new _KvU64(v[1])).value, 20 | v[2], 21 | `${v[0]} + ${v[1]} = ${v[2]}`, 22 | ) 23 | ); 24 | [[0n, 0n, 0n], [1n, 2n, 1n], [(1n << 64n) - 1n, 1n, 1n]].forEach((v) => 25 | assertEquals( 26 | new _KvU64(v[0]).min(new _KvU64(v[1])).value, 27 | v[2], 28 | `${v[0]} + ${v[1]} = ${v[2]}`, 29 | ) 30 | ); 31 | [[0n, 0n, 0n], [1n, 2n, 2n], [(1n << 64n) - 1n, 1n, (1n << 64n) - 1n]] 32 | .forEach((v) => 33 | assertEquals( 34 | new _KvU64(v[0]).max(new _KvU64(v[1])).value, 35 | v[2], 36 | `${v[0]} + ${v[1]} = ${v[2]}`, 37 | ) 38 | ); 39 | }, 40 | }); 41 | -------------------------------------------------------------------------------- /denokv/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "denokv" 3 | version = "0.13.0" 4 | description = "A self-hosted backend for Deno KV" 5 | edition.workspace = true 6 | license.workspace = true 7 | repository.workspace = true 8 | authors.workspace = true 9 | 10 | [[bin]] 11 | path = "main.rs" 12 | name = "denokv" 13 | 14 | [features] 15 | default = ["bundled-sqlite"] 16 | bundled-sqlite = ["rusqlite/bundled"] 17 | 18 | [dependencies] 19 | anyhow.workspace = true 20 | aws-config.workspace = true 21 | aws-sdk-s3.workspace = true 22 | aws-smithy-async.workspace = true 23 | aws-smithy-client.workspace = true 24 | aws-smithy-types.workspace = true 25 | axum.workspace = true 26 | chrono.workspace = true 27 | clap.workspace = true 28 | constant_time_eq.workspace = true 29 | denokv_proto.workspace = true 30 | denokv_sqlite.workspace = true 31 | denokv_timemachine.workspace = true 32 | env_logger.workspace = true 33 | futures.workspace = true 34 | hex.workspace = true 35 | hyper.workspace = true 36 | hyper-proxy.workspace = true 37 | log.workspace = true 38 | prost.workspace = true 39 | rand.workspace = true 40 | rusqlite.workspace = true 41 | serde.workspace = true 42 | thiserror.workspace = true 43 | tokio.workspace = true 44 | uuid.workspace = true 45 | deno_error.workspace = true 46 | 47 | [dev-dependencies] 48 | bytes.workspace = true 49 | denokv_remote.workspace = true 50 | http.workspace = true 51 | num-bigint.workspace = true 52 | tempfile.workspace = true 53 | reqwest.workspace = true 54 | url.workspace = true 55 | v8_valueserializer.workspace = true 56 | -------------------------------------------------------------------------------- /npm/src/proto/runtime/wire/varint.ts: -------------------------------------------------------------------------------- 1 | import Long from "../Long.ts"; 2 | 3 | export function encode(value: number | Long): Uint8Array { 4 | const result: number[] = []; 5 | const mask = 0b1111111; 6 | const head = 1 << 7; 7 | let long = typeof value === "number" ? new Long(value) : value; 8 | while (long[0] || long[1]) { 9 | const [lo, hi] = long; 10 | const chunk = lo & mask; 11 | const nextHi = hi >>> 7; 12 | const nextLo = (lo >>> 7) | ((hi & mask) << (32 - 7)); 13 | long = new Long(nextLo, nextHi); 14 | const resultChunk = !(long[0] || long[1]) ? chunk : chunk | head; 15 | result.push(resultChunk); 16 | } 17 | if (result.length < 1) return new Uint8Array(1); 18 | return Uint8Array.from(result); 19 | } 20 | 21 | export type DecodeResult = [ 22 | number, // byte count 23 | Long, // value 24 | ]; 25 | export function decode(dataview: DataView): DecodeResult { 26 | let result = new Long(0); 27 | let i = 0; 28 | while (true) { 29 | const curr = dataview.getUint8(i); 30 | result = or( 31 | result, 32 | leftshift(new Long(curr & 0b1111111), i * 7), 33 | ); 34 | ++i; 35 | if (curr >>> 7) continue; 36 | return [i, result]; 37 | } 38 | } 39 | 40 | function or(a: Long, b: Long): Long { 41 | return new Long(a[0] | b[0], a[1] | b[1]); 42 | } 43 | 44 | function leftshift(a: Long, count: number): Long { 45 | if (count === 0) return a; 46 | if (count >= 32) return new Long(0, a[0] << (count - 32)); 47 | return new Long( 48 | a[0] << count, 49 | (a[1] << count) | (a[0] >>> (32 - count)), 50 | ); 51 | } 52 | -------------------------------------------------------------------------------- /proto/schema/kv-metadata-exchange-response.v1.json: -------------------------------------------------------------------------------- 1 | { 2 | "$id": "https://deno.land/x/deno/cli/schemas/kv-metadata-exchange-response.v1.json", 3 | "$schema": "http://json-schema.org/draft-07/schema#", 4 | "definitions": { 5 | "Uuid": { 6 | "type": "string", 7 | "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" 8 | }, 9 | "DateTime": { 10 | "type": "string", 11 | "format": "date-time" 12 | }, 13 | "EndpointInfo": { 14 | "type": "object", 15 | "properties": { 16 | "url": { 17 | "type": "string" 18 | }, 19 | "consistency": { 20 | "type": "string" 21 | } 22 | }, 23 | "required": ["url", "consistency"], 24 | "additionalProperties": false 25 | }, 26 | "DatabaseMetadata": { 27 | "type": "object", 28 | "properties": { 29 | "version": { 30 | "type": "integer", 31 | "const": 1 32 | }, 33 | "databaseId": { 34 | "$ref": "#/definitions/Uuid" 35 | }, 36 | "endpoints": { 37 | "type": "array", 38 | "items": { 39 | "$ref": "#/definitions/EndpointInfo" 40 | } 41 | }, 42 | "token": { 43 | "type": "string" 44 | }, 45 | "expiresAt": { 46 | "$ref": "#/definitions/DateTime" 47 | } 48 | }, 49 | "required": ["version", "databaseId", "endpoints", "token", "expiresAt"], 50 | "additionalProperties": false 51 | } 52 | }, 53 | "$ref": "#/definitions/DatabaseMetadata" 54 | } 55 | -------------------------------------------------------------------------------- /npm/src/proto/runtime/wire/serialize.ts: -------------------------------------------------------------------------------- 1 | import { WireMessage, WireType } from "./index.ts"; 2 | import { encode } from "./varint.ts"; 3 | 4 | export default function serialize(wireMessage: WireMessage): Uint8Array { 5 | const result: Uint8Array[] = []; 6 | wireMessage.forEach(([fieldNumber, field]) => { 7 | result.push(encode((fieldNumber << 3) | field.type)); 8 | switch (field.type) { 9 | case WireType.Varint: 10 | result.push(encode(field.value)); 11 | break; 12 | case WireType.Fixed64: { 13 | const arr = new Uint8Array(8); 14 | const dataview = new DataView(arr.buffer); 15 | dataview.setUint32(0, field.value[0], true); 16 | dataview.setUint32(4, field.value[1], true); 17 | result.push(arr); 18 | break; 19 | } 20 | case WireType.LengthDelimited: 21 | result.push(encode(field.value.byteLength)); 22 | result.push(field.value); 23 | break; 24 | case WireType.Fixed32: { 25 | const arr = new Uint8Array(4); 26 | const dataview = new DataView(arr.buffer); 27 | dataview.setUint32(0, field.value, true); 28 | result.push(arr); 29 | break; 30 | } 31 | } 32 | }); 33 | return concat(result); 34 | } 35 | 36 | export function concat(arrays: Uint8Array[]): Uint8Array { 37 | const totalLength = arrays.reduce((acc, value) => { 38 | return acc + value.byteLength; 39 | }, 0); 40 | const result = new Uint8Array(totalLength); 41 | arrays.reduce((acc, array) => { 42 | result.set(array, acc); 43 | return acc + array.byteLength; 44 | }, 0); 45 | return result; 46 | } 47 | -------------------------------------------------------------------------------- /.github/workflows/docker.yml: -------------------------------------------------------------------------------- 1 | name: docker 2 | 3 | on: 4 | push: 5 | branches: ["main"] 6 | tags: ["[0-9]+.[0-9]+.[0-9]+"] 7 | pull_request: 8 | branches: ["main"] 9 | 10 | jobs: 11 | build: 12 | runs-on: ubuntu-latest 13 | permissions: 14 | packages: write 15 | contents: read 16 | 17 | steps: 18 | - name: Set up Docker Buildx 19 | uses: docker/setup-buildx-action@v2 20 | 21 | - name: Set up QEMU (for multi-platform builds) 22 | uses: docker/setup-qemu-action@v2 23 | 24 | - name: Clone repository 25 | uses: actions/checkout@v3 26 | 27 | - name: Cache Docker layers 28 | uses: actions/cache@v3 29 | with: 30 | path: /tmp/.buildx-cache 31 | key: ${{ runner.os }}-buildx-${{ github.sha }} 32 | restore-keys: | 33 | ${{ runner.os }}-buildx- 34 | 35 | - name: Build amd64 image for testing 36 | run: | 37 | docker buildx create --use 38 | docker buildx build --platform linux/amd64 \ 39 | --tag denokv:test \ 40 | --output type=docker . 41 | 42 | - name: Smoke test image 43 | run: | 44 | docker run --platform linux/amd64 -i --init denokv:test --help 45 | 46 | - name: Log in to ghcr.io 47 | if: github.repository == 'denoland/denokv' && startsWith(github.ref, 'refs/tags/') 48 | run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin 49 | 50 | - name: Build and push multi-platform image 51 | if: github.repository == 'denoland/denokv' && startsWith(github.ref, 'refs/tags/') 52 | run: | 53 | docker buildx build --platform linux/amd64,linux/arm64 \ 54 | --tag ghcr.io/denoland/denokv:${GITHUB_REF#refs/*/} \ 55 | --tag ghcr.io/denoland/denokv:latest \ 56 | --push . 57 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = ["denokv", "proto", "remote", "sqlite", "timemachine"] 3 | resolver = "2" 4 | 5 | [workspace.package] 6 | license = "MIT" 7 | repository = "https://github.com/denoland/denokv" 8 | authors = ["the Deno authors"] 9 | edition = "2021" 10 | 11 | [workspace.dependencies] 12 | denokv_proto = { version = "0.13.0", path = "./proto" } 13 | denokv_sqlite = { version = "0.13.0", path = "./sqlite" } 14 | denokv_remote = { version = "0.13.0", path = "./remote" } 15 | denokv_timemachine = { version = "0.13.0", path = "./timemachine" } 16 | 17 | anyhow = "1" 18 | async-stream = "0.3" 19 | async-trait = "0.1" 20 | aws-config = "0.55.3" 21 | aws-sdk-s3 = "0.28.0" 22 | aws-smithy-async = "0.55.3" 23 | aws-smithy-client = "0.55.3" 24 | aws-smithy-types = "0.55.3" 25 | axum = { version = "0.6", features = ["macros", "http2"] } 26 | bytes = "1" 27 | chrono = { version = "0.4", default-features = false, features = ["std", "serde"] } 28 | clap = { version = "4", features = ["derive", "env"] } 29 | constant_time_eq = "0.3" 30 | env_logger = "0.10.0" 31 | futures = "0.3.28" 32 | hex = "0.4" 33 | http = "1" 34 | hyper = { version = "0.14", features = ["client"] } 35 | hyper-proxy = { version = "0.9.1", default-features = false } 36 | log = "0.4.20" 37 | num-bigint = "0.4" 38 | prost = "0.13" 39 | prost-build = "0.13" 40 | rand = "0.8.5" 41 | reqwest = { version = "0.12.4", default-features = false, features = ["json", "stream"] } 42 | rusqlite = "0.37.0" 43 | serde = { version = "1", features = ["derive"] } 44 | serde_json = "1.0.107" 45 | tempfile = "3" 46 | thiserror = "2" 47 | deno_error = { version = "0.7.0", features = ["url", "serde_json", "serde"] } 48 | tokio = { version = "1.33.0", features = ["full"] } 49 | tokio-stream = "0.1" 50 | tokio-util = { version = "0.7", features = ["full"] } 51 | url = "2" 52 | uuid = { version = "1.4.1", features = ["v4", "serde"] } 53 | v8_valueserializer = "0.1.1" 54 | -------------------------------------------------------------------------------- /npm/src/scripts/process.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2023 the Deno authors. All rights reserved. MIT license. 2 | 3 | import { TextLineStream } from "https://deno.land/std@0.208.0/streams/text_line_stream.ts"; 4 | 5 | type Opts = { 6 | command: string; 7 | cwd?: string; 8 | args: string[]; 9 | stdinFn?: (stream: WritableStream) => Promise; 10 | stdoutFn?: (stream: ReadableStream) => Promise; 11 | signal?: AbortSignal; 12 | }; 13 | 14 | export async function run( 15 | { command, cwd, args, stdinFn, stdoutFn, signal }: Opts, 16 | ): Promise<{ code: number; millis: number }> { 17 | const start = Date.now(); 18 | const cmd = new Deno.Command(command, { 19 | cwd, 20 | args, 21 | env: { 22 | NO_COLOR: "1", 23 | }, 24 | stderr: "piped", 25 | stdout: "piped", 26 | stdin: stdinFn ? "piped" : undefined, 27 | signal, 28 | }); 29 | const p = cmd.spawn(); 30 | const errorLines: string[] = []; 31 | const readStderr = async () => { 32 | for await ( 33 | const line of p.stderr.pipeThrough(new TextDecoderStream()).pipeThrough( 34 | new TextLineStream(), 35 | ) 36 | ) { 37 | console.error(line); 38 | errorLines.push(line); 39 | } 40 | }; 41 | const readStdout = async () => { 42 | if (stdoutFn) { 43 | await stdoutFn(p.stdout); 44 | } else { 45 | for await ( 46 | const line of p.stdout.pipeThrough(new TextDecoderStream()).pipeThrough( 47 | new TextLineStream(), 48 | ) 49 | ) { 50 | console.log(line); 51 | } 52 | } 53 | }; 54 | const writeStdout = async () => { 55 | if (!stdinFn) return; 56 | await stdinFn(p.stdin); 57 | }; 58 | await Promise.all([readStderr(), readStdout(), writeStdout()]); 59 | const { code, success } = await p.status; 60 | if (!success) { 61 | throw new Error(errorLines.join("\n")); 62 | } 63 | return { code, millis: Date.now() - start }; 64 | } 65 | -------------------------------------------------------------------------------- /npm/src/proto/runtime/wire/deserialize.ts: -------------------------------------------------------------------------------- 1 | import Long from "../Long.ts"; 2 | import { WireMessage, WireType } from "./index.ts"; 3 | import { decode } from "./varint.ts"; 4 | 5 | export default function deserialize(uint8array: Uint8Array): WireMessage { 6 | let idx = 0; 7 | const offset = uint8array.byteOffset; 8 | const result: WireMessage = []; 9 | const dataview = new DataView(uint8array.buffer, offset); 10 | while (idx < uint8array.length) { 11 | const decodeResult = decode(new DataView(uint8array.buffer, offset + idx)); 12 | const key = decodeResult[1][0]; 13 | idx += decodeResult[0]; 14 | const type = (key & 0b111) as WireType; 15 | const fieldNumber = key >>> 3; 16 | switch (type) { 17 | default: 18 | throw new Error(`Unknown wire type ${type}`); 19 | case WireType.Varint: { 20 | const [len, value] = decode( 21 | new DataView(uint8array.buffer, offset + idx), 22 | ); 23 | result.push([fieldNumber, { type, value }]); 24 | idx += len; 25 | break; 26 | } 27 | case WireType.Fixed64: 28 | const lo = dataview.getUint32(idx, true); 29 | const hi = dataview.getUint32(idx += 4, true); 30 | idx += 4; 31 | result.push([fieldNumber, { 32 | type, 33 | value: new Long(lo, hi), 34 | }]); 35 | break; 36 | case WireType.LengthDelimited: { 37 | const [len, value] = decode( 38 | new DataView(uint8array.buffer, offset + idx), 39 | ); 40 | result.push([fieldNumber, { 41 | type, 42 | value: uint8array.subarray(idx += len, idx += value[0]), 43 | }]); 44 | break; 45 | } 46 | case WireType.StartGroup: 47 | case WireType.EndGroup: 48 | result.push([fieldNumber, { type }]); 49 | break; 50 | case WireType.Fixed32: 51 | result.push([fieldNumber, { 52 | type, 53 | value: dataview.getUint32(idx, true), 54 | }]); 55 | idx += 4; 56 | break; 57 | } 58 | } 59 | return result; 60 | } 61 | -------------------------------------------------------------------------------- /proto/schema/kv-metadata-exchange-response.v2.json: -------------------------------------------------------------------------------- 1 | { 2 | "$id": "https://deno.land/x/deno/cli/schemas/kv-metadata-exchange-response.v2.json", 3 | "$schema": "http://json-schema.org/draft-07/schema#", 4 | "definitions": { 5 | "Uuid": { 6 | "type": "string", 7 | "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" 8 | }, 9 | "DateTime": { 10 | "type": "string", 11 | "format": "date-time" 12 | }, 13 | "EndpointInfo": { 14 | "type": "object", 15 | "properties": { 16 | "url": { 17 | "description": "A fully qualified URL, or a URL relative to the metadata URL. The path of the URL must not end with a slash.", 18 | "type": "string", 19 | "examples": [ 20 | "https://data.example.com/v1", 21 | "/v1", 22 | "./v1" 23 | ] 24 | }, 25 | "consistency": { 26 | "description": "The consistency level of the endpoint.", 27 | "type": "string", 28 | "anyOf": [ 29 | { 30 | "const": "strong" 31 | }, 32 | { 33 | "const": "eventual" 34 | } 35 | ] 36 | } 37 | }, 38 | "required": ["url", "consistency"], 39 | "additionalProperties": false 40 | }, 41 | "DatabaseMetadata": { 42 | "type": "object", 43 | "properties": { 44 | "version": { 45 | "type": "integer", 46 | "const": 2 47 | }, 48 | "databaseId": { 49 | "$ref": "#/definitions/Uuid" 50 | }, 51 | "endpoints": { 52 | "type": "array", 53 | "items": { 54 | "$ref": "#/definitions/EndpointInfo" 55 | } 56 | }, 57 | "token": { 58 | "type": "string" 59 | }, 60 | "expiresAt": { 61 | "$ref": "#/definitions/DateTime" 62 | } 63 | }, 64 | "required": ["version", "databaseId", "endpoints", "token", "expiresAt"], 65 | "additionalProperties": false 66 | } 67 | }, 68 | "$ref": "#/definitions/DatabaseMetadata" 69 | } 70 | -------------------------------------------------------------------------------- /npm/src/proto/runtime/async/event-buffer.ts: -------------------------------------------------------------------------------- 1 | import { defer, Deferred } from "./observer.ts"; 2 | 3 | export interface EventBuffer { 4 | push(value: T): void; 5 | error(error: Error): void; 6 | finish(): void; 7 | drain(): AsyncGenerator; 8 | } 9 | export interface CreateEventBufferConfig { 10 | onDrainStart?: () => void; 11 | onDrainEnd?: () => void; 12 | } 13 | export function createEventBuffer( 14 | config?: CreateEventBufferConfig, 15 | ): EventBuffer { 16 | const queue: T[] = []; 17 | let _error: Error | undefined; 18 | let deferred: Deferred> | undefined; 19 | let finished = false; 20 | return { 21 | push(value) { 22 | if (finished) throw new Error("can't push after finish"); 23 | if (deferred) { 24 | deferred.resolve({ value, done: false }); 25 | deferred = undefined; 26 | } else { 27 | queue.push(value); 28 | } 29 | }, 30 | error(error) { 31 | if (deferred) deferred.reject(error); 32 | else _error = error; 33 | finished = true; 34 | }, 35 | finish() { 36 | deferred?.resolve({ value: undefined, done: true }); 37 | finished = true; 38 | }, 39 | drain() { 40 | config?.onDrainStart?.(); 41 | const result: AsyncGenerator = { 42 | [Symbol.asyncIterator]: () => result, 43 | next() { 44 | if (queue.length > 0) { 45 | return Promise.resolve({ 46 | value: queue.shift()!, 47 | done: false, 48 | }); 49 | } else { 50 | if (_error) return Promise.reject(_error); 51 | if (finished) { 52 | return Promise.resolve({ value: undefined, done: true }); 53 | } else { 54 | return deferred = defer(); 55 | } 56 | } 57 | }, 58 | return(value) { 59 | config?.onDrainEnd?.(); 60 | return Promise.resolve({ value, done: true }); 61 | }, 62 | throw(error) { 63 | config?.onDrainEnd?.(); 64 | return Promise.reject(error); 65 | }, 66 | }; 67 | return result; 68 | }, 69 | }; 70 | } 71 | -------------------------------------------------------------------------------- /npm/src/unraw_watch_stream.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2023 the Deno authors. All rights reserved. MIT license. 2 | 3 | import { KvEntryMaybe } from "./kv_types.ts"; 4 | import { defer } from "./proto/runtime/async/observer.ts"; 5 | 6 | // take an underlying raw kv watch stream, and create a deferred stream that dedups (by key+versionstamp) based on when it's pulled 7 | export function makeUnrawWatchStream( 8 | rawWatchStream: ReadableStream[]>, 9 | onCancel: () => void | Promise, 10 | ): ReadableStream[]> { 11 | let pulled: KvEntryMaybe[] | undefined; 12 | let latest: KvEntryMaybe[] | undefined; 13 | let signal = defer(); 14 | let cancelled = false; 15 | return new ReadableStream({ 16 | start(controller) { 17 | (async () => { 18 | const reader = rawWatchStream.getReader(); 19 | while (true) { 20 | const { done, value: entries } = await reader.read(); 21 | if (done) break; 22 | if (cancelled) break; 23 | latest = entries; 24 | signal.resolve(); 25 | signal = defer(); 26 | } 27 | await reader.cancel(); 28 | signal.resolve(); 29 | try { 30 | controller.close(); 31 | } catch { /* noop */ } 32 | })(); 33 | }, 34 | async pull(controller) { 35 | if (!latest) await signal; 36 | if (!latest) return; 37 | while (true) { 38 | let changed = false; 39 | if (pulled) { 40 | for (let i = 0; i < latest.length; i++) { 41 | if (latest[i].versionstamp === pulled[i].versionstamp) continue; 42 | changed = true; 43 | break; 44 | } 45 | } else { 46 | pulled = latest; 47 | changed = pulled.some((v) => v.versionstamp !== null); 48 | } 49 | if (changed) { 50 | pulled = latest; 51 | controller.enqueue(pulled); 52 | return; 53 | } else { 54 | await signal; 55 | } 56 | } 57 | }, 58 | async cancel() { 59 | cancelled = true; 60 | await onCancel(); 61 | }, 62 | }, { 63 | highWaterMark: 0, // ensure all pulls are user-initiated 64 | }); 65 | } 66 | -------------------------------------------------------------------------------- /proto/build.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. 2 | 3 | use std::io; 4 | 5 | #[cfg(feature = "build_protos")] 6 | mod build_protos { 7 | use std::env; 8 | use std::io; 9 | use std::path::Path; 10 | use std::path::PathBuf; 11 | fn compiled_proto_path(proto: impl AsRef) -> PathBuf { 12 | let proto = proto.as_ref(); 13 | let mut path = PathBuf::from(env::var("OUT_DIR").unwrap()); 14 | path.push(format!( 15 | "com.deno.kv.{}.rs", 16 | proto.file_stem().unwrap().to_str().unwrap() 17 | )); 18 | path 19 | } 20 | 21 | fn protobuf_module_path() -> PathBuf { 22 | PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()).join("protobuf") 23 | } 24 | 25 | fn protobuf_dest_path(compiled_proto_path: impl AsRef) -> PathBuf { 26 | let mut path = protobuf_module_path(); 27 | path.push(compiled_proto_path.as_ref().file_name().unwrap()); 28 | path 29 | } 30 | 31 | fn copy_compiled_proto(proto: impl AsRef) -> io::Result<()> { 32 | let generated = compiled_proto_path(proto); 33 | let contents = std::fs::read_to_string(&generated)?; 34 | let header = r"// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. 35 | 36 | // Generated by prost-build, enable the `build_protos` feature to regenerate."; 37 | 38 | let contents = format!("{header}\n\n{contents}"); 39 | std::fs::write(protobuf_dest_path(&generated), contents.as_bytes())?; 40 | Ok(()) 41 | } 42 | 43 | pub fn build() -> io::Result<()> { 44 | let descriptor_path = 45 | PathBuf::from(env::var("OUT_DIR").unwrap()).join("proto_descriptor.bin"); 46 | 47 | let protos = &["schema/datapath.proto", "schema/backup.proto"]; 48 | 49 | prost_build::Config::new() 50 | .file_descriptor_set_path(&descriptor_path) 51 | .compile_well_known_types() 52 | .compile_protos(protos, &["schema/"])?; 53 | 54 | for proto in protos { 55 | copy_compiled_proto(proto)?; 56 | } 57 | 58 | Ok(()) 59 | } 60 | } 61 | 62 | fn main() -> io::Result<()> { 63 | println!("cargo:rerun-if-changed=./schema/datapath.proto"); 64 | println!("cargo:rerun-if-changed=./schema/backup.proto"); 65 | #[cfg(feature = "build_protos")] 66 | { 67 | build_protos::build() 68 | } 69 | #[cfg(not(feature = "build_protos"))] 70 | { 71 | Ok(()) 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /npm/src/proto/runtime/async/observer.ts: -------------------------------------------------------------------------------- 1 | import { removeItem } from "../array.ts"; 2 | 3 | export interface Observer { 4 | next(value: T): void; 5 | error(exception?: any): void; 6 | complete(): void; 7 | } 8 | export type SubscribeFn = (observer: Observer) => UnsubscribeFn; 9 | export type UnsubscribeFn = () => void; 10 | 11 | export interface Deferred extends Promise { 12 | resolve(value: T): void; 13 | reject(reason?: any): void; 14 | } 15 | export function defer(): Deferred { 16 | const transit: any = {}; 17 | const result = new Promise( 18 | (resolve, reject) => Object.assign(transit, { resolve, reject }), 19 | ); 20 | return Object.assign(result, transit); 21 | } 22 | 23 | export interface Next { 24 | (): Promise<[T, boolean]>; 25 | } 26 | export function createSubscribeFn( 27 | next: Next, 28 | wait = Promise.resolve(), 29 | ): SubscribeFn { 30 | const observers: Observer[] = []; 31 | (async () => { 32 | try { 33 | await wait; 34 | while (observers.length) { 35 | const [value, done] = await next(); 36 | for (const observer of observers) observer.next(value); 37 | if (done) break; 38 | } 39 | } catch (err) { 40 | for (const observer of observers) observer.error(err); 41 | } finally { 42 | for (const observer of observers) observer.complete(); 43 | } 44 | })(); 45 | return (observer) => { 46 | observers.push(observer); 47 | return () => { 48 | observer.complete(); 49 | removeItem(observers, observer); 50 | }; 51 | }; 52 | } 53 | export async function* subscribeFnToAsyncGenerator( 54 | subscribe: SubscribeFn, 55 | ): AsyncGenerator { 56 | let finished = false; 57 | let deferred = defer(); 58 | const observer: Observer = { 59 | next(value) { 60 | const result = deferred; 61 | deferred = defer(); 62 | result.resolve(value); 63 | }, 64 | error(exception) { 65 | const result = deferred; 66 | deferred = defer(); 67 | result.reject(exception); 68 | }, 69 | complete() { 70 | finished = true; 71 | deferred.resolve(null as any); 72 | }, 73 | }; 74 | const unsubscribe = subscribe(observer); 75 | try { 76 | while (true) { 77 | const value = await deferred; 78 | if (finished) break; 79 | yield value; 80 | } 81 | } finally { 82 | unsubscribe(); 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /npm/src/proto/messages/com/deno/kv/datapath/WatchKey.ts: -------------------------------------------------------------------------------- 1 | // @ts-nocheck 2 | import { 3 | tsValueToJsonValueFns, 4 | jsonValueToTsValueFns, 5 | } from "../../../../../runtime/json/scalar.ts"; 6 | import { 7 | WireMessage, 8 | } from "../../../../../runtime/wire/index.ts"; 9 | import { 10 | default as serialize, 11 | } from "../../../../../runtime/wire/serialize.ts"; 12 | import { 13 | tsValueToWireValueFns, 14 | wireValueToTsValueFns, 15 | } from "../../../../../runtime/wire/scalar.ts"; 16 | import { 17 | default as deserialize, 18 | } from "../../../../../runtime/wire/deserialize.ts"; 19 | 20 | export declare namespace $.com.deno.kv.datapath { 21 | export type WatchKey = { 22 | key: Uint8Array; 23 | } 24 | } 25 | 26 | export type Type = $.com.deno.kv.datapath.WatchKey; 27 | 28 | export function getDefaultValue(): $.com.deno.kv.datapath.WatchKey { 29 | return { 30 | key: new Uint8Array(), 31 | }; 32 | } 33 | 34 | export function createValue(partialValue: Partial<$.com.deno.kv.datapath.WatchKey>): $.com.deno.kv.datapath.WatchKey { 35 | return { 36 | ...getDefaultValue(), 37 | ...partialValue, 38 | }; 39 | } 40 | 41 | export function encodeJson(value: $.com.deno.kv.datapath.WatchKey): unknown { 42 | const result: any = {}; 43 | if (value.key !== undefined) result.key = tsValueToJsonValueFns.bytes(value.key); 44 | return result; 45 | } 46 | 47 | export function decodeJson(value: any): $.com.deno.kv.datapath.WatchKey { 48 | const result = getDefaultValue(); 49 | if (value.key !== undefined) result.key = jsonValueToTsValueFns.bytes(value.key); 50 | return result; 51 | } 52 | 53 | export function encodeBinary(value: $.com.deno.kv.datapath.WatchKey): Uint8Array { 54 | const result: WireMessage = []; 55 | if (value.key !== undefined) { 56 | const tsValue = value.key; 57 | result.push( 58 | [1, tsValueToWireValueFns.bytes(tsValue)], 59 | ); 60 | } 61 | return serialize(result); 62 | } 63 | 64 | export function decodeBinary(binary: Uint8Array): $.com.deno.kv.datapath.WatchKey { 65 | const result = getDefaultValue(); 66 | const wireMessage = deserialize(binary); 67 | const wireFields = new Map(wireMessage); 68 | field: { 69 | const wireValue = wireFields.get(1); 70 | if (wireValue === undefined) break field; 71 | const value = wireValueToTsValueFns.bytes(wireValue); 72 | if (value === undefined) break field; 73 | result.key = value; 74 | } 75 | return result; 76 | } 77 | -------------------------------------------------------------------------------- /sqlite/sum_operand.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 the Deno authors. All rights reserved. MIT license. 2 | 3 | use denokv_proto::KvValue; 4 | use num_bigint::BigInt; 5 | use thiserror::Error; 6 | use v8_valueserializer::Heap; 7 | use v8_valueserializer::ParseError; 8 | use v8_valueserializer::Value; 9 | use v8_valueserializer::ValueDeserializer; 10 | use v8_valueserializer::ValueSerializer; 11 | 12 | #[derive(Clone, Debug)] 13 | pub enum SumOperand { 14 | BigInt(BigInt), 15 | Number(f64), 16 | KvU64(u64), 17 | } 18 | 19 | #[derive(Error, Debug)] 20 | pub enum InvalidSumOperandError { 21 | #[error("invalid v8 value")] 22 | InvalidV8Value(#[from] ParseError), 23 | #[error("unsupported value type")] 24 | UnsupportedValueType, 25 | #[error("operand cannot be empty")] 26 | OperandCannotBeEmpty, 27 | } 28 | 29 | impl SumOperand { 30 | pub fn variant_name(&self) -> &'static str { 31 | match self { 32 | Self::BigInt(_) => "BigInt", 33 | Self::Number(_) => "Number", 34 | Self::KvU64(_) => "KvU64", 35 | } 36 | } 37 | 38 | pub fn parse_optional( 39 | value: &KvValue, 40 | ) -> Result, InvalidSumOperandError> { 41 | match value { 42 | KvValue::V8(value) => { 43 | if value.is_empty() { 44 | return Ok(None); 45 | } 46 | let value = ValueDeserializer::default().read(value)?.0; 47 | Ok(Some(match value { 48 | Value::BigInt(x) => Self::BigInt(x), 49 | Value::Double(x) => Self::Number(x), 50 | Value::I32(x) => Self::Number(x as f64), 51 | Value::U32(x) => Self::Number(x as f64), 52 | _ => { 53 | return Err(InvalidSumOperandError::UnsupportedValueType); 54 | } 55 | })) 56 | } 57 | KvValue::U64(x) => Ok(Some(Self::KvU64(*x))), 58 | _ => Err(InvalidSumOperandError::UnsupportedValueType), 59 | } 60 | } 61 | 62 | pub fn parse(value: &KvValue) -> Result { 63 | Self::parse_optional(value)? 64 | .ok_or(InvalidSumOperandError::OperandCannotBeEmpty) 65 | } 66 | 67 | pub fn encode(self) -> KvValue { 68 | match self { 69 | Self::BigInt(x) => KvValue::V8( 70 | ValueSerializer::default() 71 | .finish(&Heap::default(), &Value::BigInt(x)) 72 | .unwrap(), 73 | ), 74 | Self::Number(x) => KvValue::V8( 75 | ValueSerializer::default() 76 | .finish(&Heap::default(), &Value::Double(x)) 77 | .unwrap(), 78 | ), 79 | Self::KvU64(x) => KvValue::U64(x), 80 | } 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /npm/src/proto/messages/com/deno/kv/datapath/Watch.ts: -------------------------------------------------------------------------------- 1 | // @ts-nocheck 2 | import { 3 | Type as WatchKey, 4 | encodeJson as encodeJson_1, 5 | decodeJson as decodeJson_1, 6 | encodeBinary as encodeBinary_1, 7 | decodeBinary as decodeBinary_1, 8 | } from "./WatchKey.ts"; 9 | import { 10 | jsonValueToTsValueFns, 11 | } from "../../../../../runtime/json/scalar.ts"; 12 | import { 13 | WireMessage, 14 | WireType, 15 | } from "../../../../../runtime/wire/index.ts"; 16 | import { 17 | default as serialize, 18 | } from "../../../../../runtime/wire/serialize.ts"; 19 | import { 20 | default as deserialize, 21 | } from "../../../../../runtime/wire/deserialize.ts"; 22 | 23 | export declare namespace $.com.deno.kv.datapath { 24 | export type Watch = { 25 | keys: WatchKey[]; 26 | } 27 | } 28 | 29 | export type Type = $.com.deno.kv.datapath.Watch; 30 | 31 | export function getDefaultValue(): $.com.deno.kv.datapath.Watch { 32 | return { 33 | keys: [], 34 | }; 35 | } 36 | 37 | export function createValue(partialValue: Partial<$.com.deno.kv.datapath.Watch>): $.com.deno.kv.datapath.Watch { 38 | return { 39 | ...getDefaultValue(), 40 | ...partialValue, 41 | }; 42 | } 43 | 44 | export function encodeJson(value: $.com.deno.kv.datapath.Watch): unknown { 45 | const result: any = {}; 46 | result.keys = value.keys.map(value => encodeJson_1(value)); 47 | return result; 48 | } 49 | 50 | export function decodeJson(value: any): $.com.deno.kv.datapath.Watch { 51 | const result = getDefaultValue(); 52 | result.keys = value.keys?.map((value: any) => decodeJson_1(value)) ?? []; 53 | return result; 54 | } 55 | 56 | export function encodeBinary(value: $.com.deno.kv.datapath.Watch): Uint8Array { 57 | const result: WireMessage = []; 58 | for (const tsValue of value.keys) { 59 | result.push( 60 | [1, { type: WireType.LengthDelimited as const, value: encodeBinary_1(tsValue) }], 61 | ); 62 | } 63 | return serialize(result); 64 | } 65 | 66 | export function decodeBinary(binary: Uint8Array): $.com.deno.kv.datapath.Watch { 67 | const result = getDefaultValue(); 68 | const wireMessage = deserialize(binary); 69 | const wireFields = new Map(wireMessage); 70 | collection: { 71 | const wireValues = wireMessage.filter(([fieldNumber]) => fieldNumber === 1).map(([, wireValue]) => wireValue); 72 | const value = wireValues.map((wireValue) => wireValue.type === WireType.LengthDelimited ? decodeBinary_1(wireValue.value) : undefined).filter(x => x !== undefined); 73 | if (!value.length) break collection; 74 | result.keys = value as any; 75 | } 76 | return result; 77 | } 78 | -------------------------------------------------------------------------------- /npm/src/proto/runtime/base64.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2021 the Deno authors. All rights reserved. MIT license. 2 | 3 | const base64abc = [ 4 | "A", 5 | "B", 6 | "C", 7 | "D", 8 | "E", 9 | "F", 10 | "G", 11 | "H", 12 | "I", 13 | "J", 14 | "K", 15 | "L", 16 | "M", 17 | "N", 18 | "O", 19 | "P", 20 | "Q", 21 | "R", 22 | "S", 23 | "T", 24 | "U", 25 | "V", 26 | "W", 27 | "X", 28 | "Y", 29 | "Z", 30 | "a", 31 | "b", 32 | "c", 33 | "d", 34 | "e", 35 | "f", 36 | "g", 37 | "h", 38 | "i", 39 | "j", 40 | "k", 41 | "l", 42 | "m", 43 | "n", 44 | "o", 45 | "p", 46 | "q", 47 | "r", 48 | "s", 49 | "t", 50 | "u", 51 | "v", 52 | "w", 53 | "x", 54 | "y", 55 | "z", 56 | "0", 57 | "1", 58 | "2", 59 | "3", 60 | "4", 61 | "5", 62 | "6", 63 | "7", 64 | "8", 65 | "9", 66 | "+", 67 | "/", 68 | ]; 69 | 70 | /** 71 | * CREDIT: https://gist.github.com/enepomnyaschih/72c423f727d395eeaa09697058238727 72 | * Encodes a given Uint8Array, ArrayBuffer or string into RFC4648 base64 representation 73 | * @param data 74 | */ 75 | export function encode(data: ArrayBuffer | string): string { 76 | const uint8 = typeof data === "string" 77 | ? new TextEncoder().encode(data) 78 | : data instanceof Uint8Array 79 | ? data 80 | : new Uint8Array(data); 81 | let result = "", 82 | i; 83 | const l = uint8.length; 84 | for (i = 2; i < l; i += 3) { 85 | result += base64abc[uint8[i - 2] >> 2]; 86 | result += base64abc[((uint8[i - 2] & 0x03) << 4) | (uint8[i - 1] >> 4)]; 87 | result += base64abc[((uint8[i - 1] & 0x0f) << 2) | (uint8[i] >> 6)]; 88 | result += base64abc[uint8[i] & 0x3f]; 89 | } 90 | if (i === l + 1) { 91 | // 1 octet yet to write 92 | result += base64abc[uint8[i - 2] >> 2]; 93 | result += base64abc[(uint8[i - 2] & 0x03) << 4]; 94 | result += "=="; 95 | } 96 | if (i === l) { 97 | // 2 octets yet to write 98 | result += base64abc[uint8[i - 2] >> 2]; 99 | result += base64abc[((uint8[i - 2] & 0x03) << 4) | (uint8[i - 1] >> 4)]; 100 | result += base64abc[(uint8[i - 1] & 0x0f) << 2]; 101 | result += "="; 102 | } 103 | return result; 104 | } 105 | 106 | /** 107 | * Decodes a given RFC4648 base64 encoded string 108 | * @param b64 109 | */ 110 | export function decode(b64: string): Uint8Array { 111 | const binString = atob(b64); 112 | const size = binString.length; 113 | const bytes = new Uint8Array(size); 114 | for (let i = 0; i < size; i++) { 115 | bytes[i] = binString.charCodeAt(i); 116 | } 117 | return bytes; 118 | } 119 | -------------------------------------------------------------------------------- /npm/src/proto/messages/com/deno/kv/datapath/SnapshotRead.ts: -------------------------------------------------------------------------------- 1 | // @ts-nocheck 2 | import { 3 | Type as ReadRange, 4 | encodeJson as encodeJson_1, 5 | decodeJson as decodeJson_1, 6 | encodeBinary as encodeBinary_1, 7 | decodeBinary as decodeBinary_1, 8 | } from "./ReadRange.ts"; 9 | import { 10 | jsonValueToTsValueFns, 11 | } from "../../../../../runtime/json/scalar.ts"; 12 | import { 13 | WireMessage, 14 | WireType, 15 | } from "../../../../../runtime/wire/index.ts"; 16 | import { 17 | default as serialize, 18 | } from "../../../../../runtime/wire/serialize.ts"; 19 | import { 20 | default as deserialize, 21 | } from "../../../../../runtime/wire/deserialize.ts"; 22 | 23 | export declare namespace $.com.deno.kv.datapath { 24 | export type SnapshotRead = { 25 | ranges: ReadRange[]; 26 | } 27 | } 28 | 29 | export type Type = $.com.deno.kv.datapath.SnapshotRead; 30 | 31 | export function getDefaultValue(): $.com.deno.kv.datapath.SnapshotRead { 32 | return { 33 | ranges: [], 34 | }; 35 | } 36 | 37 | export function createValue(partialValue: Partial<$.com.deno.kv.datapath.SnapshotRead>): $.com.deno.kv.datapath.SnapshotRead { 38 | return { 39 | ...getDefaultValue(), 40 | ...partialValue, 41 | }; 42 | } 43 | 44 | export function encodeJson(value: $.com.deno.kv.datapath.SnapshotRead): unknown { 45 | const result: any = {}; 46 | result.ranges = value.ranges.map(value => encodeJson_1(value)); 47 | return result; 48 | } 49 | 50 | export function decodeJson(value: any): $.com.deno.kv.datapath.SnapshotRead { 51 | const result = getDefaultValue(); 52 | result.ranges = value.ranges?.map((value: any) => decodeJson_1(value)) ?? []; 53 | return result; 54 | } 55 | 56 | export function encodeBinary(value: $.com.deno.kv.datapath.SnapshotRead): Uint8Array { 57 | const result: WireMessage = []; 58 | for (const tsValue of value.ranges) { 59 | result.push( 60 | [1, { type: WireType.LengthDelimited as const, value: encodeBinary_1(tsValue) }], 61 | ); 62 | } 63 | return serialize(result); 64 | } 65 | 66 | export function decodeBinary(binary: Uint8Array): $.com.deno.kv.datapath.SnapshotRead { 67 | const result = getDefaultValue(); 68 | const wireMessage = deserialize(binary); 69 | const wireFields = new Map(wireMessage); 70 | collection: { 71 | const wireValues = wireMessage.filter(([fieldNumber]) => fieldNumber === 1).map(([, wireValue]) => wireValue); 72 | const value = wireValues.map((wireValue) => wireValue.type === WireType.LengthDelimited ? decodeBinary_1(wireValue.value) : undefined).filter(x => x !== undefined); 73 | if (!value.length) break collection; 74 | result.ranges = value as any; 75 | } 76 | return result; 77 | } 78 | -------------------------------------------------------------------------------- /timemachine/src/backup.rs: -------------------------------------------------------------------------------- 1 | use std::fmt::Display; 2 | 3 | use async_trait::async_trait; 4 | 5 | use denokv_proto::backup::BackupMutationRange; 6 | use denokv_proto::backup::BackupSnapshotRange; 7 | 8 | #[async_trait] 9 | pub trait DatabaseBackupSource { 10 | async fn get_differential_versionstamp( 11 | &self, 12 | ) -> anyhow::Result>; 13 | async fn list_snapshot_ranges(&self) 14 | -> anyhow::Result>; 15 | async fn list_logs( 16 | &self, 17 | start_after: Option<&MutationRangeKey>, 18 | limit: u64, 19 | ) -> anyhow::Result>; 20 | async fn fetch_snapshot( 21 | &self, 22 | key: &SnapshotRangeKey, 23 | ) -> anyhow::Result; 24 | async fn fetch_log( 25 | &self, 26 | key: &MutationRangeKey, 27 | ) -> anyhow::Result; 28 | } 29 | 30 | #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Default)] 31 | pub struct SnapshotRangeKey { 32 | pub format_version: u16, 33 | pub monoseq: u64, 34 | pub seq: u64, 35 | } 36 | 37 | impl Display for SnapshotRangeKey { 38 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 39 | if self.format_version == 0 { 40 | let epoch = self.monoseq >> 32; 41 | let tsn = self.monoseq & 0xffff_ffff; 42 | write!(f, "{:016x}_{:016x}_{:016x}", epoch, tsn, self.seq) 43 | } else { 44 | write!( 45 | f, 46 | "{:04x}_{:016x}_{:016x}", 47 | self.format_version, self.monoseq, self.seq 48 | ) 49 | } 50 | } 51 | } 52 | 53 | #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Default)] 54 | pub struct MutationRangeKey { 55 | pub format_version: u16, 56 | pub monoseq: u64, 57 | pub first_versionstamp12: [u8; 12], 58 | pub last_versionstamp12: [u8; 12], 59 | } 60 | 61 | impl Display for MutationRangeKey { 62 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 63 | if self.format_version == 0 { 64 | let epoch = self.monoseq >> 32; 65 | let tsn = self.monoseq & 0xffff_ffff; 66 | write!( 67 | f, 68 | "{:016x}_{:016x}_{}_{}", 69 | epoch, 70 | tsn, 71 | hex::encode(self.first_versionstamp12), 72 | hex::encode(self.last_versionstamp12) 73 | ) 74 | } else { 75 | write!( 76 | f, 77 | "{:04x}_{:016x}_{}_{}", 78 | self.format_version, 79 | self.monoseq, 80 | hex::encode(self.first_versionstamp12), 81 | hex::encode(self.last_versionstamp12) 82 | ) 83 | } 84 | } 85 | } 86 | 87 | #[derive(Clone, Debug)] 88 | pub struct MutationRangeEntry { 89 | pub key: MutationRangeKey, 90 | pub last_modified_ms: u64, 91 | } 92 | -------------------------------------------------------------------------------- /npm/src/bytes.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2023 the Deno authors. All rights reserved. MIT license. 2 | 3 | export { 4 | decodeHex, 5 | encodeHex, 6 | } from "https://deno.land/std@0.208.0/encoding/hex.ts"; 7 | import { concat } from "https://deno.land/std@0.208.0/bytes/concat.ts"; 8 | 9 | export function checkEnd(bytes: Uint8Array, pos: number) { 10 | const extra = bytes.length - pos; 11 | if (extra > 0) throw new Error(`Unexpected trailing bytes: ${extra}`); 12 | } 13 | 14 | export function equalBytes(lhs: Uint8Array, rhs: Uint8Array): boolean { 15 | if (lhs.length !== rhs.length) return false; 16 | for (let i = 0; i < lhs.length; i++) { 17 | if (lhs[i] !== rhs[i]) return false; 18 | } 19 | return true; 20 | } 21 | 22 | export function flipBytes(arr: Uint8Array | number[], start = 0): void { 23 | for (let i = start; i < arr.length; i++) { 24 | arr[i] = 0xff - arr[i]; 25 | } 26 | } 27 | 28 | export function computeBigintMinimumNumberOfBytes(val: bigint): number { 29 | let n = 0; 30 | while (val !== 0n) { 31 | val >>= 8n; 32 | n++; 33 | } 34 | return n; 35 | } 36 | 37 | export function compareBytes(lhs: Uint8Array, rhs: Uint8Array): number { 38 | if (lhs === rhs) return 0; 39 | 40 | let x = lhs.length, y = rhs.length; 41 | const len = Math.min(x, y); 42 | 43 | for (let i = 0; i < len; i++) { 44 | if (lhs[i] !== rhs[i]) { 45 | x = lhs[i]; 46 | y = rhs[i]; 47 | break; 48 | } 49 | } 50 | 51 | return x < y ? -1 : y < x ? 1 : 0; 52 | } 53 | 54 | // 55 | 56 | const ZERO_BYTES = new Uint8Array(0); 57 | 58 | export class ByteReader { 59 | private readonly reader: ReadableStreamDefaultReader; 60 | 61 | private remaining = ZERO_BYTES; 62 | private done = false; 63 | 64 | constructor(reader: ReadableStreamDefaultReader) { 65 | this.reader = reader; 66 | } 67 | 68 | async read( 69 | bytes: number, 70 | ): Promise< 71 | { done: true; value: undefined } | { done: false; value: Uint8Array } 72 | > { 73 | if (bytes < 0) throw new Error(`Invalid bytes: ${bytes}`); 74 | if (bytes === 0) return { done: false, value: ZERO_BYTES }; 75 | while (this.remaining.length < bytes && !this.done) { 76 | const { done, value } = await this.reader.read(); 77 | if (done) { 78 | this.done = true; 79 | } else { 80 | this.remaining = concat([this.remaining, value]); 81 | } 82 | } 83 | const { remaining } = this; 84 | if (remaining.length < bytes && this.done) { 85 | return { done: true, value: undefined }; 86 | } 87 | const value = remaining.slice(0, bytes); 88 | this.remaining = remaining.slice(bytes); 89 | return { done: false, value }; 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /npm/src/proto/messages/com/deno/kv/datapath/ReadRangeOutput.ts: -------------------------------------------------------------------------------- 1 | // @ts-nocheck 2 | import { 3 | Type as KvEntry, 4 | encodeJson as encodeJson_1, 5 | decodeJson as decodeJson_1, 6 | encodeBinary as encodeBinary_1, 7 | decodeBinary as decodeBinary_1, 8 | } from "./KvEntry.ts"; 9 | import { 10 | jsonValueToTsValueFns, 11 | } from "../../../../../runtime/json/scalar.ts"; 12 | import { 13 | WireMessage, 14 | WireType, 15 | } from "../../../../../runtime/wire/index.ts"; 16 | import { 17 | default as serialize, 18 | } from "../../../../../runtime/wire/serialize.ts"; 19 | import { 20 | default as deserialize, 21 | } from "../../../../../runtime/wire/deserialize.ts"; 22 | 23 | export declare namespace $.com.deno.kv.datapath { 24 | export type ReadRangeOutput = { 25 | values: KvEntry[]; 26 | } 27 | } 28 | 29 | export type Type = $.com.deno.kv.datapath.ReadRangeOutput; 30 | 31 | export function getDefaultValue(): $.com.deno.kv.datapath.ReadRangeOutput { 32 | return { 33 | values: [], 34 | }; 35 | } 36 | 37 | export function createValue(partialValue: Partial<$.com.deno.kv.datapath.ReadRangeOutput>): $.com.deno.kv.datapath.ReadRangeOutput { 38 | return { 39 | ...getDefaultValue(), 40 | ...partialValue, 41 | }; 42 | } 43 | 44 | export function encodeJson(value: $.com.deno.kv.datapath.ReadRangeOutput): unknown { 45 | const result: any = {}; 46 | result.values = value.values.map(value => encodeJson_1(value)); 47 | return result; 48 | } 49 | 50 | export function decodeJson(value: any): $.com.deno.kv.datapath.ReadRangeOutput { 51 | const result = getDefaultValue(); 52 | result.values = value.values?.map((value: any) => decodeJson_1(value)) ?? []; 53 | return result; 54 | } 55 | 56 | export function encodeBinary(value: $.com.deno.kv.datapath.ReadRangeOutput): Uint8Array { 57 | const result: WireMessage = []; 58 | for (const tsValue of value.values) { 59 | result.push( 60 | [1, { type: WireType.LengthDelimited as const, value: encodeBinary_1(tsValue) }], 61 | ); 62 | } 63 | return serialize(result); 64 | } 65 | 66 | export function decodeBinary(binary: Uint8Array): $.com.deno.kv.datapath.ReadRangeOutput { 67 | const result = getDefaultValue(); 68 | const wireMessage = deserialize(binary); 69 | const wireFields = new Map(wireMessage); 70 | collection: { 71 | const wireValues = wireMessage.filter(([fieldNumber]) => fieldNumber === 1).map(([, wireValue]) => wireValue); 72 | const value = wireValues.map((wireValue) => wireValue.type === WireType.LengthDelimited ? decodeBinary_1(wireValue.value) : undefined).filter(x => x !== undefined); 73 | if (!value.length) break collection; 74 | result.values = value as any; 75 | } 76 | return result; 77 | } 78 | -------------------------------------------------------------------------------- /npm/napi/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Created by https://www.toptal.com/developers/gitignore/api/node 3 | # Edit at https://www.toptal.com/developers/gitignore?templates=node 4 | 5 | ### Node ### 6 | # Logs 7 | logs 8 | *.log 9 | npm-debug.log* 10 | yarn-debug.log* 11 | yarn-error.log* 12 | lerna-debug.log* 13 | 14 | # Diagnostic reports (https://nodejs.org/api/report.html) 15 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 16 | 17 | # Runtime data 18 | pids 19 | *.pid 20 | *.seed 21 | *.pid.lock 22 | 23 | # Directory for instrumented libs generated by jscoverage/JSCover 24 | lib-cov 25 | 26 | # Coverage directory used by tools like istanbul 27 | coverage 28 | *.lcov 29 | 30 | # nyc test coverage 31 | .nyc_output 32 | 33 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 34 | .grunt 35 | 36 | # Bower dependency directory (https://bower.io/) 37 | bower_components 38 | 39 | # node-waf configuration 40 | .lock-wscript 41 | 42 | # Compiled binary addons (https://nodejs.org/api/addons.html) 43 | build/Release 44 | 45 | # Dependency directories 46 | node_modules/ 47 | jspm_packages/ 48 | 49 | # TypeScript v1 declaration files 50 | typings/ 51 | 52 | # TypeScript cache 53 | *.tsbuildinfo 54 | 55 | # Optional npm cache directory 56 | .npm 57 | 58 | # Optional eslint cache 59 | .eslintcache 60 | 61 | # Microbundle cache 62 | .rpt2_cache/ 63 | .rts2_cache_cjs/ 64 | .rts2_cache_es/ 65 | .rts2_cache_umd/ 66 | 67 | # Optional REPL history 68 | .node_repl_history 69 | 70 | # Output of 'npm pack' 71 | *.tgz 72 | 73 | # Yarn Integrity file 74 | .yarn-integrity 75 | 76 | # dotenv environment variables file 77 | .env 78 | .env.test 79 | 80 | # parcel-bundler cache (https://parceljs.org/) 81 | .cache 82 | 83 | # Next.js build output 84 | .next 85 | 86 | # Nuxt.js build / generate output 87 | .nuxt 88 | dist 89 | 90 | # Gatsby files 91 | .cache/ 92 | # Comment in the public line in if your project uses Gatsby and not Next.js 93 | # https://nextjs.org/blog/next-9-1#public-directory-support 94 | # public 95 | 96 | # vuepress build output 97 | .vuepress/dist 98 | 99 | # Serverless directories 100 | .serverless/ 101 | 102 | # FuseBox cache 103 | .fusebox/ 104 | 105 | # DynamoDB Local files 106 | .dynamodb/ 107 | 108 | # TernJS port file 109 | .tern-port 110 | 111 | # Stores VSCode versions used for testing VSCode extensions 112 | .vscode-test 113 | 114 | # End of https://www.toptal.com/developers/gitignore/api/node 115 | 116 | 117 | #Added by cargo 118 | 119 | /target 120 | Cargo.lock 121 | 122 | *.node 123 | .pnp.* 124 | .yarn/* 125 | !.yarn/patches 126 | !.yarn/plugins 127 | !.yarn/releases 128 | !.yarn/sdks 129 | !.yarn/versions -------------------------------------------------------------------------------- /npm/src/kv_key_test.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2023 the Deno authors. All rights reserved. MIT license. 2 | 3 | import { KvKey } from "./kv_types.ts"; 4 | import { packKey, unpackKey } from "./kv_key.ts"; 5 | import { assertEquals } from "https://deno.land/std@0.208.0/assert/assert_equals.ts"; 6 | 7 | Deno.test({ 8 | name: "packKey/unpackKey", 9 | fn: () => { 10 | const tests: [KvKey, number[]][] = [ 11 | [["foo"], [2, 102, 111, 111, 0]], 12 | [["f\0o"], [2, 102, 0, 0xff, 111, 0]], 13 | [[0], [33, 128, 0, 0, 0, 0, 0, 0, 0]], 14 | [[123.456], [33, 192, 94, 221, 47, 26, 159, 190, 119]], 15 | [[-300], [33, 63, 141, 63, 255, 255, 255, 255, 255]], 16 | [[200], [33, 192, 105, 0, 0, 0, 0, 0, 0]], 17 | [[new TextEncoder().encode("foo")], [1, 102, 111, 111, 0]], 18 | [[new Uint8Array([4, 0, 4])], [1, 4, 0, 255, 4, 0]], 19 | [[false], [38]], 20 | [[true], [39]], 21 | [[0n], [20]], 22 | [[-200n], [19, 55]], 23 | [[200n], [21, 200]], 24 | [[600n], [22, 2, 88]], 25 | [[70000n], [23, 1, 17, 112]], 26 | [[305419896n], [24, 18, 52, 86, 120]], 27 | [[78187493520n], [25, 18, 52, 86, 120, 144]], 28 | [[0x112233445566n], [26, 17, 34, 51, 68, 85, 102]], 29 | [[0x11223344556677n], [27, 17, 34, 51, 68, 85, 102, 119]], 30 | [[0x1122334455667788n], [28, 17, 34, 51, 68, 85, 102, 119, 136]], 31 | [[0x112233445566778899n], [ 32 | 29, 33 | 9, 34 | 17, 35 | 34, 36 | 51, 37 | 68, 38 | 85, 39 | 102, 40 | 119, 41 | 136, 42 | 153, 43 | ]], 44 | [[0x1122334455667788990011223344556677889900n], [ 45 | 29, 46 | 20, 47 | 17, 48 | 34, 49 | 51, 50 | 68, 51 | 85, 52 | 102, 53 | 119, 54 | 136, 55 | 153, 56 | 0, 57 | 17, 58 | 34, 59 | 51, 60 | 68, 61 | 85, 62 | 102, 63 | 119, 64 | 136, 65 | 153, 66 | 0, 67 | ]], 68 | [[-0x1122334455667788990011223344556677889900n], [ 69 | 11, 70 | 235, 71 | 238, 72 | 221, 73 | 204, 74 | 187, 75 | 170, 76 | 153, 77 | 136, 78 | 119, 79 | 102, 80 | 255, 81 | 238, 82 | 221, 83 | 204, 84 | 187, 85 | 170, 86 | 153, 87 | 136, 88 | 119, 89 | 102, 90 | 255, 91 | ]], 92 | ]; 93 | for (const [key, expected] of tests) { 94 | const encoded = packKey(key); 95 | assertEquals(encoded, new Uint8Array(expected), key.join("/")); 96 | assertEquals(unpackKey(encoded), key); 97 | } 98 | }, 99 | }); 100 | -------------------------------------------------------------------------------- /npm/src/proto/runtime/json/scalar.ts: -------------------------------------------------------------------------------- 1 | import { encode as base64Encode, decode as base64Decode } from "../base64.ts"; 2 | 3 | type TsValueToJsonValue = (tsValue: T) => unknown; 4 | type JsonValueToTsValue = (jsonValue: any) => T; 5 | 6 | interface TsValueToJsonValueFns { 7 | int32: TsValueToJsonValue; 8 | int64: TsValueToJsonValue; 9 | uint32: TsValueToJsonValue; 10 | uint64: TsValueToJsonValue; 11 | sint32: TsValueToJsonValue; 12 | sint64: TsValueToJsonValue; 13 | bool: TsValueToJsonValue; 14 | double: TsValueToJsonValue; 15 | float: TsValueToJsonValue; 16 | fixed32: TsValueToJsonValue; 17 | fixed64: TsValueToJsonValue; 18 | sfixed32: TsValueToJsonValue; 19 | sfixed64: TsValueToJsonValue; 20 | string: TsValueToJsonValue; 21 | bytes: TsValueToJsonValue; 22 | enum: TsValueToJsonValue; 23 | } 24 | 25 | interface JsonValueToTsValueFns { 26 | int32: JsonValueToTsValue; 27 | int64: JsonValueToTsValue; 28 | uint32: JsonValueToTsValue; 29 | uint64: JsonValueToTsValue; 30 | sint32: JsonValueToTsValue; 31 | sint64: JsonValueToTsValue; 32 | bool: JsonValueToTsValue; 33 | double: JsonValueToTsValue; 34 | float: JsonValueToTsValue; 35 | fixed32: JsonValueToTsValue; 36 | fixed64: JsonValueToTsValue; 37 | sfixed32: JsonValueToTsValue; 38 | sfixed64: JsonValueToTsValue; 39 | string: JsonValueToTsValue; 40 | bytes: JsonValueToTsValue; 41 | enum: JsonValueToTsValue; 42 | } 43 | 44 | 45 | export const tsValueToJsonValueFns: TsValueToJsonValueFns = { 46 | int32: (tsValue) => tsValue, 47 | int64: (tsValue) => tsValue, 48 | uint32: (tsValue) => tsValue, 49 | uint64: (tsValue) => tsValue, 50 | sint32: (tsValue) => tsValue, 51 | sint64: (tsValue) => tsValue, 52 | bool: (tsValue) => tsValue, 53 | double: (tsValue) => tsValue, 54 | float: (tsValue) => tsValue, 55 | fixed32: (tsValue) => tsValue, 56 | fixed64: (tsValue) => tsValue, 57 | sfixed32: (tsValue) => tsValue, 58 | sfixed64: (tsValue) => tsValue, 59 | string: (tsValue) => tsValue, 60 | bytes: (tsValue) => base64Encode(tsValue), 61 | enum: (tsValue) => tsValue, 62 | }; 63 | 64 | 65 | export const jsonValueToTsValueFns: JsonValueToTsValueFns = { 66 | int32: (tsValue) => tsValue, 67 | int64: (tsValue) => tsValue, 68 | uint32: (tsValue) => tsValue, 69 | uint64: (tsValue) => tsValue, 70 | sint32: (tsValue) => tsValue, 71 | sint64: (tsValue) => tsValue, 72 | bool: (tsValue) => tsValue, 73 | double: (tsValue) => tsValue, 74 | float: (tsValue) => tsValue, 75 | fixed32: (tsValue) => tsValue, 76 | fixed64: (tsValue) => tsValue, 77 | sfixed32: (tsValue) => tsValue, 78 | sfixed64: (tsValue) => tsValue, 79 | string: (tsValue) => tsValue, 80 | bytes: (tsValue) => base64Decode(tsValue), 81 | enum: (tsValue) => tsValue, 82 | }; 83 | -------------------------------------------------------------------------------- /proto/protobuf/com.deno.kv.backup.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. 2 | 3 | // Generated by prost-build, enable the `build_protos` feature to regenerate. 4 | 5 | // This file is @generated by prost-build. 6 | #[derive(Clone, PartialEq, ::prost::Message)] 7 | pub struct BackupSnapshotRange { 8 | #[prost(message, repeated, tag = "1")] 9 | pub data_list: ::prost::alloc::vec::Vec, 10 | #[prost(message, repeated, tag = "2")] 11 | pub metadata_list: ::prost::alloc::vec::Vec, 12 | } 13 | #[derive(Clone, PartialEq, ::prost::Message)] 14 | pub struct BackupKvPair { 15 | #[prost(bytes = "vec", tag = "1")] 16 | pub key: ::prost::alloc::vec::Vec, 17 | #[prost(bytes = "vec", tag = "2")] 18 | pub value: ::prost::alloc::vec::Vec, 19 | } 20 | #[derive(Clone, PartialEq, ::prost::Message)] 21 | pub struct BackupMutationRange { 22 | #[prost(message, repeated, tag = "1")] 23 | pub entries: ::prost::alloc::vec::Vec, 24 | #[prost(uint64, tag = "2")] 25 | pub timestamp_ms: u64, 26 | } 27 | #[derive(Clone, PartialEq, ::prost::Message)] 28 | pub struct BackupReplicationLogEntry { 29 | #[prost(bytes = "vec", tag = "1")] 30 | pub versionstamp: ::prost::alloc::vec::Vec, 31 | #[prost(enumeration = "BackupKvMutationKind", tag = "2")] 32 | pub kind: i32, 33 | #[prost(bytes = "vec", tag = "3")] 34 | pub key: ::prost::alloc::vec::Vec, 35 | #[prost(bytes = "vec", tag = "4")] 36 | pub value: ::prost::alloc::vec::Vec, 37 | #[prost(int32, tag = "5")] 38 | pub value_encoding: i32, 39 | #[prost(uint64, tag = "6")] 40 | pub expire_at_ms: u64, 41 | } 42 | #[derive( 43 | Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration, 44 | )] 45 | #[repr(i32)] 46 | pub enum BackupKvMutationKind { 47 | MkUnspecified = 0, 48 | MkSet = 1, 49 | MkClear = 2, 50 | MkSum = 3, 51 | MkMax = 4, 52 | MkMin = 5, 53 | } 54 | impl BackupKvMutationKind { 55 | /// String value of the enum field names used in the ProtoBuf definition. 56 | /// 57 | /// The values are not transformed in any way and thus are considered stable 58 | /// (if the ProtoBuf definition does not change) and safe for programmatic use. 59 | pub fn as_str_name(&self) -> &'static str { 60 | match self { 61 | Self::MkUnspecified => "MK_UNSPECIFIED", 62 | Self::MkSet => "MK_SET", 63 | Self::MkClear => "MK_CLEAR", 64 | Self::MkSum => "MK_SUM", 65 | Self::MkMax => "MK_MAX", 66 | Self::MkMin => "MK_MIN", 67 | } 68 | } 69 | /// Creates an enum from field names used in the ProtoBuf definition. 70 | pub fn from_str_name(value: &str) -> ::core::option::Option { 71 | match value { 72 | "MK_UNSPECIFIED" => Some(Self::MkUnspecified), 73 | "MK_SET" => Some(Self::MkSet), 74 | "MK_CLEAR" => Some(Self::MkClear), 75 | "MK_SUM" => Some(Self::MkSum), 76 | "MK_MAX" => Some(Self::MkMax), 77 | "MK_MIN" => Some(Self::MkMin), 78 | _ => None, 79 | } 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /npm/napi/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@deno/kv", 3 | "version": "0.0.0-local", 4 | "description": "A Deno KV client library optimized for Node.js", 5 | "main": "index.js", 6 | "license": "MIT", 7 | "keywords": [ 8 | "deno", 9 | "kv", 10 | "napi-rs", 11 | "NAPI", 12 | "N-API", 13 | "Rust", 14 | "node-addon", 15 | "node-addon-api" 16 | ], 17 | "files": [ 18 | "index.d.ts", 19 | "index.js" 20 | ], 21 | "napi": { 22 | "name": "deno-kv-napi", 23 | "triples": { 24 | "defaults": true, 25 | "additional": [ 26 | "aarch64-apple-darwin" 27 | ] 28 | } 29 | }, 30 | "engines": { 31 | "node": ">= 18" 32 | }, 33 | "publishConfig": { 34 | "registry": "https://registry.npmjs.org/", 35 | "access": "public" 36 | }, 37 | "repository": { 38 | "type": "git", 39 | "url": "https://github.com/denoland/denokv.git", 40 | "directory": "npm/napi" 41 | }, 42 | "scripts": { 43 | "artifacts": "napi artifacts", 44 | "build": "napi build --platform --release --pipe \"prettier -w\"", 45 | "build:debug": "napi build --platform --pipe \"prettier -w\"", 46 | "format": "run-p format:prettier format:rs format:toml", 47 | "format:prettier": "prettier . -w", 48 | "format:toml": "taplo format", 49 | "format:rs": "cargo fmt", 50 | "lint": "eslint . -c ./.eslintrc.yml", 51 | "prepublishOnly": "napi prepublish --skip-gh-release", 52 | "test": "ava", 53 | "version": "napi version" 54 | }, 55 | "devDependencies": { 56 | "@napi-rs/cli": "^2.16.4", 57 | "@swc-node/register": "^1.6.8", 58 | "@swc/core": "^1.3.95", 59 | "@taplo/cli": "^0.5.2", 60 | "@typescript-eslint/eslint-plugin": "^6.9.1", 61 | "@typescript-eslint/parser": "^6.9.1", 62 | "ava": "^5.3.1", 63 | "benny": "^3.7.1", 64 | "chalk": "^5.3.0", 65 | "eslint": "^8.52.0", 66 | "eslint-config-prettier": "^9.0.0", 67 | "eslint-plugin-import": "^2.29.0", 68 | "eslint-plugin-prettier": "^5.0.1", 69 | "husky": "^8.0.3", 70 | "lint-staged": "^15.0.2", 71 | "npm-run-all": "^4.1.5", 72 | "prettier": "^3.0.3", 73 | "typescript": "^5.2.2" 74 | }, 75 | "lint-staged": { 76 | "*.@(js|ts|tsx)": [ 77 | "eslint -c .eslintrc.yml --fix" 78 | ], 79 | "*.@(js|ts|tsx|yml|yaml|md|json)": [ 80 | "prettier --write" 81 | ], 82 | "*.toml": [ 83 | "taplo format" 84 | ] 85 | }, 86 | "ava": { 87 | "require": [ 88 | "@swc-node/register" 89 | ], 90 | "extensions": [ 91 | "ts" 92 | ], 93 | "timeout": "2m", 94 | "workerThreads": false, 95 | "environmentVariables": { 96 | "TS_NODE_PROJECT": "./tsconfig.json" 97 | } 98 | }, 99 | "prettier": { 100 | "printWidth": 120, 101 | "semi": false, 102 | "trailingComma": "all", 103 | "singleQuote": true, 104 | "arrowParens": "always" 105 | }, 106 | "packageManager": "yarn@4.0.1" 107 | } 108 | -------------------------------------------------------------------------------- /npm/src/proto/messages/com/deno/kv/backup/BackupKvPair.ts: -------------------------------------------------------------------------------- 1 | // @ts-nocheck 2 | import { 3 | tsValueToJsonValueFns, 4 | jsonValueToTsValueFns, 5 | } from "../../../../../runtime/json/scalar.ts"; 6 | import { 7 | WireMessage, 8 | } from "../../../../../runtime/wire/index.ts"; 9 | import { 10 | default as serialize, 11 | } from "../../../../../runtime/wire/serialize.ts"; 12 | import { 13 | tsValueToWireValueFns, 14 | wireValueToTsValueFns, 15 | } from "../../../../../runtime/wire/scalar.ts"; 16 | import { 17 | default as deserialize, 18 | } from "../../../../../runtime/wire/deserialize.ts"; 19 | 20 | export declare namespace $.com.deno.kv.backup { 21 | export type BackupKvPair = { 22 | key: Uint8Array; 23 | value: Uint8Array; 24 | } 25 | } 26 | 27 | export type Type = $.com.deno.kv.backup.BackupKvPair; 28 | 29 | export function getDefaultValue(): $.com.deno.kv.backup.BackupKvPair { 30 | return { 31 | key: new Uint8Array(), 32 | value: new Uint8Array(), 33 | }; 34 | } 35 | 36 | export function createValue(partialValue: Partial<$.com.deno.kv.backup.BackupKvPair>): $.com.deno.kv.backup.BackupKvPair { 37 | return { 38 | ...getDefaultValue(), 39 | ...partialValue, 40 | }; 41 | } 42 | 43 | export function encodeJson(value: $.com.deno.kv.backup.BackupKvPair): unknown { 44 | const result: any = {}; 45 | if (value.key !== undefined) result.key = tsValueToJsonValueFns.bytes(value.key); 46 | if (value.value !== undefined) result.value = tsValueToJsonValueFns.bytes(value.value); 47 | return result; 48 | } 49 | 50 | export function decodeJson(value: any): $.com.deno.kv.backup.BackupKvPair { 51 | const result = getDefaultValue(); 52 | if (value.key !== undefined) result.key = jsonValueToTsValueFns.bytes(value.key); 53 | if (value.value !== undefined) result.value = jsonValueToTsValueFns.bytes(value.value); 54 | return result; 55 | } 56 | 57 | export function encodeBinary(value: $.com.deno.kv.backup.BackupKvPair): Uint8Array { 58 | const result: WireMessage = []; 59 | if (value.key !== undefined) { 60 | const tsValue = value.key; 61 | result.push( 62 | [1, tsValueToWireValueFns.bytes(tsValue)], 63 | ); 64 | } 65 | if (value.value !== undefined) { 66 | const tsValue = value.value; 67 | result.push( 68 | [2, tsValueToWireValueFns.bytes(tsValue)], 69 | ); 70 | } 71 | return serialize(result); 72 | } 73 | 74 | export function decodeBinary(binary: Uint8Array): $.com.deno.kv.backup.BackupKvPair { 75 | const result = getDefaultValue(); 76 | const wireMessage = deserialize(binary); 77 | const wireFields = new Map(wireMessage); 78 | field: { 79 | const wireValue = wireFields.get(1); 80 | if (wireValue === undefined) break field; 81 | const value = wireValueToTsValueFns.bytes(wireValue); 82 | if (value === undefined) break field; 83 | result.key = value; 84 | } 85 | field: { 86 | const wireValue = wireFields.get(2); 87 | if (wireValue === undefined) break field; 88 | const value = wireValueToTsValueFns.bytes(wireValue); 89 | if (value === undefined) break field; 90 | result.value = value; 91 | } 92 | return result; 93 | } 94 | -------------------------------------------------------------------------------- /npm/src/proto/messages/com/deno/kv/datapath/Check.ts: -------------------------------------------------------------------------------- 1 | // @ts-nocheck 2 | import { 3 | tsValueToJsonValueFns, 4 | jsonValueToTsValueFns, 5 | } from "../../../../../runtime/json/scalar.ts"; 6 | import { 7 | WireMessage, 8 | } from "../../../../../runtime/wire/index.ts"; 9 | import { 10 | default as serialize, 11 | } from "../../../../../runtime/wire/serialize.ts"; 12 | import { 13 | tsValueToWireValueFns, 14 | wireValueToTsValueFns, 15 | } from "../../../../../runtime/wire/scalar.ts"; 16 | import { 17 | default as deserialize, 18 | } from "../../../../../runtime/wire/deserialize.ts"; 19 | 20 | export declare namespace $.com.deno.kv.datapath { 21 | export type Check = { 22 | key: Uint8Array; 23 | versionstamp: Uint8Array; 24 | } 25 | } 26 | 27 | export type Type = $.com.deno.kv.datapath.Check; 28 | 29 | export function getDefaultValue(): $.com.deno.kv.datapath.Check { 30 | return { 31 | key: new Uint8Array(), 32 | versionstamp: new Uint8Array(), 33 | }; 34 | } 35 | 36 | export function createValue(partialValue: Partial<$.com.deno.kv.datapath.Check>): $.com.deno.kv.datapath.Check { 37 | return { 38 | ...getDefaultValue(), 39 | ...partialValue, 40 | }; 41 | } 42 | 43 | export function encodeJson(value: $.com.deno.kv.datapath.Check): unknown { 44 | const result: any = {}; 45 | if (value.key !== undefined) result.key = tsValueToJsonValueFns.bytes(value.key); 46 | if (value.versionstamp !== undefined) result.versionstamp = tsValueToJsonValueFns.bytes(value.versionstamp); 47 | return result; 48 | } 49 | 50 | export function decodeJson(value: any): $.com.deno.kv.datapath.Check { 51 | const result = getDefaultValue(); 52 | if (value.key !== undefined) result.key = jsonValueToTsValueFns.bytes(value.key); 53 | if (value.versionstamp !== undefined) result.versionstamp = jsonValueToTsValueFns.bytes(value.versionstamp); 54 | return result; 55 | } 56 | 57 | export function encodeBinary(value: $.com.deno.kv.datapath.Check): Uint8Array { 58 | const result: WireMessage = []; 59 | if (value.key !== undefined) { 60 | const tsValue = value.key; 61 | result.push( 62 | [1, tsValueToWireValueFns.bytes(tsValue)], 63 | ); 64 | } 65 | if (value.versionstamp !== undefined) { 66 | const tsValue = value.versionstamp; 67 | result.push( 68 | [2, tsValueToWireValueFns.bytes(tsValue)], 69 | ); 70 | } 71 | return serialize(result); 72 | } 73 | 74 | export function decodeBinary(binary: Uint8Array): $.com.deno.kv.datapath.Check { 75 | const result = getDefaultValue(); 76 | const wireMessage = deserialize(binary); 77 | const wireFields = new Map(wireMessage); 78 | field: { 79 | const wireValue = wireFields.get(1); 80 | if (wireValue === undefined) break field; 81 | const value = wireValueToTsValueFns.bytes(wireValue); 82 | if (value === undefined) break field; 83 | result.key = value; 84 | } 85 | field: { 86 | const wireValue = wireFields.get(2); 87 | if (wireValue === undefined) break field; 88 | const value = wireValueToTsValueFns.bytes(wireValue); 89 | if (value === undefined) break field; 90 | result.versionstamp = value; 91 | } 92 | return result; 93 | } 94 | -------------------------------------------------------------------------------- /denokv/config.rs: -------------------------------------------------------------------------------- 1 | use std::net::SocketAddr; 2 | 3 | use clap::Parser; 4 | 5 | #[derive(Parser)] 6 | pub struct Config { 7 | /// The path to the SQLite database KV will persist to. 8 | #[clap(long, env = "DENO_KV_SQLITE_PATH")] 9 | pub sqlite_path: String, 10 | 11 | #[command(subcommand)] 12 | pub subcommand: SubCmd, 13 | } 14 | 15 | #[derive(Parser)] 16 | pub enum SubCmd { 17 | /// Starts the Deno KV HTTP server. 18 | Serve(ServeOptions), 19 | 20 | /// Point-in-time recovery tools. 21 | Pitr(PitrOptions), 22 | } 23 | 24 | #[derive(Parser)] 25 | pub struct ServeOptions { 26 | /// The access token used by the CLI to connect to this KV instance. 27 | #[clap(long, env = "DENO_KV_ACCESS_TOKEN")] 28 | pub access_token: String, 29 | 30 | /// The address to bind the Deno KV HTTP endpoint to. 31 | #[clap(long = "addr", default_value = "0.0.0.0:4512")] 32 | pub addr: SocketAddr, 33 | 34 | /// Open in read-only mode. 35 | #[clap(long)] 36 | pub read_only: bool, 37 | 38 | /// Sync changes from S3 continuously. 39 | #[clap(long, conflicts_with = "read_only")] 40 | pub sync_from_s3: bool, 41 | 42 | /// Atomic write batch timeout. Batching is disabled if this is not set. 43 | #[clap(long, env = "DENO_KV_ATOMIC_WRITE_BATCH_TIMEOUT_MS")] 44 | pub atomic_write_batch_timeout_ms: Option, 45 | 46 | /// Number of worker threads. 47 | #[clap(long, env = "DENO_KV_NUM_WORKERS", default_value = "1")] 48 | pub num_workers: usize, 49 | 50 | #[command(flatten)] 51 | pub replica: ReplicaOptions, 52 | } 53 | 54 | #[derive(Parser)] 55 | pub struct ReplicaOptions { 56 | /// The name of the S3 bucket to sync changes from. 57 | #[clap(long, env = "DENO_KV_S3_BUCKET")] 58 | pub s3_bucket: Option, 59 | 60 | /// Prefix in the S3 bucket. 61 | #[clap(long, env = "DENO_KV_S3_PREFIX")] 62 | pub s3_prefix: Option, 63 | 64 | /// S3 endpoint URL like `https://storage.googleapis.com`. 65 | /// Only needed for S3-compatible services other than Amazon S3. 66 | #[clap(long, env = "DENO_KV_S3_ENDPOINT")] 67 | pub s3_endpoint: Option, 68 | } 69 | 70 | #[derive(Parser)] 71 | pub struct PitrOptions { 72 | #[command(subcommand)] 73 | pub subcommand: PitrSubCmd, 74 | } 75 | 76 | #[derive(Parser)] 77 | pub enum PitrSubCmd { 78 | /// Sync changes from S3 to the SQLite database, without updating the current snapshot. 79 | Sync(SyncOptions), 80 | 81 | /// List available recovery points. 82 | List(PitrListOptions), 83 | 84 | /// Show information about the current snapshot. 85 | Info, 86 | 87 | /// Checkout the snapshot at a specific versionstamp. 88 | Checkout(PitrCheckoutOptions), 89 | } 90 | 91 | #[derive(Parser)] 92 | pub struct SyncOptions { 93 | #[command(flatten)] 94 | pub replica: ReplicaOptions, 95 | } 96 | 97 | #[derive(Parser)] 98 | pub struct PitrListOptions { 99 | /// Start time in RFC3339 format, e.g. 2021-01-01T00:00:00Z 100 | #[clap(long = "start")] 101 | pub start: Option, 102 | 103 | /// End time in RFC3339 format, e.g. 2021-01-01T00:00:00Z 104 | #[clap(long = "end")] 105 | pub end: Option, 106 | } 107 | 108 | #[derive(Parser)] 109 | pub struct PitrCheckoutOptions { 110 | pub versionstamp: String, 111 | } 112 | -------------------------------------------------------------------------------- /npm/src/proto/messages/com/deno/kv/backup/BackupSnapshotRange.ts: -------------------------------------------------------------------------------- 1 | // @ts-nocheck 2 | import { 3 | Type as BackupKvPair, 4 | encodeJson as encodeJson_1, 5 | decodeJson as decodeJson_1, 6 | encodeBinary as encodeBinary_1, 7 | decodeBinary as decodeBinary_1, 8 | } from "./BackupKvPair.ts"; 9 | import { 10 | jsonValueToTsValueFns, 11 | } from "../../../../../runtime/json/scalar.ts"; 12 | import { 13 | WireMessage, 14 | WireType, 15 | } from "../../../../../runtime/wire/index.ts"; 16 | import { 17 | default as serialize, 18 | } from "../../../../../runtime/wire/serialize.ts"; 19 | import { 20 | default as deserialize, 21 | } from "../../../../../runtime/wire/deserialize.ts"; 22 | 23 | export declare namespace $.com.deno.kv.backup { 24 | export type BackupSnapshotRange = { 25 | dataList: BackupKvPair[]; 26 | metadataList: BackupKvPair[]; 27 | } 28 | } 29 | 30 | export type Type = $.com.deno.kv.backup.BackupSnapshotRange; 31 | 32 | export function getDefaultValue(): $.com.deno.kv.backup.BackupSnapshotRange { 33 | return { 34 | dataList: [], 35 | metadataList: [], 36 | }; 37 | } 38 | 39 | export function createValue(partialValue: Partial<$.com.deno.kv.backup.BackupSnapshotRange>): $.com.deno.kv.backup.BackupSnapshotRange { 40 | return { 41 | ...getDefaultValue(), 42 | ...partialValue, 43 | }; 44 | } 45 | 46 | export function encodeJson(value: $.com.deno.kv.backup.BackupSnapshotRange): unknown { 47 | const result: any = {}; 48 | result.dataList = value.dataList.map(value => encodeJson_1(value)); 49 | result.metadataList = value.metadataList.map(value => encodeJson_1(value)); 50 | return result; 51 | } 52 | 53 | export function decodeJson(value: any): $.com.deno.kv.backup.BackupSnapshotRange { 54 | const result = getDefaultValue(); 55 | result.dataList = value.dataList?.map((value: any) => decodeJson_1(value)) ?? []; 56 | result.metadataList = value.metadataList?.map((value: any) => decodeJson_1(value)) ?? []; 57 | return result; 58 | } 59 | 60 | export function encodeBinary(value: $.com.deno.kv.backup.BackupSnapshotRange): Uint8Array { 61 | const result: WireMessage = []; 62 | for (const tsValue of value.dataList) { 63 | result.push( 64 | [1, { type: WireType.LengthDelimited as const, value: encodeBinary_1(tsValue) }], 65 | ); 66 | } 67 | for (const tsValue of value.metadataList) { 68 | result.push( 69 | [2, { type: WireType.LengthDelimited as const, value: encodeBinary_1(tsValue) }], 70 | ); 71 | } 72 | return serialize(result); 73 | } 74 | 75 | export function decodeBinary(binary: Uint8Array): $.com.deno.kv.backup.BackupSnapshotRange { 76 | const result = getDefaultValue(); 77 | const wireMessage = deserialize(binary); 78 | const wireFields = new Map(wireMessage); 79 | collection: { 80 | const wireValues = wireMessage.filter(([fieldNumber]) => fieldNumber === 1).map(([, wireValue]) => wireValue); 81 | const value = wireValues.map((wireValue) => wireValue.type === WireType.LengthDelimited ? decodeBinary_1(wireValue.value) : undefined).filter(x => x !== undefined); 82 | if (!value.length) break collection; 83 | result.dataList = value as any; 84 | } 85 | collection: { 86 | const wireValues = wireMessage.filter(([fieldNumber]) => fieldNumber === 2).map(([, wireValue]) => wireValue); 87 | const value = wireValues.map((wireValue) => wireValue.type === WireType.LengthDelimited ? decodeBinary_1(wireValue.value) : undefined).filter(x => x !== undefined); 88 | if (!value.length) break collection; 89 | result.metadataList = value as any; 90 | } 91 | return result; 92 | } 93 | -------------------------------------------------------------------------------- /npm/src/proto/messages/com/deno/kv/datapath/KvValue.ts: -------------------------------------------------------------------------------- 1 | // @ts-nocheck 2 | import { 3 | Type as ValueEncoding, 4 | name2num, 5 | num2name, 6 | } from "./ValueEncoding.ts"; 7 | import { 8 | tsValueToJsonValueFns, 9 | jsonValueToTsValueFns, 10 | } from "../../../../../runtime/json/scalar.ts"; 11 | import { 12 | WireMessage, 13 | WireType, 14 | } from "../../../../../runtime/wire/index.ts"; 15 | import { 16 | default as serialize, 17 | } from "../../../../../runtime/wire/serialize.ts"; 18 | import { 19 | tsValueToWireValueFns, 20 | wireValueToTsValueFns, 21 | } from "../../../../../runtime/wire/scalar.ts"; 22 | import { 23 | default as Long, 24 | } from "../../../../../runtime/Long.ts"; 25 | import { 26 | default as deserialize, 27 | } from "../../../../../runtime/wire/deserialize.ts"; 28 | 29 | export declare namespace $.com.deno.kv.datapath { 30 | export type KvValue = { 31 | data: Uint8Array; 32 | encoding: ValueEncoding; 33 | } 34 | } 35 | 36 | export type Type = $.com.deno.kv.datapath.KvValue; 37 | 38 | export function getDefaultValue(): $.com.deno.kv.datapath.KvValue { 39 | return { 40 | data: new Uint8Array(), 41 | encoding: "VE_UNSPECIFIED", 42 | }; 43 | } 44 | 45 | export function createValue(partialValue: Partial<$.com.deno.kv.datapath.KvValue>): $.com.deno.kv.datapath.KvValue { 46 | return { 47 | ...getDefaultValue(), 48 | ...partialValue, 49 | }; 50 | } 51 | 52 | export function encodeJson(value: $.com.deno.kv.datapath.KvValue): unknown { 53 | const result: any = {}; 54 | if (value.data !== undefined) result.data = tsValueToJsonValueFns.bytes(value.data); 55 | if (value.encoding !== undefined) result.encoding = tsValueToJsonValueFns.enum(value.encoding); 56 | return result; 57 | } 58 | 59 | export function decodeJson(value: any): $.com.deno.kv.datapath.KvValue { 60 | const result = getDefaultValue(); 61 | if (value.data !== undefined) result.data = jsonValueToTsValueFns.bytes(value.data); 62 | if (value.encoding !== undefined) result.encoding = jsonValueToTsValueFns.enum(value.encoding) as ValueEncoding; 63 | return result; 64 | } 65 | 66 | export function encodeBinary(value: $.com.deno.kv.datapath.KvValue): Uint8Array { 67 | const result: WireMessage = []; 68 | if (value.data !== undefined) { 69 | const tsValue = value.data; 70 | result.push( 71 | [1, tsValueToWireValueFns.bytes(tsValue)], 72 | ); 73 | } 74 | if (value.encoding !== undefined) { 75 | const tsValue = value.encoding; 76 | result.push( 77 | [2, { type: WireType.Varint as const, value: new Long(name2num[tsValue as keyof typeof name2num]) }], 78 | ); 79 | } 80 | return serialize(result); 81 | } 82 | 83 | export function decodeBinary(binary: Uint8Array): $.com.deno.kv.datapath.KvValue { 84 | const result = getDefaultValue(); 85 | const wireMessage = deserialize(binary); 86 | const wireFields = new Map(wireMessage); 87 | field: { 88 | const wireValue = wireFields.get(1); 89 | if (wireValue === undefined) break field; 90 | const value = wireValueToTsValueFns.bytes(wireValue); 91 | if (value === undefined) break field; 92 | result.data = value; 93 | } 94 | field: { 95 | const wireValue = wireFields.get(2); 96 | if (wireValue === undefined) break field; 97 | const value = wireValue.type === WireType.Varint ? num2name[wireValue.value[0] as keyof typeof num2name] : undefined; 98 | if (value === undefined) break field; 99 | result.encoding = value; 100 | } 101 | return result; 102 | } 103 | -------------------------------------------------------------------------------- /npm/src/proto/runtime/Long.ts: -------------------------------------------------------------------------------- 1 | export const UINT16_MAX = 0xFFFF; 2 | export const UINT32_MAX = 0xFFFFFFFF; 3 | 4 | export default class Long extends Uint32Array { 5 | constructor(lo: number = 0, hi: number = 0) { 6 | super([lo, hi]); 7 | } 8 | toString(signed = true): string { 9 | const [lo, hi] = this; 10 | if (lo === 0 && hi === 0) return "0"; 11 | if (signed && (hi > 0x7FFFFFFF)) { 12 | return "-" + add(negate(this), one).toString(false); 13 | } 14 | const result = []; 15 | let tmp = new Long(lo, hi); 16 | while (compare(tmp, zero)) { 17 | const [next, remainder] = divByTen(tmp); 18 | result.push(remainder); 19 | tmp = next; 20 | } 21 | return result.reverse().join(""); 22 | } 23 | static parse(text: string): Long { 24 | const parsedValue = parseInt(text, 10); 25 | const sign = parsedValue < 0; 26 | if (Number.isNaN(parsedValue)) return new Long(0); 27 | if (text.length < 10) { 28 | if (parsedValue < 0) return add(negate(new Long(-parsedValue)), one); 29 | return new Long(parsedValue); 30 | } 31 | let result = new Long(); 32 | let powerTen = one; 33 | for (const digit of text.split("").reverse()) { 34 | if (parseInt(digit)) { 35 | result = add(result, mul(new Long(parseInt(digit)), powerTen)); 36 | } 37 | powerTen = mul(powerTen, new Long(10)); 38 | } 39 | if (!sign) return result; 40 | return add(negate(result), one); 41 | } 42 | } 43 | 44 | const zero = new Long(0); 45 | const one = new Long(1); 46 | 47 | function makeChunk(value: Long): [number, number, number, number] { 48 | const [lo, hi] = value; 49 | return [lo & UINT16_MAX, lo >>> 16, hi & UINT16_MAX, hi >>> 16]; 50 | } 51 | 52 | export function add(a: Long, b: Long): Long { 53 | const [a00, a16, a32, a48] = makeChunk(a); 54 | const [b00, b16, b32, b48] = makeChunk(b); 55 | let c48 = 0, c32 = 0, c16 = 0, c00 = 0; 56 | c00 += a00 + b00; 57 | c16 += c00 >>> 16; 58 | c00 &= UINT16_MAX; 59 | c16 += a16 + b16; 60 | c32 += c16 >>> 16; 61 | c16 &= UINT16_MAX; 62 | c32 += a32 + b32; 63 | c48 += c32 >>> 16; 64 | c32 &= UINT16_MAX; 65 | c48 += a48 + b48; 66 | c48 &= UINT16_MAX; 67 | return new Long(c16 << 16 | c00, c48 << 16 | c32); 68 | } 69 | 70 | export function sub(a: Long, b: Long): Long { 71 | return add(a, add(negate(b), one)); 72 | } 73 | 74 | export function mul(a: Long, b: Long): Long { 75 | const [a00, a16, a32, a48] = makeChunk(a); 76 | const [b00, b16, b32, b48] = makeChunk(b); 77 | let c48 = 0, c32 = 0, c16 = 0, c00 = 0; 78 | c00 += a00 * b00; 79 | c16 += c00 >>> 16; 80 | c00 &= UINT16_MAX; 81 | c16 += a00 * b16 + a16 * b00; 82 | c32 += c16 >>> 16; 83 | c16 &= UINT16_MAX; 84 | c32 += a00 * b32 + a32 * b00 + a16 * b16; 85 | c48 += c32 >>> 16; 86 | c32 &= UINT16_MAX; 87 | c48 += a00 * b48 + a16 * b32 + a32 * b16 + a48 * b00; 88 | c48 &= UINT16_MAX; 89 | return new Long(c16 << 16 | c00, c48 << 16 | c32); 90 | } 91 | 92 | export function divByTen(value: Long): [Long, number] { 93 | const [lo, hi] = value; 94 | return [ 95 | new Long( 96 | (((hi % 10) * (UINT32_MAX + 1) + lo) / 10) | 0, 97 | (hi / 10) | 0, 98 | ), 99 | ((hi % 10) * (UINT32_MAX + 1) + lo) % 10, 100 | ]; 101 | } 102 | 103 | export function compare(a: Long, b: Long): number { 104 | const [l1, h1] = a; 105 | const [l2, h2] = b; 106 | if (h1 !== h2) return h1 - h2; 107 | return l1 - l2; 108 | } 109 | 110 | function negate(value: Long): Long { 111 | const [lo, hi] = value; 112 | return new Long(~lo, ~hi); 113 | } 114 | -------------------------------------------------------------------------------- /npm/src/proto/messages/com/deno/kv/datapath/WatchKeyOutput.ts: -------------------------------------------------------------------------------- 1 | // @ts-nocheck 2 | import { 3 | Type as KvEntry, 4 | encodeJson as encodeJson_1, 5 | decodeJson as decodeJson_1, 6 | encodeBinary as encodeBinary_1, 7 | decodeBinary as decodeBinary_1, 8 | } from "./KvEntry.ts"; 9 | import { 10 | tsValueToJsonValueFns, 11 | jsonValueToTsValueFns, 12 | } from "../../../../../runtime/json/scalar.ts"; 13 | import { 14 | WireMessage, 15 | WireType, 16 | } from "../../../../../runtime/wire/index.ts"; 17 | import { 18 | default as serialize, 19 | } from "../../../../../runtime/wire/serialize.ts"; 20 | import { 21 | tsValueToWireValueFns, 22 | wireValueToTsValueFns, 23 | } from "../../../../../runtime/wire/scalar.ts"; 24 | import { 25 | default as deserialize, 26 | } from "../../../../../runtime/wire/deserialize.ts"; 27 | 28 | export declare namespace $.com.deno.kv.datapath { 29 | export type WatchKeyOutput = { 30 | changed: boolean; 31 | entryIfChanged?: KvEntry; 32 | } 33 | } 34 | 35 | export type Type = $.com.deno.kv.datapath.WatchKeyOutput; 36 | 37 | export function getDefaultValue(): $.com.deno.kv.datapath.WatchKeyOutput { 38 | return { 39 | changed: false, 40 | entryIfChanged: undefined, 41 | }; 42 | } 43 | 44 | export function createValue(partialValue: Partial<$.com.deno.kv.datapath.WatchKeyOutput>): $.com.deno.kv.datapath.WatchKeyOutput { 45 | return { 46 | ...getDefaultValue(), 47 | ...partialValue, 48 | }; 49 | } 50 | 51 | export function encodeJson(value: $.com.deno.kv.datapath.WatchKeyOutput): unknown { 52 | const result: any = {}; 53 | if (value.changed !== undefined) result.changed = tsValueToJsonValueFns.bool(value.changed); 54 | if (value.entryIfChanged !== undefined) result.entryIfChanged = encodeJson_1(value.entryIfChanged); 55 | return result; 56 | } 57 | 58 | export function decodeJson(value: any): $.com.deno.kv.datapath.WatchKeyOutput { 59 | const result = getDefaultValue(); 60 | if (value.changed !== undefined) result.changed = jsonValueToTsValueFns.bool(value.changed); 61 | if (value.entryIfChanged !== undefined) result.entryIfChanged = decodeJson_1(value.entryIfChanged); 62 | return result; 63 | } 64 | 65 | export function encodeBinary(value: $.com.deno.kv.datapath.WatchKeyOutput): Uint8Array { 66 | const result: WireMessage = []; 67 | if (value.changed !== undefined) { 68 | const tsValue = value.changed; 69 | result.push( 70 | [1, tsValueToWireValueFns.bool(tsValue)], 71 | ); 72 | } 73 | if (value.entryIfChanged !== undefined) { 74 | const tsValue = value.entryIfChanged; 75 | result.push( 76 | [2, { type: WireType.LengthDelimited as const, value: encodeBinary_1(tsValue) }], 77 | ); 78 | } 79 | return serialize(result); 80 | } 81 | 82 | export function decodeBinary(binary: Uint8Array): $.com.deno.kv.datapath.WatchKeyOutput { 83 | const result = getDefaultValue(); 84 | const wireMessage = deserialize(binary); 85 | const wireFields = new Map(wireMessage); 86 | field: { 87 | const wireValue = wireFields.get(1); 88 | if (wireValue === undefined) break field; 89 | const value = wireValueToTsValueFns.bool(wireValue); 90 | if (value === undefined) break field; 91 | result.changed = value; 92 | } 93 | field: { 94 | const wireValue = wireFields.get(2); 95 | if (wireValue === undefined) break field; 96 | const value = wireValue.type === WireType.LengthDelimited ? decodeBinary_1(wireValue.value) : undefined; 97 | if (value === undefined) break field; 98 | result.entryIfChanged = value; 99 | } 100 | return result; 101 | } 102 | -------------------------------------------------------------------------------- /npm/src/check.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2023 the Deno authors. All rights reserved. MIT license. 2 | 3 | import { KvKey } from "./kv_types.ts"; 4 | 5 | export function isRecord(obj: unknown): obj is Record { 6 | return typeof obj === "object" && obj !== null && !Array.isArray(obj) && 7 | obj.constructor === Object; 8 | } 9 | 10 | export function isDateTime(value: string): boolean { 11 | return typeof value === "string" && 12 | /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(\.\d+)?Z$/.test(value); 13 | } 14 | 15 | export function checkKeyNotEmpty(key: KvKey): void { 16 | if (key.length === 0) throw new TypeError(`Key cannot be empty`); 17 | } 18 | 19 | export function checkExpireIn(expireIn: number | undefined): void { 20 | const valid = expireIn === undefined || 21 | typeof expireIn === "number" && expireIn > 0 && 22 | Number.isSafeInteger(expireIn); 23 | if (!valid) { 24 | throw new TypeError( 25 | `Bad 'expireIn', expected optional positive integer, found ${expireIn}`, 26 | ); 27 | } 28 | } 29 | 30 | export function checkMatches( 31 | name: string, 32 | value: string, 33 | pattern: RegExp, 34 | ): RegExpMatchArray { 35 | const m = pattern.exec(value); 36 | if (!m) throw new TypeError(`Bad '${name}': ${value}`); 37 | return m; 38 | } 39 | 40 | export function checkString( 41 | name: string, 42 | value: unknown, 43 | ): asserts value is string { 44 | if (typeof value !== "string") { 45 | throw new TypeError(`Bad '${name}': expected string, found ${value}`); 46 | } 47 | } 48 | 49 | export function checkOptionalString( 50 | name: string, 51 | value: unknown, 52 | ): asserts value is string | undefined { 53 | if (!(value === undefined || typeof value === "string")) { 54 | throw new TypeError( 55 | `Bad '${name}': expected optional string, found ${value}`, 56 | ); 57 | } 58 | } 59 | 60 | export function checkRecord( 61 | name: string, 62 | value: unknown, 63 | ): asserts value is Record { 64 | if (!isRecord(value)) { 65 | throw new TypeError( 66 | `Bad '${name}': expected simple object, found ${value}`, 67 | ); 68 | } 69 | } 70 | 71 | export function checkOptionalBoolean( 72 | name: string, 73 | value: unknown, 74 | ): asserts value is boolean | undefined { 75 | if (!(value === undefined || typeof value === "boolean")) { 76 | throw new TypeError( 77 | `Bad '${name}': expected optional boolean, found ${value}`, 78 | ); 79 | } 80 | } 81 | 82 | export function checkOptionalNumber( 83 | name: string, 84 | value: unknown, 85 | ): asserts value is number | undefined { 86 | if (!(value === undefined || typeof value === "number")) { 87 | throw new TypeError( 88 | `Bad '${name}': expected optional number, found ${value}`, 89 | ); 90 | } 91 | } 92 | 93 | // deno-lint-ignore ban-types 94 | export function checkOptionalFunction( 95 | name: string, 96 | value: unknown, 97 | ): asserts value is Function | undefined { 98 | if (!(value === undefined || typeof value === "function")) { 99 | throw new TypeError( 100 | `Bad '${name}': expected optional function, found ${value}`, 101 | ); 102 | } 103 | } 104 | 105 | export function checkOptionalObject( 106 | name: string, 107 | value: unknown, 108 | ): asserts value is object { 109 | if (!(value === undefined || typeof value === "object")) { 110 | throw new TypeError( 111 | `Bad '${name}': expected optional object, found ${value}`, 112 | ); 113 | } 114 | } 115 | 116 | export function check(name: string, value: unknown, valid: boolean) { 117 | if (!valid) throw new TypeError(`Bad '${name}': ${value}`); 118 | } 119 | -------------------------------------------------------------------------------- /npm/src/proto/messages/com/deno/kv/backup/BackupMutationRange.ts: -------------------------------------------------------------------------------- 1 | // @ts-nocheck 2 | import { 3 | Type as BackupReplicationLogEntry, 4 | encodeJson as encodeJson_1, 5 | decodeJson as decodeJson_1, 6 | encodeBinary as encodeBinary_1, 7 | decodeBinary as decodeBinary_1, 8 | } from "./BackupReplicationLogEntry.ts"; 9 | import { 10 | tsValueToJsonValueFns, 11 | jsonValueToTsValueFns, 12 | } from "../../../../../runtime/json/scalar.ts"; 13 | import { 14 | WireMessage, 15 | WireType, 16 | } from "../../../../../runtime/wire/index.ts"; 17 | import { 18 | default as serialize, 19 | } from "../../../../../runtime/wire/serialize.ts"; 20 | import { 21 | tsValueToWireValueFns, 22 | wireValueToTsValueFns, 23 | } from "../../../../../runtime/wire/scalar.ts"; 24 | import { 25 | default as deserialize, 26 | } from "../../../../../runtime/wire/deserialize.ts"; 27 | 28 | export declare namespace $.com.deno.kv.backup { 29 | export type BackupMutationRange = { 30 | entries: BackupReplicationLogEntry[]; 31 | timestampMs: string; 32 | } 33 | } 34 | 35 | export type Type = $.com.deno.kv.backup.BackupMutationRange; 36 | 37 | export function getDefaultValue(): $.com.deno.kv.backup.BackupMutationRange { 38 | return { 39 | entries: [], 40 | timestampMs: "0", 41 | }; 42 | } 43 | 44 | export function createValue(partialValue: Partial<$.com.deno.kv.backup.BackupMutationRange>): $.com.deno.kv.backup.BackupMutationRange { 45 | return { 46 | ...getDefaultValue(), 47 | ...partialValue, 48 | }; 49 | } 50 | 51 | export function encodeJson(value: $.com.deno.kv.backup.BackupMutationRange): unknown { 52 | const result: any = {}; 53 | result.entries = value.entries.map(value => encodeJson_1(value)); 54 | if (value.timestampMs !== undefined) result.timestampMs = tsValueToJsonValueFns.uint64(value.timestampMs); 55 | return result; 56 | } 57 | 58 | export function decodeJson(value: any): $.com.deno.kv.backup.BackupMutationRange { 59 | const result = getDefaultValue(); 60 | result.entries = value.entries?.map((value: any) => decodeJson_1(value)) ?? []; 61 | if (value.timestampMs !== undefined) result.timestampMs = jsonValueToTsValueFns.uint64(value.timestampMs); 62 | return result; 63 | } 64 | 65 | export function encodeBinary(value: $.com.deno.kv.backup.BackupMutationRange): Uint8Array { 66 | const result: WireMessage = []; 67 | for (const tsValue of value.entries) { 68 | result.push( 69 | [1, { type: WireType.LengthDelimited as const, value: encodeBinary_1(tsValue) }], 70 | ); 71 | } 72 | if (value.timestampMs !== undefined) { 73 | const tsValue = value.timestampMs; 74 | result.push( 75 | [2, tsValueToWireValueFns.uint64(tsValue)], 76 | ); 77 | } 78 | return serialize(result); 79 | } 80 | 81 | export function decodeBinary(binary: Uint8Array): $.com.deno.kv.backup.BackupMutationRange { 82 | const result = getDefaultValue(); 83 | const wireMessage = deserialize(binary); 84 | const wireFields = new Map(wireMessage); 85 | collection: { 86 | const wireValues = wireMessage.filter(([fieldNumber]) => fieldNumber === 1).map(([, wireValue]) => wireValue); 87 | const value = wireValues.map((wireValue) => wireValue.type === WireType.LengthDelimited ? decodeBinary_1(wireValue.value) : undefined).filter(x => x !== undefined); 88 | if (!value.length) break collection; 89 | result.entries = value as any; 90 | } 91 | field: { 92 | const wireValue = wireFields.get(2); 93 | if (wireValue === undefined) break field; 94 | const value = wireValueToTsValueFns.uint64(wireValue); 95 | if (value === undefined) break field; 96 | result.timestampMs = value; 97 | } 98 | return result; 99 | } 100 | -------------------------------------------------------------------------------- /npm/src/proto/messages/com/deno/kv/datapath/WatchOutput.ts: -------------------------------------------------------------------------------- 1 | // @ts-nocheck 2 | import { 3 | Type as SnapshotReadStatus, 4 | name2num, 5 | num2name, 6 | } from "./SnapshotReadStatus.ts"; 7 | import { 8 | Type as WatchKeyOutput, 9 | encodeJson as encodeJson_1, 10 | decodeJson as decodeJson_1, 11 | encodeBinary as encodeBinary_1, 12 | decodeBinary as decodeBinary_1, 13 | } from "./WatchKeyOutput.ts"; 14 | import { 15 | tsValueToJsonValueFns, 16 | jsonValueToTsValueFns, 17 | } from "../../../../../runtime/json/scalar.ts"; 18 | import { 19 | WireMessage, 20 | WireType, 21 | } from "../../../../../runtime/wire/index.ts"; 22 | import { 23 | default as serialize, 24 | } from "../../../../../runtime/wire/serialize.ts"; 25 | import { 26 | default as Long, 27 | } from "../../../../../runtime/Long.ts"; 28 | import { 29 | default as deserialize, 30 | } from "../../../../../runtime/wire/deserialize.ts"; 31 | 32 | export declare namespace $.com.deno.kv.datapath { 33 | export type WatchOutput = { 34 | status: SnapshotReadStatus; 35 | keys: WatchKeyOutput[]; 36 | } 37 | } 38 | 39 | export type Type = $.com.deno.kv.datapath.WatchOutput; 40 | 41 | export function getDefaultValue(): $.com.deno.kv.datapath.WatchOutput { 42 | return { 43 | status: "SR_UNSPECIFIED", 44 | keys: [], 45 | }; 46 | } 47 | 48 | export function createValue(partialValue: Partial<$.com.deno.kv.datapath.WatchOutput>): $.com.deno.kv.datapath.WatchOutput { 49 | return { 50 | ...getDefaultValue(), 51 | ...partialValue, 52 | }; 53 | } 54 | 55 | export function encodeJson(value: $.com.deno.kv.datapath.WatchOutput): unknown { 56 | const result: any = {}; 57 | if (value.status !== undefined) result.status = tsValueToJsonValueFns.enum(value.status); 58 | result.keys = value.keys.map(value => encodeJson_1(value)); 59 | return result; 60 | } 61 | 62 | export function decodeJson(value: any): $.com.deno.kv.datapath.WatchOutput { 63 | const result = getDefaultValue(); 64 | if (value.status !== undefined) result.status = jsonValueToTsValueFns.enum(value.status) as SnapshotReadStatus; 65 | result.keys = value.keys?.map((value: any) => decodeJson_1(value)) ?? []; 66 | return result; 67 | } 68 | 69 | export function encodeBinary(value: $.com.deno.kv.datapath.WatchOutput): Uint8Array { 70 | const result: WireMessage = []; 71 | if (value.status !== undefined) { 72 | const tsValue = value.status; 73 | result.push( 74 | [1, { type: WireType.Varint as const, value: new Long(name2num[tsValue as keyof typeof name2num]) }], 75 | ); 76 | } 77 | for (const tsValue of value.keys) { 78 | result.push( 79 | [2, { type: WireType.LengthDelimited as const, value: encodeBinary_1(tsValue) }], 80 | ); 81 | } 82 | return serialize(result); 83 | } 84 | 85 | export function decodeBinary(binary: Uint8Array): $.com.deno.kv.datapath.WatchOutput { 86 | const result = getDefaultValue(); 87 | const wireMessage = deserialize(binary); 88 | const wireFields = new Map(wireMessage); 89 | field: { 90 | const wireValue = wireFields.get(1); 91 | if (wireValue === undefined) break field; 92 | const value = wireValue.type === WireType.Varint ? num2name[wireValue.value[0] as keyof typeof num2name] : undefined; 93 | if (value === undefined) break field; 94 | result.status = value; 95 | } 96 | collection: { 97 | const wireValues = wireMessage.filter(([fieldNumber]) => fieldNumber === 2).map(([, wireValue]) => wireValue); 98 | const value = wireValues.map((wireValue) => wireValue.type === WireType.LengthDelimited ? decodeBinary_1(wireValue.value) : undefined).filter(x => x !== undefined); 99 | if (!value.length) break collection; 100 | result.keys = value as any; 101 | } 102 | return result; 103 | } 104 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: ci 2 | 3 | on: 4 | push: 5 | branches: ["main"] 6 | tags: ["[0-9]+.[0-9]+.[0-9]+"] 7 | pull_request: 8 | branches: ["main"] 9 | 10 | env: 11 | CARGO_TERM_COLOR: always 12 | RUST_BACKTRACE: full 13 | 14 | jobs: 15 | test: 16 | runs-on: ${{ matrix.os }} 17 | strategy: 18 | matrix: 19 | os: [ubuntu-latest, macos-latest, windows-latest] 20 | fail-fast: false 21 | 22 | permissions: 23 | contents: write 24 | 25 | steps: 26 | - uses: actions/checkout@v3 27 | 28 | - name: Install Rust 29 | uses: dsherret/rust-toolchain-file@v1 30 | 31 | - uses: Swatinem/rust-cache@v2 32 | 33 | - name: Install protoc 34 | uses: arduino/setup-protoc@v2 35 | with: 36 | version: "21.12" 37 | repo-token: ${{ secrets.GITHUB_TOKEN }} 38 | 39 | - name: Check formatting 40 | run: cargo fmt -- --check 41 | 42 | - name: Check linting 43 | run: cargo clippy --release --all-targets --all-features -- -D clippy::all 44 | 45 | - name: Check generated protobuf sources are up to date 46 | if: runner.os == 'Linux' || runner.os == 'macOS' 47 | run: |- 48 | cargo check -p denokv_proto --features='build_protos' 49 | cargo fmt -p denokv_proto 50 | git diff --exit-code 51 | 52 | - name: Build 53 | run: cargo build --release --all-targets --all-features --tests -v 54 | 55 | - name: Test 56 | run: cargo test --release -- --nocapture 57 | 58 | - name: Prepare artifact (Linux) 59 | if: runner.os == 'Linux' 60 | run: |- 61 | cd target/release 62 | zip -r denokv-x86_64-unknown-linux-gnu.zip denokv 63 | 64 | - name: Prepare artifact (macOS) 65 | if: runner.os == 'macOS' 66 | run: |- 67 | cd target/release 68 | zip -r denokv-x86_64-apple-darwin.zip denokv 69 | 70 | - name: Prepare artifact (Windows) 71 | if: runner.os == 'Windows' 72 | run: |- 73 | Compress-Archive -CompressionLevel Optimal -Force -Path target/release/denokv.exe -DestinationPath target/release/denokv-x86_64-pc-windows-msvc.zip 74 | 75 | - name: Upload artifact (Linux) 76 | if: runner.os == 'Linux' 77 | uses: actions/upload-artifact@v4 78 | with: 79 | name: denokv-x86_64-unknown-linux-gnu.zip 80 | path: target/release/denokv-x86_64-unknown-linux-gnu.zip 81 | 82 | - name: Upload artifact (macOS) 83 | if: runner.os == 'macOS' 84 | uses: actions/upload-artifact@v4 85 | with: 86 | name: denokv-x86_64-apple-darwin.zip 87 | path: target/release/denokv-x86_64-apple-darwin.zip 88 | 89 | - name: Upload artifact (Windows) 90 | if: runner.os == 'Windows' 91 | uses: actions/upload-artifact@v4 92 | with: 93 | name: denokv-x86_64-pc-windows-msvc.zip 94 | path: target/release/denokv-x86_64-pc-windows-msvc.zip 95 | 96 | - name: Upload release to GitHub 97 | uses: softprops/action-gh-release@v0.1.15 98 | if: github.repository == 'denoland/denokv' && startsWith(github.ref, 'refs/tags/') 99 | env: 100 | GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" 101 | with: 102 | files: |- 103 | target/release/denokv-x86_64-pc-windows-msvc.zip 104 | target/release/denokv-x86_64-unknown-linux-gnu.zip 105 | target/release/denokv-x86_64-apple-darwin.zip 106 | draft: true 107 | 108 | - name: Publish to crates.io 109 | if: runner.os == 'Linux' && github.repository == 'denoland/denokv' && startsWith(github.ref, 'refs/tags/') 110 | env: 111 | CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} 112 | run: |- 113 | git reset --hard 114 | cargo publish -vv -p denokv_proto 115 | cargo publish -vv -p denokv_sqlite 116 | cargo publish -vv -p denokv_remote 117 | cargo publish -vv -p denokv_timemachine 118 | cargo publish -vv -p denokv 119 | -------------------------------------------------------------------------------- /npm/src/proto/runtime/rpc.ts: -------------------------------------------------------------------------------- 1 | import { createEventBuffer } from "./async/event-buffer.ts"; 2 | import { defer } from "./async/observer.ts"; 3 | 4 | export type Method< 5 | TMetadata = any, 6 | THeader = any, 7 | TTrailer = any, 8 | TServiceName extends string = string, 9 | TMethodName extends string = string, 10 | TRequestStream extends boolean = boolean, 11 | TResponseStream extends boolean = boolean, 12 | TReq = any, 13 | TRes = any, 14 | > = [ 15 | MethodDescriptor< 16 | TReq, 17 | TRes, 18 | TMethodName, 19 | TServiceName, 20 | TRequestStream, 21 | TResponseStream 22 | >, 23 | MethodImpl, 24 | ]; 25 | 26 | export type RpcClientImpl = < 27 | TReq, 28 | TRes, 29 | >( 30 | methodDescriptor: MethodDescriptor, 31 | ) => MethodImpl; 32 | 33 | export interface MethodDescriptor< 34 | TReq, 35 | TRes, 36 | TMethodName extends string = string, 37 | TServiceName extends string = string, 38 | TRequestStream extends boolean = boolean, 39 | TResponseStream extends boolean = boolean, 40 | > { 41 | methodName: TMethodName; 42 | service: { serviceName: TServiceName }; 43 | requestStream: TRequestStream; 44 | responseStream: TResponseStream; 45 | requestType: { 46 | serializeBinary: (value: TReq) => Uint8Array; 47 | deserializeBinary: (value: Uint8Array) => TReq; 48 | serializeJson: (value: TReq) => string; 49 | }; 50 | responseType: { 51 | serializeBinary: (value: TRes) => Uint8Array; 52 | deserializeBinary: (value: Uint8Array) => TRes; 53 | serializeJson: (value: TRes) => string; 54 | }; 55 | } 56 | 57 | type ThenArg = T extends Promise ? U : T; 58 | export type RpcReturnType = ( 59 | Promise : [ThenArg, ...TResArgs]> 60 | ); 61 | 62 | export interface MethodImpl< 63 | TReq, 64 | TRes, 65 | TMetadata = any, 66 | THeader = any, 67 | TTrailer = any, 68 | > { 69 | ( 70 | req: AsyncGenerator, 71 | metadata?: TMetadata, 72 | ): [AsyncGenerator, Promise, Promise]; 73 | } 74 | 75 | export interface MethodImplHandlerReq { 76 | metadata?: TMetadata; 77 | messages: AsyncGenerator; 78 | drainEnd: Promise; 79 | } 80 | export interface MethodImplHandlerRes { 81 | header(value: THeader): void; 82 | send(value: TRes): void; 83 | end(value: TTrailer): void; 84 | } 85 | export interface MethodImplHandler { 86 | ( 87 | req: MethodImplHandlerReq, 88 | res: MethodImplHandlerRes, 89 | ): void; 90 | } 91 | export function getMethodImpl< 92 | TReq, 93 | TRes, 94 | TMetadata, 95 | THeader, 96 | TTrailer, 97 | >( 98 | handler: MethodImplHandler, 99 | ): MethodImpl { 100 | return (messages: AsyncGenerator, metadata?: TMetadata) => { 101 | const headerPromise = defer(); 102 | const trailerPromise = defer(); 103 | const drainEnd = defer(); 104 | const eventBuffer = createEventBuffer({ 105 | onDrainEnd: drainEnd.resolve, 106 | }); 107 | const header = headerPromise.resolve; 108 | const send = eventBuffer.push; 109 | const end = (value: TTrailer) => { 110 | eventBuffer.finish(); 111 | trailerPromise.resolve(value); 112 | }; 113 | handler({ metadata, messages, drainEnd }, { header, send, end }); 114 | return [eventBuffer.drain(), headerPromise, trailerPromise]; 115 | }; 116 | } 117 | 118 | export function createServerImplBuilder() { 119 | const buffer = createEventBuffer>(); 120 | return { 121 | register( 122 | methodDescriptor: MethodDescriptor, 123 | handler: MethodImplHandler, 124 | ) { 125 | buffer.push([methodDescriptor, getMethodImpl(handler)]); 126 | }, 127 | finish: buffer.finish, 128 | drain: buffer.drain, 129 | }; 130 | } 131 | -------------------------------------------------------------------------------- /npm/src/proto/messages/com/deno/kv/datapath/AtomicWriteOutput.ts: -------------------------------------------------------------------------------- 1 | // @ts-nocheck 2 | import { 3 | Type as AtomicWriteStatus, 4 | name2num, 5 | num2name, 6 | } from "./AtomicWriteStatus.ts"; 7 | import { 8 | tsValueToJsonValueFns, 9 | jsonValueToTsValueFns, 10 | } from "../../../../../runtime/json/scalar.ts"; 11 | import { 12 | WireMessage, 13 | WireType, 14 | } from "../../../../../runtime/wire/index.ts"; 15 | import { 16 | default as serialize, 17 | } from "../../../../../runtime/wire/serialize.ts"; 18 | import { 19 | default as Long, 20 | } from "../../../../../runtime/Long.ts"; 21 | import { 22 | tsValueToWireValueFns, 23 | wireValueToTsValueFns, 24 | unpackFns, 25 | } from "../../../../../runtime/wire/scalar.ts"; 26 | import { 27 | default as deserialize, 28 | } from "../../../../../runtime/wire/deserialize.ts"; 29 | 30 | export declare namespace $.com.deno.kv.datapath { 31 | export type AtomicWriteOutput = { 32 | status: AtomicWriteStatus; 33 | versionstamp: Uint8Array; 34 | failedChecks: number[]; 35 | } 36 | } 37 | 38 | export type Type = $.com.deno.kv.datapath.AtomicWriteOutput; 39 | 40 | export function getDefaultValue(): $.com.deno.kv.datapath.AtomicWriteOutput { 41 | return { 42 | status: "AW_UNSPECIFIED", 43 | versionstamp: new Uint8Array(), 44 | failedChecks: [], 45 | }; 46 | } 47 | 48 | export function createValue(partialValue: Partial<$.com.deno.kv.datapath.AtomicWriteOutput>): $.com.deno.kv.datapath.AtomicWriteOutput { 49 | return { 50 | ...getDefaultValue(), 51 | ...partialValue, 52 | }; 53 | } 54 | 55 | export function encodeJson(value: $.com.deno.kv.datapath.AtomicWriteOutput): unknown { 56 | const result: any = {}; 57 | if (value.status !== undefined) result.status = tsValueToJsonValueFns.enum(value.status); 58 | if (value.versionstamp !== undefined) result.versionstamp = tsValueToJsonValueFns.bytes(value.versionstamp); 59 | result.failedChecks = value.failedChecks.map(value => tsValueToJsonValueFns.uint32(value)); 60 | return result; 61 | } 62 | 63 | export function decodeJson(value: any): $.com.deno.kv.datapath.AtomicWriteOutput { 64 | const result = getDefaultValue(); 65 | if (value.status !== undefined) result.status = jsonValueToTsValueFns.enum(value.status) as AtomicWriteStatus; 66 | if (value.versionstamp !== undefined) result.versionstamp = jsonValueToTsValueFns.bytes(value.versionstamp); 67 | result.failedChecks = value.failedChecks?.map((value: any) => jsonValueToTsValueFns.uint32(value)) ?? []; 68 | return result; 69 | } 70 | 71 | export function encodeBinary(value: $.com.deno.kv.datapath.AtomicWriteOutput): Uint8Array { 72 | const result: WireMessage = []; 73 | if (value.status !== undefined) { 74 | const tsValue = value.status; 75 | result.push( 76 | [1, { type: WireType.Varint as const, value: new Long(name2num[tsValue as keyof typeof name2num]) }], 77 | ); 78 | } 79 | if (value.versionstamp !== undefined) { 80 | const tsValue = value.versionstamp; 81 | result.push( 82 | [2, tsValueToWireValueFns.bytes(tsValue)], 83 | ); 84 | } 85 | for (const tsValue of value.failedChecks) { 86 | result.push( 87 | [4, tsValueToWireValueFns.uint32(tsValue)], 88 | ); 89 | } 90 | return serialize(result); 91 | } 92 | 93 | export function decodeBinary(binary: Uint8Array): $.com.deno.kv.datapath.AtomicWriteOutput { 94 | const result = getDefaultValue(); 95 | const wireMessage = deserialize(binary); 96 | const wireFields = new Map(wireMessage); 97 | field: { 98 | const wireValue = wireFields.get(1); 99 | if (wireValue === undefined) break field; 100 | const value = wireValue.type === WireType.Varint ? num2name[wireValue.value[0] as keyof typeof num2name] : undefined; 101 | if (value === undefined) break field; 102 | result.status = value; 103 | } 104 | field: { 105 | const wireValue = wireFields.get(2); 106 | if (wireValue === undefined) break field; 107 | const value = wireValueToTsValueFns.bytes(wireValue); 108 | if (value === undefined) break field; 109 | result.versionstamp = value; 110 | } 111 | collection: { 112 | const wireValues = wireMessage.filter(([fieldNumber]) => fieldNumber === 4).map(([, wireValue]) => wireValue); 113 | const value = Array.from(unpackFns.uint32(wireValues)); 114 | if (!value.length) break collection; 115 | result.failedChecks = value as any; 116 | } 117 | return result; 118 | } 119 | -------------------------------------------------------------------------------- /npm/src/scripts/generate_napi_index.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2023 the Deno authors. All rights reserved. MIT license. 2 | 3 | import { NAPI_FUNCTIONS } from "../napi_based.ts"; 4 | 5 | export function generateNapiIndex( 6 | { napiPackageName, napiArtifactName }: { 7 | napiPackageName: string; 8 | napiArtifactName: string; 9 | }, 10 | ) { 11 | return template(NAPI_FUNCTIONS, napiPackageName, napiArtifactName); 12 | } 13 | 14 | export const template = ( 15 | napiFunctions: string[], 16 | napiPackageName: string, 17 | napiArtifactName: string, 18 | ) => ` 19 | /* tslint:disable */ 20 | /* eslint-disable */ 21 | /* prettier-ignore */ 22 | 23 | /* auto-generated by generate_napi_index.ts, based on auto-generated index.js in NAPI-RS */ 24 | 25 | const { existsSync, readFileSync } = require('fs') 26 | const { join } = require('path') 27 | 28 | const { platform, arch } = process 29 | 30 | let nativeBinding = null 31 | let localFileExisted = false 32 | let loadError = null 33 | 34 | function isMusl() { 35 | // For Node 10 36 | if (!process.report || typeof process.report.getReport !== 'function') { 37 | try { 38 | const lddPath = require('child_process').execSync('which ldd').toString().trim() 39 | return readFileSync(lddPath, 'utf8').includes('musl') 40 | } catch (e) { 41 | return true 42 | } 43 | } else { 44 | const { glibcVersionRuntime } = process.report.getReport().header 45 | return !glibcVersionRuntime 46 | } 47 | } 48 | 49 | switch (platform) { 50 | case 'win32': 51 | switch (arch) { 52 | case 'x64': 53 | localFileExisted = existsSync(join(__dirname, '${napiArtifactName}.win32-x64-msvc.node')) 54 | try { 55 | if (localFileExisted) { 56 | nativeBinding = require('./${napiArtifactName}.win32-x64-msvc.node') 57 | } else { 58 | nativeBinding = require('${napiPackageName}-win32-x64-msvc') 59 | } 60 | } catch (e) { 61 | loadError = e 62 | } 63 | break 64 | default: 65 | throw new Error(\`Unsupported architecture on Windows: \${arch}\`) 66 | } 67 | break 68 | case 'darwin': 69 | switch (arch) { 70 | case 'x64': 71 | localFileExisted = existsSync(join(__dirname, '${napiArtifactName}.darwin-x64.node')) 72 | try { 73 | if (localFileExisted) { 74 | nativeBinding = require('./${napiArtifactName}.darwin-x64.node') 75 | } else { 76 | nativeBinding = require('${napiPackageName}-darwin-x64') 77 | } 78 | } catch (e) { 79 | loadError = e 80 | } 81 | break 82 | case 'arm64': 83 | localFileExisted = existsSync(join(__dirname, '${napiArtifactName}.darwin-arm64.node')) 84 | try { 85 | if (localFileExisted) { 86 | nativeBinding = require('./${napiArtifactName}.darwin-arm64.node') 87 | } else { 88 | nativeBinding = require('${napiPackageName}-darwin-arm64') 89 | } 90 | } catch (e) { 91 | loadError = e 92 | } 93 | break 94 | default: 95 | throw new Error(\`Unsupported architecture on macOS: \${arch}\`) 96 | } 97 | break 98 | case 'linux': 99 | switch (arch) { 100 | case 'x64': 101 | if (isMusl()) { 102 | throw new Error(\`Unsupported architecture on Linux: \${arch} (musl)\`) 103 | } else { 104 | localFileExisted = existsSync(join(__dirname, '${napiArtifactName}.linux-x64-gnu.node')) 105 | try { 106 | if (localFileExisted) { 107 | nativeBinding = require('./${napiArtifactName}.linux-x64-gnu.node') 108 | } else { 109 | nativeBinding = require('${napiPackageName}-linux-x64-gnu') 110 | } 111 | } catch (e) { 112 | loadError = e 113 | } 114 | } 115 | break 116 | default: 117 | throw new Error(\`Unsupported architecture on Linux: \${arch}\`) 118 | } 119 | break 120 | default: 121 | throw new Error(\`Unsupported OS: \${platform}, architecture: \${arch}\`) 122 | } 123 | 124 | if (!nativeBinding) { 125 | if (loadError) { 126 | throw loadError 127 | } 128 | throw new Error(\`Failed to load native binding\`) 129 | } 130 | 131 | const { ${napiFunctions.join(", ")} } = nativeBinding 132 | 133 | ${napiFunctions.map((v) => `module.exports.${v} = ${v}`).join("\n")} 134 | `; 135 | -------------------------------------------------------------------------------- /npm/src/proto/messages/com/deno/kv/datapath/ReadRange.ts: -------------------------------------------------------------------------------- 1 | // @ts-nocheck 2 | import { 3 | tsValueToJsonValueFns, 4 | jsonValueToTsValueFns, 5 | } from "../../../../../runtime/json/scalar.ts"; 6 | import { 7 | WireMessage, 8 | } from "../../../../../runtime/wire/index.ts"; 9 | import { 10 | default as serialize, 11 | } from "../../../../../runtime/wire/serialize.ts"; 12 | import { 13 | tsValueToWireValueFns, 14 | wireValueToTsValueFns, 15 | } from "../../../../../runtime/wire/scalar.ts"; 16 | import { 17 | default as deserialize, 18 | } from "../../../../../runtime/wire/deserialize.ts"; 19 | 20 | export declare namespace $.com.deno.kv.datapath { 21 | export type ReadRange = { 22 | start: Uint8Array; 23 | end: Uint8Array; 24 | limit: number; 25 | reverse: boolean; 26 | } 27 | } 28 | 29 | export type Type = $.com.deno.kv.datapath.ReadRange; 30 | 31 | export function getDefaultValue(): $.com.deno.kv.datapath.ReadRange { 32 | return { 33 | start: new Uint8Array(), 34 | end: new Uint8Array(), 35 | limit: 0, 36 | reverse: false, 37 | }; 38 | } 39 | 40 | export function createValue(partialValue: Partial<$.com.deno.kv.datapath.ReadRange>): $.com.deno.kv.datapath.ReadRange { 41 | return { 42 | ...getDefaultValue(), 43 | ...partialValue, 44 | }; 45 | } 46 | 47 | export function encodeJson(value: $.com.deno.kv.datapath.ReadRange): unknown { 48 | const result: any = {}; 49 | if (value.start !== undefined) result.start = tsValueToJsonValueFns.bytes(value.start); 50 | if (value.end !== undefined) result.end = tsValueToJsonValueFns.bytes(value.end); 51 | if (value.limit !== undefined) result.limit = tsValueToJsonValueFns.int32(value.limit); 52 | if (value.reverse !== undefined) result.reverse = tsValueToJsonValueFns.bool(value.reverse); 53 | return result; 54 | } 55 | 56 | export function decodeJson(value: any): $.com.deno.kv.datapath.ReadRange { 57 | const result = getDefaultValue(); 58 | if (value.start !== undefined) result.start = jsonValueToTsValueFns.bytes(value.start); 59 | if (value.end !== undefined) result.end = jsonValueToTsValueFns.bytes(value.end); 60 | if (value.limit !== undefined) result.limit = jsonValueToTsValueFns.int32(value.limit); 61 | if (value.reverse !== undefined) result.reverse = jsonValueToTsValueFns.bool(value.reverse); 62 | return result; 63 | } 64 | 65 | export function encodeBinary(value: $.com.deno.kv.datapath.ReadRange): Uint8Array { 66 | const result: WireMessage = []; 67 | if (value.start !== undefined) { 68 | const tsValue = value.start; 69 | result.push( 70 | [1, tsValueToWireValueFns.bytes(tsValue)], 71 | ); 72 | } 73 | if (value.end !== undefined) { 74 | const tsValue = value.end; 75 | result.push( 76 | [2, tsValueToWireValueFns.bytes(tsValue)], 77 | ); 78 | } 79 | if (value.limit !== undefined) { 80 | const tsValue = value.limit; 81 | result.push( 82 | [3, tsValueToWireValueFns.int32(tsValue)], 83 | ); 84 | } 85 | if (value.reverse !== undefined) { 86 | const tsValue = value.reverse; 87 | result.push( 88 | [4, tsValueToWireValueFns.bool(tsValue)], 89 | ); 90 | } 91 | return serialize(result); 92 | } 93 | 94 | export function decodeBinary(binary: Uint8Array): $.com.deno.kv.datapath.ReadRange { 95 | const result = getDefaultValue(); 96 | const wireMessage = deserialize(binary); 97 | const wireFields = new Map(wireMessage); 98 | field: { 99 | const wireValue = wireFields.get(1); 100 | if (wireValue === undefined) break field; 101 | const value = wireValueToTsValueFns.bytes(wireValue); 102 | if (value === undefined) break field; 103 | result.start = value; 104 | } 105 | field: { 106 | const wireValue = wireFields.get(2); 107 | if (wireValue === undefined) break field; 108 | const value = wireValueToTsValueFns.bytes(wireValue); 109 | if (value === undefined) break field; 110 | result.end = value; 111 | } 112 | field: { 113 | const wireValue = wireFields.get(3); 114 | if (wireValue === undefined) break field; 115 | const value = wireValueToTsValueFns.int32(wireValue); 116 | if (value === undefined) break field; 117 | result.limit = value; 118 | } 119 | field: { 120 | const wireValue = wireFields.get(4); 121 | if (wireValue === undefined) break field; 122 | const value = wireValueToTsValueFns.bool(wireValue); 123 | if (value === undefined) break field; 124 | result.reverse = value; 125 | } 126 | return result; 127 | } 128 | -------------------------------------------------------------------------------- /npm/src/proto/messages/com/deno/kv/datapath/AtomicWrite.ts: -------------------------------------------------------------------------------- 1 | // @ts-nocheck 2 | import { 3 | Type as Check, 4 | encodeJson as encodeJson_1, 5 | decodeJson as decodeJson_1, 6 | encodeBinary as encodeBinary_1, 7 | decodeBinary as decodeBinary_1, 8 | } from "./Check.ts"; 9 | import { 10 | Type as Mutation, 11 | encodeJson as encodeJson_2, 12 | decodeJson as decodeJson_2, 13 | encodeBinary as encodeBinary_2, 14 | decodeBinary as decodeBinary_2, 15 | } from "./Mutation.ts"; 16 | import { 17 | Type as Enqueue, 18 | encodeJson as encodeJson_3, 19 | decodeJson as decodeJson_3, 20 | encodeBinary as encodeBinary_3, 21 | decodeBinary as decodeBinary_3, 22 | } from "./Enqueue.ts"; 23 | import { 24 | jsonValueToTsValueFns, 25 | } from "../../../../../runtime/json/scalar.ts"; 26 | import { 27 | WireMessage, 28 | WireType, 29 | } from "../../../../../runtime/wire/index.ts"; 30 | import { 31 | default as serialize, 32 | } from "../../../../../runtime/wire/serialize.ts"; 33 | import { 34 | default as deserialize, 35 | } from "../../../../../runtime/wire/deserialize.ts"; 36 | 37 | export declare namespace $.com.deno.kv.datapath { 38 | export type AtomicWrite = { 39 | checks: Check[]; 40 | mutations: Mutation[]; 41 | enqueues: Enqueue[]; 42 | } 43 | } 44 | 45 | export type Type = $.com.deno.kv.datapath.AtomicWrite; 46 | 47 | export function getDefaultValue(): $.com.deno.kv.datapath.AtomicWrite { 48 | return { 49 | checks: [], 50 | mutations: [], 51 | enqueues: [], 52 | }; 53 | } 54 | 55 | export function createValue(partialValue: Partial<$.com.deno.kv.datapath.AtomicWrite>): $.com.deno.kv.datapath.AtomicWrite { 56 | return { 57 | ...getDefaultValue(), 58 | ...partialValue, 59 | }; 60 | } 61 | 62 | export function encodeJson(value: $.com.deno.kv.datapath.AtomicWrite): unknown { 63 | const result: any = {}; 64 | result.checks = value.checks.map(value => encodeJson_1(value)); 65 | result.mutations = value.mutations.map(value => encodeJson_2(value)); 66 | result.enqueues = value.enqueues.map(value => encodeJson_3(value)); 67 | return result; 68 | } 69 | 70 | export function decodeJson(value: any): $.com.deno.kv.datapath.AtomicWrite { 71 | const result = getDefaultValue(); 72 | result.checks = value.checks?.map((value: any) => decodeJson_1(value)) ?? []; 73 | result.mutations = value.mutations?.map((value: any) => decodeJson_2(value)) ?? []; 74 | result.enqueues = value.enqueues?.map((value: any) => decodeJson_3(value)) ?? []; 75 | return result; 76 | } 77 | 78 | export function encodeBinary(value: $.com.deno.kv.datapath.AtomicWrite): Uint8Array { 79 | const result: WireMessage = []; 80 | for (const tsValue of value.checks) { 81 | result.push( 82 | [1, { type: WireType.LengthDelimited as const, value: encodeBinary_1(tsValue) }], 83 | ); 84 | } 85 | for (const tsValue of value.mutations) { 86 | result.push( 87 | [2, { type: WireType.LengthDelimited as const, value: encodeBinary_2(tsValue) }], 88 | ); 89 | } 90 | for (const tsValue of value.enqueues) { 91 | result.push( 92 | [3, { type: WireType.LengthDelimited as const, value: encodeBinary_3(tsValue) }], 93 | ); 94 | } 95 | return serialize(result); 96 | } 97 | 98 | export function decodeBinary(binary: Uint8Array): $.com.deno.kv.datapath.AtomicWrite { 99 | const result = getDefaultValue(); 100 | const wireMessage = deserialize(binary); 101 | const wireFields = new Map(wireMessage); 102 | collection: { 103 | const wireValues = wireMessage.filter(([fieldNumber]) => fieldNumber === 1).map(([, wireValue]) => wireValue); 104 | const value = wireValues.map((wireValue) => wireValue.type === WireType.LengthDelimited ? decodeBinary_1(wireValue.value) : undefined).filter(x => x !== undefined); 105 | if (!value.length) break collection; 106 | result.checks = value as any; 107 | } 108 | collection: { 109 | const wireValues = wireMessage.filter(([fieldNumber]) => fieldNumber === 2).map(([, wireValue]) => wireValue); 110 | const value = wireValues.map((wireValue) => wireValue.type === WireType.LengthDelimited ? decodeBinary_2(wireValue.value) : undefined).filter(x => x !== undefined); 111 | if (!value.length) break collection; 112 | result.mutations = value as any; 113 | } 114 | collection: { 115 | const wireValues = wireMessage.filter(([fieldNumber]) => fieldNumber === 3).map(([, wireValue]) => wireValue); 116 | const value = wireValues.map((wireValue) => wireValue.type === WireType.LengthDelimited ? decodeBinary_3(wireValue.value) : undefined).filter(x => x !== undefined); 117 | if (!value.length) break collection; 118 | result.enqueues = value as any; 119 | } 120 | return result; 121 | } 122 | -------------------------------------------------------------------------------- /npm/src/proto/messages/com/deno/kv/datapath/Enqueue.ts: -------------------------------------------------------------------------------- 1 | // @ts-nocheck 2 | import { 3 | tsValueToJsonValueFns, 4 | jsonValueToTsValueFns, 5 | } from "../../../../../runtime/json/scalar.ts"; 6 | import { 7 | WireMessage, 8 | } from "../../../../../runtime/wire/index.ts"; 9 | import { 10 | default as serialize, 11 | } from "../../../../../runtime/wire/serialize.ts"; 12 | import { 13 | tsValueToWireValueFns, 14 | wireValueToTsValueFns, 15 | unpackFns, 16 | } from "../../../../../runtime/wire/scalar.ts"; 17 | import { 18 | default as deserialize, 19 | } from "../../../../../runtime/wire/deserialize.ts"; 20 | 21 | export declare namespace $.com.deno.kv.datapath { 22 | export type Enqueue = { 23 | payload: Uint8Array; 24 | deadlineMs: string; 25 | keysIfUndelivered: Uint8Array[]; 26 | backoffSchedule: number[]; 27 | } 28 | } 29 | 30 | export type Type = $.com.deno.kv.datapath.Enqueue; 31 | 32 | export function getDefaultValue(): $.com.deno.kv.datapath.Enqueue { 33 | return { 34 | payload: new Uint8Array(), 35 | deadlineMs: "0", 36 | keysIfUndelivered: [], 37 | backoffSchedule: [], 38 | }; 39 | } 40 | 41 | export function createValue(partialValue: Partial<$.com.deno.kv.datapath.Enqueue>): $.com.deno.kv.datapath.Enqueue { 42 | return { 43 | ...getDefaultValue(), 44 | ...partialValue, 45 | }; 46 | } 47 | 48 | export function encodeJson(value: $.com.deno.kv.datapath.Enqueue): unknown { 49 | const result: any = {}; 50 | if (value.payload !== undefined) result.payload = tsValueToJsonValueFns.bytes(value.payload); 51 | if (value.deadlineMs !== undefined) result.deadlineMs = tsValueToJsonValueFns.int64(value.deadlineMs); 52 | result.keysIfUndelivered = value.keysIfUndelivered.map(value => tsValueToJsonValueFns.bytes(value)); 53 | result.backoffSchedule = value.backoffSchedule.map(value => tsValueToJsonValueFns.uint32(value)); 54 | return result; 55 | } 56 | 57 | export function decodeJson(value: any): $.com.deno.kv.datapath.Enqueue { 58 | const result = getDefaultValue(); 59 | if (value.payload !== undefined) result.payload = jsonValueToTsValueFns.bytes(value.payload); 60 | if (value.deadlineMs !== undefined) result.deadlineMs = jsonValueToTsValueFns.int64(value.deadlineMs); 61 | result.keysIfUndelivered = value.keysIfUndelivered?.map((value: any) => jsonValueToTsValueFns.bytes(value)) ?? []; 62 | result.backoffSchedule = value.backoffSchedule?.map((value: any) => jsonValueToTsValueFns.uint32(value)) ?? []; 63 | return result; 64 | } 65 | 66 | export function encodeBinary(value: $.com.deno.kv.datapath.Enqueue): Uint8Array { 67 | const result: WireMessage = []; 68 | if (value.payload !== undefined) { 69 | const tsValue = value.payload; 70 | result.push( 71 | [1, tsValueToWireValueFns.bytes(tsValue)], 72 | ); 73 | } 74 | if (value.deadlineMs !== undefined) { 75 | const tsValue = value.deadlineMs; 76 | result.push( 77 | [2, tsValueToWireValueFns.int64(tsValue)], 78 | ); 79 | } 80 | for (const tsValue of value.keysIfUndelivered) { 81 | result.push( 82 | [3, tsValueToWireValueFns.bytes(tsValue)], 83 | ); 84 | } 85 | for (const tsValue of value.backoffSchedule) { 86 | result.push( 87 | [4, tsValueToWireValueFns.uint32(tsValue)], 88 | ); 89 | } 90 | return serialize(result); 91 | } 92 | 93 | export function decodeBinary(binary: Uint8Array): $.com.deno.kv.datapath.Enqueue { 94 | const result = getDefaultValue(); 95 | const wireMessage = deserialize(binary); 96 | const wireFields = new Map(wireMessage); 97 | field: { 98 | const wireValue = wireFields.get(1); 99 | if (wireValue === undefined) break field; 100 | const value = wireValueToTsValueFns.bytes(wireValue); 101 | if (value === undefined) break field; 102 | result.payload = value; 103 | } 104 | field: { 105 | const wireValue = wireFields.get(2); 106 | if (wireValue === undefined) break field; 107 | const value = wireValueToTsValueFns.int64(wireValue); 108 | if (value === undefined) break field; 109 | result.deadlineMs = value; 110 | } 111 | collection: { 112 | const wireValues = wireMessage.filter(([fieldNumber]) => fieldNumber === 3).map(([, wireValue]) => wireValue); 113 | const value = wireValues.map((wireValue) => wireValueToTsValueFns.bytes(wireValue)).filter(x => x !== undefined); 114 | if (!value.length) break collection; 115 | result.keysIfUndelivered = value as any; 116 | } 117 | collection: { 118 | const wireValues = wireMessage.filter(([fieldNumber]) => fieldNumber === 4).map(([, wireValue]) => wireValue); 119 | const value = Array.from(unpackFns.uint32(wireValues)); 120 | if (!value.length) break collection; 121 | result.backoffSchedule = value as any; 122 | } 123 | return result; 124 | } 125 | -------------------------------------------------------------------------------- /npm/src/e2e_test.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2023 the Deno authors. All rights reserved. MIT license. 2 | 3 | import { chunk } from "https://deno.land/std@0.208.0/collections/chunk.ts"; 4 | import { parseArgs as parseFlags } from "https://deno.land/std@0.208.0/cli/parse_args.ts"; 5 | import { 6 | compare as compareVersion, 7 | parse as parseVersion, 8 | } from "https://deno.land/std@0.208.0/semver/mod.ts"; 9 | import { makeRemoteService } from "./remote.ts"; 10 | import { makeNativeService } from "./native.ts"; 11 | import { endToEnd } from "./e2e.ts"; 12 | import { makeInMemoryService } from "./in_memory.ts"; 13 | import { KvKey, KvService } from "./kv_types.ts"; 14 | 15 | const flags = parseFlags(Deno.args); 16 | const debug = !!flags.debug; 17 | 18 | Deno.test({ 19 | name: "e2e-userland-in-memory", 20 | only: false, 21 | fn: async () => { 22 | await endToEnd(makeInMemoryService({ debug, maxQueueAttempts: 1 }), { 23 | type: "userland", 24 | subtype: "in-memory", 25 | path: ":memory:", 26 | }); 27 | }, 28 | }); 29 | 30 | Deno.test({ 31 | name: "e2e-deno-memory", 32 | only: false, 33 | fn: async () => { 34 | await endToEnd(makeNativeService(), { type: "deno", path: ":memory:" }); 35 | }, 36 | }); 37 | 38 | Deno.test({ 39 | name: "e2e-deno-disk", 40 | only: false, 41 | fn: async () => { 42 | const path = await Deno.makeTempFile({ 43 | prefix: "userland-e2e-tests-", 44 | suffix: ".db", 45 | }); 46 | try { 47 | await endToEnd(makeNativeService(), { type: "deno", path }); 48 | } finally { 49 | await Deno.remove(path); 50 | } 51 | }, 52 | }); 53 | 54 | // 55 | 56 | async function clear(service: KvService, path: string) { 57 | const kv = await service.openKv(path); 58 | const keys: KvKey[] = []; 59 | for await (const { key } of kv.list({ prefix: [] })) { 60 | keys.push(key); 61 | } 62 | for (const batch of chunk(keys, 1000)) { 63 | let tx = kv.atomic(); 64 | for (const key of batch) { 65 | tx = tx.delete(key); 66 | } 67 | await tx.commit(); 68 | } 69 | kv.close(); 70 | } 71 | 72 | const denoKvAccessToken = (await Deno.permissions.query({ 73 | name: "env", 74 | variable: "DENO_KV_ACCESS_TOKEN", 75 | })).state === "granted" && Deno.env.get("DENO_KV_ACCESS_TOKEN"); 76 | const denoKvDatabaseId = (await Deno.permissions.query({ 77 | name: "env", 78 | variable: "DENO_KV_DATABASE_ID", 79 | })).state === "granted" && Deno.env.get("DENO_KV_DATABASE_ID"); 80 | 81 | Deno.test({ 82 | name: "e2e-deno-remote", 83 | only: false, 84 | ignore: !(typeof denoKvAccessToken === "string" && denoKvDatabaseId), 85 | fn: async () => { 86 | const path = `https://api.deno.com/databases/${denoKvDatabaseId}/connect`; 87 | const service = makeNativeService(); 88 | await clear(service, path); 89 | try { 90 | await endToEnd(service, { type: "deno", path }); 91 | } finally { 92 | await clear(service, path); 93 | } 94 | }, 95 | }); 96 | 97 | Deno.test({ 98 | name: "e2e-userland-remote", 99 | only: false, 100 | ignore: !(typeof denoKvAccessToken === "string" && denoKvDatabaseId), 101 | fn: async () => { 102 | const path = `https://api.deno.com/databases/${denoKvDatabaseId}/connect`; 103 | const service = makeRemoteService({ 104 | accessToken: denoKvAccessToken as string, 105 | debug, 106 | }); 107 | await clear(service, path); 108 | try { 109 | await endToEnd(service, { type: "userland", subtype: "remote", path }); 110 | } finally { 111 | await clear(service, path); 112 | } 113 | }, 114 | }); 115 | 116 | const localKvUrl = 117 | (await Deno.permissions.query({ name: "env", variable: "LOCAL_KV_URL" })) 118 | .state === "granted" && Deno.env.get("LOCAL_KV_URL"); 119 | 120 | Deno.test({ 121 | only: false, 122 | ignore: 123 | compareVersion(parseVersion(Deno.version.deno), parseVersion("1.38.0")) < 124 | 0 || !(typeof denoKvAccessToken === "string" && localKvUrl), 125 | name: "e2e-deno-localkv", 126 | fn: async () => { 127 | const path = localKvUrl as string; 128 | const service = makeNativeService(); 129 | await clear(service, path); 130 | try { 131 | await endToEnd(service, { type: "deno", path }); 132 | } finally { 133 | await clear(service, path); 134 | } 135 | }, 136 | }); 137 | 138 | Deno.test({ 139 | only: false, 140 | ignore: !(typeof denoKvAccessToken === "string" && localKvUrl), 141 | name: "e2e-userland-localkv", 142 | fn: async () => { 143 | const path = localKvUrl as string; 144 | const service = makeRemoteService({ 145 | accessToken: denoKvAccessToken as string, 146 | debug, 147 | maxRetries: 0, 148 | }); 149 | await clear(service, path); 150 | try { 151 | await endToEnd(service, { type: "userland", subtype: "remote", path }); 152 | } finally { 153 | await clear(service, path); 154 | } 155 | }, 156 | }); 157 | -------------------------------------------------------------------------------- /npm/src/proto/messages/com/deno/kv/datapath/KvEntry.ts: -------------------------------------------------------------------------------- 1 | // @ts-nocheck 2 | import { 3 | Type as ValueEncoding, 4 | name2num, 5 | num2name, 6 | } from "./ValueEncoding.ts"; 7 | import { 8 | tsValueToJsonValueFns, 9 | jsonValueToTsValueFns, 10 | } from "../../../../../runtime/json/scalar.ts"; 11 | import { 12 | WireMessage, 13 | WireType, 14 | } from "../../../../../runtime/wire/index.ts"; 15 | import { 16 | default as serialize, 17 | } from "../../../../../runtime/wire/serialize.ts"; 18 | import { 19 | tsValueToWireValueFns, 20 | wireValueToTsValueFns, 21 | } from "../../../../../runtime/wire/scalar.ts"; 22 | import { 23 | default as Long, 24 | } from "../../../../../runtime/Long.ts"; 25 | import { 26 | default as deserialize, 27 | } from "../../../../../runtime/wire/deserialize.ts"; 28 | 29 | export declare namespace $.com.deno.kv.datapath { 30 | export type KvEntry = { 31 | key: Uint8Array; 32 | value: Uint8Array; 33 | encoding: ValueEncoding; 34 | versionstamp: Uint8Array; 35 | } 36 | } 37 | 38 | export type Type = $.com.deno.kv.datapath.KvEntry; 39 | 40 | export function getDefaultValue(): $.com.deno.kv.datapath.KvEntry { 41 | return { 42 | key: new Uint8Array(), 43 | value: new Uint8Array(), 44 | encoding: "VE_UNSPECIFIED", 45 | versionstamp: new Uint8Array(), 46 | }; 47 | } 48 | 49 | export function createValue(partialValue: Partial<$.com.deno.kv.datapath.KvEntry>): $.com.deno.kv.datapath.KvEntry { 50 | return { 51 | ...getDefaultValue(), 52 | ...partialValue, 53 | }; 54 | } 55 | 56 | export function encodeJson(value: $.com.deno.kv.datapath.KvEntry): unknown { 57 | const result: any = {}; 58 | if (value.key !== undefined) result.key = tsValueToJsonValueFns.bytes(value.key); 59 | if (value.value !== undefined) result.value = tsValueToJsonValueFns.bytes(value.value); 60 | if (value.encoding !== undefined) result.encoding = tsValueToJsonValueFns.enum(value.encoding); 61 | if (value.versionstamp !== undefined) result.versionstamp = tsValueToJsonValueFns.bytes(value.versionstamp); 62 | return result; 63 | } 64 | 65 | export function decodeJson(value: any): $.com.deno.kv.datapath.KvEntry { 66 | const result = getDefaultValue(); 67 | if (value.key !== undefined) result.key = jsonValueToTsValueFns.bytes(value.key); 68 | if (value.value !== undefined) result.value = jsonValueToTsValueFns.bytes(value.value); 69 | if (value.encoding !== undefined) result.encoding = jsonValueToTsValueFns.enum(value.encoding) as ValueEncoding; 70 | if (value.versionstamp !== undefined) result.versionstamp = jsonValueToTsValueFns.bytes(value.versionstamp); 71 | return result; 72 | } 73 | 74 | export function encodeBinary(value: $.com.deno.kv.datapath.KvEntry): Uint8Array { 75 | const result: WireMessage = []; 76 | if (value.key !== undefined) { 77 | const tsValue = value.key; 78 | result.push( 79 | [1, tsValueToWireValueFns.bytes(tsValue)], 80 | ); 81 | } 82 | if (value.value !== undefined) { 83 | const tsValue = value.value; 84 | result.push( 85 | [2, tsValueToWireValueFns.bytes(tsValue)], 86 | ); 87 | } 88 | if (value.encoding !== undefined) { 89 | const tsValue = value.encoding; 90 | result.push( 91 | [3, { type: WireType.Varint as const, value: new Long(name2num[tsValue as keyof typeof name2num]) }], 92 | ); 93 | } 94 | if (value.versionstamp !== undefined) { 95 | const tsValue = value.versionstamp; 96 | result.push( 97 | [4, tsValueToWireValueFns.bytes(tsValue)], 98 | ); 99 | } 100 | return serialize(result); 101 | } 102 | 103 | export function decodeBinary(binary: Uint8Array): $.com.deno.kv.datapath.KvEntry { 104 | const result = getDefaultValue(); 105 | const wireMessage = deserialize(binary); 106 | const wireFields = new Map(wireMessage); 107 | field: { 108 | const wireValue = wireFields.get(1); 109 | if (wireValue === undefined) break field; 110 | const value = wireValueToTsValueFns.bytes(wireValue); 111 | if (value === undefined) break field; 112 | result.key = value; 113 | } 114 | field: { 115 | const wireValue = wireFields.get(2); 116 | if (wireValue === undefined) break field; 117 | const value = wireValueToTsValueFns.bytes(wireValue); 118 | if (value === undefined) break field; 119 | result.value = value; 120 | } 121 | field: { 122 | const wireValue = wireFields.get(3); 123 | if (wireValue === undefined) break field; 124 | const value = wireValue.type === WireType.Varint ? num2name[wireValue.value[0] as keyof typeof num2name] : undefined; 125 | if (value === undefined) break field; 126 | result.encoding = value; 127 | } 128 | field: { 129 | const wireValue = wireFields.get(4); 130 | if (wireValue === undefined) break field; 131 | const value = wireValueToTsValueFns.bytes(wireValue); 132 | if (value === undefined) break field; 133 | result.versionstamp = value; 134 | } 135 | return result; 136 | } 137 | -------------------------------------------------------------------------------- /npm/src/proto/messages/com/deno/kv/datapath/Mutation.ts: -------------------------------------------------------------------------------- 1 | // @ts-nocheck 2 | import { 3 | Type as KvValue, 4 | encodeJson as encodeJson_1, 5 | decodeJson as decodeJson_1, 6 | encodeBinary as encodeBinary_1, 7 | decodeBinary as decodeBinary_1, 8 | } from "./KvValue.ts"; 9 | import { 10 | Type as MutationType, 11 | name2num, 12 | num2name, 13 | } from "./MutationType.ts"; 14 | import { 15 | tsValueToJsonValueFns, 16 | jsonValueToTsValueFns, 17 | } from "../../../../../runtime/json/scalar.ts"; 18 | import { 19 | WireMessage, 20 | WireType, 21 | } from "../../../../../runtime/wire/index.ts"; 22 | import { 23 | default as serialize, 24 | } from "../../../../../runtime/wire/serialize.ts"; 25 | import { 26 | tsValueToWireValueFns, 27 | wireValueToTsValueFns, 28 | } from "../../../../../runtime/wire/scalar.ts"; 29 | import { 30 | default as Long, 31 | } from "../../../../../runtime/Long.ts"; 32 | import { 33 | default as deserialize, 34 | } from "../../../../../runtime/wire/deserialize.ts"; 35 | 36 | export declare namespace $.com.deno.kv.datapath { 37 | export type Mutation = { 38 | key: Uint8Array; 39 | value?: KvValue; 40 | mutationType: MutationType; 41 | expireAtMs: string; 42 | } 43 | } 44 | 45 | export type Type = $.com.deno.kv.datapath.Mutation; 46 | 47 | export function getDefaultValue(): $.com.deno.kv.datapath.Mutation { 48 | return { 49 | key: new Uint8Array(), 50 | value: undefined, 51 | mutationType: "M_UNSPECIFIED", 52 | expireAtMs: "0", 53 | }; 54 | } 55 | 56 | export function createValue(partialValue: Partial<$.com.deno.kv.datapath.Mutation>): $.com.deno.kv.datapath.Mutation { 57 | return { 58 | ...getDefaultValue(), 59 | ...partialValue, 60 | }; 61 | } 62 | 63 | export function encodeJson(value: $.com.deno.kv.datapath.Mutation): unknown { 64 | const result: any = {}; 65 | if (value.key !== undefined) result.key = tsValueToJsonValueFns.bytes(value.key); 66 | if (value.value !== undefined) result.value = encodeJson_1(value.value); 67 | if (value.mutationType !== undefined) result.mutationType = tsValueToJsonValueFns.enum(value.mutationType); 68 | if (value.expireAtMs !== undefined) result.expireAtMs = tsValueToJsonValueFns.int64(value.expireAtMs); 69 | return result; 70 | } 71 | 72 | export function decodeJson(value: any): $.com.deno.kv.datapath.Mutation { 73 | const result = getDefaultValue(); 74 | if (value.key !== undefined) result.key = jsonValueToTsValueFns.bytes(value.key); 75 | if (value.value !== undefined) result.value = decodeJson_1(value.value); 76 | if (value.mutationType !== undefined) result.mutationType = jsonValueToTsValueFns.enum(value.mutationType) as MutationType; 77 | if (value.expireAtMs !== undefined) result.expireAtMs = jsonValueToTsValueFns.int64(value.expireAtMs); 78 | return result; 79 | } 80 | 81 | export function encodeBinary(value: $.com.deno.kv.datapath.Mutation): Uint8Array { 82 | const result: WireMessage = []; 83 | if (value.key !== undefined) { 84 | const tsValue = value.key; 85 | result.push( 86 | [1, tsValueToWireValueFns.bytes(tsValue)], 87 | ); 88 | } 89 | if (value.value !== undefined) { 90 | const tsValue = value.value; 91 | result.push( 92 | [2, { type: WireType.LengthDelimited as const, value: encodeBinary_1(tsValue) }], 93 | ); 94 | } 95 | if (value.mutationType !== undefined) { 96 | const tsValue = value.mutationType; 97 | result.push( 98 | [3, { type: WireType.Varint as const, value: new Long(name2num[tsValue as keyof typeof name2num]) }], 99 | ); 100 | } 101 | if (value.expireAtMs !== undefined) { 102 | const tsValue = value.expireAtMs; 103 | result.push( 104 | [4, tsValueToWireValueFns.int64(tsValue)], 105 | ); 106 | } 107 | return serialize(result); 108 | } 109 | 110 | export function decodeBinary(binary: Uint8Array): $.com.deno.kv.datapath.Mutation { 111 | const result = getDefaultValue(); 112 | const wireMessage = deserialize(binary); 113 | const wireFields = new Map(wireMessage); 114 | field: { 115 | const wireValue = wireFields.get(1); 116 | if (wireValue === undefined) break field; 117 | const value = wireValueToTsValueFns.bytes(wireValue); 118 | if (value === undefined) break field; 119 | result.key = value; 120 | } 121 | field: { 122 | const wireValue = wireFields.get(2); 123 | if (wireValue === undefined) break field; 124 | const value = wireValue.type === WireType.LengthDelimited ? decodeBinary_1(wireValue.value) : undefined; 125 | if (value === undefined) break field; 126 | result.value = value; 127 | } 128 | field: { 129 | const wireValue = wireFields.get(3); 130 | if (wireValue === undefined) break field; 131 | const value = wireValue.type === WireType.Varint ? num2name[wireValue.value[0] as keyof typeof num2name] : undefined; 132 | if (value === undefined) break field; 133 | result.mutationType = value; 134 | } 135 | field: { 136 | const wireValue = wireFields.get(4); 137 | if (wireValue === undefined) break field; 138 | const value = wireValueToTsValueFns.int64(wireValue); 139 | if (value === undefined) break field; 140 | result.expireAtMs = value; 141 | } 142 | return result; 143 | } 144 | -------------------------------------------------------------------------------- /npm/src/proto/messages/com/deno/kv/datapath/SnapshotReadOutput.ts: -------------------------------------------------------------------------------- 1 | // @ts-nocheck 2 | import { 3 | Type as ReadRangeOutput, 4 | encodeJson as encodeJson_1, 5 | decodeJson as decodeJson_1, 6 | encodeBinary as encodeBinary_1, 7 | decodeBinary as decodeBinary_1, 8 | } from "./ReadRangeOutput.ts"; 9 | import { 10 | Type as SnapshotReadStatus, 11 | name2num, 12 | num2name, 13 | } from "./SnapshotReadStatus.ts"; 14 | import { 15 | tsValueToJsonValueFns, 16 | jsonValueToTsValueFns, 17 | } from "../../../../../runtime/json/scalar.ts"; 18 | import { 19 | WireMessage, 20 | WireType, 21 | } from "../../../../../runtime/wire/index.ts"; 22 | import { 23 | default as serialize, 24 | } from "../../../../../runtime/wire/serialize.ts"; 25 | import { 26 | tsValueToWireValueFns, 27 | wireValueToTsValueFns, 28 | } from "../../../../../runtime/wire/scalar.ts"; 29 | import { 30 | default as Long, 31 | } from "../../../../../runtime/Long.ts"; 32 | import { 33 | default as deserialize, 34 | } from "../../../../../runtime/wire/deserialize.ts"; 35 | 36 | export declare namespace $.com.deno.kv.datapath { 37 | export type SnapshotReadOutput = { 38 | ranges: ReadRangeOutput[]; 39 | readDisabled: boolean; 40 | readIsStronglyConsistent: boolean; 41 | status: SnapshotReadStatus; 42 | } 43 | } 44 | 45 | export type Type = $.com.deno.kv.datapath.SnapshotReadOutput; 46 | 47 | export function getDefaultValue(): $.com.deno.kv.datapath.SnapshotReadOutput { 48 | return { 49 | ranges: [], 50 | readDisabled: false, 51 | readIsStronglyConsistent: false, 52 | status: "SR_UNSPECIFIED", 53 | }; 54 | } 55 | 56 | export function createValue(partialValue: Partial<$.com.deno.kv.datapath.SnapshotReadOutput>): $.com.deno.kv.datapath.SnapshotReadOutput { 57 | return { 58 | ...getDefaultValue(), 59 | ...partialValue, 60 | }; 61 | } 62 | 63 | export function encodeJson(value: $.com.deno.kv.datapath.SnapshotReadOutput): unknown { 64 | const result: any = {}; 65 | result.ranges = value.ranges.map(value => encodeJson_1(value)); 66 | if (value.readDisabled !== undefined) result.readDisabled = tsValueToJsonValueFns.bool(value.readDisabled); 67 | if (value.readIsStronglyConsistent !== undefined) result.readIsStronglyConsistent = tsValueToJsonValueFns.bool(value.readIsStronglyConsistent); 68 | if (value.status !== undefined) result.status = tsValueToJsonValueFns.enum(value.status); 69 | return result; 70 | } 71 | 72 | export function decodeJson(value: any): $.com.deno.kv.datapath.SnapshotReadOutput { 73 | const result = getDefaultValue(); 74 | result.ranges = value.ranges?.map((value: any) => decodeJson_1(value)) ?? []; 75 | if (value.readDisabled !== undefined) result.readDisabled = jsonValueToTsValueFns.bool(value.readDisabled); 76 | if (value.readIsStronglyConsistent !== undefined) result.readIsStronglyConsistent = jsonValueToTsValueFns.bool(value.readIsStronglyConsistent); 77 | if (value.status !== undefined) result.status = jsonValueToTsValueFns.enum(value.status) as SnapshotReadStatus; 78 | return result; 79 | } 80 | 81 | export function encodeBinary(value: $.com.deno.kv.datapath.SnapshotReadOutput): Uint8Array { 82 | const result: WireMessage = []; 83 | for (const tsValue of value.ranges) { 84 | result.push( 85 | [1, { type: WireType.LengthDelimited as const, value: encodeBinary_1(tsValue) }], 86 | ); 87 | } 88 | if (value.readDisabled !== undefined) { 89 | const tsValue = value.readDisabled; 90 | result.push( 91 | [2, tsValueToWireValueFns.bool(tsValue)], 92 | ); 93 | } 94 | if (value.readIsStronglyConsistent !== undefined) { 95 | const tsValue = value.readIsStronglyConsistent; 96 | result.push( 97 | [4, tsValueToWireValueFns.bool(tsValue)], 98 | ); 99 | } 100 | if (value.status !== undefined) { 101 | const tsValue = value.status; 102 | result.push( 103 | [8, { type: WireType.Varint as const, value: new Long(name2num[tsValue as keyof typeof name2num]) }], 104 | ); 105 | } 106 | return serialize(result); 107 | } 108 | 109 | export function decodeBinary(binary: Uint8Array): $.com.deno.kv.datapath.SnapshotReadOutput { 110 | const result = getDefaultValue(); 111 | const wireMessage = deserialize(binary); 112 | const wireFields = new Map(wireMessage); 113 | collection: { 114 | const wireValues = wireMessage.filter(([fieldNumber]) => fieldNumber === 1).map(([, wireValue]) => wireValue); 115 | const value = wireValues.map((wireValue) => wireValue.type === WireType.LengthDelimited ? decodeBinary_1(wireValue.value) : undefined).filter(x => x !== undefined); 116 | if (!value.length) break collection; 117 | result.ranges = value as any; 118 | } 119 | field: { 120 | const wireValue = wireFields.get(2); 121 | if (wireValue === undefined) break field; 122 | const value = wireValueToTsValueFns.bool(wireValue); 123 | if (value === undefined) break field; 124 | result.readDisabled = value; 125 | } 126 | field: { 127 | const wireValue = wireFields.get(4); 128 | if (wireValue === undefined) break field; 129 | const value = wireValueToTsValueFns.bool(wireValue); 130 | if (value === undefined) break field; 131 | result.readIsStronglyConsistent = value; 132 | } 133 | field: { 134 | const wireValue = wireFields.get(8); 135 | if (wireValue === undefined) break field; 136 | const value = wireValue.type === WireType.Varint ? num2name[wireValue.value[0] as keyof typeof num2name] : undefined; 137 | if (value === undefined) break field; 138 | result.status = value; 139 | } 140 | return result; 141 | } 142 | -------------------------------------------------------------------------------- /npm/src/v8.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2023 the Deno authors. All rights reserved. MIT license. 2 | 3 | // https://chromium.googlesource.com/v8/v8/+/refs/heads/main/src/objects/value-serializer.cc 4 | 5 | import { checkEnd } from "./bytes.ts"; 6 | 7 | /** 8 | * Returns encodeV8, decodeV8 functions that support V8-compatible serialization for limited set of types (strings, null, undefined, boolean). 9 | * 10 | * Consider using JSON.parse/stringify to marshall values to save when using this serializer. 11 | * 12 | * All other values will throw during encode/decode. 13 | */ 14 | export function makeLimitedV8Serializer() { 15 | return { encodeV8, decodeV8 }; 16 | } 17 | 18 | export function decodeV8( 19 | bytes: Uint8Array, 20 | { wrapUnknownValues = false }: { wrapUnknownValues?: boolean } = {}, 21 | ): unknown { 22 | if (bytes.length === 0) throw new Error(`decode error: empty input`); 23 | let pos = 0; 24 | const kVersion = bytes[pos++]; 25 | if (kVersion !== SerializationTag.kVersion && wrapUnknownValues) { 26 | return new UnknownV8(bytes); 27 | } 28 | if (kVersion !== SerializationTag.kVersion) { 29 | throw new Error( 30 | `decode error: Unsupported kVersion ${kVersion} [${ 31 | [...bytes].join(", ") 32 | }]`, 33 | ); 34 | } 35 | const version = bytes[pos++]; 36 | if (version !== kLatestVersion) { 37 | throw new Error(`decode error: Unsupported version ${version}`); 38 | } 39 | const tag = bytes[pos++]; 40 | if (tag === SerializationTag.kOneByteString) { 41 | const len = bytes[pos++]; 42 | const arr = bytes.subarray(pos, pos + len); 43 | const rt = new TextDecoder().decode(arr); 44 | pos += len; 45 | checkEnd(bytes, pos); 46 | return rt; 47 | } else if (tag === SerializationTag.kTwoByteString) { 48 | const len = bytes[pos++]; 49 | const arr = bytes.subarray(pos, pos + len); 50 | const rt = new TextDecoder("utf-16").decode(arr); 51 | pos += len; 52 | checkEnd(bytes, pos); 53 | return rt; 54 | } else if (tag === SerializationTag.kNull) { 55 | checkEnd(bytes, pos); 56 | return null; 57 | } else if (tag === SerializationTag.kUndefined) { 58 | checkEnd(bytes, pos); 59 | return undefined; 60 | } else if (tag === SerializationTag.kTrue) { 61 | checkEnd(bytes, pos); 62 | return true; 63 | } else if (tag === SerializationTag.kFalse) { 64 | checkEnd(bytes, pos); 65 | return false; 66 | } else if ( 67 | tag === SerializationTag.kBigInt && bytes.length === 4 && bytes[3] === 0 68 | ) { 69 | return 0n; 70 | } else if (wrapUnknownValues) { 71 | return new UnknownV8(bytes); 72 | } else { 73 | throw new Error( 74 | `decode error: Unsupported v8 tag ${tag} ('${ 75 | String.fromCharCode(tag) 76 | }') at ${pos} in [${bytes.join(", ")}]`, 77 | ); 78 | } 79 | } 80 | 81 | export function encodeV8(value: unknown): Uint8Array { 82 | if (value instanceof UnknownV8) { 83 | return value.bytes; 84 | } else if (typeof value === "string") { 85 | const chars = [...value]; 86 | if (chars.every(isOneByteChar)) { 87 | const charCodes = chars.map((v) => v.charCodeAt(0)); 88 | return new Uint8Array([ 89 | SerializationTag.kVersion, 90 | kLatestVersion, 91 | SerializationTag.kOneByteString, 92 | charCodes.length, 93 | ...charCodes, 94 | ]); 95 | } 96 | const bytes: number[] = []; 97 | for (let i = 0; i < value.length; i++) { 98 | const charCode = value.charCodeAt(i); 99 | const msb = (charCode & 0xff00) >> 8; 100 | const lsb = charCode & 0x00ff; 101 | bytes.push(lsb); 102 | bytes.push(msb); 103 | } 104 | return new Uint8Array([ 105 | SerializationTag.kVersion, 106 | kLatestVersion, 107 | SerializationTag.kTwoByteString, 108 | value.length * 2, 109 | ...bytes, 110 | ]); 111 | } else if (value === null) { 112 | return new Uint8Array([ 113 | SerializationTag.kVersion, 114 | kLatestVersion, 115 | SerializationTag.kNull, 116 | ]); 117 | } else if (value === undefined) { 118 | return new Uint8Array([ 119 | SerializationTag.kVersion, 120 | kLatestVersion, 121 | SerializationTag.kUndefined, 122 | ]); 123 | } else if (value === true) { 124 | return new Uint8Array([ 125 | SerializationTag.kVersion, 126 | kLatestVersion, 127 | SerializationTag.kTrue, 128 | ]); 129 | } else if (value === false) { 130 | return new Uint8Array([ 131 | SerializationTag.kVersion, 132 | kLatestVersion, 133 | SerializationTag.kFalse, 134 | ]); 135 | } else if (value === 0n) { 136 | return new Uint8Array([ 137 | SerializationTag.kVersion, 138 | kLatestVersion, 139 | SerializationTag.kBigInt, 140 | 0, 141 | ]); 142 | } 143 | throw new Error( 144 | `encode error: Unsupported v8 value ${typeof value} ${value}`, 145 | ); 146 | } 147 | 148 | // 149 | 150 | const kLatestVersion = 15; 151 | 152 | enum SerializationTag { 153 | kVersion = 0xff, 154 | kOneByteString = '"'.charCodeAt(0), 155 | kTwoByteString = "c".charCodeAt(0), 156 | kNull = "0".charCodeAt(0), 157 | kUndefined = "_".charCodeAt(0), 158 | kTrue = "T".charCodeAt(0), 159 | kFalse = "F".charCodeAt(0), 160 | kBigInt = "Z".charCodeAt(0), 161 | } 162 | 163 | function isOneByteChar(char: string): boolean { 164 | const cp = char.codePointAt(0)!; 165 | return cp >= 0 && cp <= 0xff; 166 | } 167 | 168 | // 169 | 170 | /** Raw V8-serialized bytes */ 171 | export class UnknownV8 { 172 | public readonly bytes: Uint8Array; 173 | 174 | constructor(bytes: Uint8Array) { 175 | this.bytes = bytes; 176 | } 177 | } 178 | -------------------------------------------------------------------------------- /npm/src/scripts/build_npm.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2023 the Deno authors. All rights reserved. MIT license. 2 | 3 | import { join } from "https://deno.land/std@0.208.0/path/join.ts"; 4 | import { 5 | build, 6 | emptyDir, 7 | LibName, 8 | } from "https://deno.land/x/dnt@0.39.0/mod.ts"; 9 | import { parseArgs as parseFlags } from "https://deno.land/std@0.208.0/cli/parse_args.ts"; 10 | import { generateNapiIndex } from "./generate_napi_index.ts"; 11 | import { run } from "./process.ts"; 12 | 13 | const flags = parseFlags(Deno.args); 14 | const tests = !!flags.tests; 15 | if (tests) console.log("including tests!"); 16 | const publish = typeof flags.publish === "string" ? flags.publish : undefined; 17 | const dryrun = !!flags["dry-run"]; 18 | if (publish) { 19 | console.log( 20 | `publish${dryrun ? ` (dryrun)` : ""} after build! (npm=${publish})`, 21 | ); 22 | } 23 | const stripLeadingV = (version: string) => version.replace(/^v/, ""); 24 | const napi = typeof flags.napi === "string" 25 | ? { 26 | packageName: "@deno/kv", 27 | packageVersion: stripLeadingV(flags.napi), 28 | artifactName: "deno-kv-napi", 29 | } 30 | : undefined; 31 | if (napi) console.log(`napi: ${JSON.stringify(napi)}`); 32 | if (!napi) throw new Error("Must provide --napi version"); 33 | const version = typeof Deno.args[0] === "string" 34 | ? stripLeadingV(Deno.args[0]) 35 | : Deno.args[0]; 36 | if (typeof version !== "string" || !/^[a-z0-9.-]+$/.test(version)) { 37 | throw new Error(`Unexpected version: ${version}`); 38 | } 39 | console.log(`version=${version}`); 40 | 41 | const outDir = await Deno.makeTempDir({ prefix: "userland-npm-" }); 42 | await emptyDir(outDir); 43 | 44 | await build({ 45 | entryPoints: [ 46 | "./src/npm.ts", 47 | ...(tests ? [{ name: "./tests", path: "./src/e2e.ts" }] : []), 48 | ], 49 | outDir, 50 | test: false, 51 | shims: { 52 | // none! 53 | }, 54 | compilerOptions: { 55 | // let's try to support Node 18+ 56 | lib: [ 57 | "ES2020", 58 | "DOM", 59 | "DOM.Iterable", 60 | "ESNext.Disposable", 61 | ...(tests ? ["ES2021.WeakRef" as LibName] : []), 62 | ], 63 | target: "ES2020", 64 | }, 65 | package: { 66 | // package.json properties 67 | name: napi.packageName, 68 | version, 69 | description: "A Deno KV client library optimized for Node.js.", 70 | license: "MIT", 71 | repository: { 72 | type: "git", 73 | url: "https://github.com/denoland/denokv.git", 74 | directory: "npm" 75 | }, 76 | bugs: { 77 | url: "https://github.com/denoland/denokv/issues", 78 | }, 79 | homepage: "https://github.com/denoland/denokv/tree/main/npm", 80 | optionalDependencies: Object.fromEntries( 81 | ["win32-x64-msvc", "darwin-x64", "linux-x64-gnu", "darwin-arm64"].map( 82 | (v) => [`${napi.packageName}-${v}`, napi.packageVersion], 83 | ), 84 | ), 85 | }, 86 | async postBuild() { 87 | // steps to run after building and before running the tests 88 | await Deno.copyFile("LICENSE", join(outDir, "LICENSE")); 89 | await Deno.copyFile( 90 | "README.md", 91 | join(outDir, "README.md"), 92 | ); 93 | const napiIndexJs = generateNapiIndex({ 94 | napiPackageName: napi.packageName, 95 | napiArtifactName: napi.artifactName, 96 | }); 97 | for (const subdir of ["script", "esm"]) { 98 | const name = "_napi_index.cjs"; // cjs to ensure 'require' works in esm mode 99 | console.log(`writing ${join(subdir, name)}`); 100 | await Deno.writeTextFile(join(outDir, subdir, name), napiIndexJs); 101 | 102 | console.log(`tweaking ${join(subdir, "napi_based.js")}`); 103 | const oldContents = await Deno.readTextFile( 104 | join(outDir, subdir, "napi_based.js"), 105 | ); 106 | const insertion = subdir === "esm" 107 | ? `await import('./${name}')` 108 | : `require('./${name}')`; 109 | 110 | const newContents = oldContents.replace( 111 | `const DEFAULT_NAPI_INTERFACE = undefined;`, 112 | `const DEFAULT_NAPI_INTERFACE = ${insertion};`, 113 | ); 114 | await Deno.writeTextFile( 115 | join(outDir, subdir, "napi_based.js"), 116 | newContents, 117 | ); 118 | } 119 | }, 120 | }); 121 | 122 | if (publish) { 123 | const updatePackageJsonVersion = async (path: string, version: string) => { 124 | console.log(`Updating ${path} version to ${version}`); 125 | const packageJson = await Deno.readTextFile(path); 126 | const newPackageJson = packageJson.replace( 127 | /("version"\s*:\s*")[0-9a-z.-]+"/, 128 | `$1${version}"`, 129 | ); 130 | if (packageJson === newPackageJson) { 131 | throw new Error(`Unable to replace version!`); 132 | } 133 | await Deno.writeTextFile(path, newPackageJson); 134 | }; 135 | const npmPublish = async (path: string) => { 136 | const next = !/^[0-9]+\.[0-9]+\.[0-9]+$/.test(version); 137 | const out = await run({ 138 | command: publish, 139 | args: [ 140 | "publish", 141 | "--access", 142 | "public", 143 | ...(next ? ["--tag", "next"] : []), 144 | ...(dryrun ? ["--dry-run"] : []), 145 | path, 146 | ], 147 | }); 148 | console.log(out); 149 | }; 150 | 151 | // first, publish the native subpackages 152 | for ( 153 | const { name: subdir } of (await Array.fromAsync(Deno.readDir("napi/npm"))) 154 | .filter((v) => v.isDirectory) 155 | ) { 156 | const path = join("napi", "npm", subdir, "package.json"); 157 | await updatePackageJsonVersion(path, version); 158 | await npmPublish(join("napi", "npm", subdir)); 159 | } 160 | // finally, publish the root package 161 | await npmPublish(outDir); 162 | } 163 | 164 | console.log(outDir); 165 | -------------------------------------------------------------------------------- /npm/napi/.eslintrc.yml: -------------------------------------------------------------------------------- 1 | parser: '@typescript-eslint/parser' 2 | 3 | parserOptions: 4 | ecmaFeatures: 5 | jsx: true 6 | ecmaVersion: latest 7 | sourceType: module 8 | project: ./tsconfig.json 9 | 10 | env: 11 | browser: true 12 | es6: true 13 | node: true 14 | jest: true 15 | 16 | ignorePatterns: ['index.js'] 17 | 18 | plugins: 19 | - import 20 | - '@typescript-eslint' 21 | 22 | extends: 23 | - eslint:recommended 24 | - plugin:prettier/recommended 25 | 26 | rules: 27 | # 0 = off, 1 = warn, 2 = error 28 | 'space-before-function-paren': 0 29 | 'no-useless-constructor': 0 30 | 'no-undef': 2 31 | 'no-console': [2, { allow: ['error', 'warn', 'info', 'assert'] }] 32 | 'comma-dangle': ['error', 'only-multiline'] 33 | 'no-unused-vars': 0 34 | 'no-var': 2 35 | 'one-var-declaration-per-line': 2 36 | 'prefer-const': 2 37 | 'no-const-assign': 2 38 | 'no-duplicate-imports': 2 39 | 'no-use-before-define': [2, { 'functions': false, 'classes': false }] 40 | 'eqeqeq': [2, 'always', { 'null': 'ignore' }] 41 | 'no-case-declarations': 0 42 | 'no-restricted-syntax': 43 | [ 44 | 2, 45 | { 46 | 'selector': 'BinaryExpression[operator=/(==|===|!=|!==)/][left.raw=true], BinaryExpression[operator=/(==|===|!=|!==)/][right.raw=true]', 47 | 'message': Don't compare for equality against boolean literals, 48 | }, 49 | ] 50 | 51 | # https://github.com/benmosher/eslint-plugin-import/pull/334 52 | 'import/no-duplicates': 2 53 | 'import/first': 2 54 | 'import/newline-after-import': 2 55 | 'import/order': 56 | [ 57 | 2, 58 | { 59 | 'newlines-between': 'always', 60 | 'alphabetize': { 'order': 'asc' }, 61 | 'groups': ['builtin', 'external', 'internal', 'parent', 'sibling', 'index'], 62 | }, 63 | ] 64 | 65 | overrides: 66 | - files: 67 | - ./**/*{.ts,.tsx} 68 | rules: 69 | 'no-unused-vars': [2, { varsIgnorePattern: '^_', argsIgnorePattern: '^_', ignoreRestSiblings: true }] 70 | 'no-undef': 0 71 | # TypeScript declare merge 72 | 'no-redeclare': 0 73 | 'no-useless-constructor': 0 74 | 'no-dupe-class-members': 0 75 | 'no-case-declarations': 0 76 | 'no-duplicate-imports': 0 77 | # TypeScript Interface and Type 78 | 'no-use-before-define': 0 79 | 80 | '@typescript-eslint/adjacent-overload-signatures': 2 81 | '@typescript-eslint/await-thenable': 2 82 | '@typescript-eslint/consistent-type-assertions': 2 83 | '@typescript-eslint/ban-types': 84 | [ 85 | 'error', 86 | { 87 | 'types': 88 | { 89 | 'String': { 'message': 'Use string instead', 'fixWith': 'string' }, 90 | 'Number': { 'message': 'Use number instead', 'fixWith': 'number' }, 91 | 'Boolean': { 'message': 'Use boolean instead', 'fixWith': 'boolean' }, 92 | 'Function': { 'message': 'Use explicit type instead' }, 93 | }, 94 | }, 95 | ] 96 | '@typescript-eslint/explicit-member-accessibility': 97 | [ 98 | 'error', 99 | { 100 | accessibility: 'explicit', 101 | overrides: 102 | { 103 | accessors: 'no-public', 104 | constructors: 'no-public', 105 | methods: 'no-public', 106 | properties: 'no-public', 107 | parameterProperties: 'explicit', 108 | }, 109 | }, 110 | ] 111 | '@typescript-eslint/method-signature-style': 2 112 | '@typescript-eslint/no-floating-promises': 2 113 | '@typescript-eslint/no-implied-eval': 2 114 | '@typescript-eslint/no-for-in-array': 2 115 | '@typescript-eslint/no-inferrable-types': 2 116 | '@typescript-eslint/no-invalid-void-type': 2 117 | '@typescript-eslint/no-misused-new': 2 118 | '@typescript-eslint/no-misused-promises': 2 119 | '@typescript-eslint/no-namespace': 2 120 | '@typescript-eslint/no-non-null-asserted-optional-chain': 2 121 | '@typescript-eslint/no-throw-literal': 2 122 | '@typescript-eslint/no-unnecessary-boolean-literal-compare': 2 123 | '@typescript-eslint/prefer-for-of': 2 124 | '@typescript-eslint/prefer-nullish-coalescing': 2 125 | '@typescript-eslint/switch-exhaustiveness-check': 2 126 | '@typescript-eslint/prefer-optional-chain': 2 127 | '@typescript-eslint/prefer-readonly': 2 128 | '@typescript-eslint/prefer-string-starts-ends-with': 0 129 | '@typescript-eslint/no-array-constructor': 2 130 | '@typescript-eslint/require-await': 2 131 | '@typescript-eslint/return-await': 2 132 | '@typescript-eslint/ban-ts-comment': 133 | [2, { 'ts-expect-error': false, 'ts-ignore': true, 'ts-nocheck': true, 'ts-check': false }] 134 | '@typescript-eslint/naming-convention': 135 | [ 136 | 2, 137 | { 138 | selector: 'memberLike', 139 | format: ['camelCase', 'PascalCase'], 140 | modifiers: ['private'], 141 | leadingUnderscore: 'forbid', 142 | }, 143 | ] 144 | '@typescript-eslint/no-unused-vars': 145 | [2, { varsIgnorePattern: '^_', argsIgnorePattern: '^_', ignoreRestSiblings: true }] 146 | '@typescript-eslint/member-ordering': 147 | [ 148 | 2, 149 | { 150 | default: 151 | [ 152 | 'public-static-field', 153 | 'protected-static-field', 154 | 'private-static-field', 155 | 'public-static-method', 156 | 'protected-static-method', 157 | 'private-static-method', 158 | 'public-instance-field', 159 | 'protected-instance-field', 160 | 'private-instance-field', 161 | 'public-constructor', 162 | 'protected-constructor', 163 | 'private-constructor', 164 | 'public-instance-method', 165 | 'protected-instance-method', 166 | 'private-instance-method', 167 | ], 168 | }, 169 | ] 170 | -------------------------------------------------------------------------------- /npm/src/kv_key.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2023 the Deno authors. All rights reserved. MIT license. 2 | 3 | import { 4 | checkEnd, 5 | computeBigintMinimumNumberOfBytes, 6 | flipBytes, 7 | } from "./bytes.ts"; 8 | import { KvKey, KvKeyPart } from "./kv_types.ts"; 9 | 10 | // https://github.com/apple/foundationdb/blob/main/design/tuple.md 11 | // limited to Uint8Array | string | number | bigint | boolean 12 | 13 | // https://github.com/denoland/deno/blob/main/ext/kv/codec.rs 14 | 15 | export function packKey(kvKey: KvKey): Uint8Array { 16 | return new Uint8Array(kvKey.flatMap((v) => [...packKeyPart(v)])); 17 | } 18 | 19 | export function packKeyPart(kvKeyPart: KvKeyPart): Uint8Array { 20 | if (kvKeyPart instanceof Uint8Array) { 21 | return new Uint8Array([ 22 | Typecode.ByteString, 23 | ...encodeZeroWithZeroFF(kvKeyPart), 24 | 0, 25 | ]); 26 | } 27 | if (typeof kvKeyPart === "string") { 28 | return new Uint8Array([ 29 | Typecode.UnicodeString, 30 | ...encodeZeroWithZeroFF(new TextEncoder().encode(kvKeyPart)), 31 | 0, 32 | ]); 33 | } 34 | if (kvKeyPart === false) return new Uint8Array([Typecode.False]); 35 | if (kvKeyPart === true) return new Uint8Array([Typecode.True]); 36 | if (typeof kvKeyPart === "bigint") { 37 | const neg = kvKeyPart < 0; 38 | const abs = neg ? -kvKeyPart : kvKeyPart; 39 | const numBytes = BigInt(computeBigintMinimumNumberOfBytes(abs)); 40 | 41 | const typecode = neg 42 | ? (numBytes <= 8n 43 | ? (Typecode.IntegerOneByteNegative - Number(numBytes) + 1) 44 | : Typecode.IntegerArbitraryByteNegative) 45 | : (numBytes <= 8n 46 | ? (Typecode.IntegerOneBytePositive + Number(numBytes) - 1) 47 | : Typecode.IntegerArbitraryBytePositive); 48 | const bytes: number[] = [typecode]; 49 | if (numBytes > 8n) bytes.push(Number(numBytes)); 50 | for (let i = 0n; i < numBytes; i++) { 51 | const mask = 0xffn << 8n * (numBytes - i - 1n); 52 | const byte = Number((abs & mask) >> (8n * (numBytes - i - 1n))); 53 | bytes.push(byte); 54 | } 55 | if (neg) flipBytes(bytes, 1); 56 | return new Uint8Array(bytes); 57 | } 58 | if (typeof kvKeyPart === "number") { 59 | const sub = new Uint8Array(8); 60 | new DataView(sub.buffer).setFloat64(0, -Math.abs(kvKeyPart), false); 61 | if (kvKeyPart < 0) flipBytes(sub); 62 | return new Uint8Array([Typecode.FloatingPointDouble, ...sub]); 63 | } 64 | throw new Error(`Unsupported keyPart: ${typeof kvKeyPart} ${kvKeyPart}`); 65 | } 66 | 67 | export function unpackKey(bytes: Uint8Array): KvKey { 68 | const rt: KvKeyPart[] = []; 69 | let pos = 0; 70 | while (pos < bytes.length) { 71 | const typecode = bytes[pos++]; 72 | if ( 73 | typecode === Typecode.ByteString || typecode === Typecode.UnicodeString 74 | ) { 75 | // Uint8Array or string 76 | const newBytes: number[] = []; 77 | while (pos < bytes.length) { 78 | const byte = bytes[pos++]; 79 | if (byte === 0 && bytes[pos] === 0xff) { 80 | pos++; 81 | } else if (byte === 0) { 82 | break; 83 | } 84 | newBytes.push(byte); 85 | } 86 | rt.push( 87 | typecode === Typecode.UnicodeString 88 | ? decoder.decode(new Uint8Array(newBytes)) 89 | : new Uint8Array(newBytes), 90 | ); 91 | } else if ( 92 | typecode >= Typecode.IntegerArbitraryByteNegative && 93 | typecode <= Typecode.IntegerArbitraryBytePositive 94 | ) { 95 | // bigint 96 | const neg = typecode < Typecode.IntegerZero; 97 | const numBytes = BigInt( 98 | (typecode === Typecode.IntegerArbitraryBytePositive || 99 | typecode === Typecode.IntegerArbitraryByteNegative) 100 | ? (neg ? (0xff - bytes[pos++]) : bytes[pos++]) 101 | : Math.abs(typecode - Typecode.IntegerZero), 102 | ); 103 | let val = 0n; 104 | for (let i = 0n; i < numBytes; i++) { 105 | let byte = bytes[pos++]; 106 | if (neg) byte = 0xff - byte; 107 | val += BigInt(byte) << ((numBytes - i - 1n) * 8n); 108 | } 109 | rt.push(neg ? -val : val); 110 | } else if (typecode === Typecode.FloatingPointDouble) { 111 | // number 112 | const sub = new Uint8Array(bytes.subarray(pos, pos + 8)); 113 | const neg = sub[0] < 128; 114 | if (neg) flipBytes(sub); 115 | const num = -new DataView(sub.buffer).getFloat64(0, false); 116 | pos += 8; 117 | rt.push(neg ? -num : num); 118 | } else if (typecode === Typecode.False) { 119 | // boolean false 120 | rt.push(false); 121 | } else if (typecode === Typecode.True) { 122 | // boolean true 123 | rt.push(true); 124 | } else { 125 | throw new Error( 126 | `Unsupported typecode: ${typecode} in key: [${ 127 | bytes.join(", ") 128 | }] after ${rt.join(", ")}`, 129 | ); 130 | } 131 | } 132 | checkEnd(bytes, pos); 133 | return rt; 134 | } 135 | 136 | // 137 | 138 | const decoder = new TextDecoder(); 139 | 140 | const enum Typecode { 141 | ByteString = 0x01, 142 | UnicodeString = 0x02, 143 | IntegerArbitraryByteNegative = 0x0b, 144 | IntegerEightByteNegative = 0x0c, 145 | IntegerSevenByteNegative = 0x0d, 146 | IntegerSixByteNegative = 0x0e, 147 | IntegerFiveByteNegative = 0x0f, 148 | IntegerFourByteNegative = 0x10, 149 | IntegerThreeByteNegative = 0x11, 150 | IntegerTwoByteNegative = 0x12, 151 | IntegerOneByteNegative = 0x13, 152 | IntegerZero = 0x14, 153 | IntegerOneBytePositive = 0x15, 154 | IntegerTwoBytePositive = 0x16, 155 | IntegerThreeBytePositive = 0x17, 156 | IntegerFourBytePositive = 0x18, 157 | IntegerFiveBytePositive = 0x19, 158 | IntegerSixBytePositive = 0x1a, 159 | IntegerSevenBytePositive = 0x1b, 160 | IntegerEightBytePositive = 0x1c, 161 | IntegerArbitraryBytePositive = 0x1d, 162 | FloatingPointDouble = 0x21, // IEEE Binary Floating Point double (64 bits) 163 | False = 0x26, 164 | True = 0x27, 165 | } 166 | 167 | function encodeZeroWithZeroFF(bytes: Uint8Array): Uint8Array { 168 | const index = bytes.indexOf(0); 169 | return index < 0 170 | ? bytes 171 | : new Uint8Array([...bytes].flatMap((v) => v === 0 ? [0, 0xff] : [v])); 172 | } 173 | -------------------------------------------------------------------------------- /npm/src/npm.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2023 the Deno authors. All rights reserved. MIT license. 2 | 3 | import { 4 | check, 5 | checkOptionalBoolean, 6 | checkOptionalString, 7 | checkRecord, 8 | } from "./check.ts"; 9 | import { makeInMemoryService } from "./in_memory.ts"; 10 | import { Kv } from "./kv_types.ts"; 11 | import { DecodeV8, EncodeV8 } from "./kv_util.ts"; 12 | import { isNapiInterface, makeNapiBasedService } from "./napi_based.ts"; 13 | import { makeNativeService } from "./native.ts"; 14 | import { makeRemoteService } from "./remote.ts"; 15 | 16 | export * from "./napi_based.ts"; 17 | export * from "./remote.ts"; 18 | export * from "./in_memory.ts"; 19 | export * from "./kv_types.ts"; 20 | export { makeLimitedV8Serializer, UnknownV8 } from "./v8.ts"; 21 | 22 | export type KvImplementation = "in-memory" | "sqlite" | "remote"; 23 | 24 | /** 25 | * Open a new {@linkcode Kv} connection to persist data. 26 | * 27 | * When an url is provided, this will connect to a remote [Deno Deploy](https://deno.com/deploy) database 28 | * or any other endpoint that supports the open [KV Connect](https://github.com/denoland/denokv/blob/main/proto/kv-connect.md) protocol. 29 | * 30 | * When a local path is provided, this will use a sqlite database on disk. Read and write access to the file is required. 31 | * 32 | * When no path is provided, this will use an ephemeral in-memory implementation. 33 | */ 34 | export async function openKv( 35 | path?: string, 36 | opts: Record & { 37 | debug?: boolean; 38 | implementation?: KvImplementation; 39 | } = {}, 40 | ): Promise { 41 | checkOptionalString("path", path); 42 | checkRecord("opts", opts); 43 | checkOptionalBoolean("opts.debug", opts.debug); 44 | check( 45 | "opts.implementation", 46 | opts.implementation, 47 | opts.implementation === undefined || 48 | ["in-memory", "sqlite", "remote"].includes(opts.implementation), 49 | ); 50 | 51 | const { debug, implementation } = opts; 52 | 53 | // use built-in native implementation if available when running on Deno 54 | if ("Deno" in globalThis && !implementation) { 55 | // deno-lint-ignore no-explicit-any 56 | const { openKv } = (globalThis as any).Deno; 57 | if (typeof openKv === "function") return makeNativeService().openKv(path); 58 | } 59 | 60 | // use in-memory implementation if no path provided 61 | if (path === undefined || path === "" || implementation === "in-memory") { 62 | const maxQueueAttempts = typeof opts.maxQueueAttempts === "number" 63 | ? opts.maxQueueAttempts 64 | : undefined; 65 | return await makeInMemoryService({ debug, maxQueueAttempts }).openKv(path); 66 | } 67 | 68 | const { encodeV8, decodeV8 } = await (async () => { 69 | const { encodeV8, decodeV8 } = opts; 70 | const defined = [encodeV8, decodeV8].filter((v) => v !== undefined).length; 71 | if (defined === 1) { 72 | throw new Error(`Provide both 'encodeV8' or 'decodeV8', or neither`); 73 | } 74 | if (defined > 0) { 75 | if (typeof encodeV8 !== "function") { 76 | throw new Error(`Unexpected 'encodeV8': ${encodeV8}`); 77 | } 78 | if (typeof decodeV8 !== "function") { 79 | throw new Error(`Unexpected 'decodeV8': ${decodeV8}`); 80 | } 81 | return { encodeV8: encodeV8 as EncodeV8, decodeV8: decodeV8 as DecodeV8 }; 82 | } 83 | if ("Bun" in globalThis) { 84 | throw new Error( 85 | `Bun provides v8.serialize/deserialize, but it uses an incompatible format (JavaScriptCore). Provide explicit 'encodeV8' and 'decodeV8' functions via options. See https://www.npmjs.com/package/@deno/kv#other-runtimes`, 86 | ); // https://discord.com/channels/876711213126520882/888937948345684008/1150135641137487892 87 | } 88 | const v8 = await import(`${"v8"}`); 89 | if (!v8) throw new Error(`Unable to import the v8 module`); 90 | const { serialize, deserialize } = v8; 91 | if (typeof serialize !== "function") { 92 | throw new Error(`Unexpected 'serialize': ${serialize}`); 93 | } 94 | if (typeof deserialize !== "function") { 95 | throw new Error(`Unexpected 'deserialize': ${deserialize}`); 96 | } 97 | return { 98 | encodeV8: serialize as EncodeV8, 99 | decodeV8: deserialize as DecodeV8, 100 | }; 101 | })(); 102 | 103 | // use remote implementation if path looks like a url 104 | if (/^https?:\/\//i.test(path) || implementation === "remote") { 105 | let accessToken = 106 | typeof opts.accessToken === "string" && opts.accessToken !== "" 107 | ? opts.accessToken 108 | : undefined; 109 | if (accessToken === undefined) { 110 | // deno-lint-ignore no-explicit-any 111 | accessToken = (globalThis as any)?.process?.env?.DENO_KV_ACCESS_TOKEN; 112 | if (typeof accessToken !== "string") { 113 | throw new Error(`Set the DENO_KV_ACCESS_TOKEN to your access token`); 114 | } 115 | } 116 | const fetcher = typeof opts.fetcher === "function" 117 | ? opts.fetcher as typeof fetch 118 | : undefined; 119 | const maxRetries = typeof opts.maxRetries === "number" 120 | ? opts.maxRetries 121 | : undefined; 122 | const wrapUnknownValues = typeof opts.wrapUnknownValues === "boolean" 123 | ? opts.wrapUnknownValues 124 | : undefined; 125 | const supportedVersions = Array.isArray(opts.supportedVersions) && 126 | opts.supportedVersions.every((v) => typeof v === "number") 127 | ? opts.supportedVersions 128 | : undefined; 129 | return await makeRemoteService({ 130 | debug, 131 | accessToken, 132 | encodeV8, 133 | decodeV8, 134 | fetcher, 135 | maxRetries, 136 | supportedVersions, 137 | wrapUnknownValues, 138 | }).openKv(path); 139 | } 140 | 141 | // else use the sqlite napi implementation 142 | const { napi } = opts; 143 | if (napi !== undefined && !isNapiInterface(napi)) { 144 | throw new Error(`Unexpected napi interface for sqlite`); 145 | } 146 | const inMemory = typeof opts.inMemory === "boolean" 147 | ? opts.inMemory 148 | : undefined; 149 | return await makeNapiBasedService({ 150 | debug, 151 | encodeV8, 152 | decodeV8, 153 | napi, 154 | inMemory, 155 | }).openKv(path); 156 | } 157 | --------------------------------------------------------------------------------