├── examples ├── mnist │ ├── .gitignore │ ├── download.ts │ ├── predict.ts │ ├── train.ts │ ├── train_batchnorm.ts │ ├── train_dropout.ts │ └── common.ts ├── filters │ └── deno.png ├── sentiment-analysis │ ├── sentiment.st │ ├── mappings.json │ ├── analyzer.ts │ └── tester.ts ├── multiple-linear │ ├── README.md │ └── student.ts ├── classification │ ├── README.md │ ├── binary_iris.ts │ ├── iris.ts │ ├── binary_iris.csv │ └── spam.ts ├── tokenizers │ └── basic.ts ├── model │ ├── run.ts │ └── train.ts ├── autoencoders │ ├── encoded.html │ └── example.ts ├── linear.ts ├── xor_cpu.ts ├── xor_gpu.ts ├── xor_auto.ts ├── xor_wasm.ts └── xor_option.ts ├── packages ├── data │ ├── datasets │ │ ├── text.ts │ │ └── csv.ts │ ├── deps.ts │ ├── mod.ts │ ├── types.ts │ └── data.ts ├── utilities │ ├── src │ │ ├── mapper │ │ │ ├── mod.ts │ │ │ └── discrete.ts │ │ ├── transformer │ │ │ ├── mod.ts │ │ │ └── tfidf.ts │ │ ├── utils │ │ │ ├── mod.ts │ │ │ ├── misc │ │ │ │ ├── mod.ts │ │ │ │ ├── argmax.ts │ │ │ │ ├── get_constructor.ts │ │ │ │ └── image.ts │ │ │ ├── array │ │ │ │ ├── mod.ts │ │ │ │ ├── unique.ts │ │ │ │ ├── range.ts │ │ │ │ └── split.ts │ │ │ ├── random │ │ │ │ ├── mod.ts │ │ │ │ ├── rng.ts │ │ │ │ ├── rearrange.ts │ │ │ │ ├── weighted.ts │ │ │ │ ├── shuffle.ts │ │ │ │ └── normal.ts │ │ │ └── math.ts │ │ ├── metrics │ │ │ ├── mod.ts │ │ │ └── regression.ts │ │ ├── mod.ts │ │ ├── text │ │ │ ├── mod.ts │ │ │ ├── vectorizer.ts │ │ │ └── cleaner.ts │ │ ├── image │ │ │ ├── mod.ts │ │ │ ├── colors │ │ │ │ ├── mod.ts │ │ │ │ ├── histogram.ts │ │ │ │ └── common.ts │ │ │ └── patches │ │ │ │ └── patch_2d.ts │ │ ├── encoding │ │ │ ├── softmax.ts │ │ │ ├── onehot.ts │ │ │ ├── multihot.ts │ │ │ ├── termfrequency.ts │ │ │ └── mod.ts │ │ └── constants │ │ │ └── stop_words.ts │ ├── examples │ │ ├── extract-colors │ │ │ ├── out.png │ │ │ ├── kagu.png │ │ │ └── extract-colors.ts │ │ └── metrics │ │ │ ├── metrics.ts │ │ │ └── binary_iris.csv │ └── mod.ts ├── core │ ├── .DS_Store │ ├── src │ │ ├── backends │ │ │ ├── .DS_Store │ │ │ ├── wasm │ │ │ │ ├── lib │ │ │ │ │ ├── netsaur_bg.wasm │ │ │ │ │ └── netsaur.generated.d.ts │ │ │ │ ├── utils.ts │ │ │ │ └── mod.ts │ │ │ ├── gpu │ │ │ │ └── util.ts │ │ │ └── cpu │ │ │ │ └── util.ts │ │ ├── plugins │ │ │ ├── mod.ts │ │ │ └── types.ts │ │ ├── core │ │ │ ├── api │ │ │ │ ├── postprocess.ts │ │ │ │ ├── scheduler.ts │ │ │ │ ├── optimizer.ts │ │ │ │ ├── error.ts │ │ │ │ ├── shape.ts │ │ │ │ └── network.ts │ │ │ └── engine.ts │ │ └── backend_loader.ts │ └── mod.ts ├── tokenizers │ ├── lib │ │ └── netsaur_tokenizers_bg.wasm │ └── mod.ts └── visualizer │ ├── types.ts │ └── mod.ts ├── .gitpod.yml ├── .DS_Store ├── deps.ts ├── .gitignore ├── crates ├── core-gpu │ ├── src │ │ ├── gpu │ │ │ ├── mod.rs │ │ │ ├── schedulers │ │ │ │ ├── decay.rs │ │ │ │ ├── oc.rs │ │ │ │ └── mod.rs │ │ │ ├── optimizers │ │ │ │ ├── sgd.rs │ │ │ │ └── adam.rs │ │ │ ├── layers │ │ │ │ ├── flatten.rs │ │ │ │ ├── dense.rs │ │ │ │ └── dropout.rs │ │ │ ├── init.rs │ │ │ ├── cost.rs │ │ │ └── activation.rs │ │ ├── lib.rs │ │ ├── util.rs │ │ └── tensor.rs │ └── Cargo.toml ├── README.md ├── core │ ├── src │ │ ├── cpu │ │ │ ├── mod.rs │ │ │ ├── schedulers │ │ │ │ ├── decay.rs │ │ │ │ ├── oc.rs │ │ │ │ └── mod.rs │ │ │ ├── postprocessing │ │ │ │ ├── step.rs │ │ │ │ └── mod.rs │ │ │ ├── optimizers │ │ │ │ ├── sgd.rs │ │ │ │ ├── rmsprop.rs │ │ │ │ ├── adam.rs │ │ │ │ └── nadam.rs │ │ │ ├── layers │ │ │ │ ├── flatten.rs │ │ │ │ ├── dropout.rs │ │ │ │ └── dense.rs │ │ │ ├── layer_norm.rs │ │ │ ├── regularizer.rs │ │ │ └── init.rs │ │ ├── lib.rs │ │ ├── util.rs │ │ └── tensor.rs │ └── Cargo.toml └── tokenizers │ ├── Cargo.toml │ └── src │ ├── lib.rs │ └── util.rs ├── Cargo.toml ├── .gitpod.Dockerfile ├── .github └── workflows │ ├── publish.yml │ └── build.yml ├── web.ts ├── CONTRIBUTING.md ├── bench ├── netsaur_cpu.ts ├── netsaur_wasm.ts └── torch_cpu.py ├── LICENSE ├── mod.ts ├── deno.lock └── tests ├── wasm.test.ts └── cpu.test.ts /examples/mnist/.gitignore: -------------------------------------------------------------------------------- 1 | *.idx -------------------------------------------------------------------------------- /packages/data/datasets/text.ts: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitpod.yml: -------------------------------------------------------------------------------- 1 | image: 2 | file: .gitpod.Dockerfile -------------------------------------------------------------------------------- /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/denosaurs/netsaur/HEAD/.DS_Store -------------------------------------------------------------------------------- /packages/data/deps.ts: -------------------------------------------------------------------------------- 1 | export { CsvParseStream } from "jsr:@std/csv@1.0.3"; 2 | -------------------------------------------------------------------------------- /packages/utilities/src/mapper/mod.ts: -------------------------------------------------------------------------------- 1 | export { DiscreteMapper } from "./discrete.ts"; 2 | -------------------------------------------------------------------------------- /packages/data/mod.ts: -------------------------------------------------------------------------------- 1 | export * from "./datasets/csv.ts"; 2 | export * from "./data.ts"; 3 | -------------------------------------------------------------------------------- /packages/utilities/src/transformer/mod.ts: -------------------------------------------------------------------------------- 1 | export { TfIdfTransformer } from "./tfidf.ts"; 2 | -------------------------------------------------------------------------------- /examples/filters/deno.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/denosaurs/netsaur/HEAD/examples/filters/deno.png -------------------------------------------------------------------------------- /packages/core/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/denosaurs/netsaur/HEAD/packages/core/.DS_Store -------------------------------------------------------------------------------- /packages/core/src/backends/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/denosaurs/netsaur/HEAD/packages/core/src/backends/.DS_Store -------------------------------------------------------------------------------- /deps.ts: -------------------------------------------------------------------------------- 1 | export { dlopen } from "jsr:@denosaurs/plug@1.0.3"; 2 | export type { FetchOptions } from "jsr:@denosaurs/plug@1.0.3"; 3 | -------------------------------------------------------------------------------- /examples/sentiment-analysis/sentiment.st: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/denosaurs/netsaur/HEAD/examples/sentiment-analysis/sentiment.st -------------------------------------------------------------------------------- /packages/utilities/src/utils/mod.ts: -------------------------------------------------------------------------------- 1 | export * from "./random/mod.ts"; 2 | export * from "./array/mod.ts"; 3 | export * from "./misc/mod.ts"; 4 | -------------------------------------------------------------------------------- /packages/tokenizers/lib/netsaur_tokenizers_bg.wasm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/denosaurs/netsaur/HEAD/packages/tokenizers/lib/netsaur_tokenizers_bg.wasm -------------------------------------------------------------------------------- /packages/utilities/examples/extract-colors/out.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/denosaurs/netsaur/HEAD/packages/utilities/examples/extract-colors/out.png -------------------------------------------------------------------------------- /examples/multiple-linear/README.md: -------------------------------------------------------------------------------- 1 | # Multiple Linear Regression 2 | 3 | This example showcases linear regression on a dataset with multiple variables. 4 | -------------------------------------------------------------------------------- /packages/core/src/backends/wasm/lib/netsaur_bg.wasm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/denosaurs/netsaur/HEAD/packages/core/src/backends/wasm/lib/netsaur_bg.wasm -------------------------------------------------------------------------------- /packages/utilities/examples/extract-colors/kagu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/denosaurs/netsaur/HEAD/packages/utilities/examples/extract-colors/kagu.png -------------------------------------------------------------------------------- /examples/classification/README.md: -------------------------------------------------------------------------------- 1 | # Binary Classification 2 | 3 | This example showcases binary classification on the Iris dataset. The 4 | `Iris Virginica` class is omitted for this example. 5 | -------------------------------------------------------------------------------- /packages/utilities/src/metrics/mod.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Metrics for Machine Learning outcomes. 3 | * @module 4 | */ 5 | 6 | export * from "./classification.ts"; 7 | export * from "./regression.ts"; 8 | -------------------------------------------------------------------------------- /packages/utilities/src/utils/misc/mod.ts: -------------------------------------------------------------------------------- 1 | export * from "./get_constructor.ts"; 2 | export * from "./image.ts"; 3 | export * from "./matrix.ts"; 4 | export * from "./tensor.ts"; 5 | export * from "./argmax.ts" -------------------------------------------------------------------------------- /packages/utilities/src/mod.ts: -------------------------------------------------------------------------------- 1 | export * from "./image/mod.ts"; 2 | export * from "./utils/mod.ts"; 3 | export * from "./metrics/mod.ts"; 4 | export * from "./encoding/mod.ts"; 5 | export * from "./text/mod.ts"; 6 | -------------------------------------------------------------------------------- /packages/utilities/src/text/mod.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Text-related utilities for Machine Learning 3 | * and Data Science. 4 | * @module 5 | */ 6 | 7 | export * from "./cleaner.ts"; 8 | export * from "./vectorizer.ts" -------------------------------------------------------------------------------- /examples/sentiment-analysis/mappings.json: -------------------------------------------------------------------------------- 1 | [["empty",0],["enthusiasm",1],["neutral",2],["sadness",3],["worry",4],["surprise",5],["love",6],["fun",7],["happiness",8],["hate",9],["boredom",10],["relief",11],["anger",12]] -------------------------------------------------------------------------------- /packages/utilities/src/image/mod.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Image-related utilities for Machine Learning 3 | * and Data Science. 4 | * @module 5 | */ 6 | export * from "./colors/mod.ts"; 7 | export * from "./patches/patch_2d.ts"; 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode/ 2 | .idea/ 3 | build/ 4 | xor_model.bin 5 | esr.bin 6 | node_modules/ 7 | output.png 8 | digit_model.bin 9 | digit_model.json 10 | package-lock.json 11 | test.ts 12 | experiments 13 | target/ 14 | 15 | *.test.bin 16 | *.test.st 17 | _test/ -------------------------------------------------------------------------------- /packages/utilities/src/utils/array/mod.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Utilities for working with arrays in appraisal. 3 | * @module 4 | */ 5 | export { useRange, useSeries } from "./range.ts"; 6 | export { useUnique } from "./unique.ts"; 7 | export { useSplit } from "./split.ts"; 8 | -------------------------------------------------------------------------------- /packages/utilities/src/utils/random/mod.ts: -------------------------------------------------------------------------------- 1 | export { useNormal, useNormalArray } from "./normal.ts"; 2 | export { useRearrange } from "./rearrange.ts"; 3 | export { useRNG } from "./rng.ts"; 4 | export { useShuffle } from "./shuffle.ts"; 5 | export { useWeighted } from "./weighted.ts"; 6 | -------------------------------------------------------------------------------- /crates/core-gpu/src/gpu/mod.rs: -------------------------------------------------------------------------------- 1 | mod activation; 2 | mod backend; 3 | mod cost; 4 | mod init; 5 | mod layers; 6 | mod optimizers; 7 | mod schedulers; 8 | 9 | pub use activation::*; 10 | pub use backend::*; 11 | pub use cost::*; 12 | pub use init::*; 13 | pub use layers::*; 14 | pub use optimizers::*; 15 | pub use schedulers::*; 16 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | package.version = "0.4.2" 3 | members = ["crates/*"] 4 | resolver = "2" 5 | 6 | [workspace.dependencies] 7 | cudarc = "0.9.14" 8 | ndarray = "0.16.1" 9 | ndarray-rand = "0.15.0" 10 | safetensors = "0.4.5" 11 | serde = { version = "1.0", features = ["derive"] } 12 | serde_json = "1.0" 13 | thiserror = "1.0.49" -------------------------------------------------------------------------------- /packages/utilities/src/utils/misc/argmax.ts: -------------------------------------------------------------------------------- 1 | export function argmax(mat: ArrayLike): number { 2 | let max = mat[0]; 3 | let index = 0; 4 | for (let i = 0; i < mat.length; i++) { 5 | if (mat[i] > max) { 6 | max = mat[i] as typeof max; 7 | index = i; 8 | } 9 | } 10 | return index; 11 | } 12 | -------------------------------------------------------------------------------- /packages/visualizer/types.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Line Type for Jupyter Notebook 3 | */ 4 | export interface Line { 5 | x: number[]; 6 | y: number[]; 7 | type?: "scatter" | "bar"; 8 | mode?: "markers" | "lines" | "lines+markers"; 9 | name?: string; 10 | line?: { 11 | color?: string; 12 | width?: number; 13 | }; 14 | } 15 | -------------------------------------------------------------------------------- /packages/core/src/plugins/mod.ts: -------------------------------------------------------------------------------- 1 | import type { Plugin } from "./types.ts"; 2 | import type { NeuralNetwork } from "../../../../mod.ts"; 3 | 4 | /** 5 | * Load a plugin into a NeuralNetwork instance. 6 | */ 7 | export const loadPlugin = ( 8 | instance: NeuralNetwork, 9 | loader: (instance: NeuralNetwork) => Plugin, 10 | ): Plugin => loader(instance); 11 | -------------------------------------------------------------------------------- /examples/tokenizers/basic.ts: -------------------------------------------------------------------------------- 1 | import { init, Tokenizer } from "../../packages/tokenizers/mod.ts"; 2 | 3 | await init(); 4 | 5 | const tokenizer = Tokenizer.fromJSON( 6 | await (await fetch( 7 | `https://huggingface.co/satvikag/chatbot/resolve/main/tokenizer.json`, 8 | )).text(), 9 | ); 10 | 11 | console.log("Hello World!", tokenizer.encode("Hello World!")); 12 | -------------------------------------------------------------------------------- /.gitpod.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM gitpod/workspace-full 2 | 3 | RUN curl -fsSL https://deno.land/x/install/install.sh | sh 4 | RUN /home/gitpod/.deno/bin/deno completions bash > /home/gitpod/.bashrc.d/90-deno && \ 5 | echo 'export DENO_INSTALL="/home/gitpod/.deno"' >> /home/gitpod/.bashrc.d/90-deno && \ 6 | echo 'export PATH="$DENO_INSTALL/bin:$PATH"' >> /home/gitpod/.bashrc.d/90-deno -------------------------------------------------------------------------------- /packages/core/src/plugins/types.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Netsaur Plugin Interface 3 | */ 4 | export interface Plugin { 5 | /** 6 | * Plugin name. 7 | */ 8 | name: string; 9 | 10 | /** 11 | * Plugin version. 12 | */ 13 | version: `${number}.${number}.${number}` | `v${number}.${number}.${number}`; 14 | // deno-lint-ignore no-explicit-any 15 | [method: string]: any; 16 | } 17 | -------------------------------------------------------------------------------- /crates/README.md: -------------------------------------------------------------------------------- 1 | # Netsaur Rust Crates 2 | 3 | This directory contains the source code for the Netsaur Rust crates. 4 | 5 | ## Crates 6 | 7 | - [core](/core) - The main crate for the Netsaur FFI and wasm bindings. 8 | - [core-gpu](/core-gpu) - The main crate for the Netsaur GPU FFI and wasm 9 | bindings. 10 | - [tokenizers](/tokenizers) - The main crate for the Netsaur tokenizers wasm 11 | bindings. 12 | -------------------------------------------------------------------------------- /crates/core-gpu/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | edition = "2021" 3 | name = "netsaur-gpu" 4 | version = { workspace = true } 5 | 6 | [lib] 7 | crate-type = ["cdylib"] 8 | 9 | [dependencies] 10 | cudarc = { workspace = true } 11 | ndarray = { workspace = true } 12 | ndarray-rand = { workspace = true } 13 | safetensors = { workspace = true } 14 | serde = { workspace = true } 15 | serde_json = { workspace = true } 16 | thiserror = { workspace = true } 17 | -------------------------------------------------------------------------------- /packages/utilities/src/utils/array/unique.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Remove duplicate values in an array. 3 | * Uses a strict = for identifying duplicates. 4 | * @param {T[]} arr Array to remove duplicates from. 5 | * @returns {T[]} Array with only unique elements. 6 | */ 7 | export function useUnique(arr: ArrayLike): T[] { 8 | const array = Array.from(arr); 9 | return [...new Set(array)] 10 | // return array.filter((x, i) => array.indexOf(x) === i); 11 | } 12 | -------------------------------------------------------------------------------- /crates/core/src/cpu/mod.rs: -------------------------------------------------------------------------------- 1 | mod activation; 2 | mod backend; 3 | mod cost; 4 | mod init; 5 | mod layers; 6 | mod layer_norm; 7 | mod optimizers; 8 | mod postprocessing; 9 | mod regularizer; 10 | mod schedulers; 11 | 12 | pub use activation::*; 13 | pub use backend::*; 14 | pub use cost::*; 15 | pub use init::*; 16 | pub use layers::*; 17 | pub use layer_norm::*; 18 | pub use optimizers::*; 19 | pub use postprocessing::*; 20 | pub use regularizer::*; 21 | pub use schedulers::*; -------------------------------------------------------------------------------- /packages/data/types.ts: -------------------------------------------------------------------------------- 1 | import type { Rank, Tensor } from "../core/mod.ts"; 2 | 3 | export interface DataLike { 4 | /** 5 | * Model input data 6 | */ 7 | train_x: Tensor; 8 | 9 | /** 10 | * Model output data / labels 11 | */ 12 | train_y: Tensor; 13 | 14 | /** 15 | * Model test input data 16 | */ 17 | test_x?: Tensor; 18 | 19 | /** 20 | * Model test output data / labels 21 | */ 22 | test_y?: Tensor; 23 | } 24 | -------------------------------------------------------------------------------- /packages/utilities/src/utils/random/rng.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Get random number from range 3 | * @param min Min value of range. 4 | * @param max Max value of range. 5 | * @param allowDecimals Whether to allow decimal point in result. 6 | * @returns Random number 7 | */ 8 | export function useRNG( 9 | min: number, 10 | max: number, 11 | allowDecimals = false, 12 | ): number { 13 | const rng = min + (Math.random() * (max - min)); 14 | return allowDecimals ? rng : Math.round(rng); 15 | } 16 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish 2 | on: 3 | workflow_dispatch: 4 | release: 5 | types: [published] 6 | 7 | jobs: 8 | publish: 9 | runs-on: ubuntu-latest 10 | 11 | permissions: 12 | contents: read 13 | id-token: write 14 | 15 | steps: 16 | - uses: actions/checkout@v4 17 | - name: Install Deno 18 | uses: denoland/setup-deno@v1 19 | with: 20 | deno-version: canary 21 | 22 | - name: Publish package 23 | run: deno publish 24 | -------------------------------------------------------------------------------- /packages/core/src/backends/wasm/utils.ts: -------------------------------------------------------------------------------- 1 | import type { Rank, Shape } from "../../core/api/shape.ts"; 2 | 3 | /** 4 | * Train Options Interface. 5 | */ 6 | export interface TrainOptions { 7 | datasets: number; 8 | inputShape: Shape; 9 | outputShape: Shape; 10 | epochs: number; 11 | batches: number; 12 | rate: number; 13 | } 14 | 15 | /** 16 | * Predict Options Interface. 17 | */ 18 | export interface PredictOptions { 19 | inputShape: Shape; 20 | outputShape: Shape; 21 | } 22 | -------------------------------------------------------------------------------- /packages/core/mod.ts: -------------------------------------------------------------------------------- 1 | export { setupBackend } from "./src/core/engine.ts"; 2 | export * from "./src/core/mod.ts"; 3 | export * from "./src/core/types.ts"; 4 | export * from "./src/core/tensor/tensor.ts"; 5 | export * from "./src/core/api/layers.ts"; 6 | export * from "./src/core/api/shape.ts"; 7 | export * from "./src/core/api/network.ts"; 8 | export * from "./src/core/api/optimizer.ts"; 9 | export * from "./src/core/api/scheduler.ts"; 10 | export * from "./src/backend_loader.ts" 11 | export * from "./src/core/api/postprocess.ts" -------------------------------------------------------------------------------- /crates/core/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | edition = "2021" 3 | name = "netsaur" 4 | version = { workspace = true } 5 | 6 | [lib] 7 | crate-type = ["cdylib"] 8 | 9 | [dependencies] 10 | ndarray = { workspace = true } 11 | ndarray-rand = { workspace = true } 12 | serde = { workspace = true } 13 | serde_json = { workspace = true } 14 | safetensors = { workspace = true } 15 | 16 | [target.'cfg(target_arch = "wasm32")'.dependencies] 17 | wasm-bindgen = "0.2.92" 18 | getrandom = { version = "0.2", features = ["js"] } 19 | js-sys = "0.3.69" -------------------------------------------------------------------------------- /packages/utilities/src/image/colors/mod.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Image-related utilities for machine learning. 3 | * @module 4 | */ 5 | 6 | import { Image } from "../../utils/mod.ts"; 7 | import type { Pixel } from "../../utils/common_types.ts"; 8 | import { quantizeByMedianCut } from "./median_cut.ts"; 9 | 10 | /** Extract colors from an image. */ 11 | export function extractColors(image: Image, nColors: number): Pixel[] { 12 | return quantizeByMedianCut(image, nColors, 5); 13 | } 14 | 15 | export { getHistogram } from "./histogram.ts"; 16 | export { Image }; 17 | -------------------------------------------------------------------------------- /web.ts: -------------------------------------------------------------------------------- 1 | export { setupBackend } from "./packages/core/src/core/engine.ts"; 2 | export * from "./packages/core/src/core/mod.ts"; 3 | export * from "./packages/core/src/core/types.ts"; 4 | export * from "./packages/core/src/core/tensor/tensor.ts"; 5 | export * from "./packages/core/src/core/api/layers.ts"; 6 | export * from "./packages/core/src/core/api/shape.ts"; 7 | export * from "./packages/core/src/core/api/network.ts"; 8 | export * from "./packages/core/src/core/api/optimizer.ts"; 9 | 10 | export { WASM } from "./packages/core/src/backends/wasm/mod.ts"; 11 | -------------------------------------------------------------------------------- /crates/tokenizers/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | edition = "2021" 3 | name = "netsaur-tokenizers" 4 | version = { workspace = true } 5 | 6 | [lib] 7 | crate-type = ["cdylib"] 8 | 9 | [dependencies] 10 | ndarray = { workspace = true } 11 | ndarray-rand = { workspace = true } 12 | serde = { workspace = true } 13 | serde_json = { workspace = true } 14 | serde-wasm-bindgen = "0.6.0" 15 | tokenizers = { version="0.20.0", default-features=false, features = ["unstable_wasm"]} 16 | wasm-bindgen = "0.2.92" 17 | getrandom = { version = "0.2", features = ["js"] } 18 | js-sys = "0.3.69" 19 | -------------------------------------------------------------------------------- /crates/core-gpu/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod gpu; 2 | mod ffi; 3 | mod tensor; 4 | mod types; 5 | mod util; 6 | 7 | pub use gpu::*; 8 | 9 | pub use ffi::*; 10 | pub use tensor::*; 11 | pub use types::*; 12 | pub use util::*; 13 | 14 | use std::cell::RefCell; 15 | 16 | pub struct Resources { 17 | pub backend: RefCell>, 18 | } 19 | 20 | impl Resources { 21 | pub fn new() -> Self { 22 | Self { 23 | backend: RefCell::new(Vec::new()), 24 | } 25 | } 26 | } 27 | 28 | thread_local! { 29 | pub static RESOURCES: Resources = Resources::new(); 30 | } 31 | -------------------------------------------------------------------------------- /packages/utilities/src/utils/random/rearrange.ts: -------------------------------------------------------------------------------- 1 | import { useShuffle } from "./shuffle.ts"; 2 | 3 | /** 4 | * Rearrange characters in a string randomly. 5 | * @param {number|string} n Number / String to rearrange. 6 | * @returns {number|string} Number / String rearranged randomly. 7 | */ 8 | export function useRearrange(n: number | string): number | string { 9 | const res = (typeof n === "number" ? n.toString() : n).split(""); 10 | const shuffled = useShuffle(res).join(""); 11 | return typeof n === "number" ? Number(shuffled) : shuffled; 12 | } 13 | export default useRearrange; 14 | -------------------------------------------------------------------------------- /crates/tokenizers/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod util; 2 | mod wasm; 3 | use tokenizers::{ModelWrapper, Tokenizer}; 4 | 5 | pub use util::*; 6 | 7 | pub use wasm::*; 8 | 9 | use std::cell::RefCell; 10 | 11 | pub struct Resources { 12 | pub tokenizer: RefCell>, 13 | pub model: RefCell>, 14 | } 15 | 16 | impl Resources { 17 | pub fn new() -> Self { 18 | Self { 19 | tokenizer: RefCell::new(Vec::new()), 20 | model: RefCell::new(Vec::new()), 21 | } 22 | } 23 | } 24 | 25 | thread_local! { 26 | pub static RESOURCES: Resources = Resources::new(); 27 | } 28 | -------------------------------------------------------------------------------- /crates/core-gpu/src/gpu/schedulers/decay.rs: -------------------------------------------------------------------------------- 1 | use crate::DecayScheduler; 2 | 3 | pub struct GPUDecayScheduler { 4 | pub rate: f32, 5 | pub step_size: usize, 6 | } 7 | 8 | impl GPUDecayScheduler { 9 | pub fn new(config: &DecayScheduler) -> Self { 10 | GPUDecayScheduler { 11 | rate: config.rate, 12 | step_size: config.step_size, 13 | } 14 | } 15 | pub fn exponential(&self, rate: f32, step: usize) -> f32 { 16 | rate * self.rate.powi((step / self.step_size) as i32) 17 | } 18 | pub fn linear(&self, rate: f32, step: usize) -> f32 { 19 | rate / (self.rate * (1 + step / self.step_size) as f32) 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /crates/core/src/cpu/schedulers/decay.rs: -------------------------------------------------------------------------------- 1 | use crate::DecayScheduler; 2 | 3 | pub struct CPUDecayScheduler { 4 | pub rate: f32, 5 | pub step_size: usize, 6 | } 7 | 8 | impl CPUDecayScheduler { 9 | pub fn new(config: &DecayScheduler) -> Self { 10 | CPUDecayScheduler { 11 | rate: config.rate, 12 | step_size: config.step_size, 13 | } 14 | } 15 | pub fn exponential(&self, rate: f32, step: usize) -> f32 { 16 | rate * self.rate.powi((step / self.step_size) as i32) 17 | } 18 | pub fn linear(&self, rate: f32, step: usize) -> f32 { 19 | rate / (self.rate * (1 + step / self.step_size) as f32) 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /crates/core/src/cpu/postprocessing/step.rs: -------------------------------------------------------------------------------- 1 | use crate::StepFunctionConfig; 2 | 3 | pub struct CPUStepFunction { 4 | thresholds: Vec, 5 | values: Vec 6 | } 7 | impl CPUStepFunction { 8 | pub fn new(config: &StepFunctionConfig) -> Self { 9 | return Self { 10 | thresholds: config.thresholds.clone(), 11 | values: config.values.clone() 12 | } 13 | } 14 | pub fn step(&self, x: f32) -> f32 { 15 | for (i, &threshold) in self.thresholds.iter().enumerate() { 16 | if x < threshold { 17 | return self.values[i]; 18 | } 19 | } 20 | return self.values.last().unwrap().clone() 21 | } 22 | } -------------------------------------------------------------------------------- /crates/core-gpu/src/gpu/optimizers/sgd.rs: -------------------------------------------------------------------------------- 1 | use std::ops::{Mul, SubAssign}; 2 | 3 | use ndarray::{ArrayViewD, ArrayViewMutD}; 4 | 5 | use crate::GPUScheduler; 6 | 7 | pub struct GPUSGDOptimizer {} 8 | 9 | impl GPUSGDOptimizer { 10 | pub fn new() -> Self { 11 | Self {} 12 | } 13 | 14 | pub fn update_grads( 15 | &mut self, 16 | mut params: Vec>, 17 | grads: Vec>, 18 | scheduler: &GPUScheduler, 19 | rate: f32, 20 | epoch: usize, 21 | ) { 22 | let eta = scheduler.eta(rate, epoch); 23 | for (param, grad) in params.iter_mut().zip(grads) { 24 | param.sub_assign(&grad.mul(eta)); 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /packages/data/data.ts: -------------------------------------------------------------------------------- 1 | import type { Rank, Tensor } from "../core/mod.ts"; 2 | import { type CsvLoaderConfig, loadCsv } from "./datasets/csv.ts"; 3 | import type { DataLike } from "./types.ts"; 4 | 5 | export class Data { 6 | /** 7 | * Model input data 8 | */ 9 | inputs: Tensor; 10 | 11 | /** 12 | * Model output data / labels 13 | */ 14 | outputs: Tensor; 15 | 16 | constructor(data: DataLike) { 17 | this.inputs = data.train_x; 18 | this.outputs = data.train_y; 19 | } 20 | 21 | /** 22 | * Load data from a CSV file or URL containing CSV data. 23 | */ 24 | static async csv(url: string | URL, config?: CsvLoaderConfig): Promise { 25 | return new Data(await loadCsv(url, config)); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /packages/core/src/core/api/postprocess.ts: -------------------------------------------------------------------------------- 1 | /** Post-processing step only occuring during prediction routine */ 2 | export type PostProcessor = 3 | | { type: "none" } 4 | | { type: "sign" } 5 | | { type: "step"; config: StepFunctionConfig }; 6 | 7 | type StepFunctionConfig = { thresholds: number[]; values: number[] }; 8 | 9 | export function PostProcess(pType: "none" | "sign"): PostProcessor; 10 | export function PostProcess( 11 | pType: "step", 12 | config: StepFunctionConfig 13 | ): PostProcessor; 14 | export function PostProcess( 15 | pType: "none" | "sign" | "step", 16 | config?: StepFunctionConfig 17 | ) { 18 | if (pType === "none" || pType === "sign") { 19 | return { type: pType }; 20 | } 21 | return { type: pType, config }; 22 | } 23 | -------------------------------------------------------------------------------- /examples/mnist/download.ts: -------------------------------------------------------------------------------- 1 | async function download(url: string, to: string) { 2 | console.log("Download", url); 3 | const f = await Deno.open(new URL(to, import.meta.url), { 4 | write: true, 5 | create: true, 6 | }); 7 | await fetch(url).then((response) => { 8 | response.body!.pipeThrough(new DecompressionStream("gzip")).pipeTo( 9 | f.writable, 10 | ); 11 | }); 12 | } 13 | 14 | await download( 15 | "train-images-idx3-ubyte.gz", 16 | "train-images.idx", 17 | ); 18 | await download( 19 | "./train-labels-idx1-ubyte.gz", 20 | "train-labels.idx", 21 | ); 22 | await download( 23 | "./t10k-images-idx3-ubyte.gz", 24 | "test-images.idx", 25 | ); 26 | await download( 27 | "./t10k-labels-idx1-ubyte.gz", 28 | "test-labels.idx", 29 | ); 30 | -------------------------------------------------------------------------------- /crates/core/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod cpu; 2 | #[cfg(not(target_arch = "wasm32"))] 3 | mod ffi; 4 | mod tensor; 5 | mod types; 6 | mod util; 7 | #[cfg(target_arch = "wasm32")] 8 | mod wasm; 9 | 10 | pub use cpu::*; 11 | 12 | 13 | #[cfg(not(target_arch = "wasm32"))] 14 | pub use ffi::*; 15 | pub use tensor::*; 16 | pub use types::*; 17 | pub use util::*; 18 | 19 | #[cfg(target_arch = "wasm32")] 20 | pub use wasm::*; 21 | 22 | use std::cell::RefCell; 23 | 24 | pub struct Resources { 25 | pub backend: RefCell>, 26 | } 27 | 28 | impl Resources { 29 | pub fn new() -> Self { 30 | Self { 31 | backend: RefCell::new(Vec::new()), 32 | } 33 | } 34 | } 35 | 36 | thread_local! { 37 | pub static RESOURCES: Resources = Resources::new(); 38 | } 39 | -------------------------------------------------------------------------------- /crates/core/src/cpu/optimizers/sgd.rs: -------------------------------------------------------------------------------- 1 | use std::ops::{Mul, SubAssign}; 2 | 3 | use ndarray::{ArrayViewD, ArrayViewMutD}; 4 | 5 | use crate::CPUScheduler; 6 | 7 | pub struct CPUSGDOptimizer {} 8 | 9 | impl CPUSGDOptimizer { 10 | pub fn new() -> Self { 11 | Self {} 12 | } 13 | 14 | pub fn update_grads( 15 | &mut self, 16 | mut params: Vec>, 17 | grads: Vec>, 18 | scheduler: &CPUScheduler, 19 | rate: f32, 20 | epoch: usize, 21 | l: Vec>, 22 | ) { 23 | let eta = scheduler.eta(rate, epoch); 24 | for ((param, grad), li) in params.iter_mut().zip(grads).zip(l) { 25 | param.sub_assign(&(&grad - &li).mul(eta)); 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | ## Building `backends/cpu` 4 | 5 | Unoptimized: 6 | 7 | ```sh 8 | cargo build --release -p netsaur 9 | ``` 10 | 11 | Optimized: 12 | 13 | ```sh 14 | deno run build:cpu 15 | ``` 16 | 17 | ## Building `backends/wasm` 18 | 19 | Unoptimized: 20 | 21 | ```sh 22 | deno -Ar jsr:@deno/wasmbuild@0.17.2 -p netsaur --out src/backends/wasm/lib --debug 23 | ``` 24 | 25 | Optimized: 26 | 27 | ```sh 28 | deno run build:wasm 29 | ``` 30 | 31 | ## Building `tokenizers` 32 | 33 | Unoptimized: 34 | 35 | ```sh 36 | deno -Ar jsr:@deno/wasmbuild@0.17.2 -p netsaur-tokenizers --out tokenizers/lib --debug 37 | ``` 38 | 39 | Optimized: 40 | 41 | ```sh 42 | deno run build:tokenizers 43 | ``` 44 | 45 | ## Building everything 46 | 47 | Optimized: 48 | 49 | ```sh 50 | deno run build 51 | ``` 52 | -------------------------------------------------------------------------------- /crates/core/src/cpu/schedulers/oc.rs: -------------------------------------------------------------------------------- 1 | use crate::OneCycleScheduler; 2 | 3 | pub struct CPUOneCycleScheduler { 4 | pub max_rate: f32, 5 | pub step_size: usize, 6 | } 7 | 8 | impl CPUOneCycleScheduler { 9 | pub fn new(config: &OneCycleScheduler) -> Self { 10 | CPUOneCycleScheduler { 11 | max_rate: config.max_rate, 12 | step_size: config.step_size, 13 | } 14 | } 15 | pub fn eta(&self, rate: f32, step: usize) -> f32 { 16 | let steps = self.step_size as f32; 17 | let step = step % (2 * self.step_size); 18 | if step < self.step_size { 19 | rate + (self.max_rate - rate) * (step as f32) / (steps) 20 | } else { 21 | self.max_rate - (self.max_rate - rate) * ((step - self.step_size) as f32) / (steps) 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /crates/core-gpu/src/gpu/schedulers/oc.rs: -------------------------------------------------------------------------------- 1 | use crate::OneCycleScheduler; 2 | 3 | pub struct GPUOneCycleScheduler { 4 | pub max_rate: f32, 5 | pub step_size: usize, 6 | } 7 | 8 | impl GPUOneCycleScheduler { 9 | pub fn new(config: &OneCycleScheduler) -> Self { 10 | GPUOneCycleScheduler { 11 | max_rate: config.max_rate, 12 | step_size: config.step_size, 13 | } 14 | } 15 | pub fn eta(&self, rate: f32, step: usize) -> f32 { 16 | let steps = self.step_size as f32; 17 | let step = step % (2 * self.step_size); 18 | if step < self.step_size { 19 | rate + (self.max_rate - rate) * (step as f32) / (steps) 20 | } else { 21 | self.max_rate - (self.max_rate - rate) * ((step - self.step_size) as f32) / (steps) 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /crates/tokenizers/src/util.rs: -------------------------------------------------------------------------------- 1 | use std::slice::from_raw_parts; 2 | 3 | use ndarray::ArrayD; 4 | use serde::Deserialize; 5 | 6 | pub struct Logger { 7 | pub log: fn(string: String) -> (), 8 | } 9 | 10 | pub fn length(shape: Vec) -> usize { 11 | return shape.iter().fold(1, |i, x| i * x); 12 | } 13 | 14 | pub fn decode_array(ptr: *const f32, shape: Vec) -> ArrayD { 15 | let buffer = unsafe { from_raw_parts(ptr, length(shape.clone())) }; 16 | let vec = Vec::from(buffer); 17 | return ArrayD::from_shape_vec(shape, vec).unwrap(); 18 | } 19 | 20 | pub fn decode_json<'a, T>(ptr: *const u8, len: usize) -> T 21 | where 22 | T: Deserialize<'a>, 23 | { 24 | let buffer = unsafe { from_raw_parts(ptr, len) }; 25 | let json = std::str::from_utf8(&buffer[0..len]).unwrap(); 26 | return serde_json::from_str(&json).unwrap(); 27 | } 28 | -------------------------------------------------------------------------------- /crates/core/src/cpu/postprocessing/mod.rs: -------------------------------------------------------------------------------- 1 | use ndarray::ArrayD; 2 | use crate::PostProcessor; 3 | 4 | mod step; 5 | use step::CPUStepFunction; 6 | 7 | pub enum CPUPostProcessor { 8 | None, 9 | Sign, 10 | Step(CPUStepFunction), 11 | } 12 | 13 | impl CPUPostProcessor { 14 | pub fn from(processor: &PostProcessor) -> Self { 15 | match processor { 16 | PostProcessor::None => CPUPostProcessor::None, 17 | PostProcessor::Sign => CPUPostProcessor::Sign, 18 | PostProcessor::Step(config) => CPUPostProcessor::Step(CPUStepFunction::new(config)), 19 | } 20 | } 21 | pub fn process(&self, x: ArrayD) -> ArrayD { 22 | match self { 23 | CPUPostProcessor::None => x, 24 | CPUPostProcessor::Sign => x.map(|y| y.signum()), 25 | CPUPostProcessor::Step(processor) => x.map(|y| processor.step(*y)), 26 | } 27 | } 28 | } -------------------------------------------------------------------------------- /examples/model/run.ts: -------------------------------------------------------------------------------- 1 | import { CPU, Sequential, setupBackend, tensor2D } from "../../packages/core/mod.ts"; 2 | 3 | /** 4 | * Setup the CPU backend. This backend is fast but doesn't work on the Edge. 5 | */ 6 | await setupBackend(CPU); 7 | 8 | const model = Sequential.loadFile("examples/model/xor.test.st"); 9 | 10 | /** 11 | * Predict the output of the XOR function for the given inputs. 12 | */ 13 | const out1 = (await model.predict(tensor2D([[0, 0]]))).data; 14 | console.log(`0 xor 0 = ${out1[0]} (should be close to 0)`); 15 | 16 | const out2 = (await model.predict(tensor2D([[1, 0]]))).data; 17 | console.log(`1 xor 0 = ${out2[0]} (should be close to 1)`); 18 | 19 | const out3 = (await model.predict(tensor2D([[0, 1]]))).data; 20 | console.log(`0 xor 1 = ${out3[0]} (should be close to 1)`); 21 | 22 | const out4 = (await model.predict(tensor2D([[1, 1]]))).data; 23 | console.log(`1 xor 1 = ${out4[0]} (should be close to 0)`); 24 | -------------------------------------------------------------------------------- /packages/utilities/src/utils/math.ts: -------------------------------------------------------------------------------- 1 | import type { DataType } from "./common_types.ts"; 2 | import { Matrix, type MatrixLike } from "./misc/matrix.ts"; 3 | 4 | /** A very basic, low-effort multiplication. */ 5 | export function multiplyDiags( 6 | x: MatrixLike, 7 | y: ArrayLike, 8 | ): Matrix { 9 | const res = new Matrix(x); 10 | if (y.length !== res.nCols) { 11 | throw new Error( 12 | `Expected diagonal vector of shape (${res.nCols}, 1). Found (${y.length}, 1).`, 13 | ); 14 | } 15 | let i = 0; 16 | while (i < res.nRows) { 17 | const offset = i * res.nCols; 18 | let j = 0; 19 | while (j < y.length) { 20 | res.setCell( 21 | i, 22 | j, 23 | // @ts-ignore types will always match 24 | x.data[offset + j] * (typeof res[0] === "bigint" ? BigInt(y[j]) : y[j]), 25 | ); 26 | j += 1; 27 | } 28 | i += 1; 29 | } 30 | return res as Matrix; 31 | } 32 | -------------------------------------------------------------------------------- /bench/netsaur_cpu.ts: -------------------------------------------------------------------------------- 1 | import { 2 | Cost, 3 | CPU, 4 | DenseLayer, 5 | Sequential, 6 | setupBackend, 7 | SigmoidLayer, 8 | tensor1D, 9 | tensor2D, 10 | } from "../mod.ts"; 11 | 12 | await setupBackend(CPU); 13 | 14 | const net = new Sequential({ 15 | size: [4, 2], 16 | silent: true, 17 | layers: [ 18 | DenseLayer({ size: [3] }), 19 | SigmoidLayer(), 20 | DenseLayer({ size: [1] }), 21 | SigmoidLayer(), 22 | ], 23 | cost: Cost.MSE, 24 | }); 25 | 26 | net.train( 27 | [ 28 | { 29 | inputs: tensor2D([ 30 | [0, 0], 31 | [1, 0], 32 | [0, 1], 33 | [1, 1], 34 | ]), 35 | outputs: tensor2D([[0], [1], [1], [0]]), 36 | }, 37 | ], 38 | 10000, 39 | ); 40 | 41 | console.log((await net.predict(tensor1D([0, 0]))).data); 42 | console.log((await net.predict(tensor1D([1, 0]))).data); 43 | console.log((await net.predict(tensor1D([0, 1]))).data); 44 | console.log((await net.predict(tensor1D([1, 1]))).data); 45 | -------------------------------------------------------------------------------- /bench/netsaur_wasm.ts: -------------------------------------------------------------------------------- 1 | import { 2 | Cost, 3 | DenseLayer, 4 | Sequential, 5 | setupBackend, 6 | SigmoidLayer, 7 | tensor1D, 8 | tensor2D, 9 | WASM, 10 | } from "../mod.ts"; 11 | 12 | await setupBackend(WASM); 13 | 14 | const net = new Sequential({ 15 | size: [4, 2], 16 | silent: true, 17 | layers: [ 18 | DenseLayer({ size: [3] }), 19 | SigmoidLayer(), 20 | DenseLayer({ size: [1] }), 21 | SigmoidLayer(), 22 | ], 23 | cost: Cost.MSE, 24 | }); 25 | 26 | net.train( 27 | [ 28 | { 29 | inputs: tensor2D([ 30 | [0, 0], 31 | [1, 0], 32 | [0, 1], 33 | [1, 1], 34 | ]), 35 | outputs: tensor2D([[0], [1], [1], [0]]), 36 | }, 37 | ], 38 | 10000, 39 | ); 40 | 41 | console.log((await net.predict(tensor1D([0, 0]))).data); 42 | console.log((await net.predict(tensor1D([1, 0]))).data); 43 | console.log((await net.predict(tensor1D([0, 1]))).data); 44 | console.log((await net.predict(tensor1D([1, 1]))).data); 45 | -------------------------------------------------------------------------------- /packages/utilities/src/utils/random/weighted.ts: -------------------------------------------------------------------------------- 1 | // Check out https://github.com/retraigo/fortuna 2 | 3 | export interface WeightedChoice { 4 | result: ItemType; 5 | chance: number; 6 | } 7 | 8 | /** 9 | * Roll one from an array of weighted choices. 10 | * @param {WeightedChoice[]} choices - Choices to roll from. 11 | * @param {number} totalChance - Sum of all chance properties. 12 | * @returns {WeightedChoice} Item rolled. 13 | */ 14 | 15 | export function useWeighted( 16 | choices: WeightedChoice[], 17 | ): WeightedChoice { 18 | const total = choices.reduce( 19 | (acc: number, val: WeightedChoice) => acc + val.chance, 20 | 0, 21 | ); 22 | const result = Math.random() * total; 23 | let going = 0.0; 24 | for (let i = 0; i < choices.length; ++i) { 25 | going += choices[i].chance; 26 | if (result < going) return choices[i]; 27 | } 28 | return choices[Math.floor(Math.random() * choices.length)]; 29 | } 30 | -------------------------------------------------------------------------------- /packages/utilities/examples/extract-colors/extract-colors.ts: -------------------------------------------------------------------------------- 1 | import { 2 | createCanvas, 3 | loadImage, 4 | } from "https://deno.land/x/canvas@v1.4.1/mod.ts"; 5 | import { extractColors, Image } from "../../mod.ts"; 6 | import { Color } from "https://deno.land/x/colors@v1.2.0/mod.ts"; 7 | 8 | const image = await loadImage("utilities/examples/extract-colors/kagu.png"); 9 | 10 | const canvas = createCanvas(image.width(), image.height()); 11 | 12 | const ctx = canvas.getContext("2d"); 13 | 14 | ctx.drawImage(image, 0, 0); 15 | 16 | const data = ctx.getImageData(0, 0, canvas.width, canvas.height); 17 | 18 | const img = new Image(data); 19 | 20 | const colors = extractColors(img, 32); 21 | 22 | const newCan = createCanvas(300, colors.length * 100); 23 | 24 | const newCtx = newCan.getContext("2d"); 25 | 26 | colors.forEach((color, i) => { 27 | newCtx.fillStyle = new Color(...color).toString(); 28 | newCtx.fillRect(0, i * 100, 300, 100); 29 | }); 30 | Deno.writeFile( 31 | "utilities/examples/extract-colors/out.png", 32 | newCan.toBuffer("image/png"), 33 | ); 34 | -------------------------------------------------------------------------------- /packages/utilities/src/utils/misc/get_constructor.ts: -------------------------------------------------------------------------------- 1 | import type { DataType, DTypeConstructor } from "../common_types.ts"; 2 | 3 | export function getConstructor
( 4 | dType: DT, 5 | ): DTypeConstructor
{ 6 | switch (dType) { 7 | case "u8": 8 | return Uint8Array as DTypeConstructor
; 9 | case "u16": 10 | return Uint16Array as DTypeConstructor
; 11 | case "u32": 12 | return Uint32Array as DTypeConstructor
; 13 | case "u64": 14 | return BigUint64Array as DTypeConstructor
; 15 | case "i8": 16 | return Int8Array as DTypeConstructor
; 17 | case "i16": 18 | return Int16Array as DTypeConstructor
; 19 | case "i32": 20 | return Int32Array as DTypeConstructor
; 21 | case "i64": 22 | return BigInt64Array as DTypeConstructor
; 23 | case "f32": 24 | return Float32Array as DTypeConstructor
; 25 | case "f64": 26 | return Float64Array as DTypeConstructor
; 27 | default: 28 | throw new Error(`Unknown data type ${dType}.`); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /packages/utilities/mod.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Machine Learning utilities for TypeScript. 3 | * 4 | * @example 5 | * ```ts 6 | * const data = [ 7 | * "twinkle twinkle little star", 8 | * "How I wonder what you are", 9 | * "up above the world so high", 10 | * "like a diamond in the sky", 11 | * ]; 12 | * 13 | * // Clean the text 14 | * const cleaner = new TextCleaner({ 15 | * lowercase: true, 16 | * stripHtml: true, 17 | * stripNewlines: true, 18 | * normalizeWhiteSpaces: true, 19 | * }); 20 | * x = cleaner.clean(x); 21 | * 22 | * // Tokenize the text 23 | * const tokenizer = new SplitTokenizer(); 24 | * tokenizer.fit(x); 25 | * const x_tokens = tokenizer.transform(x); 26 | * 27 | * // Vectorize the tokens 28 | * const vectorizer = new CountVectorizer(tokenizer.vocabulary.size); 29 | * const x_vec = vectorizer.transform(x_tokens, "f32"); 30 | * 31 | * // Apply Tf-Idf transformation 32 | * const transformer = new TfIdfTransformer(); 33 | * console.log(transformer.fit(x_vec).transform(x_vec)); 34 | * ``` 35 | * @module 36 | */ 37 | export * from "./src/mod.ts"; 38 | -------------------------------------------------------------------------------- /packages/core/src/backend_loader.ts: -------------------------------------------------------------------------------- 1 | export { GPU } from "./backends/gpu/mod.ts"; 2 | 3 | import { CPU, type CPUBackendLoader } from "./backends/cpu/mod.ts"; 4 | import { WASM, type WASMBackendLoader } from "./backends/wasm/mod.ts"; 5 | import type { BackendLoader } from "./core/engine.ts"; 6 | 7 | onerror = () => { 8 | if (typeof Deno == "undefined") { 9 | throw new Error( 10 | "Warning: Deno is not defined. Did you mean to import from ./web.ts instead of ./mod.ts?", 11 | ); 12 | } 13 | }; 14 | 15 | /** 16 | * The AUTO backend is chosen automatically based on the environment. 17 | */ 18 | const AUTO: WASMBackendLoader | CPUBackendLoader = Deno.dlopen === undefined 19 | ? WASM 20 | : CPU; 21 | 22 | /** 23 | * The OPTION function is used to choose a backend from a list of options. 24 | */ 25 | export function OPTION(...backends: BackendLoader[]): BackendLoader { 26 | for (const backend of backends) { 27 | if (backend.isSupported()) { 28 | return backend; 29 | } 30 | } 31 | throw new Error("No provided backend is supported"); 32 | } 33 | export { AUTO, CPU, WASM }; 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Denosaurs 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /crates/core/src/cpu/layers/flatten.rs: -------------------------------------------------------------------------------- 1 | use ndarray::{ArrayD, Dimension, IxDyn}; 2 | 3 | pub struct FlattenCPULayer { 4 | pub input_size: IxDyn, 5 | pub output_size: Vec, 6 | } 7 | 8 | impl FlattenCPULayer { 9 | pub fn new(size: IxDyn) -> Self { 10 | Self { 11 | input_size: size.clone(), 12 | output_size: vec![size[0], size.size() / size[0]], 13 | } 14 | } 15 | 16 | pub fn output_size(&self) -> Vec { 17 | self.output_size.clone() 18 | } 19 | 20 | pub fn reset(&mut self, batches: usize) { 21 | self.output_size[0] = batches 22 | } 23 | 24 | pub fn forward_propagate(&mut self, inputs: ArrayD) -> ArrayD { 25 | let output_size = IxDyn(&[inputs.shape()[0], self.output_size[1]]); 26 | inputs.into_shape_with_order(output_size).unwrap() 27 | } 28 | 29 | pub fn backward_propagate(&mut self, d_outputs: ArrayD) -> ArrayD { 30 | let mut current_size = self.input_size.clone(); 31 | current_size[0] = d_outputs.shape()[0]; 32 | d_outputs.to_shape(current_size).unwrap().to_owned() 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /packages/utilities/src/metrics/regression.ts: -------------------------------------------------------------------------------- 1 | /** Mean Absolute Error */ 2 | export function mae(y: ArrayLike, y1: ArrayLike): number { 3 | let err = 0; 4 | for (let i = 0; i < y.length; i += 1) { 5 | err += (y[i] - y1[i]) ** 2; 6 | } 7 | return err / y.length; 8 | } 9 | 10 | /** Mean Square Error */ 11 | export function mse(y: ArrayLike, y1: ArrayLike): number { 12 | let err = 0; 13 | for (let i = 0; i < y.length; i += 1) { 14 | err += (y[i] - y1[i]) ** 2; 15 | } 16 | return err / y.length; 17 | } 18 | 19 | /** Root Mean Square Error */ 20 | export function rmse(y: ArrayLike, y1: ArrayLike): number { 21 | return Math.sqrt(mse(y, y1)); 22 | } 23 | 24 | /** R2 Score for regression */ 25 | export function r2(y: ArrayLike, y1: ArrayLike): number { 26 | let mean = 0; 27 | for (let i = 0; i < y.length; i += 1) { 28 | mean += y[i]; 29 | } 30 | mean /= y.length; 31 | let ssr = 0, sst = 0; 32 | for (let i = 0; i < y.length; i += 1) { 33 | ssr += Math.pow(y1[i] - mean, 2); 34 | sst += Math.pow(y[i] - mean, 2); 35 | } 36 | return ssr / sst; 37 | } 38 | -------------------------------------------------------------------------------- /packages/utilities/src/encoding/softmax.ts: -------------------------------------------------------------------------------- 1 | import { Matrix, type MatrixLike } from "../mod.ts"; 2 | import type { DataType, DType, DTypeValue } from "../utils/common_types.ts"; 3 | 4 | /** 5 | * Convert a softmax output into one-hot vectors. 6 | * Mutates the input. 7 | */ 8 | export function transformSoftmaxMut
( 9 | targets: MatrixLike
10 | ): Matrix
{ 11 | const matrix = new Matrix(targets); 12 | for (let i = 0; i < matrix.nRows; i += 1) { 13 | const max = matrix 14 | .row(i) 15 | // @ts-ignore It can reduce. 16 | .reduce( 17 | (acc: number, curr: DTypeValue
, i: number, arr: DType
) => 18 | arr[acc] > curr ? acc : i, 19 | 0 20 | ); 21 | if ( 22 | targets.data instanceof BigInt64Array || 23 | targets.data instanceof BigUint64Array 24 | ) { 25 | const newR = new Array(matrix.nCols).fill(0n); 26 | newR[max] = 1n; 27 | matrix.setRow(i, newR); 28 | } else { 29 | const newR = new Array(matrix.nCols).fill(0); 30 | newR[max] = 1; 31 | matrix.setRow(i, newR); 32 | } 33 | } 34 | return matrix; 35 | } 36 | -------------------------------------------------------------------------------- /crates/core-gpu/src/util.rs: -------------------------------------------------------------------------------- 1 | use std::slice::from_raw_parts; 2 | 3 | use ndarray::ArrayD; 4 | use safetensors::tensor::TensorView; 5 | use serde::Deserialize; 6 | 7 | pub struct Logger { 8 | pub log: fn(string: String) -> (), 9 | } 10 | 11 | pub fn length(shape: Vec) -> usize { 12 | return shape.iter().fold(1, |i, x| i * x); 13 | } 14 | 15 | pub fn decode_array(ptr: *const f32, shape: Vec) -> ArrayD { 16 | let buffer = unsafe { from_raw_parts(ptr, length(shape.clone())) }; 17 | let vec = Vec::from(buffer); 18 | return ArrayD::from_shape_vec(shape, vec).unwrap(); 19 | } 20 | 21 | pub fn decode_json<'a, T>(ptr: *const u8, len: usize) -> T 22 | where 23 | T: Deserialize<'a>, 24 | { 25 | let buffer = unsafe { from_raw_parts(ptr, len) }; 26 | let json = std::str::from_utf8(&buffer[0..len]).unwrap(); 27 | return serde_json::from_str(&json).unwrap(); 28 | } 29 | 30 | pub fn to_arr(view: TensorView) -> ArrayD { 31 | let slice: &[f32] = 32 | unsafe { from_raw_parts(view.data().as_ptr() as *const f32, view.data().len() / 4) }; 33 | return ArrayD::from_shape_vec(view.shape(), slice.to_vec()).unwrap(); 34 | } 35 | -------------------------------------------------------------------------------- /examples/sentiment-analysis/analyzer.ts: -------------------------------------------------------------------------------- 1 | import { CPU, setupBackend, tensor, Sequential, Tensor } from "../../mod.ts"; 2 | 3 | import Mappings from "./mappings.json" with { type: "json" }; 4 | import Vocab from "./vocab.json" with { type: "json" }; 5 | import { TextVectorizer, CategoricalEncoder } from "../../packages/utilities/mod.ts"; 6 | 7 | const vocab = new Map(); 8 | 9 | for (const entry of Vocab.vocab) { 10 | vocab.set(entry[0], entry[1]); 11 | } 12 | 13 | const vectorizer = new TextVectorizer("indices"); 14 | vectorizer.mapper.mapping = vocab; 15 | vectorizer.maxLength = Vocab.maxLength 16 | 17 | const encoder = new CategoricalEncoder(); 18 | const mappings = new Map(); 19 | 20 | for (const entry of Mappings) { 21 | mappings.set(entry[0], entry[1]); 22 | } 23 | 24 | encoder.mapper.mapping = mappings; 25 | 26 | await setupBackend(CPU); 27 | 28 | const net = Sequential.loadFile("examples/sentiment-analysis/sentiment.st"); 29 | 30 | const text = prompt("Text to analyze?") || "hello world"; 31 | 32 | const predYSoftmax = await net.predict( 33 | tensor(vectorizer.transform([text], "f32")), 34 | ); 35 | 36 | const predY = encoder.untransform(predYSoftmax as Tensor<2>); 37 | 38 | console.log(`The sentiment predicted is ${predY[0]}`); 39 | -------------------------------------------------------------------------------- /packages/utilities/src/encoding/onehot.ts: -------------------------------------------------------------------------------- 1 | import { Matrix, type MatrixLike } from "../mod.ts"; 2 | import type { DataType } from "../utils/common_types.ts"; 3 | 4 | /** 5 | * Convert an array of indices into one-hot encoded vectors. 6 | */ 7 | export class OneHotEncoder { 8 | /** Size of one-hot encoded vectors. */ 9 | mappingSize: number; 10 | constructor(mappingSize: number) { 11 | this.mappingSize = mappingSize; 12 | } 13 | /** One-hot encoding of values */ 14 | transform
(targets: number[], dType: DT): Matrix
{ 15 | const res = new Matrix
(dType, [targets.length, this.mappingSize]); 16 | let i = 0; 17 | while (i < targets.length) { 18 | const index = targets[i]; 19 | if (index >= this.mappingSize) { 20 | i += 1; 21 | continue; 22 | } 23 | res.setCell(i, index, 1); 24 | i += 1; 25 | } 26 | return res; 27 | } 28 | untransform
(data: MatrixLike
): number[] { 29 | const matrix = new Matrix(data); 30 | const res = new Array(matrix.nRows); 31 | for (let i = 0; i < res.length; i += 1) { 32 | const idx = matrix.row(i).findIndex((x) => x === 1); 33 | res[i] = idx; 34 | } 35 | return res; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /crates/core/src/util.rs: -------------------------------------------------------------------------------- 1 | use std::slice::from_raw_parts; 2 | 3 | use ndarray::ArrayD; 4 | use safetensors::tensor::TensorView; 5 | use serde::Deserialize; 6 | 7 | #[derive(Clone)] 8 | pub struct Logger { 9 | pub log: fn(string: String) -> (), 10 | } 11 | 12 | #[derive(Clone)] 13 | pub struct Timer { 14 | pub now: fn() -> u128, 15 | } 16 | 17 | pub fn length(shape: Vec) -> usize { 18 | return shape.iter().fold(1, |i, x| i * x); 19 | } 20 | 21 | pub fn decode_array(ptr: *const f32, shape: Vec) -> ArrayD { 22 | let buffer = unsafe { from_raw_parts(ptr, length(shape.clone())) }; 23 | let vec = Vec::from(buffer); 24 | return ArrayD::from_shape_vec(shape, vec).unwrap(); 25 | } 26 | 27 | pub fn decode_json<'a, T>(ptr: *const u8, len: usize) -> T 28 | where 29 | T: Deserialize<'a>, 30 | { 31 | let buffer = unsafe { from_raw_parts(ptr, len) }; 32 | let json = std::str::from_utf8(&buffer[0..len]).unwrap(); 33 | return serde_json::from_str(&json).unwrap(); 34 | } 35 | 36 | pub fn to_arr(view: TensorView) -> ArrayD { 37 | let slice: &[f32] = 38 | unsafe { from_raw_parts(view.data().as_ptr() as *const f32, view.data().len() / 4) }; 39 | return ArrayD::from_shape_vec(view.shape(), slice.to_vec()).unwrap(); 40 | } 41 | -------------------------------------------------------------------------------- /examples/mnist/predict.ts: -------------------------------------------------------------------------------- 1 | import { 2 | CPU, 3 | type Rank, 4 | Sequential, 5 | setupBackend, 6 | type Shape, 7 | type Tensor, 8 | tensor, 9 | } from "../../packages/core/mod.ts"; 10 | import { loadDataset } from "./common.ts"; 11 | 12 | await setupBackend(CPU); 13 | 14 | const network = Sequential.loadFile("examples/mnist/mnist.test.st"); 15 | 16 | const testSet = loadDataset("test-images.idx", "test-labels.idx", 0, 1000); 17 | testSet.map((_, i) => (testSet[i].inputs.shape = [1, 28, 28])); 18 | 19 | function argmax(mat: Tensor) { 20 | let max = -Infinity; 21 | let index = -1; 22 | for (let i = 0; i < mat.data.length; i++) { 23 | if (mat.data[i] > max) { 24 | max = mat.data[i]; 25 | index = i; 26 | } 27 | } 28 | return index; 29 | } 30 | 31 | let correct = 0; 32 | for (const test of testSet) { 33 | const prediction = argmax( 34 | await network.predict( 35 | tensor(test.inputs.data, [1, ...test.inputs.shape] as Shape), 36 | ), 37 | ); 38 | const expected = argmax(test.outputs as Tensor); 39 | if (expected === prediction) { 40 | correct += 1; 41 | } 42 | } 43 | 44 | console.log(`${correct} / ${testSet.length} correct`); 45 | console.log(`accuracy: ${((correct / testSet.length) * 100).toFixed(2)}%`); 46 | -------------------------------------------------------------------------------- /packages/utilities/src/utils/random/shuffle.ts: -------------------------------------------------------------------------------- 1 | import { useSeries } from "../array/range.ts"; 2 | 3 | /** 4 | * Shuffle a given array in-place. 5 | * @param arr Array to shuffle 6 | */ 7 | export function useShuffle(arr: T[]): T[]; 8 | /** Get a shuffled array of numbers from 0 to the given number */ 9 | export function useShuffle(max: number): number[]; 10 | /** Get a shuffled array of numbers between the given range */ 11 | export function useShuffle(min: number, max: number): number[]; 12 | export function useShuffle( 13 | maybeArr: number | T[], 14 | max?: number, 15 | ): T[] | number[] { 16 | if (Array.isArray(maybeArr)) { 17 | const idx = useShuffle(0, maybeArr.length); 18 | const res = new Array(maybeArr.length); 19 | let i = 0; 20 | while (i < maybeArr.length) { 21 | res[i] = maybeArr[idx[i]]; 22 | i += 1; 23 | } 24 | return res; 25 | } else { 26 | const min = typeof max !== "undefined" ? maybeArr : 0; 27 | const max1 = typeof max !== "undefined" ? max : maybeArr; 28 | 29 | const arr = useSeries(min, max1); 30 | 31 | let i = arr.length - 1; 32 | while (i >= 1) { 33 | const j = Math.floor(Math.random() * i); 34 | [arr[i], arr[j]] = [arr[j], arr[i]]; 35 | i -= 1; 36 | } 37 | return arr; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /packages/utilities/src/utils/array/range.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Get n evenly distributed numbers in a range. 3 | * @param n Number of numbers to generate. 4 | * @param min Lower limit of range (inclusive). 5 | * @param max Upper limit of range (exclusive). 6 | * @returns Array of n evenly distributed numbers. 7 | */ 8 | export function useRange(n: number, min = 0, max = 1): number[] { 9 | const res = new Array(n); 10 | let i = 0; 11 | while (i < n) { 12 | res[i] = min + ((i * (max - min)) / n); 13 | i += 1; 14 | } 15 | return res; 16 | } 17 | 18 | /** 19 | * Get an array of numbers between a given range, 20 | * incremented by a step. 21 | * @param min Lower limit of range (inclusive). 22 | * @param max Upper limit of range (exclusive). 23 | * @param step step to increment by 24 | * @returns Array of numbers 25 | */ 26 | export function useSeries(max: number): number[]; 27 | export function useSeries(min: number, max: number, step?: number): number[]; 28 | export function useSeries(min: number, max?: number, step = 1): number[] { 29 | if (typeof max === "undefined") [min, max] = [0, min]; 30 | const res = new Array(~~((max - min) / step)); 31 | res[0] = min; 32 | let i = 1; 33 | while (i < res.length) { 34 | res[i] = res[i - 1] + step; 35 | i += 1; 36 | } 37 | return res; 38 | } 39 | -------------------------------------------------------------------------------- /crates/core/src/cpu/schedulers/mod.rs: -------------------------------------------------------------------------------- 1 | mod decay; 2 | mod oc; 3 | 4 | use crate::Scheduler; 5 | 6 | pub use decay::*; 7 | pub use oc::*; 8 | pub enum CPUScheduler { 9 | None, 10 | LinearDecay(CPUDecayScheduler), 11 | ExponentialDecay(CPUDecayScheduler), 12 | OneCycle(CPUOneCycleScheduler), 13 | } 14 | 15 | impl CPUScheduler { 16 | pub fn from(scheduler: &Scheduler) -> Self { 17 | match scheduler { 18 | Scheduler::None => CPUScheduler::None, 19 | Scheduler::LinearDecay(config) => { 20 | CPUScheduler::LinearDecay(CPUDecayScheduler::new(config)) 21 | } 22 | Scheduler::ExponentialDecay(config) => { 23 | CPUScheduler::ExponentialDecay(CPUDecayScheduler::new(config)) 24 | } 25 | Scheduler::OneCycle(config) => { 26 | CPUScheduler::OneCycle(CPUOneCycleScheduler::new(config)) 27 | } 28 | } 29 | } 30 | pub fn eta(&self, rate: f32, step: usize) -> f32 { 31 | match self { 32 | CPUScheduler::None => rate, 33 | CPUScheduler::LinearDecay(scheduler) => scheduler.linear(rate, step), 34 | CPUScheduler::ExponentialDecay(scheduler) => scheduler.exponential(rate, step), 35 | CPUScheduler::OneCycle(scheduler) => scheduler.eta(rate, step), 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /crates/core-gpu/src/gpu/schedulers/mod.rs: -------------------------------------------------------------------------------- 1 | mod decay; 2 | mod oc; 3 | 4 | use crate::Scheduler; 5 | 6 | pub use decay::*; 7 | pub use oc::*; 8 | pub enum GPUScheduler { 9 | None, 10 | LinearDecay(GPUDecayScheduler), 11 | ExponentialDecay(GPUDecayScheduler), 12 | OneCycle(GPUOneCycleScheduler), 13 | } 14 | 15 | impl GPUScheduler { 16 | pub fn from(scheduler: &Scheduler) -> Self { 17 | match scheduler { 18 | Scheduler::None => GPUScheduler::None, 19 | Scheduler::LinearDecay(config) => { 20 | GPUScheduler::LinearDecay(GPUDecayScheduler::new(config)) 21 | } 22 | Scheduler::ExponentialDecay(config) => { 23 | GPUScheduler::ExponentialDecay(GPUDecayScheduler::new(config)) 24 | } 25 | Scheduler::OneCycle(config) => { 26 | GPUScheduler::OneCycle(GPUOneCycleScheduler::new(config)) 27 | } 28 | } 29 | } 30 | pub fn eta(&self, rate: f32, step: usize) -> f32 { 31 | match self { 32 | GPUScheduler::None => rate, 33 | GPUScheduler::LinearDecay(scheduler) => scheduler.linear(rate, step), 34 | GPUScheduler::ExponentialDecay(scheduler) => scheduler.exponential(rate, step), 35 | GPUScheduler::OneCycle(scheduler) => scheduler.eta(rate, step), 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /crates/core-gpu/src/gpu/layers/flatten.rs: -------------------------------------------------------------------------------- 1 | use ndarray::{ArrayD, Dimension, IxDyn}; 2 | 3 | use crate::FlattenLayer; 4 | 5 | pub struct FlattenGPULayer { 6 | pub input_size: IxDyn, 7 | pub output_size: Vec, 8 | } 9 | 10 | impl FlattenGPULayer { 11 | pub fn new(config: FlattenLayer, size: IxDyn) -> Self { 12 | let mut new_size = config.size.clone(); 13 | new_size.insert(0, size[0]); 14 | let output_size = IxDyn(&new_size); 15 | if output_size.size() != size.size() { 16 | panic!( 17 | "Shape {:#?} is incompatible with shape {:#?}", 18 | output_size, size 19 | ) 20 | } 21 | Self { 22 | input_size: size, 23 | output_size: new_size, 24 | } 25 | } 26 | 27 | pub fn output_size(&self) -> Vec { 28 | self.output_size.clone() 29 | } 30 | 31 | pub fn reset(&mut self, batches: usize) { 32 | self.output_size[0] = batches 33 | } 34 | 35 | pub fn forward_propagate(&mut self, inputs: ArrayD) -> ArrayD { 36 | let output_size = IxDyn(&self.output_size); 37 | inputs.into_shape(output_size).unwrap() 38 | } 39 | 40 | pub fn backward_propagate(&mut self, d_outputs: ArrayD) -> ArrayD { 41 | d_outputs.into_shape(self.input_size.clone()).unwrap() 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /packages/core/src/core/api/scheduler.ts: -------------------------------------------------------------------------------- 1 | import { SchedulerType } from "../types.ts"; 2 | 3 | export type Scheduler = 4 | | { type: SchedulerType.None } 5 | | { 6 | type: SchedulerType.LinearDecay | SchedulerType.ExponentialDecay; 7 | config: DecaySchedulerConfig; 8 | } 9 | | { type: SchedulerType.OneCycle; config: OneCycleSchedulerConfig }; 10 | 11 | export interface DecaySchedulerConfig { 12 | rate?: number; 13 | step_size?: number; 14 | } 15 | 16 | export interface OneCycleSchedulerConfig { 17 | max_rate?: number; 18 | step_size?: number; 19 | } 20 | 21 | export function NoScheduler(): Scheduler { 22 | return { type: SchedulerType.None }; 23 | } 24 | 25 | export function LinearDecay(config: DecaySchedulerConfig = {}): Scheduler { 26 | config.rate = config.rate || 0.99; 27 | config.step_size = config.step_size || 100; 28 | return { type: SchedulerType.LinearDecay, config }; 29 | } 30 | 31 | export function ExponentialDecay(config: DecaySchedulerConfig = {}): Scheduler { 32 | config.rate = config.rate || 0.99; 33 | config.step_size = config.step_size || 100; 34 | return { type: SchedulerType.ExponentialDecay, config }; 35 | } 36 | 37 | export function OneCycle(config: OneCycleSchedulerConfig = {}): Scheduler { 38 | config.max_rate = config.max_rate || 0.01; 39 | config.step_size = config.step_size || 100; 40 | return { type: SchedulerType.OneCycle, config }; 41 | } 42 | -------------------------------------------------------------------------------- /packages/utilities/src/encoding/multihot.ts: -------------------------------------------------------------------------------- 1 | import { Matrix } from "../mod.ts"; 2 | import type { DataType } from "../utils/common_types.ts"; 3 | 4 | /** 5 | * Convert 2D array of indices into multi-hot encoded vectors. 6 | */ 7 | export class MultiHotEncoder { 8 | /** Size of encoded vectors. */ 9 | mappingSize: number; 10 | constructor(mappingSize: number) { 11 | this.mappingSize = mappingSize; 12 | } 13 | /** Encoding values into multi-hot vectors */ 14 | transform
(targets: Matrix
): Matrix
; 15 | transform
(targets: number[][], dType: DT): Matrix
; 16 | transform
( 17 | targets: number[][] | Matrix
, 18 | dType?: DT 19 | ): Matrix
{ 20 | if (!dType && !(targets instanceof Matrix)) 21 | throw new Error("dType required when not dealing with matrices."); 22 | const dataType = dType || (targets as Matrix
).dType; 23 | const res = new Matrix
(dataType, [targets.length, this.mappingSize]); 24 | let i = 0; 25 | while (i < targets.length) { 26 | const row = targets instanceof Matrix ? targets.row(i) : targets[i]; 27 | let j = 0; 28 | while (j < row.length) { 29 | if (Number(row[j]) >= row.length) { 30 | j += 1; 31 | continue; 32 | } 33 | res.setCell(i, Number(row[j]), 1); 34 | j += 1; 35 | } 36 | 37 | i += 1; 38 | } 39 | return res; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /crates/core/src/cpu/layer_norm.rs: -------------------------------------------------------------------------------- 1 | extern crate ndarray; 2 | use ndarray::{Array1, ArrayD, Axis}; 3 | 4 | pub struct LayerNorm { 5 | pub gamma: Array1, 6 | pub beta: Array1, 7 | pub epsilon: f32, 8 | } 9 | 10 | impl LayerNorm { 11 | pub fn new(hidden_size: usize, epsilon: f32) -> Self { 12 | LayerNorm { 13 | gamma: Array1::ones(hidden_size), 14 | beta: Array1::zeros(hidden_size), 15 | epsilon, 16 | } 17 | } 18 | 19 | pub fn forward(&self, input: ArrayD) -> ArrayD { 20 | let shape = input.shape(); 21 | let last_axis = shape.len() - 1; 22 | 23 | let mean = input.mean_axis(Axis(last_axis)).unwrap(); 24 | let variance = input.var_axis(Axis(last_axis), 0.0); 25 | 26 | let mut normalized_input = input.clone(); 27 | normalized_input 28 | .axis_iter_mut(Axis(last_axis)) 29 | .enumerate() 30 | .for_each(|(i, mut row)| { 31 | let mean_i = mean[i]; 32 | let var_i = variance[i].sqrt() + self.epsilon; 33 | row -= mean_i; 34 | row /= var_i; 35 | }); 36 | 37 | normalized_input 38 | .axis_iter_mut(Axis(last_axis)) 39 | .for_each(|mut item| { 40 | let new = &item * &self.gamma + &self.beta; 41 | item.assign(&new); 42 | }); 43 | normalized_input 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /packages/utilities/src/transformer/tfidf.ts: -------------------------------------------------------------------------------- 1 | import type { DataType } from "../utils/common_types.ts"; 2 | import type { Matrix, MatrixLike } from "../mod.ts"; 3 | import { multiplyDiags } from "../utils/math.ts"; 4 | 5 | /** Convert tf features (Count) into tf-idf features. */ 6 | export class TfIdfTransformer { 7 | idf: null | Float32Array; 8 | constructor({ idf }: { idf?: Float32Array } = {}) { 9 | this.idf = idf ?? null; 10 | } 11 | /** 12 | * Get idf matrix from tf features. 13 | * @param data tf features from CountVectorizer 14 | * @returns Tf-Idf transformer 15 | */ 16 | fit(data: Matrix): TfIdfTransformer { 17 | const shape = { 18 | features: data.nCols, 19 | samples: data.nRows, 20 | }; 21 | const freq = data.rowSum(); 22 | 23 | const idf = new Float32Array(freq.length); 24 | 25 | let i = 0; 26 | while (i < idf.length) { 27 | idf[i] = Math.log(shape.samples / Number(freq[i])) + 1; 28 | i += 1; 29 | } 30 | this.idf = idf; 31 | return this; 32 | } 33 | /** 34 | * Transform an tf features into tf-idf features. 35 | * Mutates the input. 36 | * @param data tf features from CountVectorizer 37 | * @returns Sparse matrix of Tf-Idf features 38 | */ 39 | transform(data: MatrixLike): Matrix { 40 | if (this.idf === null) throw new Error("IDF not initialized yet."); 41 | return multiplyDiags(data, this.idf); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /examples/mnist/train.ts: -------------------------------------------------------------------------------- 1 | import { 2 | Conv2DLayer, 3 | Cost, 4 | CPU, 5 | DenseLayer, 6 | FlattenLayer, 7 | Init, 8 | MaxPool2DLayer, 9 | ReluLayer, 10 | Sequential, 11 | setupBackend, 12 | SoftmaxLayer, 13 | } from "../../packages/core/mod.ts"; 14 | 15 | import { loadDataset } from "./common.ts"; 16 | 17 | await setupBackend(CPU); 18 | 19 | // training 20 | const network = new Sequential({ 21 | size: [32, 1, 28, 28], 22 | layers: [ 23 | Conv2DLayer({ kernelSize: [6, 1, 5, 5], padding: [2, 2] }), 24 | ReluLayer(), 25 | MaxPool2DLayer({ strides: [2, 2] }), 26 | Conv2DLayer({ kernelSize: [16, 6, 5, 5] }), 27 | ReluLayer(), 28 | MaxPool2DLayer({ strides: [2, 2] }), 29 | Conv2DLayer({ kernelSize: [120, 16, 5, 5] }), 30 | ReluLayer(), 31 | FlattenLayer(), 32 | DenseLayer({ size: [84], init: Init.Kaiming }), 33 | ReluLayer(), 34 | DenseLayer({ size: [10], init: Init.Kaiming }), 35 | SoftmaxLayer(), 36 | ], 37 | cost: Cost.CrossEntropy, 38 | }); 39 | 40 | console.log("Loading training dataset..."); 41 | const trainSet = loadDataset( 42 | "train-images.idx", 43 | "train-labels.idx", 44 | 0, 45 | 10000, 46 | 32, 47 | ); 48 | 49 | const epochs = 10; 50 | console.log("Training (" + epochs + " epochs)..."); 51 | const start = performance.now(); 52 | network.train(trainSet, epochs, 1, 0.005); 53 | console.log("Training complete!", performance.now() - start); 54 | 55 | network.saveFile("examples/mnist/mnist.test.st"); 56 | -------------------------------------------------------------------------------- /packages/core/src/core/api/optimizer.ts: -------------------------------------------------------------------------------- 1 | import { OptimizerType } from "../types.ts"; 2 | 3 | export type Optimizer = 4 | | { type: OptimizerType.SGD } 5 | | { type: OptimizerType.Adam; config: AdamOptimizerConfig } 6 | | { type: OptimizerType.Nadam; config: AdamOptimizerConfig } 7 | | { type: OptimizerType.RMSProp; config: RMSPropOptimizerConfig }; 8 | 9 | export interface AdamOptimizerConfig { 10 | beta1?: number; 11 | beta2?: number; 12 | epsilon?: number; 13 | } 14 | 15 | export interface RMSPropOptimizerConfig { 16 | decayRate?: number; 17 | epsilon?: number; 18 | } 19 | 20 | export function SGDOptimizer(): Optimizer { 21 | return { type: OptimizerType.SGD }; 22 | } 23 | 24 | export function AdamOptimizer(config: AdamOptimizerConfig = {}): Optimizer { 25 | config.beta1 = config.beta1 || 0.9; 26 | config.beta2 = config.beta2 || 0.999; 27 | config.epsilon = config.epsilon || 1e-8; 28 | return { type: OptimizerType.Adam, config }; 29 | } 30 | 31 | export function NadamOptimizer(config: AdamOptimizerConfig = {}): Optimizer { 32 | config.beta1 = config.beta1 || 0.9; 33 | config.beta2 = config.beta2 || 0.999; 34 | config.epsilon = config.epsilon || 1e-8; 35 | return { type: OptimizerType.Nadam, config }; 36 | } 37 | 38 | export function RMSPropOptimizer( 39 | config: RMSPropOptimizerConfig = {}, 40 | ): Optimizer { 41 | config.decayRate = config.decayRate || 0.9; 42 | config.epsilon = config.epsilon || 1e-8; 43 | return { type: OptimizerType.RMSProp, config }; 44 | } 45 | -------------------------------------------------------------------------------- /packages/utilities/src/utils/random/normal.ts: -------------------------------------------------------------------------------- 1 | const TWO_PI = Math.PI * 2; 2 | 3 | /** 4 | * A **Normal** or **Gaussian** distribution is a type of 5 | * continuous probability distribution dependent on two 6 | * parameters: 7 | * 8 | * **μ** - The **mean** 9 | * **σ** - The **standard deviation** 10 | * 11 | * This implementation makes use of the popular Box-Muller transform. 12 | */ 13 | 14 | /** 15 | * Generate a normal random variate. 16 | * @param mean Mean of the distribution μ. 17 | * @param stddev Standard Deviation of the distribution σ. 18 | * @returns A normal random variate. 19 | */ 20 | export function useNormal( 21 | mean: number, 22 | stddev: number, 23 | ): [number, number] { 24 | const u = [Math.random(), Math.random()]; 25 | 26 | const m = Math.sqrt(-2.0 * Math.log(u[0])); 27 | return [ 28 | (stddev * m * Math.cos(TWO_PI * u[1])) + mean, 29 | (stddev * m * Math.sin(TWO_PI * u[1])) + mean, 30 | ]; 31 | } 32 | 33 | /** 34 | * Generate a normally distributed array. 35 | * @param mean Mean of the distribution μ. 36 | * @param variance Variance of the distribution σ^2. 37 | * @returns A normally distributed array. 38 | */ 39 | 40 | export function useNormalArray( 41 | num: number, 42 | mean: number, 43 | variance: number, 44 | ): Float32Array { 45 | const result = new Float32Array(num); 46 | let i = 0; 47 | const stddev = Math.sqrt(variance); 48 | while (i < num) { 49 | result[i] = useNormal(mean, stddev)[0]; 50 | ++i; 51 | } 52 | return result; 53 | } 54 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [v1, main] 6 | pull_request: 7 | 8 | jobs: 9 | build: 10 | name: Build ${{ matrix.os }} 11 | runs-on: ${{ matrix.os }} 12 | strategy: 13 | matrix: 14 | os: [macos-latest, ubuntu-latest, windows-latest] 15 | fail-fast: false 16 | steps: 17 | - uses: actions/checkout@v2 18 | 19 | - name: Install rust 20 | uses: hecrj/setup-rust-action@v1 21 | with: 22 | rust-version: nightly 23 | 24 | - uses: denoland/setup-deno@v1 25 | with: 26 | deno-version: canary 27 | 28 | - name: Build 29 | shell: bash 30 | run: | 31 | set -xeuo pipefail 32 | rustc --version 33 | cargo --version 34 | cargo build --release -p netsaur 35 | deno run -Ar jsr:@deno/wasmbuild@0.17.2 -p netsaur --out src/backends/wasm/lib 36 | deno run -Ar jsr:@deno/wasmbuild@0.17.2 -p netsaur-tokenizers --out tokenizers/lib 37 | - name: Release 38 | uses: softprops/action-gh-release@master 39 | env: 40 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 41 | with: 42 | tag_name: "release draft" 43 | draft: true 44 | files: | 45 | target/release/libnetsaur.so 46 | target/release/libnetsaur.dylib 47 | target/release/netsaur.dll 48 | src/backends/wasm/lib/netsaur_bg.wasm 49 | tokenizers/lib/netsaur_tokenizers_bg.wasm 50 | -------------------------------------------------------------------------------- /packages/utilities/src/encoding/termfrequency.ts: -------------------------------------------------------------------------------- 1 | import { Matrix } from "../mod.ts"; 2 | import type { DataType } from "../utils/common_types.ts"; 3 | 4 | /** 5 | * Convert 2D array of indices into multi-hot encoded vectors 6 | * where each index contains the number of times the respective 7 | * value appears in a sample (term frequency encoder). 8 | */ 9 | export class TfEncoder { 10 | /** Size of encoded vectors. */ 11 | mappingSize: number; 12 | constructor(mappingSize: number) { 13 | this.mappingSize = mappingSize; 14 | } 15 | /** Encoding values into count vectors */ 16 | transform
(targets: Matrix
): Matrix
; 17 | transform
(targets: number[][], dType: DT): Matrix
; 18 | transform
( 19 | targets: number[][] | Matrix
, 20 | dType?: DT 21 | ): Matrix
{ 22 | if (!dType && !(targets instanceof Matrix)) 23 | throw new Error("dType required when not dealing with matrices."); 24 | const dataType = dType || (targets as Matrix
).dType; 25 | const res = new Matrix
(dataType, [targets.length, this.mappingSize]); 26 | let i = 0; 27 | while (i < targets.length) { 28 | const row = targets instanceof Matrix ? targets.row(i) : targets[i]; 29 | let j = 0; 30 | while (j < row.length) { 31 | if (Number(row[j]) >= row.length) { 32 | j += 1; 33 | continue; 34 | } 35 | res.setAdd(i, Number(row[j]), 1); 36 | j += 1; 37 | } 38 | 39 | i += 1; 40 | } 41 | return res; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /packages/utilities/src/utils/array/split.ts: -------------------------------------------------------------------------------- 1 | import type { Sliceable } from "../common_types.ts"; 2 | import { Matrix } from "../mod.ts"; 3 | import { useShuffle } from "../random/shuffle.ts"; 4 | 5 | interface SplitOptions { 6 | ratio: [number, number]; 7 | shuffle?: boolean; 8 | } 9 | 10 | /** Split arrays by their first axis. Only usable on Matrices and Arrays. */ 11 | export function useSplit( 12 | options: SplitOptions = { ratio: [7, 3], shuffle: false }, 13 | ...arr: T 14 | ): [typeof arr, typeof arr] { 15 | if (!arr.every((x) => x.length === arr[0].length)) { 16 | throw new Error("All arrays must have equal length!"); 17 | } 18 | const { ratio, shuffle } = options; 19 | const idx = Math.floor(arr[0].length * (ratio[0] / (ratio[0] + ratio[1]))); 20 | if (!shuffle) { 21 | return [arr.map((x) => x.slice(0, idx)), arr.map((x) => x.slice(idx))] as [ 22 | T, 23 | T 24 | ]; 25 | } else { 26 | const shuffled = useShuffle(0, arr[0].length); 27 | const x1 = shuffled.slice(0, idx); 28 | const x2 = shuffled.slice(idx); 29 | return [ 30 | arr.map((x) => { 31 | if (x instanceof Matrix) { 32 | return x.rowSelect(x1); 33 | } else { 34 | return x1.map((idx) => (x as unknown[])[idx]); 35 | } 36 | }) as unknown as typeof arr, 37 | arr.map((x) => { 38 | if (x instanceof Matrix) { 39 | return x.rowSelect(x2); 40 | } else { 41 | return x2.map((idx) => (x as unknown[])[idx]); 42 | } 43 | }) as unknown as typeof arr, 44 | ]; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /examples/mnist/train_batchnorm.ts: -------------------------------------------------------------------------------- 1 | import { 2 | BatchNorm2DLayer, 3 | Conv2DLayer, 4 | Cost, 5 | CPU, 6 | DenseLayer, 7 | FlattenLayer, 8 | Init, 9 | MaxPool2DLayer, 10 | ReluLayer, 11 | Sequential, 12 | setupBackend, 13 | SoftmaxLayer, 14 | } from "../../packages/core/mod.ts"; 15 | 16 | import { loadDataset } from "./common.ts"; 17 | 18 | await setupBackend(CPU); 19 | 20 | // training 21 | const network = new Sequential({ 22 | size: [32, 1, 28, 28], 23 | layers: [ 24 | Conv2DLayer({ kernelSize: [6, 1, 5, 5], padding: [2, 2] }), 25 | BatchNorm2DLayer({ momentum: 0.9 }), 26 | ReluLayer(), 27 | MaxPool2DLayer({ strides: [2, 2] }), 28 | Conv2DLayer({ kernelSize: [16, 6, 5, 5] }), 29 | BatchNorm2DLayer({ momentum: 0.9 }), 30 | ReluLayer(), 31 | MaxPool2DLayer({ strides: [2, 2] }), 32 | Conv2DLayer({ kernelSize: [120, 16, 5, 5] }), 33 | ReluLayer(), 34 | FlattenLayer({ size: [120] }), 35 | DenseLayer({ size: [84], init: Init.Kaiming }), 36 | ReluLayer(), 37 | DenseLayer({ size: [10], init: Init.Kaiming }), 38 | SoftmaxLayer(), 39 | ], 40 | cost: Cost.CrossEntropy, 41 | }); 42 | 43 | console.log("Loading training dataset..."); 44 | const trainSet = loadDataset( 45 | "train-images.idx", 46 | "train-labels.idx", 47 | 0, 48 | 5000, 49 | 32, 50 | ); 51 | 52 | const epochs = 1; 53 | console.log("Training (" + epochs + " epochs)..."); 54 | const start = performance.now(); 55 | network.train(trainSet, epochs, 1, 0.01); 56 | console.log("Training complete!", performance.now() - start); 57 | 58 | network.saveFile("examples/mnist/mnist.test.st"); 59 | -------------------------------------------------------------------------------- /examples/mnist/train_dropout.ts: -------------------------------------------------------------------------------- 1 | import { 2 | Conv2DLayer, 3 | Cost, 4 | CPU, 5 | DenseLayer, 6 | Dropout2DLayer, 7 | FlattenLayer, 8 | Init, 9 | MaxPool2DLayer, 10 | ReluLayer, 11 | Sequential, 12 | setupBackend, 13 | SoftmaxLayer, 14 | } from "../../packages/core/mod.ts"; 15 | 16 | import { loadDataset } from "./common.ts"; 17 | 18 | await setupBackend(CPU); 19 | 20 | // training 21 | const network = new Sequential({ 22 | size: [32, 1, 28, 28], 23 | layers: [ 24 | Conv2DLayer({ kernelSize: [6, 1, 5, 5], padding: [2, 2] }), 25 | Dropout2DLayer({ probability: 0.01 }), 26 | ReluLayer(), 27 | MaxPool2DLayer({ strides: [2, 2] }), 28 | Conv2DLayer({ kernelSize: [16, 6, 5, 5] }), 29 | Dropout2DLayer({ probability: 0.01 }), 30 | ReluLayer(), 31 | MaxPool2DLayer({ strides: [2, 2] }), 32 | Conv2DLayer({ kernelSize: [120, 16, 5, 5] }), 33 | ReluLayer(), 34 | FlattenLayer({ size: [120] }), 35 | DenseLayer({ size: [84], init: Init.Kaiming }), 36 | ReluLayer(), 37 | DenseLayer({ size: [10], init: Init.Kaiming }), 38 | SoftmaxLayer(), 39 | ], 40 | cost: Cost.CrossEntropy, 41 | }); 42 | 43 | console.log("Loading training dataset..."); 44 | const trainSet = loadDataset( 45 | "train-images.idx", 46 | "train-labels.idx", 47 | 0, 48 | 5000, 49 | 32, 50 | ); 51 | 52 | const epochs = 1; 53 | console.log("Training (" + epochs + " epochs)..."); 54 | const start = performance.now(); 55 | network.train(trainSet, epochs, 1, 0.005); 56 | console.log("Training complete!", performance.now() - start); 57 | 58 | network.saveFile("examples/mnist/mnist.test.st"); 59 | -------------------------------------------------------------------------------- /packages/core/src/core/api/error.ts: -------------------------------------------------------------------------------- 1 | import type { BackendType } from "../types.ts"; 2 | import type { Rank, Shape, Shape2D } from "./shape.ts"; 3 | 4 | /** 5 | * Incompatible Rank Error is thrown when a tensor is incompatible with a layer. 6 | */ 7 | export class IncompatibleRankError extends Error { 8 | constructor(shape: number, expected: number) { 9 | super( 10 | `Layer of rank ${expected} is incompatible with tensor of rank ${shape}`, 11 | ); 12 | } 13 | } 14 | 15 | /** 16 | * Invalid Flatten Error is thrown when a tensor cannot be flattened. 17 | */ 18 | export class InvalidFlattenError extends Error { 19 | constructor(input: Shape, output: Shape) { 20 | super(`Cannot flatten tensor of shape ${input} to shape ${output}`); 21 | } 22 | } 23 | 24 | /** 25 | * No backend error is thrown when a backend is not initialized. 26 | */ 27 | export class NoBackendError extends Error { 28 | constructor(type: BackendType) { 29 | super( 30 | `${type} backend not initialized. Help: Did you forget to call setupBackend()?`, 31 | ); 32 | } 33 | } 34 | 35 | /** 36 | * Invalid Pool Error is thrown when a tensor cannot be pooled. 37 | */ 38 | export class InvalidPoolError extends Error { 39 | constructor(size: Shape, stride: Shape2D) { 40 | super(`Cannot pool shape ${size} with stride ${stride}`); 41 | } 42 | } 43 | 44 | /** 45 | * Invalid Activation Error is thrown when an activation function is invalid. 46 | */ 47 | export class ActivationError extends Error { 48 | constructor(activation: string) { 49 | super(`Unknown activation function: ${activation}.`); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /packages/core/src/backends/gpu/util.ts: -------------------------------------------------------------------------------- 1 | import type { Rank, Shape } from "../../core/api/shape.ts"; 2 | import type { DataSet } from "../../core/types.ts"; 3 | 4 | export class Buffer { 5 | buffer = new Uint8Array(); 6 | allocBuffer = new Deno.UnsafeCallback({ 7 | parameters: ["usize"], 8 | result: "buffer", 9 | }, (length) => { 10 | this.buffer = new Uint8Array(Number(length)); 11 | return this.buffer; 12 | }).pointer; 13 | } 14 | 15 | /** 16 | * Train Options Interface. 17 | */ 18 | export type TrainOptions = { 19 | datasets: number; 20 | inputShape: Shape; 21 | outputShape: Shape; 22 | epochs: number; 23 | batches: number; 24 | rate: number; 25 | }; 26 | 27 | /** 28 | * Predict Options Interface. 29 | */ 30 | export type PredictOptions = { 31 | inputShape: Shape; 32 | outputShape: Shape; 33 | }; 34 | 35 | /** 36 | * Encode JSON data. 37 | */ 38 | export function encodeJSON(json: unknown): Uint8Array { 39 | return new TextEncoder().encode(JSON.stringify(json)); 40 | } 41 | 42 | /** 43 | * Returns the BigInt value of a pointer. 44 | */ 45 | export function pointer(arr: BufferSource): bigint { 46 | return BigInt(Deno.UnsafePointer.value(Deno.UnsafePointer.of(arr))); 47 | } 48 | 49 | /** 50 | * Encode datasets. 51 | */ 52 | export function encodeDatasets(datasets: DataSet[]): BigUint64Array { 53 | const pointers: bigint[] = []; 54 | for (const dataset of datasets) { 55 | pointers.push(pointer(dataset.inputs.data as Float32Array)); 56 | pointers.push(pointer(dataset.outputs.data as Float32Array)); 57 | } 58 | return new BigUint64Array(pointers); 59 | } 60 | -------------------------------------------------------------------------------- /crates/core/src/cpu/regularizer.rs: -------------------------------------------------------------------------------- 1 | use ndarray::ArrayD; 2 | 3 | pub struct CPURegularizer { 4 | l1_strength: f32, 5 | l2_strength: f32, 6 | } 7 | 8 | impl CPURegularizer { 9 | pub fn from(c: f32, l1_ratio: f32) -> Self { 10 | if c == 0.0 { 11 | return CPURegularizer { 12 | l1_strength: 0.0, 13 | l2_strength: 0.0 14 | } 15 | } 16 | let strength = 1.0 / c; 17 | if l1_ratio == 1.0 { 18 | CPURegularizer { 19 | l1_strength: strength, 20 | l2_strength: 0.0, 21 | } 22 | } else if l1_ratio == 0.0 { 23 | CPURegularizer { 24 | l1_strength: 0.0, 25 | l2_strength: strength, 26 | } 27 | } else { 28 | let l1_strength = strength * l1_ratio; 29 | let l2_strength = strength - l1_strength; 30 | CPURegularizer { 31 | l1_strength, 32 | l2_strength, 33 | } 34 | } 35 | } 36 | pub fn l1_coeff(&self, x: &ArrayD) -> ArrayD { 37 | if self.l1_strength == 0.0 { 38 | ArrayD::zeros(x.shape()) 39 | } else { 40 | self.l1_strength * x.map(|w| w.abs()) 41 | } 42 | } 43 | pub fn l2_coeff(&self, x: &ArrayD) -> ArrayD { 44 | if self.l2_strength == 0.0 { 45 | ArrayD::zeros(x.shape()) 46 | } else { 47 | self.l2_strength * x.map(|w| w * w) 48 | } 49 | } 50 | pub fn coeff(&self, x: &ArrayD) -> ArrayD { 51 | self.l1_coeff(x) + self.l2_coeff(x) 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /bench/torch_cpu.py: -------------------------------------------------------------------------------- 1 | # ported from https://medium.com/mlearning-ai/learning-xor-with-pytorch-c1c11d67ba8e 2 | 3 | import torch 4 | import torch.nn as nn 5 | from torch.autograd import Variable 6 | import time 7 | 8 | # create data 9 | Xs = torch.Tensor([[0., 0.], 10 | [1., 0.], 11 | [0., 1.], 12 | [1., 1.]]) 13 | 14 | y = torch.Tensor([0., 1., 1., 0.]).reshape(Xs.shape[0], 1) 15 | 16 | 17 | class XOR(nn.Module): 18 | def __init__(self): 19 | super(XOR, self).__init__() 20 | self.linear = nn.Linear(2, 2) 21 | self.Sigmoid = nn.Sigmoid() 22 | self.linear2 = nn.Linear(2, 1) 23 | 24 | def forward(self, input): 25 | x = self.linear(input) 26 | sig = self.Sigmoid(x) 27 | yh = self.linear2(sig) 28 | return yh 29 | 30 | 31 | xor_network = XOR() 32 | epochs = 10000 33 | mseloss = nn.MSELoss() 34 | optimizer = torch.optim.Adam(xor_network.parameters(), lr=0.03) 35 | 36 | start = time.perf_counter() 37 | 38 | for epoch in range(epochs): 39 | 40 | # input training example and return the prediction 41 | yhat = xor_network.forward(Xs) 42 | 43 | # calculate MSE loss 44 | loss = mseloss(yhat, y) 45 | 46 | # backpropogate through the loss gradiants 47 | loss.backward() 48 | 49 | # update model weights 50 | optimizer.step() 51 | 52 | # remove current gradients for next iteration 53 | optimizer.zero_grad() 54 | 55 | print((time.perf_counter()-start)*1000, " miliseconds") 56 | 57 | print(xor_network(torch.tensor([0., 0.]))) 58 | print(xor_network(torch.tensor([1., 0.]))) 59 | print(xor_network(torch.tensor([0., 1.]))) 60 | print(xor_network(torch.tensor([1., 1.]))) -------------------------------------------------------------------------------- /crates/core-gpu/src/gpu/init.rs: -------------------------------------------------------------------------------- 1 | use ndarray::{ArrayD, IxDyn}; 2 | use ndarray_rand::{ 3 | rand_distr::{Normal, Uniform}, 4 | RandomExt, 5 | }; 6 | 7 | use crate::Init; 8 | 9 | pub struct GPUInit { 10 | pub init: Init, 11 | } 12 | 13 | impl GPUInit { 14 | pub fn from(init: Init) -> Self { 15 | Self { init } 16 | } 17 | 18 | pub fn init(&self, size: IxDyn, input_size: usize, output_size: usize) -> ArrayD { 19 | match self.init { 20 | Init::Uniform => uniform(size), 21 | Init::Xavier => xavier(size, input_size), 22 | Init::XavierN => xaviern(size, input_size, output_size), 23 | Init::Kaiming => kaiming(size, input_size), 24 | } 25 | } 26 | 27 | pub fn from_default(init: Option, default: Init) -> Self { 28 | if let Some(init) = init { 29 | return Self { init }; 30 | } 31 | Self { init: default } 32 | } 33 | } 34 | 35 | pub fn uniform(size: IxDyn) -> ArrayD { 36 | ArrayD::random(size, Uniform::new(-1.0, 1.0)) 37 | } 38 | 39 | pub fn xavier(size: IxDyn, input_size: usize) -> ArrayD { 40 | let bounds = 1.0 / (input_size as f32).sqrt(); 41 | ArrayD::random(size, Uniform::new(-bounds, bounds)) 42 | } 43 | 44 | pub fn xaviern(size: IxDyn, input_size: usize, output_size: usize) -> ArrayD { 45 | let bounds = (6.0 as f32).sqrt() / ((input_size + output_size) as f32).sqrt(); 46 | ArrayD::random(size, Uniform::new(-bounds, bounds)) 47 | } 48 | 49 | pub fn kaiming(size: IxDyn, input_size: usize) -> ArrayD { 50 | let deviation = (2.0 / (input_size as f32)).sqrt(); 51 | ArrayD::random(size, Normal::new(0.0, deviation).unwrap()) 52 | } 53 | -------------------------------------------------------------------------------- /crates/core/src/cpu/init.rs: -------------------------------------------------------------------------------- 1 | use ndarray::{ArrayD, IxDyn}; 2 | use ndarray_rand::{ 3 | rand_distr::{Normal, Uniform}, 4 | RandomExt, 5 | }; 6 | 7 | use crate::Init; 8 | 9 | pub struct CPUInit { 10 | pub init: Init, 11 | } 12 | 13 | impl CPUInit { 14 | pub fn from(init: Init) -> Self { 15 | Self { init } 16 | } 17 | 18 | pub fn init(&self, size: IxDyn, input_size: usize, output_size: usize) -> ArrayD { 19 | match self.init { 20 | Init::Uniform => uniform(size), 21 | Init::Xavier => xavier(size, input_size), 22 | Init::XavierN => xaviern(size, input_size, output_size), 23 | Init::Kaiming => kaiming(size, input_size), 24 | } 25 | } 26 | 27 | pub fn from_default(init: Option, default: Init) -> Self { 28 | if let Some(init) = init { 29 | return Self { init }; 30 | } 31 | Self { init: default } 32 | } 33 | } 34 | 35 | pub fn uniform(size: IxDyn) -> ArrayD { 36 | ArrayD::random(size, Uniform::new(-1.0, 1.0)) 37 | } 38 | 39 | pub fn xavier(size: IxDyn, input_size: usize) -> ArrayD { 40 | let bounds = 1.0 / (input_size as f32).sqrt(); 41 | ArrayD::random(size, Uniform::new(-bounds, bounds)) 42 | } 43 | 44 | pub fn xaviern(size: IxDyn, input_size: usize, output_size: usize) -> ArrayD { 45 | let bounds = (6.0 as f32).sqrt() / ((input_size + output_size) as f32).sqrt(); 46 | ArrayD::random(size, Uniform::new(-bounds, bounds)) 47 | } 48 | 49 | pub fn kaiming(size: IxDyn, input_size: usize) -> ArrayD { 50 | let deviation = (2.0 / (input_size as f32)).sqrt(); 51 | ArrayD::random(size, Normal::new(0.0, deviation).unwrap()) 52 | } 53 | -------------------------------------------------------------------------------- /mod.ts: -------------------------------------------------------------------------------- 1 | export { setupBackend } from "./packages/core/src/core/engine.ts"; 2 | export * from "./packages/core/src/core/mod.ts"; 3 | export * from "./packages/core/src/core/types.ts"; 4 | export * from "./packages/core/src/core/tensor/tensor.ts"; 5 | export * from "./packages/core/src/core/api/layers.ts"; 6 | export * from "./packages/core/src/core/api/shape.ts"; 7 | export * from "./packages/core/src/core/api/network.ts"; 8 | export * from "./packages/core/src/core/api/optimizer.ts"; 9 | export * from "./packages/core/src/core/api/scheduler.ts"; 10 | export { GPU } from "./packages/core/src/backends/gpu/mod.ts"; 11 | 12 | import { 13 | CPU, 14 | type CPUBackendLoader, 15 | } from "./packages/core/src/backends/cpu/mod.ts"; 16 | import { 17 | WASM, 18 | type WASMBackendLoader, 19 | } from "./packages/core/src/backends/wasm/mod.ts"; 20 | import type { BackendLoader } from "./packages/core/src/core/engine.ts"; 21 | 22 | onerror = () => { 23 | if (typeof Deno == "undefined") { 24 | throw new Error( 25 | "Warning: Deno is not defined. Did you mean to import from ./web.ts instead of ./mod.ts?", 26 | ); 27 | } 28 | }; 29 | 30 | /** 31 | * The AUTO backend is chosen automatically based on the environment. 32 | */ 33 | const AUTO: WASMBackendLoader | CPUBackendLoader = Deno.dlopen === undefined 34 | ? WASM 35 | : CPU; 36 | 37 | /** 38 | * The OPTION function is used to choose a backend from a list of options. 39 | */ 40 | export function OPTION(...backends: BackendLoader[]): BackendLoader { 41 | for (const backend of backends) { 42 | if (backend.isSupported()) { 43 | return backend; 44 | } 45 | } 46 | throw new Error("No provided backend is supported"); 47 | } 48 | export { AUTO, CPU, WASM }; 49 | -------------------------------------------------------------------------------- /packages/core/src/backends/cpu/util.ts: -------------------------------------------------------------------------------- 1 | import type { Rank, Shape } from "../../core/api/shape.ts"; 2 | import type { DataSet } from "../../core/types.ts"; 3 | 4 | export class Buffer { 5 | buffer: Uint8Array = new Uint8Array(); 6 | allocBuffer: Deno.PointerObject<{ 7 | readonly parameters: readonly ["usize"]; 8 | readonly result: "buffer"; 9 | }> = new Deno.UnsafeCallback({ 10 | parameters: ["usize"], 11 | result: "buffer", 12 | }, (length) => { 13 | this.buffer = new Uint8Array(Number(length)); 14 | return this.buffer; 15 | }).pointer; 16 | } 17 | 18 | /** 19 | * Train Options Interface. 20 | */ 21 | export type TrainOptions = { 22 | datasets: number; 23 | inputShape: Shape; 24 | outputShape: Shape; 25 | epochs: number; 26 | batches: number; 27 | rate: number; 28 | }; 29 | 30 | /** 31 | * Predict Options Interface. 32 | */ 33 | export type PredictOptions = { 34 | inputShape: Shape; 35 | outputShape: Shape; 36 | }; 37 | 38 | /** 39 | * Encode JSON data. 40 | */ 41 | export function encodeJSON(json: unknown): Uint8Array { 42 | return new TextEncoder().encode(JSON.stringify(json)); 43 | } 44 | 45 | /** 46 | * Returns the BigInt value of a pointer. 47 | */ 48 | export function pointer(arr: BufferSource): bigint { 49 | return BigInt(Deno.UnsafePointer.value(Deno.UnsafePointer.of(arr))); 50 | } 51 | 52 | /** 53 | * Encode datasets. 54 | */ 55 | export function encodeDatasets(datasets: DataSet[]): BigUint64Array { 56 | const pointers: bigint[] = []; 57 | for (const dataset of datasets) { 58 | pointers.push(pointer(dataset.inputs.data as Float32Array)); 59 | pointers.push(pointer(dataset.outputs.data as Float32Array)); 60 | } 61 | return new BigUint64Array(pointers); 62 | } 63 | -------------------------------------------------------------------------------- /crates/core/src/cpu/optimizers/rmsprop.rs: -------------------------------------------------------------------------------- 1 | use std::ops::{Add, Div, Mul, SubAssign, Sub}; 2 | 3 | use ndarray::{ArrayD, ArrayViewD, ArrayViewMutD}; 4 | 5 | use crate::{CPUScheduler, RMSPropOptimizer}; 6 | 7 | pub struct CPURMSPropOptimizer { 8 | decay_rate: f32, 9 | epsilon: f32, 10 | acc_sg: Vec>>, 11 | } 12 | 13 | impl CPURMSPropOptimizer { 14 | pub fn new(config: RMSPropOptimizer, params: Vec>>) -> Self { 15 | let mut acc_sg = Vec::new(); 16 | for params in params { 17 | acc_sg.push( 18 | params 19 | .iter() 20 | .map(|param| ArrayD::zeros(param.dim())) 21 | .collect(), 22 | ); 23 | } 24 | Self { 25 | acc_sg, 26 | decay_rate: config.decay_rate, 27 | epsilon: config.epsilon, 28 | } 29 | } 30 | 31 | pub fn update_grads( 32 | &mut self, 33 | mut params: Vec>, 34 | grads: Vec>, 35 | idx: usize, 36 | scheduler: &CPUScheduler, 37 | rate: f32, 38 | epoch: usize, 39 | l: Vec>, 40 | ) { 41 | for (j, ((param, grad), li)) in params.iter_mut().zip(grads).zip(l).enumerate() { 42 | self.acc_sg[idx][j] = self 43 | .decay_rate 44 | .mul(&self.acc_sg[idx][j]) 45 | .add((1.0 - self.decay_rate).mul(&grad.map(|x| x.powi(2)))); 46 | 47 | let rate = scheduler.eta(rate, epoch); 48 | 49 | param.sub_assign( 50 | &rate 51 | .mul(&grad) 52 | .div(self.acc_sg[idx][j].map(|x| x.sqrt()).add(self.epsilon)) 53 | .sub(&li), 54 | ) 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /packages/utilities/src/encoding/mod.ts: -------------------------------------------------------------------------------- 1 | import { DiscreteMapper } from "../mapper/mod.ts"; 2 | import { argmax, Matrix, MatrixLike } from "../mod.ts"; 3 | import type { DataType, DType } from "../utils/common_types.ts"; 4 | import { MultiHotEncoder } from "./multihot.ts"; 5 | import { OneHotEncoder } from "./onehot.ts"; 6 | import { TfEncoder } from "./termfrequency.ts"; 7 | 8 | export class CategoricalEncoder { 9 | mapper: DiscreteMapper; 10 | encoder?: OneHotEncoder; 11 | maxLength: number; 12 | constructor(mappings?: Map) { 13 | this.mapper = new DiscreteMapper(mappings); 14 | this.maxLength = 0; 15 | } 16 | fit(document: T[]): CategoricalEncoder { 17 | this.mapper.fit(document); 18 | this.encoder = new OneHotEncoder(this.mapper.mapping.size); 19 | return this; 20 | } 21 | transform
(document: T[], dType: DT): Matrix
{ 22 | if (!this.mapper.mapping.size) 23 | throw new Error("Categorical Encoder not trained yet. Use .fit() first."); 24 | const tokens = this.mapper.transform(document); 25 | if (!this.encoder) 26 | throw new Error("Categorical Encoder not trained yet. Use .fit() first."); 27 | const encoded = this.encoder.transform(tokens, dType); 28 | return encoded; 29 | } 30 | getMapping
(data: DType
): T | undefined { 31 | const max = argmax(data); 32 | return this.mapper.getOg(max); 33 | } 34 | untransform
(data: MatrixLike
): T[] { 35 | const matrix = new Matrix(data); 36 | const res = new Array(matrix.nRows); 37 | for (let i = 0; i < matrix.nRows; i += 1) { 38 | res[i] = this.getMapping(matrix.row(i)); 39 | } 40 | return res; 41 | } 42 | } 43 | 44 | export { transformSoftmaxMut } from "./softmax.ts"; 45 | export { OneHotEncoder, MultiHotEncoder, TfEncoder }; 46 | -------------------------------------------------------------------------------- /crates/core-gpu/src/tensor.rs: -------------------------------------------------------------------------------- 1 | use std::{borrow::Cow, slice::from_raw_parts}; 2 | 3 | use ndarray::{ArrayD, ArrayViewD}; 4 | use safetensors::{Dtype, View}; 5 | 6 | pub struct Tensor<'a> { 7 | pub data: ArrayViewD<'a, f32>, 8 | } 9 | 10 | impl<'a> Tensor<'a> { 11 | pub fn new(data: ArrayViewD<'a, f32>) -> Self { 12 | Self { data } 13 | } 14 | } 15 | 16 | impl<'a> View for Tensor<'a> { 17 | fn dtype(&self) -> Dtype { 18 | Dtype::F32 19 | } 20 | 21 | fn shape(&self) -> &[usize] { 22 | self.data.shape() 23 | } 24 | 25 | fn data(&self) -> Cow<[u8]> { 26 | let slice = self.data.as_slice().expect("Non contiguous tensors"); 27 | let new_slice: &[u8] = 28 | unsafe { from_raw_parts(slice.as_ptr() as *const u8, slice.len() * 4) }; 29 | Cow::from(new_slice) 30 | } 31 | 32 | fn data_len(&self) -> usize { 33 | self.data.len() * 4 34 | } 35 | } 36 | 37 | #[derive(Debug)] 38 | pub struct DenseTensors { 39 | pub weights: ArrayD, 40 | pub biases: ArrayD, 41 | } 42 | 43 | #[derive(Debug)] 44 | pub struct ConvTensors { 45 | pub weights: ArrayD, 46 | pub biases: ArrayD, 47 | } 48 | 49 | #[derive(Debug)] 50 | pub struct BatchNormTensors { 51 | pub gamma: ArrayD, 52 | pub beta: ArrayD, 53 | pub running_mean: ArrayD, 54 | pub running_var: ArrayD, 55 | } 56 | 57 | #[derive(Debug)] 58 | pub enum Tensors { 59 | Dense(DenseTensors), 60 | Conv(ConvTensors), 61 | BatchNorm(BatchNormTensors), 62 | } 63 | 64 | pub trait GetTensor { 65 | fn get(&mut self) -> Option; 66 | } 67 | 68 | impl GetTensor for Option> { 69 | fn get(&mut self) -> Option { 70 | if let Some(tensors) = self { 71 | return Some(tensors.remove(0)); 72 | } 73 | None 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /crates/core/src/tensor.rs: -------------------------------------------------------------------------------- 1 | use std::{borrow::Cow, slice::from_raw_parts}; 2 | 3 | use ndarray::{ArrayD, ArrayViewD}; 4 | use safetensors::{Dtype, View}; 5 | 6 | pub struct Tensor<'a> { 7 | pub data: ArrayViewD<'a, f32>, 8 | } 9 | 10 | impl<'a> Tensor<'a> { 11 | pub fn new(data: ArrayViewD<'a, f32>) -> Self { 12 | Self { data } 13 | } 14 | } 15 | 16 | impl<'a> View for Tensor<'a> { 17 | fn dtype(&self) -> Dtype { 18 | Dtype::F32 19 | } 20 | 21 | fn shape(&self) -> &[usize] { 22 | self.data.shape() 23 | } 24 | 25 | fn data(&self) -> Cow<[u8]> { 26 | let slice = self.data.as_slice().expect("Non contiguous tensors"); 27 | let new_slice: &[u8] = 28 | unsafe { from_raw_parts(slice.as_ptr() as *const u8, slice.len() * 4) }; 29 | Cow::from(new_slice) 30 | } 31 | 32 | fn data_len(&self) -> usize { 33 | self.data.len() * 4 34 | } 35 | } 36 | 37 | #[derive(Debug)] 38 | pub struct DenseTensors { 39 | pub weights: ArrayD, 40 | pub biases: ArrayD, 41 | } 42 | 43 | #[derive(Debug)] 44 | pub struct ConvTensors { 45 | pub weights: ArrayD, 46 | pub biases: ArrayD, 47 | } 48 | 49 | #[derive(Debug)] 50 | pub struct BatchNormTensors { 51 | pub gamma: ArrayD, 52 | pub beta: ArrayD, 53 | pub running_mean: ArrayD, 54 | pub running_var: ArrayD, 55 | } 56 | 57 | #[derive(Debug)] 58 | pub enum Tensors { 59 | Dense(DenseTensors), 60 | Conv(ConvTensors), 61 | BatchNorm(BatchNormTensors), 62 | } 63 | 64 | pub trait GetTensor { 65 | fn get(&mut self) -> Option; 66 | } 67 | 68 | impl GetTensor for Option> { 69 | fn get(&mut self) -> Option { 70 | if let Some(tensors) = self { 71 | return Some(tensors.remove(0)); 72 | } 73 | None 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /examples/autoencoders/encoded.html: -------------------------------------------------------------------------------- 1 | 2 | idx01
0-0.78356546163558960.6262750625610352
118.180845260620117-15.350777626037598
2-0.74291682243347170.5920295715332031
313.401049613952637-11.3239164352417
4-0.78356546163558960.6262750625610352
5-0.38311046361923220.28890150785446167
6-0.78356546163558960.6262750625610352
70.36166173219680786-0.3385509252548218
8-0.78356546163558960.6262750625610352
9-0.78356546163558960.6262750625610352
10-0.78356546163558960.6262750625610352
11-0.78356546163558960.6262750625610352
12-0.78356546163558960.6262750625610352
130.11124086380004883-0.12757742404937744
1452.79803466796875-44.514915466308594
1547.98031234741211-40.45610046386719
1632.84283447265625-27.703144073486328
1710.527204513549805-8.902771949768066
18-0.78356546163558960.6262750625610352
1914.474761009216309-12.22849178314209
-------------------------------------------------------------------------------- /packages/data/datasets/csv.ts: -------------------------------------------------------------------------------- 1 | import { tensor2D } from "../../core/mod.ts"; 2 | import type { DataLike } from "../types.ts"; 3 | import { CsvParseStream } from "../deps.ts"; 4 | 5 | export interface CsvColumnConfig { 6 | /** 7 | * Whether this column is a label column. 8 | */ 9 | label?: boolean; 10 | } 11 | 12 | export interface CsvLoaderConfig { 13 | columns?: Record; 14 | } 15 | 16 | export async function loadCsv( 17 | url: string | URL, 18 | config: CsvLoaderConfig = {}, 19 | ): Promise { 20 | const data = await fetch(url).then((res) => 21 | res 22 | .body!.pipeThrough(new TextDecoderStream()) 23 | .pipeThrough(new CsvParseStream()) 24 | ); 25 | const colConfigs = Object.entries(config.columns ?? {}); 26 | const train_x: number[][] = []; 27 | const train_y: number[][] = []; 28 | const labelCols = colConfigs.filter(([, col]) => col.label); 29 | if (labelCols.length === 0) { 30 | throw new Error("No label column was set"); 31 | } 32 | let columnNames!: string[]; 33 | let columnIndices!: Record; 34 | for await (const row of data) { 35 | if (!columnNames) { 36 | columnNames = row as string[]; 37 | columnIndices = columnNames.reduce((acc, col, i) => { 38 | acc[col] = i; 39 | return acc; 40 | }, {} as Record); 41 | continue; 42 | } 43 | const x = []; 44 | const y = []; 45 | for (const col in columnIndices) { 46 | const colConfig = config.columns?.[col]; 47 | const i = columnIndices[col]; 48 | const value = (row as string[])[i]; 49 | if (colConfig?.label) { 50 | y.push(Number(value)); 51 | } else { 52 | x.push(Number(value)); 53 | } 54 | } 55 | train_x.push(x); 56 | train_y.push(y); 57 | } 58 | return { 59 | train_x: tensor2D(train_x), 60 | train_y: tensor2D(train_y), 61 | }; 62 | } 63 | -------------------------------------------------------------------------------- /examples/model/train.ts: -------------------------------------------------------------------------------- 1 | import { 2 | Cost, 3 | CPU, 4 | DenseLayer, 5 | Sequential, 6 | setupBackend, 7 | SigmoidLayer, 8 | tensor2D, 9 | } from "../../packages/core/mod.ts"; 10 | 11 | /** 12 | * Setup the CPU backend. This backend is fast but doesn't work on the Edge. 13 | */ 14 | await setupBackend(CPU); 15 | 16 | /** 17 | * Creates a sequential neural network. 18 | */ 19 | const net = new Sequential({ 20 | /** 21 | * The number of minibatches is set to 4 and the output size is set to 2. 22 | */ 23 | size: [4, 2], 24 | 25 | /** 26 | * The silent option is set to true, which means that the network will not output any logs during trainin 27 | */ 28 | silent: true, 29 | 30 | /** 31 | * Defines the layers of a neural network in the XOR function example. 32 | * The neural network has two input neurons and one output neuron. 33 | * The layers are defined as follows: 34 | * - A dense layer with 3 neurons. 35 | * - sigmoid activation layer. 36 | * - A dense layer with 1 neuron. 37 | * -A sigmoid activation layer. 38 | */ 39 | layers: [ 40 | DenseLayer({ size: [3] }), 41 | SigmoidLayer(), 42 | DenseLayer({ size: [1] }), 43 | SigmoidLayer(), 44 | ], 45 | 46 | /** 47 | * The cost function used for training the network is the mean squared error (MSE). 48 | */ 49 | cost: Cost.MSE, 50 | }); 51 | 52 | const time = performance.now(); 53 | 54 | /** 55 | * Train the network on the given data. 56 | */ 57 | net.train( 58 | [ 59 | { 60 | inputs: tensor2D([ 61 | [0, 0], 62 | [1, 0], 63 | [0, 1], 64 | [1, 1], 65 | ]), 66 | outputs: tensor2D([[0], [1], [1], [0]]), 67 | }, 68 | ], 69 | /** 70 | * The number of iterations is set to 1,000,000. 71 | */ 72 | 1000000, 73 | ); 74 | 75 | console.log(`training time: ${performance.now() - time}ms`); 76 | 77 | net.saveFile("examples/model/xor.test.st"); 78 | -------------------------------------------------------------------------------- /packages/core/src/core/api/shape.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Shape Type 3 | */ 4 | export type Shape = [number, ...number[]] & { length: R }; 5 | /** 6 | * 1st dimentional shape. 7 | */ 8 | export type Shape1D = Shape<1>; 9 | 10 | /** 11 | * 2nd dimentional shape. 12 | */ 13 | export type Shape2D = Shape<2>; 14 | 15 | /** 16 | * 3th dimentional shape. 17 | */ 18 | export type Shape3D = Shape<3>; 19 | 20 | /** 21 | * 4th dimentional shape. 22 | */ 23 | export type Shape4D = Shape<4>; 24 | 25 | /** 26 | * 5th dimentional shape. 27 | */ 28 | export type Shape5D = Shape<5>; 29 | 30 | /** 31 | * 6th dimentional shape. 32 | */ 33 | export type Shape6D = Shape<6>; 34 | 35 | /** 36 | * Rank Types. 37 | */ 38 | export enum Rank { 39 | /** 40 | * Scalar (magnitude only). 41 | */ 42 | R1 = 1, 43 | 44 | /** 45 | * Vector (magnitude and direction). 46 | */ 47 | R2 = 2, 48 | 49 | /** 50 | * Matrix (table of numbers). 51 | */ 52 | R3 = 3, 53 | 54 | /** 55 | * 3-Tensor (cube of numbers) 56 | */ 57 | R4 = 4, 58 | 59 | /** 60 | * Rank 5 Tensor 61 | */ 62 | R5 = 5, 63 | 64 | /** 65 | * Rank 6 Tensor 66 | */ 67 | R6 = 6, 68 | } 69 | 70 | /** 71 | * Array Map Types. 72 | */ 73 | export type ArrayMap = 74 | | Array1D 75 | | Array2D 76 | | Array3D 77 | | Array4D 78 | | Array5D 79 | | Array6D; 80 | 81 | /** 82 | * 1D Array. 83 | */ 84 | export type Array1D = number[]; 85 | 86 | /** 87 | * 2D Array. 88 | */ 89 | export type Array2D = number[][]; 90 | 91 | /** 92 | * 3D Array. 93 | */ 94 | export type Array3D = number[][][]; 95 | 96 | /** 97 | * 4D Array. 98 | */ 99 | export type Array4D = number[][][][]; 100 | 101 | /** 102 | * 5D Array. 103 | */ 104 | export type Array5D = number[][][][][]; 105 | 106 | /** 107 | * 6D Array. 108 | */ 109 | export type Array6D = number[][][][][][]; 110 | -------------------------------------------------------------------------------- /packages/utilities/src/mapper/discrete.ts: -------------------------------------------------------------------------------- 1 | /** Map discrete values into numbers */ 2 | export class DiscreteMapper { 3 | /** Map categories to indices */ 4 | mapping: Map; 5 | /** An internal counter for remembering the last index in mapping. */ 6 | #lastToken: Uint32Array; 7 | constructor(mapping?: Map) { 8 | this.mapping = mapping || new Map(); 9 | this.#lastToken = new Uint32Array(1); 10 | if (mapping?.size) { 11 | this.#lastToken[0] = Array.from(mapping.values()).reduce( 12 | (acc, val) => (acc > val ? acc : val), 13 | 0 14 | ); 15 | } 16 | } 17 | /** Construct a mapping from a given set of text. */ 18 | fit(targets: T[]): this { 19 | let i = 0; 20 | while (i < targets.length) { 21 | if (!this.mapping.has(targets[i])) { 22 | const token = this.#incrementToken(); 23 | this.mapping.set(targets[i], token); 24 | } 25 | i += 1; 26 | } 27 | return this; 28 | } 29 | /** 30 | * Encode values into their respective mappings. 31 | * Returns -1 in case of missing mapping. 32 | */ 33 | transform(targets: T[]): number[] { 34 | const res = new Array(targets.length); 35 | let i = 0; 36 | while (i < targets.length) { 37 | const index = this.mapping.get(targets[i]) ?? -1; 38 | res[i] = index; 39 | i += 1; 40 | } 41 | return res; 42 | } 43 | /** Convert mapped numbers into actual values */ 44 | untransform(data: number[]): T[] { 45 | const res = new Array(data.length); 46 | for (let i = 0; i < res.length; i += 1) { 47 | res[i] = this.getOg(data[i]) || "__unknown__"; 48 | } 49 | return res; 50 | } 51 | getOg(data: number): T | undefined { 52 | for (const [k, v] of this.mapping.entries()) { 53 | if (v === data) { 54 | return k; 55 | } 56 | } 57 | return undefined; 58 | } 59 | #incrementToken(): number { 60 | return Atomics.add(this.#lastToken, 0, 1); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /packages/core/src/core/engine.ts: -------------------------------------------------------------------------------- 1 | import { WASM } from "../backends/wasm/mod.ts"; 2 | import type { Sequential } from "./mod.ts"; 3 | import type { Backend, BackendType, NetworkConfig } from "./types.ts"; 4 | 5 | export interface BackendInstance { 6 | /** 7 | * Initialize the backend. 8 | */ 9 | init(): Promise; 10 | } 11 | 12 | /** 13 | * The Backend Loader is responsible for loading the backend and setting it up. 14 | */ 15 | export interface BackendLoader { 16 | /** 17 | * Whether the backend is supported. 18 | * 19 | * ```ts 20 | * import { WASM } from "https://deno.land/x/netsaur/mod.ts"; 21 | * 22 | * console.log(WASM.isSupported()); 23 | * ``` 24 | */ 25 | isSupported(): boolean; 26 | 27 | /** 28 | * Setup the backend. Returns true if the backend was successfully setup. 29 | */ 30 | setup(silent: boolean): Promise; 31 | 32 | /** 33 | * Load the backend from a config. 34 | */ 35 | loadBackend(config: NetworkConfig): Backend; 36 | 37 | /** 38 | * Load a model from a safe tensors file path. 39 | */ 40 | loadFile(path: string): Sequential; 41 | 42 | /** 43 | * Load a model from Uint8Array data. 44 | */ 45 | load(data: Uint8Array): Sequential; 46 | } 47 | 48 | /** 49 | * setupBackend loads the backend and sets it up. 50 | * ```ts 51 | * import { setupBackend, CPU } from "https://deno.land/x/netsaur/mod.ts"; 52 | * 53 | * await setupBackend(CPU); 54 | * ``` 55 | */ 56 | export async function setupBackend( 57 | loader: BackendLoader, 58 | silent = false, 59 | ): Promise { 60 | const success = await loader.setup(silent); 61 | if (!success) { 62 | if (!silent) console.log("Defaulting to CPU Backend"); 63 | await WASM.setup(silent); 64 | } 65 | Engine.backendLoader = loader; 66 | } 67 | 68 | /** 69 | * the Engine manages the backend and the backend loader. 70 | */ 71 | export class Engine { 72 | static backendLoader: BackendLoader; 73 | 74 | static type: BackendType; 75 | } 76 | -------------------------------------------------------------------------------- /examples/mnist/common.ts: -------------------------------------------------------------------------------- 1 | import { Tensor, type DataSet } from "../../packages/core/mod.ts"; 2 | 3 | export function assert(condition: boolean, message?: string) { 4 | if (!condition) { 5 | throw new Error(message); 6 | } 7 | } 8 | 9 | export function loadDataset( 10 | imagesFile: string, 11 | labelsFile: string, 12 | start: number, 13 | end: number, 14 | minibatch = 1, 15 | ) { 16 | const images = Deno.readFileSync(new URL(imagesFile, import.meta.url)); 17 | const labels = Deno.readFileSync(new URL(labelsFile, import.meta.url)); 18 | 19 | const imageView = new DataView(images.buffer); 20 | const labelView = new DataView(labels.buffer); 21 | 22 | assert(imageView.getUint32(0) === 0x803, "Invalid image file"); 23 | assert(labelView.getUint32(0) === 0x801, "Invalid label file"); 24 | 25 | const count = imageView.getUint32(4); 26 | assert(count === labelView.getUint32(4), "Image and label count mismatch"); 27 | 28 | const inputs: Float32Array[] = []; 29 | let mean = 0; 30 | let sd = 0; 31 | for (let i = 0; i < count / minibatch; i++) { 32 | const input = new Float32Array(784 * minibatch); 33 | for (let j = 0; j < 784 * minibatch; j++) { 34 | input[j] = imageView.getUint8(16 + i * (784 * minibatch) + j); 35 | mean += input[j]; 36 | sd += Math.pow(input[j], 2); 37 | } 38 | inputs.push(input); 39 | } 40 | 41 | mean /= count * 784; 42 | sd /= count * 784; 43 | sd -= Math.pow(mean, 2); 44 | sd = Math.sqrt(sd); 45 | 46 | const results: DataSet[] = []; 47 | 48 | for (let i = start; i < end / minibatch; i++) { 49 | for (let j = 0; j < 784 * minibatch; j++) { 50 | inputs[i][j] -= mean; 51 | inputs[i][j] /= sd; 52 | } 53 | 54 | const outputs = new Float32Array(10 * minibatch); 55 | for (let j = 0; j < minibatch; j++) { 56 | outputs[labelView.getUint8(8 + i * minibatch + j) + j * 10] = 1; 57 | } 58 | 59 | results.push({ 60 | inputs: new Tensor(inputs[i], [minibatch, 1, 28, 28]), 61 | outputs: new Tensor(outputs, [minibatch, 10]), 62 | }); 63 | } 64 | 65 | return results; 66 | } 67 | -------------------------------------------------------------------------------- /packages/utilities/src/image/colors/histogram.ts: -------------------------------------------------------------------------------- 1 | import type { Image } from "../../utils/mod.ts"; 2 | import type { Pixel } from "../../utils/common_types.ts"; 3 | 4 | /** 5 | * Histogram of colors with reduced space 6 | * Effectively quantizes the image into 32768 colors 7 | */ 8 | export class ColorHistogram { 9 | #data: Uint32Array; 10 | #quantizeBy: number; 11 | sigBits: number; 12 | constructor(sigBits: number) { 13 | this.sigBits = sigBits; 14 | this.#quantizeBy = 8 - sigBits; 15 | this.#data = new Uint32Array(1 << (sigBits * 3)); 16 | } 17 | #getIndex([r, g, b]: [number, number, number, number?]) { 18 | // ignore alpha 19 | const index = ((r >> this.#quantizeBy) << (this.sigBits << 1)) + 20 | ((g >> this.#quantizeBy) << this.sigBits) + 21 | (b >> this.#quantizeBy); 22 | return index; 23 | } 24 | get(color: Pixel): number { 25 | const index = this.#getIndex(color); 26 | return this.#data[index]; 27 | } 28 | getQuantized(color: Pixel): number { 29 | const index = (color[0] << 10) + (color[1] << 5) + color[2]; 30 | return this.#data[index]; 31 | } 32 | add(color: Pixel, amount: number): number { 33 | const index = this.#getIndex(color); 34 | return Atomics.add(this.#data, index, amount); 35 | } 36 | get raw(): Uint32Array { 37 | return this.#data; 38 | } 39 | get length(): number { 40 | return this.#data.filter((x) => x).length; 41 | } 42 | static getColor(index: number, sigBits: number): Pixel { 43 | const quantizeBy = 8 - sigBits; 44 | const ri = index >> 10; 45 | const gi = (index - (ri << 10)) >> 5; 46 | const bi = index - (ri << 10) - (gi << 5); 47 | return [ri << quantizeBy, gi << quantizeBy, bi << quantizeBy, 255]; 48 | } 49 | } 50 | 51 | /** Get a histogram of frequency of colors. */ 52 | export function getHistogram(image: Image, sigBits = 5): ColorHistogram { 53 | const histo = new ColorHistogram(sigBits); 54 | let i = 0; 55 | while (i < image.pixels) { 56 | const hIndex = image.getNthPixel(i); 57 | histo.add(hIndex, 1); 58 | i += 1; 59 | } 60 | return histo; 61 | } 62 | -------------------------------------------------------------------------------- /deno.lock: -------------------------------------------------------------------------------- 1 | { 2 | "version": "4", 3 | "specifiers": { 4 | "jsr:@denosaurs/netsaur@0.4.0": "0.4.0", 5 | "jsr:@denosaurs/plug@1.0.3": "1.0.3", 6 | "jsr:@std/assert@~0.213.1": "0.213.1", 7 | "jsr:@std/csv@1.0.3": "1.0.3", 8 | "jsr:@std/encoding@0.213.1": "0.213.1", 9 | "jsr:@std/fmt@0.213.1": "0.213.1", 10 | "jsr:@std/fmt@1.0.2": "1.0.2", 11 | "jsr:@std/fs@0.213.1": "0.213.1", 12 | "jsr:@std/path@0.213.1": "0.213.1", 13 | "jsr:@std/path@~0.213.1": "0.213.1" 14 | }, 15 | "jsr": { 16 | "@denosaurs/netsaur@0.4.0": { 17 | "integrity": "91192ee3ffffc2db3eac38a3f01abb7d52d9fd1053591eddb70852423d44f8ae", 18 | "dependencies": [ 19 | "jsr:@denosaurs/plug" 20 | ] 21 | }, 22 | "@denosaurs/plug@1.0.3": { 23 | "integrity": "b010544e386bea0ff3a1d05e0c88f704ea28cbd4d753439c2f1ee021a85d4640", 24 | "dependencies": [ 25 | "jsr:@std/encoding", 26 | "jsr:@std/fmt@0.213.1", 27 | "jsr:@std/fs", 28 | "jsr:@std/path@0.213.1" 29 | ] 30 | }, 31 | "@std/assert@0.213.1": { 32 | "integrity": "24c28178b30c8e0782c18e8e94ea72b16282207569cdd10ffb9d1d26f2edebfe" 33 | }, 34 | "@std/csv@1.0.3": { 35 | "integrity": "623acf0dcb88d62ba727c3611ad005df7f109ede8cac833e3986f540744562e5" 36 | }, 37 | "@std/encoding@0.213.1": { 38 | "integrity": "fcbb6928713dde941a18ca5db88ca1544d0755ec8fb20fe61e2dc8144b390c62" 39 | }, 40 | "@std/fmt@0.213.1": { 41 | "integrity": "a06d31777566d874b9c856c10244ac3e6b660bdec4c82506cd46be052a1082c3" 42 | }, 43 | "@std/fmt@1.0.2": { 44 | "integrity": "87e9dfcdd3ca7c066e0c3c657c1f987c82888eb8103a3a3baa62684ffeb0f7a7" 45 | }, 46 | "@std/fs@0.213.1": { 47 | "integrity": "fbcaf099f8a85c27ab0712b666262cda8fe6d02e9937bf9313ecaea39a22c501", 48 | "dependencies": [ 49 | "jsr:@std/assert", 50 | "jsr:@std/path@~0.213.1" 51 | ] 52 | }, 53 | "@std/path@0.213.1": { 54 | "integrity": "f187bf278a172752e02fcbacf6bd78a335ed320d080a7ed3a5a59c3e88abc673", 55 | "dependencies": [ 56 | "jsr:@std/assert" 57 | ] 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /examples/linear.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * This example trains a neural network to predict the output of the function y = 2x + 1 3 | */ 4 | 5 | import { 6 | Cost, 7 | CPU, 8 | DenseLayer, 9 | Sequential, 10 | setupBackend, 11 | tensor2D, 12 | } from "../packages/core/mod.ts"; 13 | 14 | /** 15 | * The test data used for predicting the output of the function y = 2x + 1 16 | */ 17 | const testData = [20, 40, 43, 87, 43]; 18 | 19 | /** 20 | * Setup the CPU backend. This backend is fast but doesn't work on the Edge. 21 | */ 22 | await setupBackend(CPU); 23 | 24 | /** 25 | * Creates a sequential neural network. 26 | */ 27 | const network = new Sequential({ 28 | /** 29 | * The number of minibatches is set to 4 and the output size is set to 1. 30 | */ 31 | size: [4, 1], 32 | 33 | /** 34 | * The silent option is set to true, which means that the network will not output any logs during trainin 35 | */ 36 | silent: true, 37 | 38 | /** 39 | * Creates two dense layers, with the first layer having 3 neurons and the second layer having 1 neuron. 40 | */ 41 | layers: [DenseLayer({ size: [3] }), DenseLayer({ size: [1] })], 42 | 43 | /** 44 | * The cost function used for training the network is the mean squared error (MSE). 45 | */ 46 | cost: Cost.MSE, 47 | }); 48 | 49 | const start = performance.now(); 50 | 51 | /** 52 | * Train the network on the given data. 53 | */ 54 | network.train( 55 | [ 56 | { 57 | // y = 2x + 1 58 | inputs: tensor2D([[1], [2], [3], [4]]), 59 | outputs: tensor2D([[3], [5], [7], [9]]), 60 | }, 61 | ], 62 | /** 63 | * The number of iterations is set to 400. 64 | */ 65 | 400, 66 | /** 67 | * The number of batches is set to 1. 68 | */ 69 | 1, 70 | /** 71 | * The learning rate is set to 0.01. 72 | */ 73 | 0.01, 74 | ); 75 | 76 | console.log("training time", performance.now() - start, " milliseconds"); 77 | console.log("y = 2x + 1"); 78 | 79 | /** 80 | * Make a prediction on the test data. 81 | */ 82 | const predicted = await network.predict(tensor2D(testData.map((x) => [x]))); 83 | for (const [i, res] of predicted.data.entries()) { 84 | console.log( 85 | `input: ${testData[i]}\noutput: ${res.toFixed(2)}\nexpected: ${ 86 | 2 * testData[i] + 1 87 | }\n`, 88 | ); 89 | } 90 | -------------------------------------------------------------------------------- /packages/utilities/examples/metrics/metrics.ts: -------------------------------------------------------------------------------- 1 | import { 2 | Cost, 3 | CPU, 4 | DenseLayer, 5 | Sequential, 6 | setupBackend, 7 | SigmoidLayer, 8 | tensor2D, 9 | } from "https://deno.land/x/netsaur@0.3.0/mod.ts"; 10 | 11 | import { parse } from "jsr:@std/csv@1.0.3/parse"; 12 | 13 | // Import helpers for metrics 14 | import { 15 | // Metrics 16 | ClassificationReport, 17 | // Split the dataset 18 | useSplit, 19 | } from "../../mod.ts"; 20 | 21 | // Define classes 22 | const classes = ["Setosa", "Versicolor"]; 23 | 24 | // Read the training dataset 25 | const _data = Deno.readTextFileSync("examples/metrics/binary_iris.csv"); 26 | const data = parse(_data); 27 | 28 | // Get the predictors (x) and targets (y) 29 | const x = data.map((fl) => fl.slice(0, 4).map(Number)); 30 | const y = data.map((fl) => classes.indexOf(fl[4])); 31 | 32 | // Split the dataset for training and testing 33 | const [train, test] = useSplit({ ratio: [7, 3], shuffle: true }, x, y) as [ 34 | [typeof x, typeof y], 35 | [typeof x, typeof y], 36 | ]; 37 | 38 | // Setup the CPU backend for Netsaur 39 | await setupBackend(CPU); 40 | 41 | // Create a sequential neural network 42 | const net = new Sequential({ 43 | // Set number of minibatches to 4 44 | // Set size of output to 4 45 | size: [4, 4], 46 | 47 | // Disable logging during training 48 | silent: true, 49 | 50 | // Define each layer of the network 51 | layers: [ 52 | // A dense layer with 4 neurons 53 | DenseLayer({ size: [4] }), 54 | // A sigmoid activation layer 55 | SigmoidLayer(), 56 | // A dense layer with 1 neuron 57 | DenseLayer({ size: [1] }), 58 | // Another sigmoid layer 59 | SigmoidLayer(), 60 | ], 61 | // We are using MSE for finding cost 62 | cost: Cost.MSE, 63 | }); 64 | 65 | const time = performance.now(); 66 | 67 | // Train the network 68 | net.train( 69 | [ 70 | { 71 | inputs: tensor2D(train[0]), 72 | outputs: tensor2D(train[1].map((x) => [x])), 73 | }, 74 | ], 75 | // Train for 10000 epochs 76 | 10000, 77 | ); 78 | 79 | console.log(`training time: ${performance.now() - time}ms`); 80 | 81 | const res = await net.predict(tensor2D(test[0])); 82 | const y1 = res.data.map( 83 | (x) => (x < 0.5 ? 0 : 1), 84 | ); 85 | const cMatrix = new ClassificationReport(test[1], y1); 86 | console.log("Confusion Matrix: ", cMatrix); 87 | -------------------------------------------------------------------------------- /examples/autoencoders/example.ts: -------------------------------------------------------------------------------- 1 | import { Matrix } from "../../packages/utilities/mod.ts"; 2 | import { 3 | AdamOptimizer, 4 | Cost, 5 | CPU, 6 | DenseLayer, 7 | ReluLayer, 8 | Sequential, 9 | setupBackend, 10 | type Shape2D, 11 | tensor, 12 | } from "../../packages/core/mod.ts"; 13 | 14 | import { parse } from "jsr:@std/csv@1.0.3/parse"; 15 | 16 | const data = parse( 17 | Deno.readTextFileSync("examples/autoencoders/winequality-red.csv") 18 | ); 19 | data.shift(); 20 | 21 | const x_data = data.slice(0, 20).map((fl) => fl.slice(0, 11).map(Number)); 22 | const X = new Matrix(x_data, "f32"); 23 | 24 | await setupBackend(CPU); 25 | 26 | const net = new Sequential({ 27 | size: [4, X.nCols], 28 | silent: false, 29 | layers: [ 30 | // Encoder 31 | DenseLayer({ size: [8] }), 32 | ReluLayer(), 33 | DenseLayer({ size: [4] }), 34 | ReluLayer(), 35 | DenseLayer({ size: [2] }), 36 | // Decoder 37 | DenseLayer({ size: [4] }), 38 | ReluLayer(), 39 | DenseLayer({ size: [8] }), 40 | ReluLayer(), 41 | DenseLayer({ size: [X.nCols] }), 42 | ], 43 | cost: Cost.MSE, 44 | patience: 50, 45 | optimizer: AdamOptimizer(), 46 | // scheduler: OneCycle() 47 | }); 48 | 49 | const input = tensor(X); 50 | 51 | const timeStart = performance.now(); 52 | net.train([{ inputs: input, outputs: tensor(Float32Array.from(input.data), input.shape) }], 10000, 1, 0.001); 53 | console.log(`Trained in ${performance.now() - timeStart}ms`); 54 | 55 | function saveTable(name: string, data: Matrix<"f32">) { 56 | Deno.writeTextFileSync(`examples/autoencoders/${name}.html`, data.html); 57 | } 58 | 59 | saveTable("input", X); 60 | 61 | console.log("Running Whole Net"); 62 | const output = await net.predict(input); 63 | 64 | const output_mat = new Matrix<"f32">(output.data, output.shape as Shape2D); 65 | 66 | saveTable("output", output_mat); 67 | 68 | console.log("Running Encoder"); 69 | const encoded = await net.predict(input, { layers: [0, 5] }); 70 | 71 | const encoded_mat = new Matrix<"f32">(encoded.data, encoded.shape as Shape2D); 72 | 73 | saveTable("encoded", encoded_mat); 74 | 75 | console.log("Running Decoder"); 76 | const decoded = await net.predict(tensor(encoded_mat), { layers: [5, 10] }); 77 | 78 | const decoded_mat = new Matrix<"f32">(decoded.data, decoded.shape as Shape2D); 79 | 80 | saveTable("decoded", decoded_mat); 81 | -------------------------------------------------------------------------------- /crates/core-gpu/src/gpu/optimizers/adam.rs: -------------------------------------------------------------------------------- 1 | use std::ops::{Add, Div, Mul, SubAssign}; 2 | 3 | use ndarray::{ArrayD, ArrayViewD, ArrayViewMutD}; 4 | 5 | use crate::{AdamOptimizer, GPUScheduler}; 6 | 7 | pub struct GPUAdamOptimizer { 8 | pub beta1: f32, 9 | pub beta2: f32, 10 | pub epsilon: f32, 11 | pub m: Vec>>, 12 | pub v: Vec>>, 13 | pub t: f32, 14 | } 15 | 16 | impl GPUAdamOptimizer { 17 | pub fn new(config: AdamOptimizer, params: Vec>>) -> Self { 18 | let mut m = Vec::new(); 19 | let mut v = Vec::new(); 20 | for params in params { 21 | m.push( 22 | params 23 | .iter() 24 | .map(|param| ArrayD::zeros(param.dim())) 25 | .collect(), 26 | ); 27 | v.push( 28 | params 29 | .iter() 30 | .map(|param| ArrayD::zeros(param.dim())) 31 | .collect(), 32 | ); 33 | } 34 | Self { 35 | beta1: config.beta1, 36 | beta2: config.beta2, 37 | epsilon: config.epsilon, 38 | m, 39 | v, 40 | t: 0.0, 41 | } 42 | } 43 | 44 | pub fn update_grads( 45 | &mut self, 46 | mut params: Vec>, 47 | grads: Vec>, 48 | idx: usize, 49 | scheduler: &GPUScheduler, 50 | rate: f32, 51 | ) { 52 | for (j, (param, grad)) in params.iter_mut().zip(grads).enumerate() { 53 | self.m[idx][j] = self 54 | .beta1 55 | .mul(&self.m[idx][j]) 56 | .add((1.0 - self.beta1).mul(&grad)); 57 | self.v[idx][j] = self 58 | .beta2 59 | .mul(&self.v[idx][j]) 60 | .add((1.0 - self.beta2).mul(&grad.map(|x| x.powi(2)))); 61 | 62 | let m_hat = self.m[idx][j].view().div(1.0 - self.beta1.powf(self.t)); 63 | let v_hat = self.v[idx][j].view().div(1.0 - self.beta2.powf(self.t)); 64 | 65 | let rate = scheduler.eta(rate, self.t as usize); 66 | 67 | param.sub_assign( 68 | &rate 69 | .mul(m_hat) 70 | .div(v_hat.map(|x| x.sqrt()).add(self.epsilon)), 71 | ) 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /packages/core/src/core/api/network.ts: -------------------------------------------------------------------------------- 1 | import type { Backend, DataSet, NetworkConfig } from "../types.ts"; 2 | import type { Tensor } from "../tensor/tensor.ts"; 3 | import type { Rank } from "./shape.ts"; 4 | 5 | /** 6 | * Base Neural Network Structure. All Neural Networks should implement this. 7 | */ 8 | export interface NeuralNetwork { 9 | /** 10 | * The backend used by the Neural Network. 11 | */ 12 | backend: Backend; 13 | 14 | /** 15 | * The configuration of the Neural Network. 16 | */ 17 | config: NetworkConfig; 18 | 19 | /** 20 | * The train method is a function that trains a neural network using a set of training data. 21 | * It takes in an array of DataSet objects, the number of epochs to train for, and the learning rate. 22 | * The method modifies the weights and biases of the network to minimize the cost function and improve its accuracy on the training data. 23 | * 24 | * ```ts 25 | * network.train([{ 26 | * inputs: tensor2D([ 27 | * [0, 0], 28 | * [1, 0], 29 | * [0, 1], 30 | * [1, 1], 31 | * ]), 32 | * outputs: tensor2D([[0], [1], [1], [0]]), 33 | * }]); 34 | * ``` 35 | */ 36 | train(datasets: DataSet[], epochs?: number, rate?: number): void; 37 | 38 | /** 39 | * The predict method is a function that takes in a Tensor object 40 | * representing the input to the neural network and returns a Promise that resolves to a Tensor object representing the output of the network. 41 | * This method is used to make predictions on new data after the network has been trained. 42 | * 43 | * ```ts 44 | * const prediction = await net.predict(tensor1D([0, 0])); 45 | * console.log(prediction.data[0]); 46 | * ``` 47 | */ 48 | predict(data: Tensor): Promise>; 49 | 50 | /** 51 | * The save method saves the network to a Uint8Array. 52 | * This method is used to save the network after it has been trained. 53 | * 54 | * ```ts 55 | * const modelData = network.save(); 56 | * Deno.writeFileSync("model.st", modelData); 57 | * ``` 58 | */ 59 | save(): Uint8Array; 60 | 61 | /** 62 | * The saveFile method takes in a string representing the path to a file to the safetensors format and saves the network to that file. 63 | * This method is used to save the network after it has been trained. 64 | * 65 | * ```ts 66 | * network.saveFile("model.st"); 67 | * ``` 68 | */ 69 | saveFile(path: string): void; 70 | } 71 | -------------------------------------------------------------------------------- /crates/core/src/cpu/optimizers/adam.rs: -------------------------------------------------------------------------------- 1 | use std::ops::{Add, Div, Mul, SubAssign, Sub}; 2 | 3 | use ndarray::{ArrayD, ArrayViewD, ArrayViewMutD}; 4 | 5 | use crate::{AdamOptimizer, CPUScheduler}; 6 | 7 | pub struct CPUAdamOptimizer { 8 | pub beta1: f32, 9 | pub beta2: f32, 10 | pub epsilon: f32, 11 | pub m: Vec>>, 12 | pub v: Vec>>, 13 | pub t: f32, 14 | } 15 | 16 | impl CPUAdamOptimizer { 17 | pub fn new(config: AdamOptimizer, params: Vec>>) -> Self { 18 | let mut m = Vec::new(); 19 | let mut v = Vec::new(); 20 | for params in params { 21 | m.push( 22 | params 23 | .iter() 24 | .map(|param| ArrayD::zeros(param.dim())) 25 | .collect(), 26 | ); 27 | v.push( 28 | params 29 | .iter() 30 | .map(|param| ArrayD::zeros(param.dim())) 31 | .collect(), 32 | ); 33 | } 34 | Self { 35 | beta1: config.beta1, 36 | beta2: config.beta2, 37 | epsilon: config.epsilon, 38 | m, 39 | v, 40 | t: 0.0, 41 | } 42 | } 43 | 44 | pub fn update_grads( 45 | &mut self, 46 | mut params: Vec>, 47 | grads: Vec>, 48 | idx: usize, 49 | scheduler: &CPUScheduler, 50 | rate: f32, 51 | l: Vec>, 52 | ) { 53 | for (j, ((param, grad), li)) in params.iter_mut().zip(grads).zip(l).enumerate() { 54 | self.m[idx][j] = self 55 | .beta1 56 | .mul(&self.m[idx][j]) 57 | .add((1.0 - self.beta1).mul(&grad)); 58 | self.v[idx][j] = self 59 | .beta2 60 | .mul(&self.v[idx][j]) 61 | .add((1.0 - self.beta2).mul(&grad.map(|x| x.powi(2)))); 62 | 63 | let m_hat = self.m[idx][j].view().div(1.0 - self.beta1.powf(self.t)); 64 | let v_hat = self.v[idx][j].view().div(1.0 - self.beta2.powf(self.t)); 65 | let rate = scheduler.eta(rate, self.t as usize); 66 | 67 | param.sub_assign( 68 | &rate 69 | .mul(m_hat) 70 | .div(v_hat.map(|x| x.sqrt()).add(self.epsilon)) 71 | .sub(&li), 72 | ); 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /packages/utilities/src/image/colors/common.ts: -------------------------------------------------------------------------------- 1 | import type { Image } from "../../utils/mod.ts"; 2 | import type { Pixel } from "../../utils/common_types.ts"; 3 | import type { ColorHistogram } from "./histogram.ts"; 4 | 5 | export function getAverageColor( 6 | vbox: ColorRange, 7 | histo: ColorHistogram, 8 | ): Pixel { 9 | let total = 0; 10 | let totalR = 0, totalG = 0, totalB = 0; 11 | let ri = vbox.r.min; 12 | while (ri <= vbox.r.max) { 13 | let gi = vbox.g.min; 14 | while (gi <= vbox.g.max) { 15 | let bi = vbox.b.min; 16 | while (bi <= vbox.b.max) { 17 | const count = histo.getQuantized([ri, gi, bi]) || 0; 18 | total += count; 19 | totalR += count * (ri + 0.5) * 8; 20 | totalG += count * (gi + 0.5) * 8; 21 | totalB += count * (bi + 0.5) * 8; 22 | bi += 1; 23 | } 24 | gi += 1; 25 | } 26 | ri += 1; 27 | } 28 | if (total) { 29 | return [ 30 | ~~(totalR / total), 31 | ~~(totalG / total), 32 | ~~(totalB / total), 33 | 255, 34 | ]; 35 | } 36 | // In case box is empty 37 | return [ 38 | Math.trunc(8 * (vbox.r.min + vbox.r.max + 1) / 2), 39 | Math.trunc(8 * (vbox.g.min + vbox.g.max + 1) / 2), 40 | Math.trunc(8 * (vbox.b.min + vbox.b.max + 1) / 2), 41 | 255, 42 | ]; 43 | } 44 | 45 | /** The vbox */ 46 | export interface ColorRange { 47 | r: { min: number; max: number }; 48 | g: { min: number; max: number }; 49 | b: { min: number; max: number }; 50 | } 51 | 52 | /** Get the minimum and maximum RGB values. */ 53 | export function getColorRange( 54 | image: Image, 55 | sigBits = 5, 56 | ): ColorRange { 57 | const quantizeBy = 8 - sigBits; 58 | const range = { 59 | r: { min: 1000, max: 0 }, 60 | g: { min: 1000, max: 0 }, 61 | b: { min: 1000, max: 0 }, 62 | }; 63 | let i = 0; 64 | while (i < image.pixels) { 65 | const pixel = image.getNthPixel(i).map((x) => x ? x >> quantizeBy : 0); 66 | if (pixel[0] < range.r.min) { 67 | range.r.min = pixel[0]; 68 | } 69 | if (pixel[0] > range.r.max) { 70 | range.r.max = pixel[0]; 71 | } 72 | 73 | if (pixel[1] < range.g.min) { 74 | range.g.min = pixel[1]; 75 | } 76 | if (pixel[1] > range.g.max) { 77 | range.g.max = pixel[1]; 78 | } 79 | 80 | if (pixel[2] < range.b.min) { 81 | range.b.min = pixel[2]; 82 | } 83 | if (pixel[2] > range.b.max) { 84 | range.b.max = pixel[2]; 85 | } 86 | 87 | i += 1; 88 | } 89 | return range; 90 | } 91 | -------------------------------------------------------------------------------- /examples/sentiment-analysis/tester.ts: -------------------------------------------------------------------------------- 1 | import { CPU, setupBackend, tensor } from "jsr:@denosaurs/netsaur@0.4.0"; 2 | import { Sequential } from "jsr:@denosaurs/netsaur@0.4.0/core"; 3 | 4 | import { 5 | ClassificationReport, 6 | type MatrixLike, 7 | useSplit, 8 | } from "jsr:@denosaurs/netsaur@0.4.0/utilities"; 9 | 10 | import { CategoricalEncoder } from "jsr:@denosaurs/netsaur@0.4.0/utilities/encoding"; 11 | import { 12 | CountVectorizer, 13 | SplitTokenizer, 14 | TfIdfTransformer, 15 | } from "jsr:@denosaurs/netsaur@0.4.0/utilities/text"; 16 | 17 | import Mappings from "./mappings.json" with { type: "json" }; 18 | import Vocab from "./vocab.json" with { type: "json" }; 19 | import Idf from "./tfidf.json" with { type: "json" }; 20 | 21 | import { parse as parseCsv } from "jsr:@std/csv@1.0.3/parse"; 22 | 23 | const file = Deno.readTextFileSync( 24 | "examples/sentiment-analysis/text_emotion.csv", 25 | ); 26 | 27 | const data = parseCsv(file, { skipFirstRow: true }) as { 28 | sentiment: string; 29 | content: string; 30 | }[]; 31 | const text = data.map((x) => x.content); 32 | const labels = data.map((x) => x.sentiment); 33 | 34 | const [[_trainX, _trainY], [testX, testY]] = useSplit( 35 | { shuffle: true, ratio: [7, 3] }, 36 | text, 37 | labels, 38 | ); 39 | 40 | const vocab = new Map(); 41 | 42 | for (const entry of Vocab) { 43 | vocab.set(entry[0], entry[1]); 44 | } 45 | 46 | const tokenizer = new SplitTokenizer({ 47 | skipWords: "english", 48 | vocabulary: vocab, 49 | standardize: { lowercase: true, stripNewlines: true }, 50 | }); 51 | 52 | const vectorizer = new CountVectorizer(tokenizer.vocabulary.size); 53 | 54 | const transformer = new TfIdfTransformer({ idf: Float64Array.from(Idf) }); 55 | 56 | const encoder = new CategoricalEncoder(); 57 | const mappings = new Map(); 58 | 59 | for (const entry of Mappings) { 60 | mappings.set(entry[0], entry[1]); 61 | } 62 | 63 | encoder.mapping = mappings; 64 | 65 | await setupBackend(CPU); 66 | 67 | const net = Sequential.loadFile("examples/sentiment-analysis/sentiment.st"); 68 | 69 | const predYSoftmax = await net.predict( 70 | tensor( 71 | transformer.transform<"f32">( 72 | vectorizer.transform(tokenizer.transform(testX), "f32"), 73 | ), 74 | ), 75 | ); 76 | 77 | CategoricalEncoder.fromSoftmax<"f32">(predYSoftmax as MatrixLike<"f32">); 78 | const predY = encoder.untransform(predYSoftmax as MatrixLike<"f32">); 79 | 80 | console.log(new ClassificationReport(testY, predY)); 81 | -------------------------------------------------------------------------------- /examples/classification/binary_iris.ts: -------------------------------------------------------------------------------- 1 | import { 2 | Cost, 3 | DenseLayer, 4 | Sequential, 5 | setupBackend, 6 | SigmoidLayer, 7 | tensor2D, 8 | AdamOptimizer, 9 | WASM, 10 | PostProcess, 11 | } from "../../packages/core/mod.ts"; 12 | 13 | import { parse } from "jsr:@std/csv@1.0.3/parse"; 14 | 15 | // Import helpers for metrics 16 | import { 17 | ClassificationReport, 18 | // Split the dataset 19 | useSplit, 20 | } from "../../packages/utilities/mod.ts"; 21 | 22 | // Define classes 23 | const classes = ["Setosa", "Versicolor"]; 24 | 25 | // Read the training dataset 26 | const _data = Deno.readTextFileSync("examples/classification/binary_iris.csv"); 27 | const data = parse(_data); 28 | 29 | // Get the predictors (x) and targets (y) 30 | const x = data.map((fl) => fl.slice(0, 4).map(Number)); 31 | const y = data.map((fl) => classes.indexOf(fl[4])); 32 | 33 | // Split the dataset for training and testing 34 | const [train, test] = useSplit({ ratio: [7, 3], shuffle: true }, x, y); 35 | 36 | // Setup the CPU backend for Netsaur 37 | await setupBackend(WASM); 38 | 39 | // Create a sequential neural network 40 | const net = new Sequential({ 41 | // Set number of minibatches to 4 42 | // Set size of output to 4 43 | size: [4, 4], 44 | 45 | // Disable logging during training 46 | silent: false, 47 | 48 | // Define each layer of the network 49 | layers: [ 50 | // A dense layer with 4 neurons 51 | DenseLayer({ size: [4] }), 52 | // A sigmoid activation layer 53 | SigmoidLayer(), 54 | // A dense layer with 1 neuron 55 | DenseLayer({ size: [1] }), 56 | // Another sigmoid layer 57 | SigmoidLayer(), 58 | ], 59 | // We are using Log Loss for finding cost 60 | cost: Cost.BinCrossEntropy, 61 | optimizer: AdamOptimizer(), 62 | }); 63 | 64 | const time = performance.now(); 65 | 66 | // Train the network 67 | net.train( 68 | [ 69 | { 70 | inputs: tensor2D(train[0]), 71 | outputs: tensor2D(train[1].map((x) => [x])), 72 | }, 73 | ], 74 | // Train for 150 epochs 75 | 100, 76 | 1, 77 | // Use a smaller learning rate 78 | 0.02 79 | ); 80 | 81 | console.log(`training time: ${performance.now() - time}ms`); 82 | 83 | const res = await net.predict(tensor2D(test[0]), { 84 | postProcess: PostProcess("step", { thresholds: [0.5], values: [0, 1] }), 85 | }); 86 | 87 | const cMatrix = new ClassificationReport( 88 | test[1].map((x) => classes[x]), 89 | Array.from(res.data).map((x) => classes[x]) 90 | ); 91 | console.log("Confusion Matrix: ", cMatrix); 92 | -------------------------------------------------------------------------------- /packages/core/src/backends/wasm/lib/netsaur.generated.d.ts: -------------------------------------------------------------------------------- 1 | // deno-lint-ignore-file 2 | // deno-fmt-ignore-file 3 | 4 | export interface InstantiateResult { 5 | instance: WebAssembly.Instance; 6 | exports: { 7 | wasm_backend_create: typeof wasm_backend_create; 8 | wasm_backend_train: typeof wasm_backend_train; 9 | wasm_backend_predict: typeof wasm_backend_predict; 10 | wasm_backend_save: typeof wasm_backend_save; 11 | wasm_backend_load: typeof wasm_backend_load 12 | }; 13 | } 14 | 15 | /** Gets if the Wasm module has been instantiated. */ 16 | export function isInstantiated(): boolean; 17 | 18 | /** Options for instantiating a Wasm instance. */ 19 | export interface InstantiateOptions { 20 | /** Optional url to the Wasm file to instantiate. */ 21 | url?: URL; 22 | /** Callback to decompress the raw Wasm file bytes before instantiating. */ 23 | decompress?: (bytes: Uint8Array) => Uint8Array; 24 | } 25 | 26 | /** Instantiates an instance of the Wasm module returning its functions. 27 | * @remarks It is safe to call this multiple times and once successfully 28 | * loaded it will always return a reference to the same object. */ 29 | export function instantiate(opts?: InstantiateOptions): Promise; 30 | 31 | /** Instantiates an instance of the Wasm module along with its exports. 32 | * @remarks It is safe to call this multiple times and once successfully 33 | * loaded it will always return a reference to the same object. */ 34 | export function instantiateWithInstance(opts?: InstantiateOptions): Promise; 35 | 36 | /** 37 | * @param {string} config 38 | * @param {Array} shape 39 | * @returns {number} 40 | */ 41 | export function wasm_backend_create(config: string, shape: Array): number; 42 | /** 43 | * @param {number} id 44 | * @param {(Float32Array)[]} buffers 45 | * @param {string} options 46 | */ 47 | export function wasm_backend_train(id: number, buffers: (Float32Array)[], options: string): void; 48 | /** 49 | * @param {number} id 50 | * @param {Float32Array} buffer 51 | * @param {string} options 52 | * @returns {Float32Array} 53 | */ 54 | export function wasm_backend_predict(id: number, buffer: Float32Array, options: string): Float32Array; 55 | /** 56 | * @param {number} id 57 | * @returns {Uint8Array} 58 | */ 59 | export function wasm_backend_save(id: number): Uint8Array; 60 | /** 61 | * @param {Uint8Array} buffer 62 | * @param {Array} shape 63 | * @returns {number} 64 | */ 65 | export function wasm_backend_load(buffer: Uint8Array, shape: Array): number; 66 | -------------------------------------------------------------------------------- /examples/xor_cpu.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * This example shows how to train a neural network to predict the output of the XOR function. 3 | */ 4 | 5 | import { 6 | Cost, 7 | CPU, 8 | DenseLayer, 9 | Sequential, 10 | setupBackend, 11 | SigmoidLayer, 12 | tensor2D, 13 | } from "../packages/core/mod.ts"; 14 | 15 | /** 16 | * Setup the CPU backend. This backend is fast but doesn't work on the Edge. 17 | */ 18 | await setupBackend(CPU); 19 | 20 | /** 21 | * Creates a sequential neural network. 22 | */ 23 | const net = new Sequential({ 24 | /** 25 | * The number of minibatches is set to 4 and the output size is set to 2. 26 | */ 27 | size: [4, 2], 28 | 29 | /** 30 | * The silent option is set to true, which means that the network will not output any logs during trainin 31 | */ 32 | silent: true, 33 | 34 | /** 35 | * Defines the layers of a neural network in the XOR function example. 36 | * The neural network has two input neurons and one output neuron. 37 | * The layers are defined as follows: 38 | * - A dense layer with 3 neurons. 39 | * - sigmoid activation layer. 40 | * - A dense layer with 1 neuron. 41 | * -A sigmoid activation layer. 42 | */ 43 | layers: [ 44 | DenseLayer({ size: [3] }), 45 | SigmoidLayer(), 46 | DenseLayer({ size: [1] }), 47 | SigmoidLayer(), 48 | ], 49 | 50 | /** 51 | * The cost function used for training the network is the mean squared error (MSE). 52 | */ 53 | cost: Cost.MSE, 54 | }); 55 | 56 | const time = performance.now(); 57 | 58 | /** 59 | * Train the network on the given data. 60 | */ 61 | net.train( 62 | [ 63 | { 64 | inputs: tensor2D([ 65 | [0, 0], 66 | [1, 0], 67 | [0, 1], 68 | [1, 1], 69 | ]), 70 | outputs: tensor2D([[0], [1], [1], [0]]), 71 | }, 72 | ], 73 | /** 74 | * The number of iterations is set to 10000. 75 | */ 76 | 10000, 77 | ); 78 | 79 | console.log(`training time: ${performance.now() - time}ms`); 80 | 81 | /** 82 | * Predict the output of the XOR function for the given inputs. 83 | */ 84 | const out1 = (await net.predict(tensor2D([[0, 0]]))).data; 85 | console.log(`0 xor 0 = ${out1[0]} (should be close to 0)`); 86 | 87 | const out2 = (await net.predict(tensor2D([[1, 0]]))).data; 88 | console.log(`1 xor 0 = ${out2[0]} (should be close to 1)`); 89 | 90 | const out3 = (await net.predict(tensor2D([[0, 1]]))).data; 91 | console.log(`0 xor 1 = ${out3[0]} (should be close to 1)`); 92 | 93 | const out4 = (await net.predict(tensor2D([[1, 1]]))).data; 94 | console.log(`1 xor 1 = ${out4[0]} (should be close to 0)`); 95 | -------------------------------------------------------------------------------- /examples/xor_gpu.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * This example shows how to train a neural network to predict the output of the XOR function. 3 | */ 4 | 5 | import { 6 | Cost, 7 | DenseLayer, 8 | GPU, 9 | Sequential, 10 | setupBackend, 11 | SigmoidLayer, 12 | tensor2D, 13 | } from "../packages/core/mod.ts"; 14 | 15 | /** 16 | * Setup the GPU backend. This backend is fast but doesn't work on the Edge. 17 | */ 18 | await setupBackend(GPU); 19 | 20 | /** 21 | * Creates a sequential neural network. 22 | */ 23 | const net = new Sequential({ 24 | /** 25 | * The number of minibatches is set to 4 and the output size is set to 2. 26 | */ 27 | size: [4, 2], 28 | 29 | /** 30 | * The silent option is set to true, which means that the network will not output any logs during trainin 31 | */ 32 | silent: true, 33 | 34 | /** 35 | * Defines the layers of a neural network in the XOR function example. 36 | * The neural network has two input neurons and one output neuron. 37 | * The layers are defined as follows: 38 | * - A dense layer with 3 neurons. 39 | * - sigmoid activation layer. 40 | * - A dense layer with 1 neuron. 41 | * -A sigmoid activation layer. 42 | */ 43 | layers: [ 44 | DenseLayer({ size: [3] }), 45 | SigmoidLayer(), 46 | DenseLayer({ size: [1] }), 47 | SigmoidLayer(), 48 | ], 49 | 50 | /** 51 | * The cost function used for training the network is the mean squared error (MSE). 52 | */ 53 | cost: Cost.MSE, 54 | }); 55 | 56 | const time = performance.now(); 57 | 58 | /** 59 | * Train the network on the given data. 60 | */ 61 | net.train( 62 | [ 63 | { 64 | inputs: tensor2D([ 65 | [0, 0], 66 | [1, 0], 67 | [0, 1], 68 | [1, 1], 69 | ]), 70 | outputs: tensor2D([[0], [1], [1], [0]]), 71 | }, 72 | ], 73 | /** 74 | * The number of iterations is set to 10000. 75 | */ 76 | 10000, 77 | ); 78 | 79 | console.log(`training time: ${performance.now() - time}ms`); 80 | 81 | /** 82 | * Predict the output of the XOR function for the given inputs. 83 | */ 84 | const out1 = (await net.predict(tensor2D([[0, 0]]))).data; 85 | console.log(`0 xor 0 = ${out1[0]} (should be close to 0)`); 86 | 87 | const out2 = (await net.predict(tensor2D([[1, 0]]))).data; 88 | console.log(`1 xor 0 = ${out2[0]} (should be close to 1)`); 89 | 90 | const out3 = (await net.predict(tensor2D([[0, 1]]))).data; 91 | console.log(`0 xor 1 = ${out3[0]} (should be close to 1)`); 92 | 93 | const out4 = (await net.predict(tensor2D([[1, 1]]))).data; 94 | console.log(`1 xor 1 = ${out4[0]} (should be close to 0)`); 95 | -------------------------------------------------------------------------------- /examples/xor_auto.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * This example shows how to train a neural network to predict the output of the XOR function. 3 | */ 4 | 5 | import { 6 | AUTO, 7 | Cost, 8 | DenseLayer, 9 | Sequential, 10 | setupBackend, 11 | SigmoidLayer, 12 | tensor2D, 13 | } from "../packages/core/mod.ts"; 14 | 15 | /** 16 | * Setup the AUTO backend. This backend is chosen automatically based on the environment. 17 | */ 18 | await setupBackend(AUTO); 19 | 20 | /** 21 | * Creates a sequential neural network. 22 | */ 23 | const net = new Sequential({ 24 | /** 25 | * The number of minibatches is set to 4 and the output size is set to 2. 26 | */ 27 | size: [4, 2], 28 | 29 | /** 30 | * The silent option is set to true, which means that the network will not output any logs during trainin 31 | */ 32 | silent: true, 33 | 34 | /** 35 | * Defines the layers of a neural network in the XOR function example. 36 | * The neural network has two input neurons and one output neuron. 37 | * The layers are defined as follows: 38 | * - A dense layer with 3 neurons. 39 | * - sigmoid activation layer. 40 | * - A dense layer with 1 neuron. 41 | * -A sigmoid activation layer. 42 | */ 43 | layers: [ 44 | DenseLayer({ size: [3] }), 45 | SigmoidLayer(), 46 | DenseLayer({ size: [1] }), 47 | SigmoidLayer(), 48 | ], 49 | 50 | /** 51 | * The cost function used for training the network is the mean squared error (MSE). 52 | */ 53 | cost: Cost.MSE, 54 | }); 55 | 56 | const time = performance.now(); 57 | 58 | /** 59 | * Train the network on the given data. 60 | */ 61 | net.train( 62 | [ 63 | { 64 | inputs: tensor2D([ 65 | [0, 0], 66 | [1, 0], 67 | [0, 1], 68 | [1, 1], 69 | ]), 70 | outputs: tensor2D([[0], [1], [1], [0]]), 71 | }, 72 | ], 73 | /** 74 | * The number of iterations is set to 10000. 75 | */ 76 | 10000, 77 | ); 78 | 79 | console.log(`training time: ${performance.now() - time}ms`); 80 | 81 | /** 82 | * Predict the output of the XOR function for the given inputs. 83 | */ 84 | const out1 = (await net.predict(tensor2D([[0, 0]]))).data; 85 | console.log(`0 xor 0 = ${out1[0]} (should be close to 0)`); 86 | 87 | const out2 = (await net.predict(tensor2D([[1, 0]]))).data; 88 | console.log(`1 xor 0 = ${out2[0]} (should be close to 1)`); 89 | 90 | const out3 = (await net.predict(tensor2D([[0, 1]]))).data; 91 | console.log(`0 xor 1 = ${out3[0]} (should be close to 1)`); 92 | 93 | const out4 = (await net.predict(tensor2D([[1, 1]]))).data; 94 | console.log(`1 xor 1 = ${out4[0]} (should be close to 0)`); 95 | -------------------------------------------------------------------------------- /examples/xor_wasm.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * This example shows how to train a neural network to predict the output of the XOR function. 3 | */ 4 | 5 | import { 6 | Cost, 7 | DenseLayer, 8 | Sequential, 9 | setupBackend, 10 | SigmoidLayer, 11 | tensor2D, 12 | WASM, 13 | } from "../packages/core/mod.ts"; 14 | 15 | /** 16 | * Setup the WASM backend. This backend is slower than the CPU backend but works on the Edge. 17 | */ 18 | await setupBackend(WASM); 19 | 20 | /** 21 | * Creates a sequential neural network. 22 | */ 23 | const net = new Sequential({ 24 | /** 25 | * The number of minibatches is set to 4 and the output size is set to 2. 26 | */ 27 | size: [4, 2], 28 | 29 | /** 30 | * The silent option is set to true, which means that the network will not output any logs during trainin 31 | */ 32 | silent: true, 33 | 34 | /** 35 | * Defines the layers of a neural network in the XOR function example. 36 | * The neural network has two input neurons and one output neuron. 37 | * The layers are defined as follows: 38 | * - A dense layer with 3 neurons. 39 | * - sigmoid activation layer. 40 | * - A dense layer with 1 neuron. 41 | * -A sigmoid activation layer. 42 | */ 43 | layers: [ 44 | DenseLayer({ size: [3] }), 45 | SigmoidLayer(), 46 | DenseLayer({ size: [1] }), 47 | SigmoidLayer(), 48 | ], 49 | 50 | /** 51 | * The cost function used for training the network is the mean squared error (MSE). 52 | */ 53 | cost: Cost.MSE, 54 | }); 55 | 56 | const time = performance.now(); 57 | 58 | /** 59 | * Train the network on the given data. 60 | */ 61 | net.train( 62 | [ 63 | { 64 | inputs: tensor2D([ 65 | [0, 0], 66 | [1, 0], 67 | [0, 1], 68 | [1, 1], 69 | ]), 70 | outputs: tensor2D([[0], [1], [1], [0]]), 71 | }, 72 | ], 73 | /** 74 | * The number of iterations is set to 10000. 75 | */ 76 | 10000, 77 | ); 78 | 79 | console.log(`training time: ${performance.now() - time}ms`); 80 | 81 | /** 82 | * Predict the output of the XOR function for the given inputs. 83 | */ 84 | const out1 = (await net.predict(tensor2D([[0, 0]]))).data; 85 | console.log(`0 xor 0 = ${out1[0]} (should be close to 0)`); 86 | 87 | const out2 = (await net.predict(tensor2D([[1, 0]]))).data; 88 | console.log(`1 xor 0 = ${out2[0]} (should be close to 1)`); 89 | 90 | const out3 = (await net.predict(tensor2D([[0, 1]]))).data; 91 | console.log(`0 xor 1 = ${out3[0]} (should be close to 1)`); 92 | 93 | const out4 = (await net.predict(tensor2D([[1, 1]]))).data; 94 | console.log(`1 xor 1 = ${out4[0]} (should be close to 0)`); 95 | -------------------------------------------------------------------------------- /crates/core/src/cpu/optimizers/nadam.rs: -------------------------------------------------------------------------------- 1 | use std::ops::{Add, Div, Mul, SubAssign, Sub}; 2 | 3 | use ndarray::{ArrayD, ArrayViewD, ArrayViewMutD}; 4 | 5 | use crate::{NadamOptimizer, CPUScheduler}; 6 | 7 | pub struct CPUNadamOptimizer { 8 | pub beta1: f32, 9 | pub beta2: f32, 10 | pub epsilon: f32, 11 | pub m: Vec>>, 12 | pub n: Vec>>, 13 | pub t: f32, 14 | } 15 | 16 | impl CPUNadamOptimizer { 17 | pub fn new(config: NadamOptimizer, params: Vec>>) -> Self { 18 | let mut m = Vec::new(); 19 | let mut n = Vec::new(); 20 | for params in params { 21 | m.push( 22 | params 23 | .iter() 24 | .map(|param| ArrayD::zeros(param.dim())) 25 | .collect(), 26 | ); 27 | n.push( 28 | params 29 | .iter() 30 | .map(|param| ArrayD::zeros(param.dim())) 31 | .collect(), 32 | ); 33 | } 34 | Self { 35 | beta1: config.beta1, 36 | beta2: config.beta2, 37 | epsilon: config.epsilon, 38 | m, 39 | n, 40 | t: 0.0, 41 | } 42 | } 43 | 44 | pub fn update_grads( 45 | &mut self, 46 | mut params: Vec>, 47 | grads: Vec>, 48 | idx: usize, 49 | scheduler: &CPUScheduler, 50 | rate: f32, 51 | l: Vec>, 52 | ) { 53 | for (j, ((param, grad), li)) in params.iter_mut().zip(grads).zip(l).enumerate() { 54 | self.m[idx][j] = self 55 | .beta1 56 | .mul(&self.m[idx][j]) 57 | .add((1.0 - self.beta1).mul(&grad)); 58 | self.n[idx][j] = self 59 | .beta2 60 | .mul(&self.n[idx][j]) 61 | .add((1.0 - self.beta2).mul(&grad.map(|x| x.powi(2)))); 62 | 63 | let m_hat = self.m[idx][j].view(); 64 | let n_hat = self.n[idx][j].view().div(1.0 - self.beta2.powf(self.t)); 65 | 66 | let nestrov_m_hat = self.beta1.mul(&m_hat).add((1.0 - self.beta1).mul(&grad)).div(1.0 - self.beta1.powf(self.t)); 67 | 68 | let rate = scheduler.eta(rate, self.t as usize); 69 | 70 | param.sub_assign( 71 | &rate 72 | .mul(nestrov_m_hat) 73 | .div(n_hat.map(|x| x.sqrt()).add(self.epsilon)) 74 | .sub(&li), 75 | ) 76 | } 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /packages/core/src/backends/wasm/mod.ts: -------------------------------------------------------------------------------- 1 | import { WASMBackend } from "./backend.ts"; 2 | import { NoBackendError } from "../../core/api/error.ts"; 3 | import { type BackendLoader, Engine } from "../../core/engine.ts"; 4 | import { 5 | type Backend, 6 | BackendType, 7 | Cost, 8 | type NetworkConfig, 9 | SchedulerType, 10 | } from "../../core/types.ts"; 11 | import { instantiate } from "./lib/netsaur.generated.js"; 12 | import { Sequential } from "../../core/mod.ts"; 13 | 14 | /** 15 | * Web Assembly backend instance. 16 | */ 17 | export class WASMInstance { 18 | static initialized = false; 19 | 20 | static async init(silent = false): Promise { 21 | if (WASMInstance.initialized) return true; 22 | await instantiate({ 23 | url: new URL(import.meta.url).protocol !== "file:" 24 | ? new URL( 25 | "https://github.com/denosaurs/netsaur/releases/download/0.4.2/netsaur_bg.wasm", 26 | import.meta.url, 27 | ) 28 | : undefined, 29 | }); 30 | WASMInstance.initialized = true; 31 | if (!silent) console.log("WASM Backend Initialized"); 32 | return true; 33 | } 34 | } 35 | 36 | /** 37 | * Web Assembly Backend Loader. 38 | */ 39 | export class WASMBackendLoader implements BackendLoader { 40 | backend?: WASMBackend; 41 | 42 | isSupported(): boolean { 43 | return true; 44 | } 45 | 46 | async setup(silent = false): Promise { 47 | Engine.type = BackendType.WASM; 48 | return await WASMInstance.init(silent); 49 | } 50 | 51 | loadBackend(config: NetworkConfig): Backend { 52 | if (!WASMInstance.initialized) { 53 | throw new NoBackendError(BackendType.WASM); 54 | } 55 | return this.backend ? this.backend : WASMBackend.create(config); 56 | } 57 | 58 | load(buffer: Uint8Array): Sequential { 59 | this.backend = WASMBackend.load(buffer); 60 | const net = new Sequential({ 61 | size: [0], 62 | layers: [], 63 | cost: Cost.MSE, 64 | scheduler: { 65 | type: SchedulerType.None, 66 | }, 67 | }); 68 | this.backend = undefined; 69 | return net; 70 | } 71 | 72 | loadFile(path: string): Sequential { 73 | this.backend = WASMBackend.loadFile(path); 74 | const net = new Sequential({ 75 | size: [0], 76 | layers: [], 77 | cost: Cost.MSE, 78 | scheduler: { 79 | type: SchedulerType.None, 80 | }, 81 | }); 82 | this.backend = undefined; 83 | return net; 84 | } 85 | } 86 | 87 | /** 88 | * Web Assembly Backend written in Rust & compiled to Web Assembly. 89 | */ 90 | export const WASM: WASMBackendLoader = new WASMBackendLoader(); 91 | -------------------------------------------------------------------------------- /examples/multiple-linear/student.ts: -------------------------------------------------------------------------------- 1 | import { 2 | AdamOptimizer, 3 | Cost, 4 | CPU, 5 | DenseLayer, 6 | OneCycle, 7 | Sequential, 8 | setupBackend, 9 | tensor2D, 10 | } from "../../packages/core/mod.ts"; 11 | 12 | import { parse } from "jsr:@std/csv@1.0.3/parse"; 13 | 14 | // Import helpers for splitting dataset 15 | import { useSplit } from "../../packages/utilities/mod.ts"; 16 | 17 | // Read the training dataset 18 | const _data = Deno.readTextFileSync("examples/multiple-linear/student.csv"); 19 | const data = parse(_data); 20 | 21 | // Get the independent variables (x) and map text to numbers 22 | const x = data.map((fl) => 23 | [fl[0], fl[1], fl[2] === "Yes" ? 1 : 0, fl[3], fl[4]].map(Number) 24 | ); 25 | // Get dependent variables (y) 26 | const y = data.map((fl) => Number(fl[5])); 27 | 28 | // Split the data for training and testing 29 | // Cast to original types because useSplit returns a weird type 30 | const [train, test] = useSplit({ ratio: [7, 3], shuffle: true }, x, y) as [ 31 | [typeof x, typeof y], 32 | [typeof x, typeof y], 33 | ]; 34 | 35 | // Setup the CPU backend for Netsaur 36 | await setupBackend(CPU); 37 | 38 | console.log(train); 39 | 40 | // Create a sequential neural network 41 | const net = new Sequential({ 42 | // Set number of minibatches to 4 43 | // Set size of output to 5 44 | size: [4, 5], 45 | 46 | // Disable logging during training 47 | silent: true, 48 | 49 | // Set up a simple linear model 50 | layers: [ 51 | // A dense layer with 8 neurons 52 | DenseLayer({ size: [8] }), 53 | // A dense layer with 1 neuron 54 | DenseLayer({ size: [1] }), 55 | ], 56 | // We are using Adam as the optimizer 57 | optimizer: AdamOptimizer(), 58 | // The one cycle scheduler cycles between max_rate and 59 | // the learning rate supplied during training. 60 | scheduler: OneCycle({ max_rate: 0.05, step_size: 100 }), 61 | // We are using MSE for finding cost 62 | cost: Cost.MSE, 63 | }); 64 | 65 | const time = performance.now(); 66 | 67 | // Train the network 68 | net.train( 69 | [ 70 | { 71 | inputs: tensor2D(train[0]), 72 | outputs: tensor2D(train[1].map((x) => [x])), 73 | }, 74 | ], 75 | // Train for 1000 epochs 76 | 1000, 77 | 1, 78 | 0.02, 79 | ); 80 | 81 | console.log(`training time: ${performance.now() - time}ms`); 82 | 83 | // Compute RMSE 84 | let err = 0; 85 | const y_test = await net.predict(tensor2D(test[0])); 86 | for (const i in test[0]) { 87 | err += (test[1][i] - y_test.data[i]) ** 2; 88 | console.log(`\nOutput: ${y_test.data[i]}\nExpected: ${test[1][i]}`); 89 | } 90 | console.log("RMSE:", Math.sqrt(err / test[0].length)); 91 | -------------------------------------------------------------------------------- /tests/wasm.test.ts: -------------------------------------------------------------------------------- 1 | import { 2 | Cost, 3 | DenseLayer, 4 | Sequential, 5 | setupBackend, 6 | SigmoidLayer, 7 | tensor1D, 8 | tensor2D, 9 | WASM, 10 | } from "../mod.ts"; 11 | 12 | await setupBackend(WASM); 13 | 14 | Deno.test("save wasm", () => { 15 | const network = new Sequential({ 16 | size: [4, 1], 17 | silent: true, 18 | layers: [ 19 | DenseLayer({ size: [3] }), 20 | ], 21 | cost: Cost.MSE, 22 | }); 23 | 24 | network.saveFile("./save_test_wasm.test.st"); 25 | }); 26 | 27 | Deno.test("cost(wasm): hinge", async () => { 28 | const net = new Sequential({ 29 | size: [4, 2], 30 | silent: true, 31 | layers: [ 32 | DenseLayer({ size: [3] }), 33 | SigmoidLayer(), 34 | DenseLayer({ size: [1] }), 35 | SigmoidLayer(), 36 | ], 37 | cost: Cost.Hinge, 38 | }); 39 | 40 | net.train([{ 41 | inputs: tensor2D([ 42 | [0, 0], 43 | [1, 0], 44 | [0, 1], 45 | [1, 1], 46 | ]), 47 | outputs: tensor2D([[0], [1], [1], [0]]), 48 | }], 50000); 49 | 50 | const out1 = (await net.predict(tensor1D([0, 0]))).data; 51 | console.log(`0 xor 0 = ${out1[0]} (should be close to 0)`); 52 | 53 | const out2 = (await net.predict(tensor1D([1, 0]))).data; 54 | console.log(`1 xor 0 = ${out2[0]} (should be close to 1)`); 55 | 56 | const out3 = (await net.predict(tensor1D([0, 1]))).data; 57 | console.log(`0 xor 1 = ${out3[0]} (should be close to 1)`); 58 | 59 | const out4 = (await net.predict(tensor1D([1, 1]))).data; 60 | console.log(`1 xor 1 = ${out4[0]} (should be close to 0)`); 61 | }); 62 | 63 | Deno.test("cost(wasm): mse", async () => { 64 | const net = new Sequential({ 65 | size: [4, 2], 66 | silent: true, 67 | layers: [ 68 | DenseLayer({ size: [3] }), 69 | SigmoidLayer(), 70 | DenseLayer({ size: [1] }), 71 | SigmoidLayer(), 72 | ], 73 | cost: Cost.MSE, 74 | }); 75 | 76 | net.train([{ 77 | inputs: tensor2D([ 78 | [0, 0], 79 | [1, 0], 80 | [0, 1], 81 | [1, 1], 82 | ]), 83 | outputs: tensor2D([[0], [1], [1], [0]]), 84 | }], 5000); 85 | 86 | const out1 = (await net.predict(tensor1D([0, 0]))).data; 87 | console.log(`0 xor 0 = ${out1[0]} (should be close to 0)`); 88 | 89 | const out2 = (await net.predict(tensor1D([1, 0]))).data; 90 | console.log(`1 xor 0 = ${out2[0]} (should be close to 1)`); 91 | 92 | const out3 = (await net.predict(tensor1D([0, 1]))).data; 93 | console.log(`0 xor 1 = ${out3[0]} (should be close to 1)`); 94 | 95 | const out4 = (await net.predict(tensor1D([1, 1]))).data; 96 | console.log(`1 xor 1 = ${out4[0]} (should be close to 0)`); 97 | }); 98 | -------------------------------------------------------------------------------- /packages/utilities/src/text/vectorizer.ts: -------------------------------------------------------------------------------- 1 | import { TfEncoder } from "../encoding/termfrequency.ts"; 2 | import { DiscreteMapper } from "../mapper/discrete.ts"; 3 | import { Matrix } from "../mod.ts"; 4 | import { TfIdfTransformer } from "../transformer/tfidf.ts"; 5 | import type { DataType } from "../utils/common_types.ts"; 6 | 7 | export class TextVectorizer { 8 | mode: "tf" | "tfidf" | "indices"; 9 | mapper: DiscreteMapper; 10 | encoder?: TfEncoder; 11 | transformer?: TfIdfTransformer; 12 | maxLength: number; 13 | constructor(mode: "tf" | "tfidf" | "indices" = "indices") { 14 | this.mode = mode; 15 | this.mapper = new DiscreteMapper(); 16 | this.maxLength = 0; 17 | } 18 | get vocabSize(): number { 19 | return this.mapper.mapping.size; 20 | } 21 | fit(document: string | string[]): TextVectorizer { 22 | this.mapper.fit( 23 | (Array.isArray(document) ? document.join(" ") : document).split(" ") 24 | ); 25 | const tokens = Array.isArray(document) 26 | ? document.map((x) => this.mapper.transform(x.split(" "))) 27 | : [this.mapper.transform(document.split(" "))]; 28 | this.maxLength = Math.max( 29 | Math.max(...tokens.map((x) => x.length)), 30 | this.maxLength 31 | ); 32 | if (this.mode === "tf" || this.mode === "tfidf") { 33 | this.encoder = new TfEncoder(this.mapper.mapping.size); 34 | if (this.mode === "tfidf") { 35 | this.transformer = new TfIdfTransformer(); 36 | this.transformer.fit(this.encoder.transform(tokens, "f32")); 37 | } 38 | } 39 | return this; 40 | } 41 | transform
( 42 | document: string | string[], 43 | dType: DT 44 | ): Matrix
{ 45 | if (!this.mapper.mapping.size) 46 | throw new Error("Text Vectorizer not trained yet. Use .fit() first."); 47 | const tokens = Array.isArray(document) 48 | ? document.map((x) => this.mapper.transform(x.split(" "))) 49 | : [this.mapper.transform(document.split(" "))]; 50 | if (this.mode === "indices") { 51 | const res = new Matrix(dType, [tokens.length, this.maxLength]); 52 | for (let i = 0; i < res.nRows; i += 1) { 53 | res.setRow(i, tokens[i]); 54 | } 55 | return res; 56 | } 57 | if (!this.encoder) 58 | throw new Error("Text Vectorizer not trained yet. Use .fit() first."); 59 | const encoded = this.encoder.transform(tokens, dType); 60 | if (this.mode === "tf") return encoded; 61 | else { 62 | if (!this.transformer) 63 | throw new Error("Text Vectorizer not trained yet. Use .fit() first."); 64 | return this.transformer.transform
(encoded); 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /crates/core-gpu/src/gpu/layers/dense.rs: -------------------------------------------------------------------------------- 1 | use ndarray::{Array1, Array2, ArrayD, Axis, Dimension, Ix1, Ix2, IxDyn}; 2 | use std::ops::Add; 3 | 4 | use crate::{GPUInit, DenseLayer, Init, Tensors}; 5 | 6 | pub struct DenseGPULayer { 7 | // cache 8 | pub output_size: Ix2, 9 | pub inputs: Array2, 10 | 11 | // parameters 12 | pub weights: Array2, 13 | pub biases: Array1, 14 | 15 | // gradients 16 | pub d_weights: Array2, 17 | pub d_biases: Array1, 18 | } 19 | 20 | impl DenseGPULayer { 21 | pub fn new(config: DenseLayer, size: IxDyn, tensors: Option) -> Self { 22 | let init = GPUInit::from_default(config.init, Init::Uniform); 23 | let input_size = Ix2(size[0], size[1]); 24 | let weight_size = Ix2(size[1], config.size[0]); 25 | let output_size = Ix2(size[0], config.size[0]); 26 | 27 | let (weights, biases) = if let Some(Tensors::Dense(tensors)) = tensors { 28 | (tensors.weights, tensors.biases) 29 | } else { 30 | let weights = init.init(weight_size.into_dyn(), size[1], config.size[0]); 31 | let biases = ArrayD::zeros(config.size.clone()); 32 | (weights, biases) 33 | }; 34 | 35 | Self { 36 | output_size, 37 | inputs: Array2::zeros(input_size), 38 | weights: weights.into_dimensionality::().unwrap(), 39 | biases: biases.into_dimensionality::().unwrap(), 40 | d_weights: Array2::zeros(weight_size), 41 | d_biases: Array1::zeros(config.size[0]), 42 | } 43 | } 44 | 45 | pub fn output_size(&self) -> Vec { 46 | self.output_size.as_array_view().to_vec() 47 | } 48 | 49 | pub fn reset(&mut self, batches: usize) { 50 | let input_size = self.inputs.dim().1; 51 | self.inputs = Array2::zeros((batches, input_size)); 52 | self.output_size[0] = batches; 53 | } 54 | 55 | pub fn forward_propagate(&mut self, inputs: ArrayD) -> ArrayD { 56 | self.inputs = inputs.into_dimensionality::().unwrap(); 57 | self.inputs.dot(&self.weights).add(&self.biases).into_dyn() 58 | } 59 | 60 | pub fn backward_propagate(&mut self, d_outputs: ArrayD) -> ArrayD { 61 | let d_outputs = d_outputs.into_dimensionality::().unwrap(); 62 | let mut weights_t = self.weights.view(); 63 | weights_t.swap_axes(0, 1); 64 | let d_inputs = d_outputs.dot(&weights_t); 65 | let mut inputs_t = self.inputs.view(); 66 | inputs_t.swap_axes(0, 1); 67 | self.d_weights = inputs_t.dot(&d_outputs); 68 | self.d_biases = d_outputs.sum_axis(Axis(0)); 69 | d_inputs.into_dyn() 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /packages/utilities/src/image/patches/patch_2d.ts: -------------------------------------------------------------------------------- 1 | import type { Image } from "../../utils/mod.ts"; 2 | import type { Patch2d, PatchCollection } from "../../utils/common_types.ts"; 3 | 4 | /** 5 | * Extract patches from a 2d image. 6 | */ 7 | 8 | /** 9 | * Get the row and column of the nth element of a 10 | * 2d array. 11 | * @param n Current Position 12 | * @param width Width of a row 13 | * @returns [row, column] 14 | */ 15 | function clamp(n: number, width: number): [number, number] { 16 | if (n >= width) return [~~(n / width), n % width]; 17 | return [0, n]; 18 | } 19 | 20 | /** Private function to extract patches */ 21 | function extract(image: Image, options: Patch2d): [Uint8ClampedArray, number] { 22 | /** Get number of possible patches in each dimension */ 23 | const nX = image.width - options.width + 1; 24 | const nY = image.height - options.height + 1; 25 | /** Total number of patches is nX * nY */ 26 | const nPatches = nX * nY; 27 | /** Area of each patch, used to calculate offset of resulting array */ 28 | const patchArea = options.width * options.height; 29 | 30 | const res = new Uint8ClampedArray( 31 | options.width * options.height * image.channels * nPatches, 32 | ); 33 | 34 | let i = 0; 35 | while (i < nPatches) { 36 | const [row, col] = clamp(i, nX); 37 | /** Starting index of the current patch */ 38 | const offset = row * image.width + col; 39 | let j = 0; 40 | while (j < options.height) { 41 | /** Starting index of the current subrow */ 42 | const patchRow = j * image.width; 43 | /** Copy the entire patchArea-size patch into the resulting array */ 44 | res.set( 45 | image.data.slice( 46 | (offset + patchRow) * image.channels, 47 | (offset + options.width + patchRow) * image.channels, 48 | ), 49 | (i * patchArea + j * options.width) * image.channels, 50 | ); 51 | j += 1; 52 | } 53 | i += 1; 54 | } 55 | return [res, nPatches]; 56 | } 57 | 58 | /** 59 | * Extract patches from a 2d image 60 | * @param image Source image to extract patches from 61 | * @param options Dimensions of a single patch 62 | * @returns A collection of patches as a single Uint8ClampedArray 63 | */ 64 | export function patch2d(image: Image, options: Patch2d): PatchCollection { 65 | if (image.width < options.width) { 66 | throw new Error("Patch width cannot be greater than image width."); 67 | } 68 | if (image.height < options.height) { 69 | throw new Error("Patch height cannot be greater than image width."); 70 | } 71 | 72 | const [patches, n] = extract(image, options); 73 | 74 | return { 75 | width: options.width, 76 | height: options.height, 77 | channels: image.channels, 78 | size: n, 79 | data: patches, 80 | }; 81 | } 82 | -------------------------------------------------------------------------------- /tests/cpu.test.ts: -------------------------------------------------------------------------------- 1 | // import { Cost, CPU, DenseLayer, Sequential, setupBackend, SigmoidLayer, tensor1D, tensor2D } from "../mod.ts"; 2 | 3 | // await setupBackend(CPU); 4 | 5 | // Deno.test("save cpu", () => { 6 | // const network = new Sequential({ 7 | // size: [4, 1], 8 | // silent: true, 9 | // layers: [ 10 | // DenseLayer({ size: [3] }), 11 | // ], 12 | // cost: Cost.MSE, 13 | // }); 14 | 15 | // network.saveFile("./save_test_cpu.test.st"); 16 | // }); 17 | 18 | // Deno.test("cost(cpu): hinge", async () => { 19 | // const net = new Sequential({ 20 | // size: [4, 2], 21 | // silent: true, 22 | // layers: [ 23 | // DenseLayer({ size: [3] }), 24 | // SigmoidLayer(), 25 | // DenseLayer({ size: [1] }), 26 | // SigmoidLayer(), 27 | // ], 28 | // cost: Cost.Hinge, 29 | // }); 30 | 31 | // net.train([{ 32 | // inputs: tensor2D([ 33 | // [0, 0], 34 | // [1, 0], 35 | // [0, 1], 36 | // [1, 1], 37 | // ]), 38 | // outputs: tensor2D([[0], [1], [1], [0]]), 39 | // }]); 40 | 41 | // const out1 = (await net.predict(tensor1D([0, 0]))).data; 42 | // console.log(`0 xor 0 = ${out1[0]} (should be close to 0)`); 43 | 44 | // const out2 = (await net.predict(tensor1D([1, 0]))).data; 45 | // console.log(`1 xor 0 = ${out2[0]} (should be close to 1)`); 46 | 47 | // const out3 = (await net.predict(tensor1D([0, 1]))).data; 48 | // console.log(`0 xor 1 = ${out3[0]} (should be close to 1)`); 49 | 50 | // const out4 = (await net.predict(tensor1D([1, 1]))).data; 51 | // console.log(`1 xor 1 = ${out4[0]} (should be close to 0)`); 52 | // }); 53 | 54 | // Deno.test("cost(cpu): mse", async () => { 55 | // const net = new Sequential({ 56 | // size: [4, 2], 57 | // silent: true, 58 | // layers: [ 59 | // DenseLayer({ size: [3] }), 60 | // SigmoidLayer(), 61 | // DenseLayer({ size: [1] }), 62 | // SigmoidLayer(), 63 | // ], 64 | // cost: Cost.MSE, 65 | // }); 66 | 67 | // net.train([{ 68 | // inputs: tensor2D([ 69 | // [0, 0], 70 | // [1, 0], 71 | // [0, 1], 72 | // [1, 1], 73 | // ]), 74 | // outputs: tensor2D([[0], [1], [1], [0]]), 75 | // }]); 76 | 77 | // const out1 = (await net.predict(tensor1D([0, 0]))).data; 78 | // console.log(`0 xor 0 = ${out1[0]} (should be close to 0)`); 79 | 80 | // const out2 = (await net.predict(tensor1D([1, 0]))).data; 81 | // console.log(`1 xor 0 = ${out2[0]} (should be close to 1)`); 82 | 83 | // const out3 = (await net.predict(tensor1D([0, 1]))).data; 84 | // console.log(`0 xor 1 = ${out3[0]} (should be close to 1)`); 85 | 86 | // const out4 = (await net.predict(tensor1D([1, 1]))).data; 87 | // console.log(`1 xor 1 = ${out4[0]} (should be close to 0)`); 88 | // }); 89 | -------------------------------------------------------------------------------- /examples/classification/iris.ts: -------------------------------------------------------------------------------- 1 | import { 2 | Cost, 3 | WASM, 4 | DenseLayer, 5 | ReluLayer, 6 | RMSPropOptimizer, 7 | Sequential, 8 | setupBackend, 9 | SoftmaxLayer, 10 | type Tensor, 11 | tensor, 12 | tensor2D, 13 | } from "../../packages/core/mod.ts"; 14 | 15 | import { parse } from "jsr:@std/csv@1.0.3/parse"; 16 | 17 | // Import helpers for metrics 18 | import { 19 | // One-hot encoding of targets 20 | CategoricalEncoder, 21 | ClassificationReport, 22 | // Split the dataset 23 | useSplit, 24 | } from "../../packages/utilities/mod.ts"; 25 | 26 | // Read the training dataset 27 | const _data = Deno.readTextFileSync("examples/classification/iris.csv"); 28 | const data = parse(_data); 29 | 30 | // Get the predictors (x) and targets (y) 31 | const x = data.map((fl) => fl.slice(0, 4).map(Number)); 32 | const y_pre = data.map((fl) => fl[4]); 33 | 34 | const encoder = new CategoricalEncoder(); 35 | 36 | const y = encoder.fit(y_pre).transform(y_pre, "f32"); 37 | 38 | // Split the dataset for training and testing 39 | // @ts-ignore Matrices can be split 40 | const [train, test] = useSplit({ ratio: [7, 3], shuffle: true }, x, y); 41 | 42 | // Setup the WebAssembly backend for Netsaur 43 | await setupBackend(WASM); 44 | 45 | console.log(train[1].head); 46 | 47 | // Create a sequential neural network 48 | const net = new Sequential({ 49 | // Set number of minibatches to 4 50 | // Set size of output to 4 51 | size: [4, 4], 52 | 53 | // Disable logging during training 54 | silent: false, 55 | 56 | // Define each layer of the network 57 | layers: [ 58 | // A dense layer with 8 neurons 59 | DenseLayer({ size: [8] }), 60 | // A ReLu activation layer 61 | ReluLayer(), 62 | // A dense layer with 8 neurons 63 | DenseLayer({ size: [8] }), 64 | // A ReLu activation layer 65 | ReluLayer(), 66 | // A dense layer with 3 neurons 67 | DenseLayer({ size: [3] }), 68 | // A Softmax activation layer 69 | SoftmaxLayer(), 70 | ], 71 | optimizer: RMSPropOptimizer(), 72 | // We are using CrossEntropy for finding cost 73 | cost: Cost.CrossEntropy, 74 | }); 75 | 76 | const time = performance.now(); 77 | 78 | // Train the network 79 | net.train( 80 | [ 81 | { 82 | inputs: tensor2D(train[0]), 83 | outputs: tensor(train[1]), 84 | }, 85 | ], 86 | // Train for 300 epochs 87 | 400, 88 | 1, 89 | 0.02 90 | ); 91 | 92 | console.log(`training time: ${performance.now() - time}ms`); 93 | 94 | // Calculate metrics 95 | const res = await net.predict(tensor2D(test[0])); 96 | const y1 = encoder.untransform(res as Tensor<2>); 97 | const y0 = encoder.untransform(test[1]); 98 | 99 | const report = new ClassificationReport(y0, y1); 100 | console.log(report); 101 | -------------------------------------------------------------------------------- /packages/tokenizers/mod.ts: -------------------------------------------------------------------------------- 1 | import { 2 | instantiate, 3 | wasm_tokenizer_decode, 4 | wasm_tokenizer_encode, 5 | wasm_tokenizer_from_json, 6 | wasm_tokenizer_get_vocab, 7 | wasm_tokenizer_get_vocab_size, 8 | wasm_tokenizer_id_to_token, 9 | wasm_tokenizer_save, 10 | wasm_tokenizer_token_to_id, 11 | } from "./lib/netsaur_tokenizers.generated.js"; 12 | 13 | let initialized = false; 14 | export async function init(): Promise { 15 | if (initialized) return; 16 | await instantiate({ 17 | url: new URL(import.meta.url).protocol !== "file:" 18 | ? new URL( 19 | "https://github.com/denosaurs/netsaur/releases/download/0.4.2/netsaur_tokenizers_bg.wasm", 20 | import.meta.url, 21 | ) 22 | : undefined, 23 | }); 24 | initialized = true; 25 | } 26 | 27 | /** 28 | * Tokenizer class 29 | */ 30 | export class Tokenizer { 31 | #id: number; 32 | constructor(id: number) { 33 | this.#id = id; 34 | } 35 | 36 | /** 37 | * Get the vocab size 38 | */ 39 | getVocabSize(withAddedTokens = true): number { 40 | return wasm_tokenizer_get_vocab_size(this.#id, withAddedTokens); 41 | } 42 | 43 | /** 44 | * Get the vocab 45 | */ 46 | // deno-lint-ignore no-explicit-any 47 | getVocab(withAddedTokens = true): any { 48 | return wasm_tokenizer_get_vocab(this.#id, withAddedTokens); 49 | } 50 | 51 | /** 52 | * Get the token from an id 53 | */ 54 | idToToken(id: number): string { 55 | return wasm_tokenizer_id_to_token(this.#id, id); 56 | } 57 | 58 | /** 59 | * Get the id from a token 60 | */ 61 | tokenToId(token: string): number { 62 | return wasm_tokenizer_token_to_id(this.#id, token); 63 | } 64 | 65 | /** 66 | * Encode a sentence to tokens 67 | * @param sentence sentence to tokenize 68 | * @returns 69 | */ 70 | encode(sentence: string): Uint32Array { 71 | return wasm_tokenizer_encode(this.#id, sentence); 72 | } 73 | 74 | /** 75 | * Decode a sentence from its encoded tokens to a string 76 | * @param tokens tokens to decode 77 | * @returns 78 | */ 79 | decode(ids: Uint32Array, skipSpecialTokens = false): string { 80 | return wasm_tokenizer_decode(this.#id, ids, skipSpecialTokens); 81 | } 82 | 83 | /** 84 | * Save the tokenizer as json 85 | */ 86 | save(): string; 87 | /** 88 | * Save the tokenizer as json 89 | * @param pretty pretty print the json 90 | */ 91 | save(pretty: boolean): string; 92 | save(...args: [boolean?]): string { 93 | return wasm_tokenizer_save(this.#id, args[0] ?? false); 94 | } 95 | /** 96 | * Load a tokenizer from json data 97 | * @param json string 98 | * @returns 99 | */ 100 | static fromJSON(json: string): Tokenizer { 101 | return new Tokenizer(wasm_tokenizer_from_json(json)); 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /packages/utilities/src/constants/stop_words.ts: -------------------------------------------------------------------------------- 1 | /** List obtained from NLTK */ 2 | export const DefaultIgnoreList = [ 3 | "i", 4 | "me", 5 | "my", 6 | "myself", 7 | "we", 8 | "our", 9 | "ours", 10 | "ourselves", 11 | "you", 12 | "you're", 13 | "you've", 14 | "you'll", 15 | "you'd", 16 | "your", 17 | "yours", 18 | "yourself", 19 | "yourselves", 20 | "he", 21 | "him", 22 | "his", 23 | "himself", 24 | "she", 25 | "she's", 26 | "her", 27 | "hers", 28 | "herself", 29 | "it", 30 | "it's", 31 | "its", 32 | "itself", 33 | "they", 34 | "them", 35 | "their", 36 | "theirs", 37 | "themselves", 38 | "what", 39 | "which", 40 | "who", 41 | "whom", 42 | "this", 43 | "that", 44 | "that'll", 45 | "these", 46 | "those", 47 | "am", 48 | "is", 49 | "are", 50 | "was", 51 | "were", 52 | "be", 53 | "been", 54 | "being", 55 | "have", 56 | "has", 57 | "had", 58 | "having", 59 | "do", 60 | "does", 61 | "did", 62 | "doing", 63 | "a", 64 | "an", 65 | "the", 66 | "and", 67 | "but", 68 | "if", 69 | "or", 70 | "because", 71 | "as", 72 | "until", 73 | "while", 74 | "of", 75 | "at", 76 | "by", 77 | "for", 78 | "with", 79 | "about", 80 | "against", 81 | "between", 82 | "into", 83 | "through", 84 | "during", 85 | "before", 86 | "after", 87 | "above", 88 | "below", 89 | "to", 90 | "from", 91 | "up", 92 | "down", 93 | "in", 94 | "out", 95 | "on", 96 | "off", 97 | "over", 98 | "under", 99 | "again", 100 | "further", 101 | "then", 102 | "once", 103 | "here", 104 | "there", 105 | "when", 106 | "where", 107 | "why", 108 | "how", 109 | "all", 110 | "any", 111 | "both", 112 | "each", 113 | "few", 114 | "more", 115 | "most", 116 | "other", 117 | "some", 118 | "such", 119 | "no", 120 | "nor", 121 | "not", 122 | "only", 123 | "own", 124 | "same", 125 | "so", 126 | "than", 127 | "too", 128 | "very", 129 | "s", 130 | "t", 131 | "can", 132 | "will", 133 | "just", 134 | "don", 135 | "don't", 136 | "should", 137 | "should've", 138 | "now", 139 | "d", 140 | "ll", 141 | "m", 142 | "o", 143 | "re", 144 | "ve", 145 | "y", 146 | "ain", 147 | "aren", 148 | "aren't", 149 | "couldn", 150 | "couldn't", 151 | "didn", 152 | "didn't", 153 | "doesn", 154 | "doesn't", 155 | "hadn", 156 | "hadn't", 157 | "hasn", 158 | "hasn't", 159 | "haven", 160 | "haven't", 161 | "isn", 162 | "isn't", 163 | "ma", 164 | "mightn", 165 | "mightn't", 166 | "mustn", 167 | "mustn't", 168 | "needn", 169 | "needn't", 170 | "shan", 171 | "shan't", 172 | "shouldn", 173 | "shouldn't", 174 | "wasn", 175 | "wasn't", 176 | "weren", 177 | "weren't", 178 | "won", 179 | "won't", 180 | "wouldn", 181 | "wouldn't", 182 | ]; 183 | -------------------------------------------------------------------------------- /crates/core/src/cpu/layers/dropout.rs: -------------------------------------------------------------------------------- 1 | use std::ops::Mul; 2 | 3 | use ndarray::{Array2, Array4, ArrayD, Axis, IxDyn}; 4 | use ndarray_rand::{rand_distr::Uniform, RandomExt}; 5 | 6 | use crate::DropoutLayer; 7 | 8 | pub struct Dropout1DCPULayer { 9 | mask: ArrayD, 10 | probability: f32, 11 | } 12 | 13 | impl Dropout1DCPULayer { 14 | pub fn new(config: DropoutLayer, size: IxDyn) -> Self { 15 | Self { 16 | mask: ArrayD::zeros(size), 17 | probability: config.probability, 18 | } 19 | } 20 | 21 | pub fn output_size(&self) -> Vec { 22 | self.mask.shape().to_vec() 23 | } 24 | 25 | pub fn reset(&mut self, batches: usize) { 26 | let mut output_size = self.mask.shape().to_vec(); 27 | output_size[0] = batches; 28 | self.mask = ArrayD::zeros(output_size); 29 | } 30 | 31 | pub fn forward_propagate(&mut self, inputs: ArrayD, training: bool) -> ArrayD { 32 | if training { 33 | self.mask = ArrayD::random(inputs.dim(), Uniform::new(0.0, 1.0)) 34 | .map(|x| (if x > &self.probability { 1.0 } else { 0.0 })); 35 | inputs.mul(&self.mask).mul(1.0 / 1.0 - self.probability) 36 | } else { 37 | inputs 38 | } 39 | } 40 | 41 | pub fn backward_propagate(&mut self, d_outputs: ArrayD) -> ArrayD { 42 | d_outputs.mul(&self.mask).mul(1.0 / 1.0 - self.probability) 43 | } 44 | } 45 | 46 | pub struct Dropout2DCPULayer { 47 | mask: Array4, 48 | probability: f32, 49 | } 50 | 51 | impl Dropout2DCPULayer { 52 | pub fn new(config: DropoutLayer, size: IxDyn) -> Self { 53 | Self { 54 | mask: Array4::zeros([size[0], size[1], size[2], size[3]]), 55 | probability: config.probability, 56 | } 57 | } 58 | 59 | pub fn output_size(&self) -> Vec { 60 | self.mask.shape().to_vec() 61 | } 62 | 63 | pub fn reset(&mut self, batches: usize) { 64 | let size = self.mask.dim(); 65 | self.mask = Array4::zeros([batches, size.1, size.2, size.3]); 66 | } 67 | 68 | pub fn forward_propagate(&mut self, inputs: ArrayD, training: bool) -> ArrayD { 69 | if training { 70 | let size = self.mask.dim(); 71 | self.mask = Array2::random([size.0, size.1], Uniform::new(0.0, 1.0)) 72 | .map(|x| (if x > &self.probability { 1.0 } else { 0.0 })) 73 | .insert_axis(Axis(2)) 74 | .insert_axis(Axis(3)) 75 | .broadcast(size) 76 | .unwrap() 77 | .to_owned(); 78 | inputs.mul(&self.mask).mul(1.0 / 1.0 - self.probability) 79 | } else { 80 | inputs 81 | } 82 | } 83 | 84 | pub fn backward_propagate(&mut self, d_outputs: ArrayD) -> ArrayD { 85 | d_outputs.mul(&self.mask).mul(1.0 / 1.0 - self.probability) 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /crates/core-gpu/src/gpu/layers/dropout.rs: -------------------------------------------------------------------------------- 1 | use std::ops::Mul; 2 | 3 | use ndarray::{Array2, Array4, ArrayD, Axis, IxDyn}; 4 | use ndarray_rand::{rand_distr::Uniform, RandomExt}; 5 | 6 | use crate::DropoutLayer; 7 | 8 | pub struct Dropout1DGPULayer { 9 | mask: ArrayD, 10 | probability: f32, 11 | } 12 | 13 | impl Dropout1DGPULayer { 14 | pub fn new(config: DropoutLayer, size: IxDyn) -> Self { 15 | Self { 16 | mask: ArrayD::zeros(size), 17 | probability: config.probability, 18 | } 19 | } 20 | 21 | pub fn output_size(&self) -> Vec { 22 | self.mask.shape().to_vec() 23 | } 24 | 25 | pub fn reset(&mut self, batches: usize) { 26 | let mut output_size = self.mask.shape().to_vec(); 27 | output_size[0] = batches; 28 | self.mask = ArrayD::zeros(output_size); 29 | } 30 | 31 | pub fn forward_propagate(&mut self, inputs: ArrayD, training: bool) -> ArrayD { 32 | if training { 33 | self.mask = ArrayD::random(self.mask.dim(), Uniform::new(0.0, 1.0)) 34 | .map(|x| (if x > &self.probability { 1.0 } else { 0.0 })); 35 | inputs.mul(&self.mask).mul(1.0 / 1.0 - self.probability) 36 | } else { 37 | inputs 38 | } 39 | } 40 | 41 | pub fn backward_propagate(&mut self, d_outputs: ArrayD) -> ArrayD { 42 | d_outputs.mul(&self.mask).mul(1.0 / 1.0 - self.probability) 43 | } 44 | } 45 | 46 | pub struct Dropout2DGPULayer { 47 | mask: Array4, 48 | probability: f32, 49 | } 50 | 51 | impl Dropout2DGPULayer { 52 | pub fn new(config: DropoutLayer, size: IxDyn) -> Self { 53 | Self { 54 | mask: Array4::zeros([size[0], size[1], size[2], size[3]]), 55 | probability: config.probability, 56 | } 57 | } 58 | 59 | pub fn output_size(&self) -> Vec { 60 | self.mask.shape().to_vec() 61 | } 62 | 63 | pub fn reset(&mut self, batches: usize) { 64 | let size = self.mask.dim(); 65 | self.mask = Array4::zeros([batches, size.1, size.2, size.3]); 66 | } 67 | 68 | pub fn forward_propagate(&mut self, inputs: ArrayD, training: bool) -> ArrayD { 69 | if training { 70 | let size = self.mask.dim(); 71 | self.mask = Array2::random([size.0, size.1], Uniform::new(0.0, 1.0)) 72 | .map(|x| (if x > &self.probability { 1.0 } else { 0.0 })) 73 | .insert_axis(Axis(2)) 74 | .insert_axis(Axis(3)) 75 | .broadcast(size) 76 | .unwrap() 77 | .to_owned(); 78 | inputs.mul(&self.mask).mul(1.0 / 1.0 - self.probability) 79 | } else { 80 | inputs 81 | } 82 | } 83 | 84 | pub fn backward_propagate(&mut self, d_outputs: ArrayD) -> ArrayD { 85 | d_outputs.mul(&self.mask).mul(1.0 / 1.0 - self.probability) 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /examples/xor_option.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * This example shows how to train a neural network to predict the output of the XOR function. 3 | */ 4 | 5 | import { 6 | Cost, 7 | CPU, 8 | DenseLayer, 9 | OPTION, 10 | Sequential, 11 | setupBackend, 12 | SigmoidLayer, 13 | tensor2D, 14 | WASM, 15 | } from "../packages/core/mod.ts"; 16 | 17 | /** 18 | * Setup the backend with the OPTION function. The backend is chosen automatically based on the environment. 19 | */ 20 | await setupBackend(OPTION( 21 | /** 22 | * The CPU backend is chosen if running with the unstable flag. 23 | */ 24 | CPU, 25 | /** 26 | * The WASM backend is chosen if running without the unstable flag. 27 | */ 28 | WASM, 29 | )); 30 | 31 | /** 32 | * Creates a sequential neural network. 33 | */ 34 | const net = new Sequential({ 35 | /** 36 | * The number of minibatches is set to 4 and the output size is set to 2. 37 | */ 38 | size: [4, 2], 39 | 40 | /** 41 | * The silent option is set to true, which means that the network will not output any logs during trainin 42 | */ 43 | silent: true, 44 | 45 | /** 46 | * Defines the layers of a neural network in the XOR function example. 47 | * The neural network has two input neurons and one output neuron. 48 | * The layers are defined as follows: 49 | * - A dense layer with 3 neurons. 50 | * - sigmoid activation layer. 51 | * - A dense layer with 1 neuron. 52 | * -A sigmoid activation layer. 53 | */ 54 | layers: [ 55 | DenseLayer({ size: [3] }), 56 | SigmoidLayer(), 57 | DenseLayer({ size: [1] }), 58 | SigmoidLayer(), 59 | ], 60 | 61 | /** 62 | * The cost function used for training the network is the mean squared error (MSE). 63 | */ 64 | cost: Cost.MSE, 65 | }); 66 | 67 | const time = performance.now(); 68 | 69 | /** 70 | * Train the network on the given data. 71 | */ 72 | net.train( 73 | [ 74 | { 75 | inputs: tensor2D([ 76 | [0, 0], 77 | [1, 0], 78 | [0, 1], 79 | [1, 1], 80 | ]), 81 | outputs: tensor2D([[0], [1], [1], [0]]), 82 | }, 83 | ], 84 | /** 85 | * The number of iterations is set to 10000. 86 | */ 87 | 10000, 88 | ); 89 | 90 | console.log(`training time: ${performance.now() - time}ms`); 91 | 92 | /** 93 | * Predict the output of the XOR function for the given inputs. 94 | */ 95 | const out1 = (await net.predict(tensor2D([[0, 0]]))).data; 96 | console.log(`0 xor 0 = ${out1[0]} (should be close to 0)`); 97 | 98 | const out2 = (await net.predict(tensor2D([[1, 0]]))).data; 99 | console.log(`1 xor 0 = ${out2[0]} (should be close to 1)`); 100 | 101 | const out3 = (await net.predict(tensor2D([[0, 1]]))).data; 102 | console.log(`0 xor 1 = ${out3[0]} (should be close to 1)`); 103 | 104 | const out4 = (await net.predict(tensor2D([[1, 1]]))).data; 105 | console.log(`1 xor 1 = ${out4[0]} (should be close to 0)`); 106 | -------------------------------------------------------------------------------- /crates/core-gpu/src/gpu/cost.rs: -------------------------------------------------------------------------------- 1 | use std::ops::{Div, Mul, Sub}; 2 | 3 | use ndarray::{s, ArrayD, ArrayViewD}; 4 | 5 | use crate::Cost; 6 | 7 | pub struct GPUCost { 8 | pub cost: for<'a> fn(y_hat: ArrayViewD<'a, f32>, y: ArrayViewD<'a, f32>) -> f32, 9 | pub prime: for<'a> fn(y_hat: ArrayViewD<'a, f32>, y: ArrayViewD<'a, f32>) -> ArrayD, 10 | } 11 | 12 | impl GPUCost { 13 | pub fn from(cost: Cost) -> GPUCost { 14 | match cost { 15 | Cost::MSE => GPUCost { 16 | cost: mse, 17 | prime: mse_prime, 18 | }, 19 | Cost::CrossEntropy => GPUCost { 20 | cost: cross_entropy, 21 | prime: cross_entropy_prime, 22 | }, 23 | Cost::BinCrossEntropy => GPUCost { 24 | cost: bin_cross_entropy, 25 | prime: bin_cross_entropy_prime, 26 | }, 27 | Cost::Hinge => GPUCost { 28 | cost: hinge, 29 | prime: hinge_prime, 30 | }, 31 | } 32 | } 33 | } 34 | 35 | fn mse<'a>(y_hat: ArrayViewD<'a, f32>, y: ArrayViewD<'a, f32>) -> f32 { 36 | let sub = y.sub(&y_hat); 37 | return sub.clone().mul(sub).sum(); 38 | } 39 | 40 | fn mse_prime<'a>(y_hat: ArrayViewD<'a, f32>, y: ArrayViewD<'a, f32>) -> ArrayD { 41 | return y.sub(&y_hat); 42 | } 43 | 44 | fn cross_entropy<'a>(y_hat: ArrayViewD<'a, f32>, y: ArrayViewD<'a, f32>) -> f32 { 45 | let batches = y_hat.dim()[0]; 46 | let mut total = 0.0; 47 | for b in 0..batches { 48 | total -= y_hat.slice(s![b, ..]).mul(&y.slice(s![b, ..])).sum().ln() 49 | } 50 | return total / batches as f32; 51 | } 52 | 53 | fn cross_entropy_prime<'a>(y_hat: ArrayViewD<'a, f32>, y: ArrayViewD<'a, f32>) -> ArrayD { 54 | return -y_hat.div(&y); 55 | } 56 | 57 | fn bin_cross_entropy<'a>(y_hat: ArrayViewD<'a, f32>, y: ArrayViewD<'a, f32>) -> f32 { 58 | return -y_hat 59 | .mul(y.map(|x| x.ln())) 60 | .sub(((1.0).sub(&y_hat)).mul(y.map(|x| 1.0 - x.ln()))) 61 | .sum() 62 | / y.len() as f32; 63 | } 64 | 65 | fn bin_cross_entropy_prime<'a>(y_hat: ArrayViewD<'a, f32>, y: ArrayViewD<'a, f32>) -> ArrayD { 66 | return y.sub(&y_hat).div(y.mul(1.0.sub(&y))); 67 | } 68 | 69 | fn hinge<'a>(y_hat: ArrayViewD<'a, f32>, y: ArrayViewD<'a, f32>) -> f32 { 70 | let mut sum = 0.0; 71 | for (y_hat_i, y_i) in y_hat.iter().zip(y.iter()) { 72 | let margin = 1.0 - y_hat_i * y_i; 73 | if margin > 0.0 { 74 | sum += margin; 75 | } 76 | } 77 | return sum; 78 | } 79 | 80 | fn hinge_prime<'a>(y_hat: ArrayViewD<'a, f32>, y: ArrayViewD<'a, f32>) -> ArrayD { 81 | let mut result = ArrayD::zeros(y_hat.shape()); 82 | for ((result_i, y_hat_i), y_i) in result.iter_mut().zip(y_hat.iter()).zip(y.iter()) { 83 | let margin = 1.0 - y_hat_i * y_i; 84 | if margin > 0.0 { 85 | *result_i = -y_i; 86 | } 87 | } 88 | return result; 89 | } 90 | -------------------------------------------------------------------------------- /packages/utilities/src/text/cleaner.ts: -------------------------------------------------------------------------------- 1 | import { DefaultIgnoreList } from "../constants/stop_words.ts"; 2 | 3 | interface StandardizeConfig { 4 | /** Whether to convert everything to lowercase before fitting / transforming */ 5 | lowercase?: boolean; 6 | /** Whether to strip HTML tags */ 7 | stripHtml?: boolean; 8 | /** Whether to replace multiple whitespaces. */ 9 | normalizeWhiteSpaces?: boolean; 10 | /** Strip Newlines */ 11 | stripNewlines?: boolean; 12 | removeMentions?: boolean; 13 | keepOnlyAlphaNumeric?: boolean; 14 | /** Remove stop words from text */ 15 | removeStopWords?: "english" | false | string[]; 16 | } 17 | 18 | /** Simple text cleaner */ 19 | export class TextCleaner implements StandardizeConfig { 20 | stripHtml: boolean; 21 | lowercase: boolean; 22 | normalizeWhiteSpaces: boolean; 23 | stripNewlines: boolean; 24 | removeStopWords: false | "english" | string[]; 25 | removeMentions: boolean; 26 | keepOnlyAlphaNumeric: boolean; 27 | constructor({ 28 | stripHtml = false, 29 | lowercase = false, 30 | normalizeWhiteSpaces = true, 31 | stripNewlines = true, 32 | removeStopWords = false, 33 | removeMentions = false, 34 | keepOnlyAlphaNumeric = false, 35 | }: StandardizeConfig = {}) { 36 | this.stripHtml = stripHtml; 37 | this.lowercase = lowercase; 38 | this.normalizeWhiteSpaces = normalizeWhiteSpaces; 39 | this.stripNewlines = stripNewlines; 40 | this.removeStopWords = removeStopWords; 41 | this.keepOnlyAlphaNumeric = keepOnlyAlphaNumeric; 42 | this.removeMentions = removeMentions; 43 | } 44 | clean(text: string): string; 45 | clean(text: string[]): string[]; 46 | clean(text: string | string[]) { 47 | if (Array.isArray(text)) { 48 | return text.map((line) => preprocess(line, this)); 49 | } 50 | return preprocess(text, this); 51 | } 52 | } 53 | 54 | /** Function for quick cleaning of text */ 55 | export function preprocess( 56 | text: string, 57 | { 58 | stripHtml = false, 59 | lowercase = false, 60 | normalizeWhiteSpaces = true, 61 | stripNewlines = true, 62 | removeStopWords = false, 63 | removeMentions = false, 64 | keepOnlyAlphaNumeric = false, 65 | }: StandardizeConfig = {} 66 | ): string { 67 | if (lowercase) { 68 | text = text.toLowerCase(); 69 | } 70 | if (stripHtml) { 71 | text = text.replace(/<([^>]+)>/g, " "); 72 | } 73 | if (stripNewlines) { 74 | text = text.replace(/\n/g, " "); 75 | } 76 | if (normalizeWhiteSpaces) { 77 | text = text.replace(/\s\s+/g, " "); 78 | } 79 | if (removeStopWords) { 80 | const stopWords = 81 | removeStopWords === "english" ? DefaultIgnoreList : removeStopWords; 82 | text = text 83 | .split(" ") 84 | .filter((x) => !stopWords.includes(x)) 85 | .join(" "); 86 | } 87 | if (removeMentions) { 88 | text = text.replace(/@\w+/g, "") 89 | } 90 | if (keepOnlyAlphaNumeric) { 91 | text = text.replace(/[^a-zA-Z0-9 ]+/g, "") 92 | } 93 | return text; 94 | } 95 | -------------------------------------------------------------------------------- /packages/visualizer/mod.ts: -------------------------------------------------------------------------------- 1 | import type { NeuralNetwork, Rank, Tensor } from "../../mod.ts"; 2 | import type { Line } from "./types.ts"; 3 | 4 | /** 5 | * Visualizer for Neural Networks in Jupyter Notebook 6 | */ 7 | export class Visualizer { 8 | #title: string; 9 | 10 | constructor(title: string) { 11 | this.#title = title; 12 | } 13 | 14 | /** 15 | * Graph the results of a Neural Network 16 | */ 17 | async graph( 18 | net: NeuralNetwork, 19 | inputs: Tensor[], 20 | expectedResults: Tensor[], 21 | ): Promise< 22 | { 23 | [x: symbol]: () => { 24 | "application/vnd.plotly.v1+json": { 25 | data: Line[]; 26 | layout: { title: string }; 27 | }; 28 | }; 29 | } 30 | > { 31 | const expected: Line = { 32 | x: [], 33 | y: [], 34 | type: "scatter", 35 | mode: "lines+markers", 36 | name: "Expected", 37 | line: { 38 | color: "blue", 39 | width: 3, 40 | }, 41 | }; 42 | const results: Line = { 43 | x: [], 44 | y: [], 45 | type: "scatter", 46 | mode: "lines+markers", 47 | name: "Results", 48 | line: { 49 | color: "red", 50 | width: 3, 51 | }, 52 | }; 53 | 54 | for (let i = 0; i < inputs.length; i++) { 55 | expected.x.push(i + 1); 56 | results.x.push(i + 1); 57 | const output = (await net.predict(inputs[i])).data; 58 | expected.y.push(expectedResults[i].data[0]); 59 | results.y.push(output[0]); 60 | } 61 | const title = this.#title; 62 | 63 | return { 64 | [Symbol.for("Jupyter.display")]() { 65 | return { 66 | "application/vnd.plotly.v1+json": { 67 | data: [expected, results], 68 | layout: { 69 | title, 70 | }, 71 | }, 72 | }; 73 | }, 74 | }; 75 | } 76 | 77 | /** 78 | * Graph the loss of a Neural Network during training 79 | */ 80 | graphLoss(loss: number[]): { 81 | [x: symbol]: () => { 82 | "application/vnd.plotly.v1+json": { 83 | data: Line[]; 84 | layout: { title: string }; 85 | }; 86 | }; 87 | } { 88 | const lossLine: Line = { 89 | x: [], 90 | y: [], 91 | type: "scatter", 92 | mode: "lines+markers", 93 | name: "Loss", 94 | line: { 95 | color: "blue", 96 | width: 3, 97 | }, 98 | }; 99 | 100 | for (let i = 0; i < loss.length; i++) { 101 | lossLine.x.push(i + 1); 102 | lossLine.y.push(loss[i]); 103 | } 104 | 105 | const title = this.#title; 106 | 107 | return { 108 | [Symbol.for("Jupyter.display")]() { 109 | return { 110 | "application/vnd.plotly.v1+json": { 111 | data: [lossLine], 112 | layout: { 113 | title, 114 | }, 115 | }, 116 | }; 117 | }, 118 | }; 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /examples/classification/binary_iris.csv: -------------------------------------------------------------------------------- 1 | 5.1,3.5,1.4,.2,"Setosa" 2 | 4.9,3,1.4,.2,"Setosa" 3 | 4.7,3.2,1.3,.2,"Setosa" 4 | 4.6,3.1,1.5,.2,"Setosa" 5 | 5,3.6,1.4,.2,"Setosa" 6 | 5.4,3.9,1.7,.4,"Setosa" 7 | 4.6,3.4,1.4,.3,"Setosa" 8 | 5,3.4,1.5,.2,"Setosa" 9 | 4.4,2.9,1.4,.2,"Setosa" 10 | 4.9,3.1,1.5,.1,"Setosa" 11 | 5.4,3.7,1.5,.2,"Setosa" 12 | 4.8,3.4,1.6,.2,"Setosa" 13 | 4.8,3,1.4,.1,"Setosa" 14 | 4.3,3,1.1,.1,"Setosa" 15 | 5.8,4,1.2,.2,"Setosa" 16 | 5.7,4.4,1.5,.4,"Setosa" 17 | 5.4,3.9,1.3,.4,"Setosa" 18 | 5.1,3.5,1.4,.3,"Setosa" 19 | 5.7,3.8,1.7,.3,"Setosa" 20 | 5.1,3.8,1.5,.3,"Setosa" 21 | 5.4,3.4,1.7,.2,"Setosa" 22 | 5.1,3.7,1.5,.4,"Setosa" 23 | 4.6,3.6,1,.2,"Setosa" 24 | 5.1,3.3,1.7,.5,"Setosa" 25 | 4.8,3.4,1.9,.2,"Setosa" 26 | 5,3,1.6,.2,"Setosa" 27 | 5,3.4,1.6,.4,"Setosa" 28 | 5.2,3.5,1.5,.2,"Setosa" 29 | 5.2,3.4,1.4,.2,"Setosa" 30 | 4.7,3.2,1.6,.2,"Setosa" 31 | 4.8,3.1,1.6,.2,"Setosa" 32 | 5.4,3.4,1.5,.4,"Setosa" 33 | 5.2,4.1,1.5,.1,"Setosa" 34 | 5.5,4.2,1.4,.2,"Setosa" 35 | 4.9,3.1,1.5,.2,"Setosa" 36 | 5,3.2,1.2,.2,"Setosa" 37 | 5.5,3.5,1.3,.2,"Setosa" 38 | 4.9,3.6,1.4,.1,"Setosa" 39 | 4.4,3,1.3,.2,"Setosa" 40 | 5.1,3.4,1.5,.2,"Setosa" 41 | 5,3.5,1.3,.3,"Setosa" 42 | 4.5,2.3,1.3,.3,"Setosa" 43 | 4.4,3.2,1.3,.2,"Setosa" 44 | 5,3.5,1.6,.6,"Setosa" 45 | 5.1,3.8,1.9,.4,"Setosa" 46 | 4.8,3,1.4,.3,"Setosa" 47 | 5.1,3.8,1.6,.2,"Setosa" 48 | 4.6,3.2,1.4,.2,"Setosa" 49 | 5.3,3.7,1.5,.2,"Setosa" 50 | 5,3.3,1.4,.2,"Setosa" 51 | 7,3.2,4.7,1.4,"Versicolor" 52 | 6.4,3.2,4.5,1.5,"Versicolor" 53 | 6.9,3.1,4.9,1.5,"Versicolor" 54 | 5.5,2.3,4,1.3,"Versicolor" 55 | 6.5,2.8,4.6,1.5,"Versicolor" 56 | 5.7,2.8,4.5,1.3,"Versicolor" 57 | 6.3,3.3,4.7,1.6,"Versicolor" 58 | 4.9,2.4,3.3,1,"Versicolor" 59 | 6.6,2.9,4.6,1.3,"Versicolor" 60 | 5.2,2.7,3.9,1.4,"Versicolor" 61 | 5,2,3.5,1,"Versicolor" 62 | 5.9,3,4.2,1.5,"Versicolor" 63 | 6,2.2,4,1,"Versicolor" 64 | 6.1,2.9,4.7,1.4,"Versicolor" 65 | 5.6,2.9,3.6,1.3,"Versicolor" 66 | 6.7,3.1,4.4,1.4,"Versicolor" 67 | 5.6,3,4.5,1.5,"Versicolor" 68 | 5.8,2.7,4.1,1,"Versicolor" 69 | 6.2,2.2,4.5,1.5,"Versicolor" 70 | 5.6,2.5,3.9,1.1,"Versicolor" 71 | 5.9,3.2,4.8,1.8,"Versicolor" 72 | 6.1,2.8,4,1.3,"Versicolor" 73 | 6.3,2.5,4.9,1.5,"Versicolor" 74 | 6.1,2.8,4.7,1.2,"Versicolor" 75 | 6.4,2.9,4.3,1.3,"Versicolor" 76 | 6.6,3,4.4,1.4,"Versicolor" 77 | 6.8,2.8,4.8,1.4,"Versicolor" 78 | 6.7,3,5,1.7,"Versicolor" 79 | 6,2.9,4.5,1.5,"Versicolor" 80 | 5.7,2.6,3.5,1,"Versicolor" 81 | 5.5,2.4,3.8,1.1,"Versicolor" 82 | 5.5,2.4,3.7,1,"Versicolor" 83 | 5.8,2.7,3.9,1.2,"Versicolor" 84 | 6,2.7,5.1,1.6,"Versicolor" 85 | 5.4,3,4.5,1.5,"Versicolor" 86 | 6,3.4,4.5,1.6,"Versicolor" 87 | 6.7,3.1,4.7,1.5,"Versicolor" 88 | 6.3,2.3,4.4,1.3,"Versicolor" 89 | 5.6,3,4.1,1.3,"Versicolor" 90 | 5.5,2.5,4,1.3,"Versicolor" 91 | 5.5,2.6,4.4,1.2,"Versicolor" 92 | 6.1,3,4.6,1.4,"Versicolor" 93 | 5.8,2.6,4,1.2,"Versicolor" 94 | 5,2.3,3.3,1,"Versicolor" 95 | 5.6,2.7,4.2,1.3,"Versicolor" 96 | 5.7,3,4.2,1.2,"Versicolor" 97 | 5.7,2.9,4.2,1.3,"Versicolor" 98 | 6.2,2.9,4.3,1.3,"Versicolor" 99 | 5.1,2.5,3,1.1,"Versicolor" 100 | 5.7,2.8,4.1,1.3,"Versicolor" -------------------------------------------------------------------------------- /packages/utilities/examples/metrics/binary_iris.csv: -------------------------------------------------------------------------------- 1 | 5.1,3.5,1.4,.2,"Setosa" 2 | 4.9,3,1.4,.2,"Setosa" 3 | 4.7,3.2,1.3,.2,"Setosa" 4 | 4.6,3.1,1.5,.2,"Setosa" 5 | 5,3.6,1.4,.2,"Setosa" 6 | 5.4,3.9,1.7,.4,"Setosa" 7 | 4.6,3.4,1.4,.3,"Setosa" 8 | 5,3.4,1.5,.2,"Setosa" 9 | 4.4,2.9,1.4,.2,"Setosa" 10 | 4.9,3.1,1.5,.1,"Setosa" 11 | 5.4,3.7,1.5,.2,"Setosa" 12 | 4.8,3.4,1.6,.2,"Setosa" 13 | 4.8,3,1.4,.1,"Setosa" 14 | 4.3,3,1.1,.1,"Setosa" 15 | 5.8,4,1.2,.2,"Setosa" 16 | 5.7,4.4,1.5,.4,"Setosa" 17 | 5.4,3.9,1.3,.4,"Setosa" 18 | 5.1,3.5,1.4,.3,"Setosa" 19 | 5.7,3.8,1.7,.3,"Setosa" 20 | 5.1,3.8,1.5,.3,"Setosa" 21 | 5.4,3.4,1.7,.2,"Setosa" 22 | 5.1,3.7,1.5,.4,"Setosa" 23 | 4.6,3.6,1,.2,"Setosa" 24 | 5.1,3.3,1.7,.5,"Setosa" 25 | 4.8,3.4,1.9,.2,"Setosa" 26 | 5,3,1.6,.2,"Setosa" 27 | 5,3.4,1.6,.4,"Setosa" 28 | 5.2,3.5,1.5,.2,"Setosa" 29 | 5.2,3.4,1.4,.2,"Setosa" 30 | 4.7,3.2,1.6,.2,"Setosa" 31 | 4.8,3.1,1.6,.2,"Setosa" 32 | 5.4,3.4,1.5,.4,"Setosa" 33 | 5.2,4.1,1.5,.1,"Setosa" 34 | 5.5,4.2,1.4,.2,"Setosa" 35 | 4.9,3.1,1.5,.2,"Setosa" 36 | 5,3.2,1.2,.2,"Setosa" 37 | 5.5,3.5,1.3,.2,"Setosa" 38 | 4.9,3.6,1.4,.1,"Setosa" 39 | 4.4,3,1.3,.2,"Setosa" 40 | 5.1,3.4,1.5,.2,"Setosa" 41 | 5,3.5,1.3,.3,"Setosa" 42 | 4.5,2.3,1.3,.3,"Setosa" 43 | 4.4,3.2,1.3,.2,"Setosa" 44 | 5,3.5,1.6,.6,"Setosa" 45 | 5.1,3.8,1.9,.4,"Setosa" 46 | 4.8,3,1.4,.3,"Setosa" 47 | 5.1,3.8,1.6,.2,"Setosa" 48 | 4.6,3.2,1.4,.2,"Setosa" 49 | 5.3,3.7,1.5,.2,"Setosa" 50 | 5,3.3,1.4,.2,"Setosa" 51 | 7,3.2,4.7,1.4,"Versicolor" 52 | 6.4,3.2,4.5,1.5,"Versicolor" 53 | 6.9,3.1,4.9,1.5,"Versicolor" 54 | 5.5,2.3,4,1.3,"Versicolor" 55 | 6.5,2.8,4.6,1.5,"Versicolor" 56 | 5.7,2.8,4.5,1.3,"Versicolor" 57 | 6.3,3.3,4.7,1.6,"Versicolor" 58 | 4.9,2.4,3.3,1,"Versicolor" 59 | 6.6,2.9,4.6,1.3,"Versicolor" 60 | 5.2,2.7,3.9,1.4,"Versicolor" 61 | 5,2,3.5,1,"Versicolor" 62 | 5.9,3,4.2,1.5,"Versicolor" 63 | 6,2.2,4,1,"Versicolor" 64 | 6.1,2.9,4.7,1.4,"Versicolor" 65 | 5.6,2.9,3.6,1.3,"Versicolor" 66 | 6.7,3.1,4.4,1.4,"Versicolor" 67 | 5.6,3,4.5,1.5,"Versicolor" 68 | 5.8,2.7,4.1,1,"Versicolor" 69 | 6.2,2.2,4.5,1.5,"Versicolor" 70 | 5.6,2.5,3.9,1.1,"Versicolor" 71 | 5.9,3.2,4.8,1.8,"Versicolor" 72 | 6.1,2.8,4,1.3,"Versicolor" 73 | 6.3,2.5,4.9,1.5,"Versicolor" 74 | 6.1,2.8,4.7,1.2,"Versicolor" 75 | 6.4,2.9,4.3,1.3,"Versicolor" 76 | 6.6,3,4.4,1.4,"Versicolor" 77 | 6.8,2.8,4.8,1.4,"Versicolor" 78 | 6.7,3,5,1.7,"Versicolor" 79 | 6,2.9,4.5,1.5,"Versicolor" 80 | 5.7,2.6,3.5,1,"Versicolor" 81 | 5.5,2.4,3.8,1.1,"Versicolor" 82 | 5.5,2.4,3.7,1,"Versicolor" 83 | 5.8,2.7,3.9,1.2,"Versicolor" 84 | 6,2.7,5.1,1.6,"Versicolor" 85 | 5.4,3,4.5,1.5,"Versicolor" 86 | 6,3.4,4.5,1.6,"Versicolor" 87 | 6.7,3.1,4.7,1.5,"Versicolor" 88 | 6.3,2.3,4.4,1.3,"Versicolor" 89 | 5.6,3,4.1,1.3,"Versicolor" 90 | 5.5,2.5,4,1.3,"Versicolor" 91 | 5.5,2.6,4.4,1.2,"Versicolor" 92 | 6.1,3,4.6,1.4,"Versicolor" 93 | 5.8,2.6,4,1.2,"Versicolor" 94 | 5,2.3,3.3,1,"Versicolor" 95 | 5.6,2.7,4.2,1.3,"Versicolor" 96 | 5.7,3,4.2,1.2,"Versicolor" 97 | 5.7,2.9,4.2,1.3,"Versicolor" 98 | 6.2,2.9,4.3,1.3,"Versicolor" 99 | 5.1,2.5,3,1.1,"Versicolor" 100 | 5.7,2.8,4.1,1.3,"Versicolor" -------------------------------------------------------------------------------- /crates/core-gpu/src/gpu/activation.rs: -------------------------------------------------------------------------------- 1 | use crate::Activation; 2 | pub struct GPUActivation { 3 | pub activation: Activation, 4 | pub activate: ActivationFn, 5 | pub prime: ActivationFn, 6 | } 7 | 8 | type ActivationFn = fn(x: &f32) -> f32; 9 | 10 | impl GPUActivation { 11 | pub fn from(activation: Activation) -> Self { 12 | let (activate, prime): (ActivationFn, ActivationFn) = match activation { 13 | Activation::Elu => (elu, elu_prime), 14 | Activation::LeakyRelu => (leaky_relu, leaky_relu_prime), 15 | Activation::Linear => (linear, linear_prime), 16 | Activation::Relu => (relu, relu_prime), 17 | Activation::Relu6 => (relu6, relu6_prime), 18 | Activation::Selu => (selu, selu_prime), 19 | Activation::Sigmoid => (sigmoid, sigmoid_prime), 20 | Activation::Tanh => (tanh, tanh_prime), 21 | }; 22 | 23 | Self { 24 | activation, 25 | activate, 26 | prime, 27 | } 28 | } 29 | 30 | pub fn from_option(activation: Option) -> Option { 31 | if let Some(activation) = activation { 32 | Some(GPUActivation::from(activation)) 33 | } else { 34 | None 35 | } 36 | } 37 | 38 | pub fn memoize_output(activation: &GPUActivation) -> bool { 39 | match activation.activation { 40 | Activation::Sigmoid | Activation::Tanh => true, 41 | _ => true, 42 | } 43 | } 44 | } 45 | 46 | fn sigmoid(x: &f32) -> f32 { 47 | return 1.0 / (1.0 + (-x).exp()); 48 | } 49 | 50 | fn sigmoid_prime(x: &f32) -> f32 { 51 | return x * (1.0 - x); 52 | } 53 | 54 | fn tanh(x: &f32) -> f32 { 55 | return x.tanh(); 56 | } 57 | 58 | fn tanh_prime(x: &f32) -> f32 { 59 | return 1.0 - tanh(x).powi(2); 60 | } 61 | 62 | fn linear(x: &f32) -> f32 { 63 | return *x; 64 | } 65 | 66 | fn linear_prime(_x: &f32) -> f32 { 67 | return 1.0; 68 | } 69 | 70 | fn relu(x: &f32) -> f32 { 71 | return x.max(0.0); 72 | } 73 | 74 | fn relu_prime(x: &f32) -> f32 { 75 | return if *x > 0.0 { 1.0 } else { 0.0 }; 76 | } 77 | 78 | fn relu6(x: &f32) -> f32 { 79 | return x.max(0.0).min(6.0); 80 | } 81 | 82 | fn relu6_prime(x: &f32) -> f32 { 83 | return if *x > 0.0 && *x < 6.0 { 1.0 } else { 0.0 }; 84 | } 85 | 86 | fn leaky_relu(x: &f32) -> f32 { 87 | return if *x > 0.0 { *x } else { x.max(0.01 * x) }; 88 | } 89 | 90 | fn leaky_relu_prime(x: &f32) -> f32 { 91 | return if *x > 0.0 { 1.0 } else { 0.01 }; 92 | } 93 | 94 | fn elu(x: &f32) -> f32 { 95 | return if *x >= 0.0 { *x } else { x.exp() - 1.0 }; 96 | } 97 | 98 | fn elu_prime(x: &f32) -> f32 { 99 | return if *x > 0.0 { 1.0 } else { x.exp() }; 100 | } 101 | 102 | fn selu(x: &f32) -> f32 { 103 | return if *x >= 0.0 { 104 | *x 105 | } else { 106 | 1.0507 * (x.exp() - 1.0) 107 | }; 108 | } 109 | 110 | fn selu_prime(x: &f32) -> f32 { 111 | return if *x > 0.0 { 1.0 } else { 1.0507 * x.exp() }; 112 | } 113 | -------------------------------------------------------------------------------- /examples/classification/spam.ts: -------------------------------------------------------------------------------- 1 | import { parse } from "jsr:@std/csv@1.0.3/parse"; 2 | import { 3 | Cost, 4 | CPU, 5 | DenseLayer, 6 | NadamOptimizer, 7 | PostProcess, 8 | ReluLayer, 9 | Sequential, 10 | setupBackend, 11 | tensor, 12 | tensor2D, 13 | } from "../../packages/core/mod.ts"; 14 | 15 | // Import helpers for metrics 16 | import { 17 | ClassificationReport, 18 | TextCleaner, 19 | TextVectorizer, 20 | // Split the dataset 21 | useSplit, 22 | } from "../../packages/utilities/mod.ts"; 23 | import { 24 | EmbeddingLayer, 25 | FlattenLayer, 26 | Init, 27 | LinearDecay, 28 | LSTMLayer, 29 | SigmoidLayer, 30 | } from "../../mod.ts"; 31 | 32 | // Define classes 33 | const ymap = ["spam", "ham"]; 34 | 35 | // Read the training dataset 36 | const _data = Deno.readTextFileSync("examples/classification/spam.csv"); 37 | const data = parse(_data); 38 | 39 | // Get the predictors (messages) 40 | const x = data.map((msg) => msg[1]); 41 | 42 | // Get the classes 43 | const y = data.map((msg) => (ymap.indexOf(msg[0]) === 0 ? -1 : 1)); 44 | 45 | // Split the dataset for training and testing 46 | const [train, test] = useSplit({ ratio: [7, 3], shuffle: true }, x, y); 47 | 48 | // Vectorize the text messages 49 | 50 | const textCleaner = new TextCleaner({ 51 | lowercase: true, 52 | stripHtml: true, 53 | normalizeWhiteSpaces: true, 54 | removeStopWords: "english", 55 | stripNewlines: true, 56 | keepOnlyAlphaNumeric: true, 57 | }); 58 | 59 | train[0] = textCleaner.clean(train[0]); 60 | 61 | const vec = new TextVectorizer("indices").fit(train[0]); 62 | 63 | const x_vec = vec.transform(train[0], "f32"); 64 | 65 | // Setup the CPU backend for Netsaur 66 | await setupBackend(CPU); 67 | 68 | const net = new Sequential({ 69 | size: [4, x_vec.nCols], 70 | layers: [ 71 | EmbeddingLayer({ vocabSize: vec.vocabSize, embeddingSize: 50 }), 72 | LSTMLayer({ size: 128, init: Init.Kaiming, returnSequences: true, }), 73 | LSTMLayer({ size: 128, init: Init.Kaiming }), 74 | //FlattenLayer(), 75 | // A dense layer with 1 neuron 76 | DenseLayer({ size: [1], init: Init.XavierN }), 77 | ], 78 | 79 | // We are using Log Loss for finding cost 80 | cost: Cost.Hinge, 81 | patience: 50, 82 | optimizer: NadamOptimizer(), 83 | scheduler: LinearDecay({ rate: 2, step_size: 5 }) 84 | }); 85 | 86 | const inputs = tensor(x_vec); 87 | 88 | const time = performance.now(); 89 | // Train the network 90 | net.train( 91 | [ 92 | { 93 | inputs: inputs, 94 | outputs: tensor2D(train[1].map((x) => [x])), 95 | }, 96 | ], 97 | // Train for 20 epochs 98 | 10, 99 | 2, 100 | 0.005 101 | ); 102 | 103 | console.log(`training time: ${performance.now() - time}ms`); 104 | 105 | const x_vec_test = vec.transform(test[0], "f32"); 106 | 107 | // Calculate metrics 108 | const res = await net.predict(tensor(x_vec_test), {postProcess: PostProcess("sign")}); 109 | const y1 = Array.from(res.data); 110 | const cMatrix = new ClassificationReport( 111 | test[1].map((x) => ymap[x < 0 ? 0 : 1]), 112 | y1.map((x) => ymap[x < 0 ? 0 : 1]) 113 | ); 114 | console.log("Confusion Matrix: ", cMatrix); 115 | -------------------------------------------------------------------------------- /crates/core/src/cpu/layers/dense.rs: -------------------------------------------------------------------------------- 1 | use ndarray::{Array1, Array2, ArrayD, Axis, Dimension, Ix1, Ix2, IxDyn}; 2 | use std::ops::Add; 3 | 4 | use crate::{CPUInit, CPURegularizer, DenseLayer, Init, Tensors}; 5 | 6 | pub struct DenseCPULayer { 7 | // cache 8 | pub output_size: Ix2, 9 | pub inputs: Array2, 10 | 11 | // parameters 12 | pub weights: Array2, 13 | pub biases: Array1, 14 | 15 | // gradients 16 | pub d_weights: Array2, 17 | pub d_biases: Array1, 18 | 19 | // regularization 20 | pub l_weights: Array2, 21 | pub l_biases: Array1, 22 | 23 | pub regularizer: CPURegularizer, 24 | } 25 | 26 | impl DenseCPULayer { 27 | pub fn new(config: DenseLayer, size: IxDyn, tensors: Option) -> Self { 28 | let init = CPUInit::from_default(config.init, Init::Uniform); 29 | let input_size = Ix2(size[0], size[1]); 30 | let weight_size = Ix2(size[1], config.size[0]); 31 | let output_size = Ix2(size[0], config.size[0]); 32 | 33 | let (weights, biases) = if let Some(Tensors::Dense(tensors)) = tensors { 34 | (tensors.weights, tensors.biases) 35 | } else { 36 | let weights = init.init(weight_size.into_dyn(), size[1], config.size[0]); 37 | let biases = ArrayD::zeros(config.size.clone()); 38 | (weights, biases) 39 | }; 40 | 41 | Self { 42 | output_size, 43 | inputs: Array2::zeros(input_size), 44 | weights: weights.into_dimensionality::().unwrap(), 45 | biases: biases.into_dimensionality::().unwrap(), 46 | d_weights: Array2::zeros(weight_size), 47 | d_biases: Array1::zeros(config.size[0]), 48 | l_weights: Array2::zeros(weight_size), 49 | l_biases: Array1::zeros(config.size[0]), 50 | regularizer: CPURegularizer::from(config.c.unwrap_or(0.0), config.l1_ratio.unwrap_or(1.0)) 51 | } 52 | } 53 | 54 | pub fn output_size(&self) -> Vec { 55 | self.output_size.as_array_view().to_vec() 56 | } 57 | 58 | pub fn reset(&mut self, batches: usize) { 59 | let input_size = self.inputs.dim().1; 60 | self.inputs = Array2::zeros((batches, input_size)); 61 | self.output_size[0] = batches; 62 | } 63 | 64 | pub fn forward_propagate(&mut self, inputs: ArrayD) -> ArrayD { 65 | self.inputs = inputs.into_dimensionality::().unwrap(); 66 | self.inputs.dot(&self.weights).add(&self.biases).into_dyn() 67 | } 68 | 69 | pub fn backward_propagate(&mut self, d_outputs: ArrayD) -> ArrayD { 70 | let d_outputs = d_outputs.into_dimensionality::().unwrap(); 71 | let mut weights_t = self.weights.view(); 72 | weights_t.swap_axes(0, 1); 73 | let d_inputs = d_outputs.dot(&weights_t); 74 | let mut inputs_t = self.inputs.view(); 75 | inputs_t.swap_axes(0, 1); 76 | self.d_weights = inputs_t.dot(&d_outputs); 77 | self.d_biases = d_outputs.sum_axis(Axis(0)); 78 | 79 | self.l_weights = self.regularizer.coeff(&self.weights.clone().into_dyn()).into_dimensionality::().unwrap(); 80 | self.l_biases = self.regularizer.coeff(&self.biases.clone().into_dyn()).into_dimensionality::().unwrap(); 81 | d_inputs.into_dyn() 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /packages/utilities/src/utils/misc/image.ts: -------------------------------------------------------------------------------- 1 | import type { Pixel } from "../common_types.ts"; 2 | 3 | export type ImageData = { 4 | data: Uint8ClampedArray; 5 | width: number; 6 | height: number; 7 | channels: number; // always 4, for compat with retraigo/vectorizer 8 | colorSpace: "srgb" | "display-p3"; 9 | }; 10 | 11 | type ImageOptions = { 12 | data: Uint8ClampedArray; 13 | width: number; 14 | height?: number; 15 | channels?: number; 16 | }; 17 | 18 | export class Image implements ImageData { 19 | data: Uint8ClampedArray; 20 | width: number; 21 | height: number; 22 | channels: number; // always 4, for compat with retraigo/vectorizer 23 | colorSpace: "srgb" | "display-p3"; 24 | constructor(data: ImageOptions) { 25 | this.data = Uint8ClampedArray.from(data.data); 26 | // N-channels is always 4 27 | this.channels = 4; 28 | this.width = data.width; 29 | this.height = data.height ?? 30 | this.data.length / (this.width * this.channels); 31 | // If height is not an integer or width is incorrect 32 | if (this.height !== ~~this.height) { 33 | throw new TypeError( 34 | `Height must be an integer. Received ${this.height}.`, 35 | ); 36 | } 37 | // Only srgb is supported 38 | this.colorSpace = "srgb"; 39 | } 40 | get pixels(): number { 41 | return this.width * this.height; 42 | } 43 | getNthPixel(n: number): [number, number, number, number] { 44 | const offset = n << 2; 45 | return [ 46 | this.data[offset], 47 | this.data[offset + 1], 48 | this.data[offset + 2], 49 | this.data[offset + 3], 50 | ]; 51 | } 52 | getPixel(row: number, col: number): Pixel { 53 | if (row >= this.height) { 54 | throw new RangeError( 55 | `Requested row ${row} is outside of bounds 0..${this.height}.`, 56 | ); 57 | } 58 | if (col >= this.width) { 59 | throw new RangeError( 60 | `Requested column ${col} is outside of bounds 0..${this.width}.`, 61 | ); 62 | } 63 | const offset = row * this.width + col; 64 | const [r, g, b, a] = this.data.slice(offset, offset + 4); 65 | return [r, g, b, a]; 66 | } 67 | setPixel(row: number, col: number, [r, g, b, a]: Pixel) { 68 | if (row >= this.height) { 69 | throw new RangeError( 70 | `Requested row ${row} is outside of bounds 0..${this.height}.`, 71 | ); 72 | } 73 | if (col >= this.width) { 74 | throw new RangeError( 75 | `Requested column ${col} is outside of bounds 0..${this.width}.`, 76 | ); 77 | } 78 | const offset = row * this.width + col; 79 | this.data.set(typeof a !== "undefined" ? [r, g, b, a] : [r, g, b], offset); 80 | } 81 | updatePixel( 82 | row: number, 83 | col: number, 84 | color: Pixel, 85 | ) { 86 | if (row >= this.height) { 87 | throw new RangeError( 88 | `Requested row ${row} is outside of bounds 0..${this.height}.`, 89 | ); 90 | } 91 | if (col >= this.width) { 92 | throw new RangeError( 93 | `Requested column ${col} is outside of bounds 0..${this.width}.`, 94 | ); 95 | } 96 | const offset = row * this.width + col; 97 | for (let i = 0; i < color.length; i += 1) { 98 | this.data[offset + i] += color[i] ?? 0; 99 | } 100 | } 101 | } 102 | --------------------------------------------------------------------------------