9 |
10 | We are on a mission to bring fast, cross platform GPU accelerated inference on native + browser.
11 |
12 | > [!NOTE]
13 | > Ratchet is currently in active development. We are working on the engine, adding more models and improving compatibility. Please, reach out if you'd like to help!
14 |
15 | ## Getting Started
16 |
17 | The easiest way to experience Ratchet is to check out our [Hugging Face spaces](https://huggingface.co/FL33TW00D-HF):
18 | - [Whisper](https://huggingface.co/spaces/FL33TW00D-HF/ratchet-whisper)
19 | - [Phi](https://huggingface.co/spaces/FL33TW00D-HF/ratchet-phi)
20 |
21 | To dig deeper, check out the [examples](https://github.com/FL33TW00D/ratchet/tree/master/examples)
22 |
23 | We welcome contributions from the community. If you have any ideas or suggestions, please feel free to open an issue or pull request.
24 |
25 | ### Javascript
26 |
27 | ```javascript
28 | // Asynchronous loading & caching with IndexedDB
29 | let model = await Model.load(AvailableModels.WHISPER_TINY, Quantization.Q8, (p: number) => setProgress(p))
30 | let result = await model.run({ input });
31 | ```
32 |
33 | ### Rust
34 |
35 | Rust crate & CLI coming soon...
36 |
37 | ## Philosophy
38 |
39 | We want a toolkit for developers to make integrating performant AI functionality into existing production applications easy.
40 | The following principles will help us accomplish this:
41 | 1. **Inference only**
42 | 2. **WebGPU/CPU only**
43 | 3. First class quantization support
44 | 4. Lazy computation
45 | 5. Inplace by default
46 |
47 | ## Supported Models
48 | - Whisper
49 | - Phi 2 & 3
50 | - Moondream
51 |
52 | ## Upcoming Models
53 | - Gemini 2 2B
54 |
--------------------------------------------------------------------------------
/config/webdriver-linux.json:
--------------------------------------------------------------------------------
1 | {
2 | "goog:chromeOptions": {
3 | "args": [
4 | "--no-sandbox",
5 | "--headless=new",
6 | "--use-angle=vulkan",
7 | "--enable-features=Vulkan",
8 | "--enable-unsafe-webgpu"
9 | ]
10 | }
11 | }
12 |
--------------------------------------------------------------------------------
/config/webdriver-macos.json:
--------------------------------------------------------------------------------
1 | {
2 | "goog:chromeOptions": {
3 | "args": [
4 | "--no-sandbox",
5 | "--headless=new",
6 | "--use-angle=metal",
7 | "--enable-features=Metal",
8 | "--enable-unsafe-webgpu"
9 | ]
10 | }
11 | }
12 |
--------------------------------------------------------------------------------
/config/webdriver-win.json:
--------------------------------------------------------------------------------
1 | {
2 | "goog:chromeOptions": {
3 | "args": [
4 | "--no-sandbox",
5 | "--headless=new",
6 | "--use-angle=d3d12",
7 | "--enable-features=D3D12",
8 | "--enable-unsafe-webgpu"
9 | ]
10 | }
11 | }
12 |
--------------------------------------------------------------------------------
/crates/ratchet-cli/Cargo.toml:
--------------------------------------------------------------------------------
1 | [package]
2 | name = "ratchet-cli"
3 | version = "0.1.0"
4 | edition = "2021"
5 |
6 | [[bin]]
7 | name = "ratchet"
8 | path = "src/bin/cli.rs"
9 |
10 | [dependencies]
11 | ratchet = { path = "../ratchet-core" }
12 | ratchet-loader = { path = "../ratchet-loader" }
13 | ratchet-models = { path = "../ratchet-models" }
14 | ratchet-hub = { path = "../ratchet-hub" }
15 | ratchet-nn = { path = "../ratchet-nn" }
16 | log.workspace = true
17 | clap = { workspace = true, features = ["derive"] }
18 | hf-hub = { workspace = true }
19 | serde_json = { workspace = true }
20 | env_logger = { workspace = true }
21 | fern = { workspace = true }
22 | chrono = { workspace = true }
23 | tokenizers = { workspace = true }
24 | ndarray = { workspace = true }
25 | ndarray-stats = { workspace = true }
26 | anyhow.workspace = true
27 |
--------------------------------------------------------------------------------
/crates/ratchet-cli/src/lib.rs:
--------------------------------------------------------------------------------
1 |
2 |
--------------------------------------------------------------------------------
/crates/ratchet-core/Cargo.toml:
--------------------------------------------------------------------------------
1 | [package]
2 | name = "ratchet"
3 | version = "0.1.0"
4 | edition = "2021"
5 |
6 | [features]
7 | default = ["rand", "testing"]
8 | gpu-profiling = ["dep:tabled", "dep:itertools"]
9 | rand = ["dep:rand", "dep:rand_distr"]
10 | plotting = ["dep:dot3", "dep:tempfile"]
11 | testing = ["dep:npyz", "dep:ndarray"]
12 | pyo3 = ["dep:pyo3", "dep:numpy", "dep:regex"]
13 | debug = [] #dump every node
14 |
15 | [dependencies]
16 | ratchet-macros = { path = "../ratchet-macros" }
17 | inline-wgsl = { git = "https://github.com/FL33TW00D/inline-wgsl.git", branch = "master" }
18 | wgpu = { workspace = true }
19 | bytemuck = { workspace = true }
20 | half = { workspace = true }
21 | derive-new = { workspace = true }
22 | num-traits = { workspace = true }
23 | log = { workspace = true }
24 | thiserror = { workspace = true }
25 | serde = { workspace = true, features = ["derive"] }
26 | anyhow.workspace = true
27 |
28 | rustc-hash = { workspace = true }
29 | slotmap = { workspace = true }
30 | parking_lot = { workspace = true }
31 | smallvec = { workspace = true }
32 | encase = { workspace = true, features = ["smallvec", "glam"] }
33 | pollster = { workspace = true }
34 | getrandom = { workspace = true, features = [
35 | "js",
36 | ] } # Needed for wasm support in `num` trait
37 | num = { workspace = true }
38 | rand_distr = { workspace = true, optional = true }
39 | rand = { workspace = true, optional = true }
40 | glam = { workspace = true }
41 | npyz = { workspace = true, optional = true }
42 | ndarray = { workspace = true, optional = true }
43 |
44 | strum = { workspace = true }
45 | strum_macros = { workspace = true }
46 |
47 | #Plotting
48 | dot3 = { workspace = true, optional = true }
49 | tempfile = { workspace = true, optional = true }
50 |
51 | # Profiling
52 | tabled = { workspace = true, optional = true }
53 | itertools = { workspace = true, optional = true }
54 |
55 | pyo3 = { workspace = true, features = ["auto-initialize"], optional = true }
56 | regex = { workspace = true, optional = true }
57 | numpy = { workspace = true, optional = true, features = ["half"] }
58 | gemm = { version = "0.18.0", features = ["nightly", "wasm-simd128-enable"] }
59 |
60 | [target.'cfg(target_arch = "wasm32")'.dependencies]
61 | wasm-bindgen.workspace = true
62 | futures-intrusive.workspace = true
63 | wasm-bindgen-futures.workspace = true
64 |
65 | async-trait = "0.1.77"
66 | smallvec = { workspace = true, features = ["serde"] }
67 |
68 | [dev-dependencies]
69 | env_logger = { workspace = true }
70 | rand = { workspace = true }
71 | test-strategy = { workspace = true }
72 | ndarray = { workspace = true }
73 | proptest = { workspace = true }
74 |
--------------------------------------------------------------------------------
/crates/ratchet-core/src/compiled_op.rs:
--------------------------------------------------------------------------------
1 | use crate::gpu::{
2 | BindGroupDescriptor, BindGroupLayoutHandle, ComputePipelineHandle, GpuBindGroup, WgpuDevice,
3 | WorkgroupCount,
4 | };
5 | use crate::{drvec, rvec, KernelKey, OperationError, RVec, Tensor};
6 | use derive_new::new;
7 | use wgpu::DynamicOffset;
8 |
9 | //Compiled op represents a single kernel invocation
10 | //TODO: We need to be more general here, enum with encoder.copy_buffer_to_buffer as a COPY, and
11 | //compiledOp as compute
12 | #[derive(Debug, new)]
13 | pub struct CompiledOp {
14 | pipeline_handle: ComputePipelineHandle,
15 | workgroup_count: WorkgroupCount,
16 | storage_groups: RVec