├── VERSION ├── bindings ├── python │ ├── noop.c │ ├── sqlite_vss │ │ ├── version.py │ │ └── __init__.py │ ├── version.py.tmpl │ ├── setup.py │ ├── README.md │ └── .gitignore ├── elixir │ ├── VERSION │ ├── test │ │ ├── test_helper.exs │ │ └── sqlite_vss_test.exs │ ├── lib │ │ ├── mix │ │ │ └── tasks │ │ │ │ └── sqlite_vss.install.ex │ │ └── sqlite_vss.ex │ ├── sqlite-vss-checksum.exs.tmpl │ ├── demo │ │ └── demo.exs │ ├── mix.exs │ ├── README.md │ └── mix.lock ├── ruby │ ├── .gitignore │ ├── Rakefile │ ├── lib │ │ ├── version.rb │ │ ├── version.rb.tmpl │ │ └── sqlite_vss.rb │ ├── Gemfile │ └── sqlite_vss.gemspec ├── rust │ ├── .gitignore │ ├── Cargo.toml │ ├── Cargo.toml.tmpl │ ├── src │ │ └── lib.rs │ └── build.rs ├── node │ ├── sqlite-vss-darwin-x64 │ │ ├── lib │ │ │ └── .gitkeep │ │ ├── package.json │ │ └── README.md │ ├── sqlite-vss-linux-x64 │ │ ├── lib │ │ │ └── .gitkeep │ │ ├── package.json │ │ └── README.md │ ├── sqlite-vss-darwin-arm64 │ │ ├── lib │ │ │ └── .gitkeep │ │ ├── package.json │ │ └── README.md │ ├── README.md │ ├── platform-package.package.json.tmpl │ ├── sqlite-vss │ │ ├── src │ │ │ ├── index.d.ts │ │ │ └── index.js │ │ ├── package.json │ │ ├── package.json.tmpl │ │ ├── test.js │ │ └── README.md │ ├── platform-package.README.md.tmpl │ └── .gitignore ├── datasette │ ├── version.py.tmpl │ ├── datasette_sqlite_vss │ │ ├── version.py │ │ └── __init__.py │ ├── README.md │ ├── tests │ │ └── test_sqlite_vector.py │ ├── setup.py │ └── .gitignore ├── sqlite-utils │ ├── sqlite_utils_sqlite_vss │ │ ├── version.py │ │ ├── version.py.tmpl │ │ └── __init__.py │ ├── README.md │ ├── pyproject.toml │ ├── pyproject.toml.tmpl │ └── .gitignore ├── deno │ ├── deno.json │ ├── deno.json.tmpl │ ├── test.ts │ ├── README.md │ ├── README.md.tmpl │ ├── mod.ts │ └── deno.lock └── go │ ├── lib.go │ ├── README.md │ ├── vss │ ├── sqlite-vss.h │ └── vss.go │ ├── vector │ ├── vector.go │ └── sqlite-vector.h │ └── go.mod ├── examples ├── sift │ ├── README.md │ ├── .gitignore │ └── build.sql ├── go │ ├── .gitignore │ ├── Makefile │ ├── README.md │ ├── go.mod │ ├── go.sum │ └── demo.go ├── c │ ├── .gitignore │ ├── README.md │ ├── Makefile │ └── demo.c ├── rust │ ├── .gitignore │ ├── build.rs │ ├── Cargo.toml │ └── demo.rs ├── headlines │ ├── .dockerignore │ ├── build.sh │ ├── plugins │ │ └── plugin.py │ ├── bench │ │ ├── query-flat.sql │ │ └── query-ivfflat.sql │ ├── build │ │ ├── import_articles.sql │ │ ├── add_vector_search.sql │ │ └── add_embeddings.py │ ├── fly.toml │ ├── Dockerfile │ └── README.md ├── go-cybertron │ ├── .gitignore │ ├── go.mod │ └── demo-cybertron.go ├── simonw_blog │ ├── up.sh │ ├── README.md │ └── build.sql └── mcu-wiki │ ├── build │ ├── add_vss.sql │ ├── extract_plots.ts │ ├── build.sql │ └── add_embeddings.py │ └── README.md ├── site ├── see-also.md ├── using │ ├── turso.md │ ├── loadable.md │ ├── rust.md │ ├── elixir.md │ ├── ruby.md │ ├── datasette.md │ ├── go.md │ ├── nodejs.md │ ├── deno.md │ └── python.md ├── .gitignore ├── guides │ ├── cohere.md │ ├── openai-python.md │ ├── openai-to-sqlite.md │ ├── huggingface.md │ └── sentence-transformers-python.md ├── hero.png ├── demo_base.png ├── demo_q1.png ├── demo_q2.png ├── demo_base_dark.png ├── .vitepress │ ├── theme │ │ ├── index.ts │ │ └── style.css │ └── config.ts ├── public │ └── favicon.svg ├── project.data.ts ├── package.json ├── index.md ├── compare.md ├── getting-started.md └── building-source.md ├── .github ├── FUNDING.yml └── workflows │ ├── site.yaml │ ├── upload-deno-assets.js │ ├── upload.js │ └── test.yaml ├── benchmarks ├── sift1m │ ├── bench.sh │ ├── cleanup.sh │ ├── load-flat.sql │ ├── build.sql │ ├── README.md │ ├── load-ivfflat.sql │ ├── query-ivfflat.sql │ └── query-flat.sql └── gist1m │ └── build.sql ├── .gitmodules ├── vendor └── get_sqlite.sh ├── scripts ├── deno_generate_package.sh ├── publish_release.sh ├── elixir_generate_checksum.sh ├── npm_generate_platform_packages.sh ├── rename-wheels.py └── openmp_static.patch ├── .gitignore ├── tests ├── np.py └── test-python.py ├── src ├── sqlite-vss.h.in └── sqlite-vector.h.in ├── LICENSE ├── NOTES └── CMakeLists.txt /VERSION: -------------------------------------------------------------------------------- 1 | 0.1.2 -------------------------------------------------------------------------------- /bindings/python/noop.c: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/sift/README.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /bindings/elixir/VERSION: -------------------------------------------------------------------------------- 1 | 0.1.2 -------------------------------------------------------------------------------- /examples/go/.gitignore: -------------------------------------------------------------------------------- 1 | demo -------------------------------------------------------------------------------- /examples/c/.gitignore: -------------------------------------------------------------------------------- 1 | demo 2 | -------------------------------------------------------------------------------- /examples/sift/.gitignore: -------------------------------------------------------------------------------- 1 | data/ -------------------------------------------------------------------------------- /site/see-also.md: -------------------------------------------------------------------------------- 1 | # See also 2 | -------------------------------------------------------------------------------- /bindings/ruby/.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | -------------------------------------------------------------------------------- /bindings/rust/.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | -------------------------------------------------------------------------------- /examples/rust/.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: [asg017] 2 | -------------------------------------------------------------------------------- /bindings/node/sqlite-vss-darwin-x64/lib/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /bindings/node/sqlite-vss-linux-x64/lib/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /bindings/node/sqlite-vss-darwin-arm64/lib/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /site/using/turso.md: -------------------------------------------------------------------------------- 1 | # `sqlite-vss` with Turso 2 | -------------------------------------------------------------------------------- /bindings/elixir/test/test_helper.exs: -------------------------------------------------------------------------------- 1 | ExUnit.start() 2 | -------------------------------------------------------------------------------- /examples/headlines/.dockerignore: -------------------------------------------------------------------------------- 1 | *.db 2 | *.json 3 | venv/ -------------------------------------------------------------------------------- /site/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .vitepress/cache 3 | .vitepress/dist -------------------------------------------------------------------------------- /site/guides/cohere.md: -------------------------------------------------------------------------------- 1 | # Using `sqlite-vss` with Cohere's Embeddings API 2 | -------------------------------------------------------------------------------- /site/guides/openai-python.md: -------------------------------------------------------------------------------- 1 | # Using `sqlite-vss` with OpenAI's Python API 2 | -------------------------------------------------------------------------------- /site/hero.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mthli/sqlite-vss/main/site/hero.png -------------------------------------------------------------------------------- /bindings/ruby/Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | task :default => :spec 3 | -------------------------------------------------------------------------------- /site/guides/openai-to-sqlite.md: -------------------------------------------------------------------------------- 1 | # Using `sqlite-vss` with `openai-to-sqlite` 2 | -------------------------------------------------------------------------------- /site/demo_base.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mthli/sqlite-vss/main/site/demo_base.png -------------------------------------------------------------------------------- /site/demo_q1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mthli/sqlite-vss/main/site/demo_q1.png -------------------------------------------------------------------------------- /site/demo_q2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mthli/sqlite-vss/main/site/demo_q2.png -------------------------------------------------------------------------------- /site/guides/huggingface.md: -------------------------------------------------------------------------------- 1 | # Using `sqlite-vss` with the Huggingface Inference API 2 | -------------------------------------------------------------------------------- /examples/go-cybertron/.gitignore: -------------------------------------------------------------------------------- 1 | models/ 2 | demo-cybertron 3 | sentence-transformers 4 | -------------------------------------------------------------------------------- /site/demo_base_dark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mthli/sqlite-vss/main/site/demo_base_dark.png -------------------------------------------------------------------------------- /site/guides/sentence-transformers-python.md: -------------------------------------------------------------------------------- 1 | # Using `sqlite-vss` with Sentence Transformers in Python 2 | -------------------------------------------------------------------------------- /bindings/datasette/version.py.tmpl: -------------------------------------------------------------------------------- 1 | __version__ = "${VERSION}" 2 | __version_info__ = tuple(__version__.split(".")) 3 | -------------------------------------------------------------------------------- /bindings/python/sqlite_vss/version.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.1.2" 2 | __version_info__ = tuple(__version__.split(".")) 3 | -------------------------------------------------------------------------------- /bindings/python/version.py.tmpl: -------------------------------------------------------------------------------- 1 | __version__ = "${VERSION}" 2 | __version_info__ = tuple(__version__.split(".")) 3 | -------------------------------------------------------------------------------- /bindings/datasette/datasette_sqlite_vss/version.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.1.2" 2 | __version_info__ = tuple(__version__.split(".")) 3 | -------------------------------------------------------------------------------- /bindings/ruby/lib/version.rb: -------------------------------------------------------------------------------- 1 | # automatically generated, do not edit by hand. 2 | module SqliteVss 3 | VERSION = "0.1.2" 4 | end 5 | -------------------------------------------------------------------------------- /bindings/sqlite-utils/sqlite_utils_sqlite_vss/version.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.1.2" 2 | __version_info__ = tuple(__version__.split(".")) 3 | -------------------------------------------------------------------------------- /bindings/ruby/lib/version.rb.tmpl: -------------------------------------------------------------------------------- 1 | # automatically generated, do not edit by hand. 2 | module SqliteVss 3 | VERSION = "${VERSION}" 4 | end 5 | -------------------------------------------------------------------------------- /bindings/sqlite-utils/README.md: -------------------------------------------------------------------------------- 1 | # `sqlite-utils-sqlite-vss` 2 | 3 | A `sqlite-utils` plugin that registers the `sqlite-vss` extension. 4 | -------------------------------------------------------------------------------- /bindings/sqlite-utils/sqlite_utils_sqlite_vss/version.py.tmpl: -------------------------------------------------------------------------------- 1 | __version__ = "${VERSION}" 2 | __version_info__ = tuple(__version__.split(".")) 3 | -------------------------------------------------------------------------------- /bindings/ruby/Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | git_source(:github) {|repo_name| "https://github.com/#{repo_name}" } 4 | 5 | gemspec 6 | -------------------------------------------------------------------------------- /examples/go/Makefile: -------------------------------------------------------------------------------- 1 | 2 | 3 | CGO_LDFLAGS="-L$(SQLITE_VSS_LIB_DIR)" 4 | 5 | demo: demo.go 6 | CGO_LDFLAGS="$(CGO_LDFLAGS)" \ 7 | go build -o $@ 8 | -------------------------------------------------------------------------------- /benchmarks/sift1m/bench.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | hyperfine --cleanup=./cleanup.sh \ 4 | 'sqlite3 test.db ".read load-flat.sql"' \ 5 | 'sqlite3 test.db ".read load-ivfflat.sql"' -------------------------------------------------------------------------------- /benchmarks/sift1m/cleanup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | sqlite3 test.db '.load ./vector0' '.load ./vss0' \ 3 | 'drop table if exists x_flat;' \ 4 | 'drop table if exists x_ivfflat;' -------------------------------------------------------------------------------- /benchmarks/gist1m/build.sql: -------------------------------------------------------------------------------- 1 | create table gist as 2 | select 3 | rowid, vector_to_blob(vector) as vector 4 | from vector_fvecs_each(readfile('../../examples/sift/data/gist/gist_base.fvecs')); 5 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "vendor/faiss"] 2 | path = vendor/faiss 3 | url = https://github.com/facebookresearch/faiss 4 | [submodule "vendor/json"] 5 | path = vendor/json 6 | url = https://github.com/nlohmann/json.git 7 | -------------------------------------------------------------------------------- /bindings/deno/deno.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sqlite-vss", 3 | "version": "0.1.2", 4 | "github": "https://github.com/asg017/sqlite-vss", 5 | "tasks": { 6 | "test": "deno test --unstable -A test.ts" 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /examples/headlines/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | DB=$1 4 | 5 | sqlite3 $DB '.read build/import_articles.sql' 6 | python3 build/add_embeddings.py $DB 7 | sqlite3 $DB '.read build/add_vector_search.sql' 8 | -------------------------------------------------------------------------------- /bindings/deno/deno.json.tmpl: -------------------------------------------------------------------------------- 1 | { 2 | "name": "${PACKAGE_NAME}", 3 | "version": "${VERSION}", 4 | "github": "https://github.com/asg017/${PACKAGE_NAME}", 5 | "tasks": { 6 | "test": "deno test --unstable -A test.ts" 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /site/using/loadable.md: -------------------------------------------------------------------------------- 1 | # `sqlite-vss` as a Loadable Extension 2 | 3 | ![GitHub tag (latest SemVer pre-release)](https://img.shields.io/github/v/tag/asg017/sqlite-vss?color=lightgrey&include_prereleases&label=Github+release&logo=github) 4 | -------------------------------------------------------------------------------- /benchmarks/sift1m/load-flat.sql: -------------------------------------------------------------------------------- 1 | .bail on 2 | 3 | .load ./vector0 4 | .load ./vss0 5 | .timer on 6 | 7 | create virtual table x_flat using vss0(v(128)); 8 | 9 | -- 10s 10 | insert into x_flat(rowid, v) 11 | select rowid, vector from sift; -------------------------------------------------------------------------------- /examples/simonw_blog/up.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | LD_LIBRARY_PATH=/root/tmp/py/sqlite-snapshot-202302081247/.libs datasette test.db --host 0.0.0.0 \ 3 | --load-extension=../../../sqlite-vector/build_release/vector0 \ 4 | --load-extension=../../build_release/vss0 5 | -------------------------------------------------------------------------------- /bindings/datasette/datasette_sqlite_vss/__init__.py: -------------------------------------------------------------------------------- 1 | from datasette import hookimpl 2 | import sqlite_vss 3 | 4 | @hookimpl 5 | def prepare_connection(conn): 6 | conn.enable_load_extension(True) 7 | sqlite_vss.load(conn) 8 | conn.enable_load_extension(False) -------------------------------------------------------------------------------- /examples/go/README.md: -------------------------------------------------------------------------------- 1 | # `sqlite-vss` in Go example 2 | 3 | To run: 4 | 5 | ``` 6 | make demo SQLITE_VSS_LIB_DIR=../../dist/debug 7 | ./demo 8 | ``` 9 | 10 | Where `SQLITE_VSS_LIB_DIR` is a path to a directory containing pre-compiled `sqlite-vss` static files. 11 | -------------------------------------------------------------------------------- /bindings/go/lib.go: -------------------------------------------------------------------------------- 1 | // asdflakdjf 2 | package vss 3 | 4 | import ( 5 | vector "github.com/asg017/sqlite-vss/bindings/go/vector" 6 | vss "github.com/asg017/sqlite-vss/bindings/go/vss" 7 | ) 8 | 9 | // please?? 10 | func init() { 11 | vector.Auto() 12 | vss.Auto() 13 | } 14 | -------------------------------------------------------------------------------- /examples/c/README.md: -------------------------------------------------------------------------------- 1 | # `sqlite-vss` in C Example 2 | 3 | To build: 4 | 5 | ``` 6 | make demo 7 | ``` 8 | 9 | On Linux machines you'll need to link to many other libraries, which you can do with: 10 | 11 | ``` 12 | make demo LDFLAGS="-lgomp -latlas -lblas -llapack -lm -lstdc++" 13 | ``` 14 | -------------------------------------------------------------------------------- /examples/rust/build.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | if cfg!(target_os = "macos") { 3 | println!("cargo:rustc-link-arg=-Wl,-undefined,dynamic_lookup,-lomp"); 4 | } else if cfg!(target_os = "linux") { 5 | println!("cargo:rustc-link-arg=-Wl,-undefined,dynamic_lookup,-lstdc++"); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /vendor/get_sqlite.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | VENDOR_DIR=$(dirname "$0") 5 | 6 | wget https://www.sqlite.org/2022/sqlite-autoconf-3400100.tar.gz -O $VENDOR_DIR/sqlite.tar.gz 7 | tar -xvzf $VENDOR_DIR/sqlite.tar.gz 8 | rm $VENDOR_DIR/sqlite.tar.gz 9 | mv sqlite-autoconf-3400100/ $VENDOR_DIR/sqlite -------------------------------------------------------------------------------- /examples/go/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/asg017/sqlite-vss/examples/go 2 | 3 | go 1.20 4 | 5 | require github.com/mattn/go-sqlite3 v1.14.17 6 | 7 | require github.com/asg017/sqlite-vss/bindings/go v0.0.0-20230529051321-8bd578f11697 8 | 9 | replace github.com/asg017/sqlite-vss/bindings/go => ../../bindings/go 10 | -------------------------------------------------------------------------------- /examples/rust/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "demo" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | rusqlite = {version="0.29.0", features=["bundled"]} 8 | sqlite-vss = { path = "../../bindings/rust", features=["download-libs"] } 9 | 10 | [[bin]] 11 | name="demo" 12 | path="demo.rs" 13 | -------------------------------------------------------------------------------- /benchmarks/sift1m/build.sql: -------------------------------------------------------------------------------- 1 | 2 | .load ./vector0 3 | 4 | create table sift as 5 | select 6 | rowid, vector_to_blob(vector) as vector 7 | from vector_fvecs_each(readfile('../../examples/sift/data/sift/sift_base.fvecs')); 8 | 9 | 10 | --select rowid, vector_length(vector_from_blob(vector)) from sift; 11 | -------------------------------------------------------------------------------- /bindings/elixir/lib/mix/tasks/sqlite_vss.install.ex: -------------------------------------------------------------------------------- 1 | defmodule Mix.Tasks.SqliteVss.Install do 2 | @shortdoc "Installs SqliteVss extension files" 3 | use Mix.Task 4 | 5 | @impl true 6 | def run(_args) do 7 | Application.ensure_all_started(:sqlite_vss) 8 | 9 | SqliteVss.install() 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /bindings/elixir/sqlite-vss-checksum.exs.tmpl: -------------------------------------------------------------------------------- 1 | %{ 2 | "${LOADABLE_ASSET_LINUX_X86_64_NAME}" => "sha256:${LOADABLE_ASSET_LINUX_X86_64_SHA256}", 3 | "${LOADABLE_ASSET_MACOS_X86_64_NAME}" => "sha256:${LOADABLE_ASSET_MACOS_X86_64_SHA256}", 4 | "${LOADABLE_ASSET_MACOS_AARCH64_NAME}" => "sha256:${LOADABLE_ASSET_MACOS_AARCH64_SHA256}", 5 | } 6 | -------------------------------------------------------------------------------- /benchmarks/sift1m/README.md: -------------------------------------------------------------------------------- 1 | 2 | http://corpus-texmex.irisa.fr/ 3 | 4 | sift_base.fvecs 516 MB 5 | 6 | select name, sum(pgsize) from dbstat group by 1; 7 | 8 | sift|586616832 -> 586.6MB 9 | x_flat_data|8024064 -> 8MB 10 | x_flat_index|520511488 -> 520.5MB 11 | 12 | x_ivfflat_data|8024064 -> 8 MB 13 | x_ivfflat_index|529584128 -> 529.5MB -------------------------------------------------------------------------------- /bindings/sqlite-utils/sqlite_utils_sqlite_vss/__init__.py: -------------------------------------------------------------------------------- 1 | from sqlite_utils import hookimpl 2 | import sqlite_vss 3 | 4 | from sqlite_utils_sqlite_vss.version import __version_info__, __version__ 5 | 6 | 7 | @hookimpl 8 | def prepare_connection(conn): 9 | conn.enable_load_extension(True) 10 | sqlite_vss.load(conn) 11 | conn.enable_load_extension(False) 12 | -------------------------------------------------------------------------------- /examples/go/go.sum: -------------------------------------------------------------------------------- 1 | github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= 2 | github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= 3 | github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM= 4 | github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= 5 | -------------------------------------------------------------------------------- /site/using/rust.md: -------------------------------------------------------------------------------- 1 | # `sqlite-vss` with Rust 2 | 3 | [![Crates.io](https://img.shields.io/crates/v/sqlite-vss?logo=rust)](https://crates.io/crates/sqlite-vss) 4 | 5 | ::: warning 6 | The Rust bindings for `sqlite-vss` are still in beta and are subject to change. If you come across problems, [please comment on the Rust tracking issue](https://github.com/asg017/sqlite-vss/issues/50). 7 | ::: 8 | -------------------------------------------------------------------------------- /examples/c/Makefile: -------------------------------------------------------------------------------- 1 | # -L/usr/local/opt/sqlite/lib 2 | # /usr/local/opt/sqlite/lib/libsqlite3.a 3 | demo: demo.c 4 | gcc $< \ 5 | -Os \ 6 | -I../../dist/debug \ 7 | -L../../dist/debug \ 8 | -L/usr/local/opt/sqlite/lib \ 9 | -DSQLITE_CORE \ 10 | -Wl,-undefined,dynamic_lookup \ 11 | -lsqlite3 -lsqlite_vector0 -lsqlite_vss0 -lfaiss_avx2 -lomp \ 12 | $(LDFLAGS) \ 13 | -o $@ 14 | -------------------------------------------------------------------------------- /site/using/elixir.md: -------------------------------------------------------------------------------- 1 | # `sqlite-vss` with Elixir 2 | 3 | [![Hex.pm](https://img.shields.io/hexpm/v/sqlite_vss?color=purple&logo=elixir)](https://hex.pm/packages/sqlite_vss) 4 | 5 | ::: warning 6 | The Elixir bindings for `sqlite-vss` are still in beta and are subject to change. If you come across problems, [please comment on the Elixir tracking issue](https://github.com/asg017/sqlite-vss/issues/51). 7 | ::: 8 | -------------------------------------------------------------------------------- /benchmarks/sift1m/load-ivfflat.sql: -------------------------------------------------------------------------------- 1 | .bail on 2 | 3 | .load ./vector0 4 | .load ./vss0 5 | .timer on 6 | 7 | create virtual table x_ivfflat using vss0(v(128) factory="IVF4096,Flat,IDMap2"); 8 | 9 | -- 2048: 146s, 4096: 550s 10 | insert into x_ivfflat(operation, rowid, v) 11 | select 'training', rowid, vector from sift; 12 | 13 | -- 2048: 42s, 4096: 68s 14 | insert into x_ivfflat(rowid, v) 15 | select rowid, vector from sift; -------------------------------------------------------------------------------- /scripts/deno_generate_package.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | export PACKAGE_NAME="sqlite-vss" 6 | export EXTENSION_NAME="vss0" 7 | export VERSION=$(cat VERSION) 8 | 9 | envsubst < bindings/deno/deno.json.tmpl > bindings/deno/deno.json 10 | echo "✅ generated bindings/deno/deno.json" 11 | 12 | envsubst < bindings/deno/README.md.tmpl > bindings/deno/README.md 13 | echo "✅ generated bindings/deno/README.md" 14 | -------------------------------------------------------------------------------- /examples/headlines/plugins/plugin.py: -------------------------------------------------------------------------------- 1 | import json 2 | from datasette import hookimpl 3 | 4 | from sentence_transformers import SentenceTransformer 5 | model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2") 6 | 7 | def st_encode(x): 8 | result = model.encode([x]) 9 | return json.dumps(result.tolist()[0]) 10 | 11 | @hookimpl 12 | def prepare_connection(conn): 13 | conn.create_function('st_encode', 1, st_encode) -------------------------------------------------------------------------------- /site/.vitepress/theme/index.ts: -------------------------------------------------------------------------------- 1 | // https://vitepress.dev/guide/custom-theme 2 | import { h } from "vue"; 3 | import Theme from "vitepress/theme"; 4 | import "./style.css"; 5 | 6 | export default { 7 | ...Theme, 8 | Layout: () => { 9 | return h(Theme.Layout, null, { 10 | //"home-hero-image": () => h("div", null, "yo"), 11 | }); 12 | }, 13 | enhanceApp({ app, router, siteData }) { 14 | // ... 15 | }, 16 | }; 17 | -------------------------------------------------------------------------------- /examples/mcu-wiki/build/add_vss.sql: -------------------------------------------------------------------------------- 1 | .bail on 2 | .timer on 3 | 4 | .load ./vss0 5 | .load ./vector0 6 | 7 | create virtual table vss_plot_sentences using vss_index( 8 | contents_embedding(384) with "Flat,IDMap2" 9 | ); 10 | 11 | insert into vss_plot_sentences(rowid, contents_embedding) 12 | select 13 | rowid, 14 | vector_from_blob(contents_embedding) 15 | from plot_sentences; 16 | 17 | select count(*) from vss_plot_sentences_index; -------------------------------------------------------------------------------- /bindings/node/README.md: -------------------------------------------------------------------------------- 1 | # sqlite-vss on npm 2 | 3 | `sqlite-vss` is also available for download through [`npm`](https://www.npmjs.com/) for Node.js developers. See the [`sqlite-vss` NPM package README](./sqlite-vss/README.md) for details. 4 | 5 | The other NPM packages in this folder (`sqlite-vss-darwin-x64`, `sqlite-vss-linux-x64` etc.) are autogenerated platform-specific packages. See [Supported Platforms](./sqlite-vss/README.md#supported-platforms) for details. 6 | -------------------------------------------------------------------------------- /site/public/favicon.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /examples/simonw_blog/README.md: -------------------------------------------------------------------------------- 1 | 2 | ```sql 3 | .param set :id 8217 4 | with similar_matches as ( 5 | select rowid, distance 6 | from vss_blog_entry 7 | where vss_search( 8 | body_embedding, 9 | (select body_embedding from blog_entry where id = :id limit 1) 10 | ) 11 | limit 10 12 | ) 13 | select 14 | similar_matches.*, 15 | blog_entry.title, 16 | blog_entry.body 17 | from similar_matches 18 | left join blog_entry on blog_entry.id = similar_matches.rowid; 19 | ``` -------------------------------------------------------------------------------- /site/project.data.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from "vitepress"; 2 | import { readFileSync } from "node:fs"; 3 | import { join, dirname } from "node:path"; 4 | import { fileURLToPath } from "node:url"; 5 | 6 | const PROJECT = "sqlite-vss"; 7 | const VERSION = readFileSync( 8 | join(dirname(fileURLToPath(import.meta.url)), "..", "VERSION"), 9 | "utf8" 10 | ); 11 | 12 | export default { 13 | load() { 14 | return { 15 | PROJECT, 16 | VERSION, 17 | }; 18 | }, 19 | }; 20 | -------------------------------------------------------------------------------- /bindings/node/sqlite-vss-linux-x64/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "//": "Autogenerated by the npm_generate_platform_packages.sh script, do not edit by hand", 3 | "name": "sqlite-vss-linux-x64", 4 | "version": "0.1.2", 5 | "repository": { 6 | "type": "git", 7 | "url": "https://github.com/asg017/sqlite-vss.git", 8 | "directory": "npm/sqlite-vss-linux-x64" 9 | }, 10 | "author": "Alex Garcia ", 11 | "os": [ 12 | "linux" 13 | ], 14 | "cpu": [ 15 | "x64" 16 | ] 17 | } -------------------------------------------------------------------------------- /bindings/node/sqlite-vss-darwin-x64/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "//": "Autogenerated by the npm_generate_platform_packages.sh script, do not edit by hand", 3 | "name": "sqlite-vss-darwin-x64", 4 | "version": "0.1.2", 5 | "repository": { 6 | "type": "git", 7 | "url": "https://github.com/asg017/sqlite-vss.git", 8 | "directory": "npm/sqlite-vss-darwin-x64" 9 | }, 10 | "author": "Alex Garcia ", 11 | "os": [ 12 | "darwin" 13 | ], 14 | "cpu": [ 15 | "x64" 16 | ] 17 | } -------------------------------------------------------------------------------- /bindings/node/sqlite-vss-darwin-arm64/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "//": "Autogenerated by the npm_generate_platform_packages.sh script, do not edit by hand", 3 | "name": "sqlite-vss-darwin-arm64", 4 | "version": "0.1.2", 5 | "repository": { 6 | "type": "git", 7 | "url": "https://github.com/asg017/sqlite-vss.git", 8 | "directory": "npm/sqlite-vss-darwin-arm64" 9 | }, 10 | "author": "Alex Garcia ", 11 | "os": [ 12 | "darwin" 13 | ], 14 | "cpu": [ 15 | "arm64" 16 | ] 17 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | /build 3 | /build_release 4 | News_Category_Dataset_v3.json 5 | *.faissindex 6 | *.db 7 | *.so 8 | *.dylib 9 | *.dll 10 | *.xml 11 | *.zip 12 | *.index 13 | vendor/sqlite/ 14 | vendor/sqlite-snapshot* 15 | vendor_old 16 | 17 | CMakeLists.txt.user 18 | CMakeCache.txt 19 | CMakeFiles 20 | CMakeScripts 21 | Testing 22 | 23 | cmake_install.cmake 24 | install_manifest.txt 25 | compile_commands.json 26 | CTestTestfile.cmake 27 | _deps 28 | 29 | venv/ 30 | dist/ 31 | 32 | deps/ 33 | _build/ 34 | *.parquet 35 | 36 | .cache -------------------------------------------------------------------------------- /bindings/node/platform-package.package.json.tmpl: -------------------------------------------------------------------------------- 1 | { 2 | "//": "Autogenerated by the npm_generate_platform_packages.sh script, do not edit by hand", 3 | "name": "${PACKAGE_NAME}", 4 | "version": "${VERSION}", 5 | "repository": { 6 | "type": "git", 7 | "url": "https://github.com/asg017/${PACKAGE_NAME_BASE}.git", 8 | "directory": "npm/${PACKAGE_NAME}" 9 | }, 10 | "author": "Alex Garcia ", 11 | "os": [ 12 | "${PLATFORM_OS}" 13 | ], 14 | "cpu": [ 15 | "${PLATFORM_ARCH}" 16 | ] 17 | } -------------------------------------------------------------------------------- /examples/headlines/bench/query-flat.sql: -------------------------------------------------------------------------------- 1 | .load ../../../sqlite-vector/build_release/vector0 2 | .load ../../build_release/vss0 3 | .timer on 4 | 5 | select rowid from vss_articles limit 1; 6 | 7 | with similar_matches as ( 8 | select 9 | articles.rowid as base, 10 | vss_articles.rowid 11 | from (select * from articles limit 128) as articles 12 | join vss_articles 13 | where vss_search( 14 | vss_articles.headline_embedding, 15 | vss_search_params(articles.headline_embedding, 128) 16 | ) 17 | ) 18 | select count(*) from similar_matches; -------------------------------------------------------------------------------- /site/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "site", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1", 8 | "dev": "vitepress dev", 9 | "build": "vitepress build", 10 | "preview": "vitepress preview" 11 | }, 12 | "keywords": [], 13 | "author": "", 14 | "license": "ISC", 15 | "devDependencies": { 16 | "@types/node": "^20.0.0", 17 | "vitepress": "^1.0.0-beta.1" 18 | }, 19 | "dependencies": { 20 | "markdown-it-container": "^3.0.0" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /benchmarks/sift1m/query-ivfflat.sql: -------------------------------------------------------------------------------- 1 | .bail on 2 | 3 | .load ./vector0 4 | .load ./vss0 5 | .timer on 6 | 7 | select rowid from x_ivfflat limit 1; 8 | 9 | 10 | -- 1ms 11 | select count(*) 12 | from x_ivfflat 13 | where vss_search( 14 | v, 15 | vss_search_params( 16 | vector_from_blob((select vector from sift where rowid = 5 limit 1)), 17 | 128 18 | ) 19 | ); 20 | 21 | -- 26ms 22 | select count(*) 23 | from (select vector from sift limit 128) as x 24 | join x_ivfflat 25 | where vss_search( 26 | v, 27 | vss_search_params( 28 | vector, 29 | 128 30 | ) 31 | ); -------------------------------------------------------------------------------- /bindings/ruby/lib/sqlite_vss.rb: -------------------------------------------------------------------------------- 1 | require "version" 2 | 3 | module SqliteVss 4 | class Error < StandardError; end 5 | def self.vss_loadable_path 6 | File.expand_path('../vss0', __FILE__) 7 | end 8 | def self.vector_loadable_path 9 | File.expand_path('../vector0', __FILE__) 10 | end 11 | def self.load_vector(db) 12 | db.load_extension(self.vector_loadable_path) 13 | end 14 | def self.load_vss(db) 15 | db.load_extension(self.vss_loadable_path) 16 | end 17 | def self.load(db) 18 | self.load_vector(db) 19 | self.load_vss(db) 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /bindings/elixir/demo/demo.exs: -------------------------------------------------------------------------------- 1 | Mix.install([ 2 | {:sqlite_vss, path: "../"}, 3 | {:exqlite, "~> 0.13.0"} 4 | ], verbose: true) 5 | 6 | Mix.Task.run("sqlite_vss.install") 7 | 8 | alias Exqlite.Basic 9 | 10 | {:ok, conn} = Basic.open("example.db") 11 | 12 | :ok = Exqlite.Basic.enable_load_extension(conn) 13 | Exqlite.Basic.load_extension(conn, SqliteVss.loadable_path_vector0()) 14 | Exqlite.Basic.load_extension(conn, SqliteVss.loadable_path_vss0()) 15 | 16 | {:ok, [[version]], [_]} = Basic.exec(conn, "select vss_version()") |> Basic.rows() 17 | 18 | IO.puts("version: #{version}") 19 | -------------------------------------------------------------------------------- /benchmarks/sift1m/query-flat.sql: -------------------------------------------------------------------------------- 1 | .bail on 2 | 3 | .load ./vector0 4 | .load ./vss0 5 | .timer on 6 | 7 | select rowid from x_flat limit 1; 8 | 9 | -- point, k=128 on nq=1 80ms 10 | select count(*) 11 | from x_flat 12 | where vss_search( 13 | v, 14 | vss_search_params( 15 | (select vector from sift where rowid = 5 limit 1), 16 | 128 17 | ) 18 | ); 19 | 20 | 21 | -- many, k=128 on nq=128, 11s! 22 | select count(*) 23 | from (select vector from sift limit 128) as x 24 | join x_flat 25 | where vss_search( 26 | v, 27 | vss_search_params( 28 | vector, 29 | 128 30 | ) 31 | ); -------------------------------------------------------------------------------- /examples/headlines/bench/query-ivfflat.sql: -------------------------------------------------------------------------------- 1 | .load ../../../sqlite-vector/build_release/vector0 2 | .load ../../build_release/vss0 3 | .timer on 4 | 5 | select rowid from vss_articles limit 1; 6 | 7 | with similar_matches as ( 8 | select 9 | articles.rowid as base, 10 | vss_ivf_articles.rowid 11 | from (select rowid, headline_embedding from articles limit 128) as articles 12 | join vss_ivf_articles 13 | where vss_search( 14 | vss_ivf_articles.headline_embedding, 15 | vss_search_params(articles.headline_embedding, 128) 16 | ) 17 | ) 18 | select count(*) from similar_matches; -------------------------------------------------------------------------------- /tests/np.py: -------------------------------------------------------------------------------- 1 | # https://github.com/facebookresearch/faiss/blob/74ee67aefc21859e02388f32f4876a0cbd21cabd/contrib/vecs_io.py#L20 2 | 3 | import numpy as np 4 | 5 | def ivecs_read(fname): 6 | a = np.fromfile(fname, dtype='int32') 7 | d = a[0] 8 | return a.reshape(-1, d + 1)[:, 1:].copy() 9 | 10 | 11 | def fvecs_read(fname): 12 | return ivecs_read(fname).view('float32') 13 | 14 | if __name__ == "__main__": 15 | fv = fvecs_read('tests/data/siftsmall/siftsmall_base.fvecs') 16 | print(fv.shape) 17 | print(fv[0]) 18 | print(fv[1]) 19 | for i, row in enumerate(fv[:10]): 20 | print(i, sum(row)) -------------------------------------------------------------------------------- /bindings/sqlite-utils/pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "sqlite-utils-sqlite-vss" 3 | version = "0.1.2" 4 | description = "TODO" 5 | readme = "README.md" 6 | authors = [{name = "Alex Garcia"}] 7 | license = {text = "Apache-2.0"} 8 | classifiers = [] 9 | 10 | dependencies = [ 11 | "sqlite-utils", 12 | "sqlite-vss" 13 | ] 14 | 15 | [project.urls] 16 | Homepage = "https://github.com/asg017/sqlite-vss" 17 | Changelog = "https://github.com/asg017/sqlite-vss/releases" 18 | Issues = "https://github.com/asg017/sqlite-vss/issues" 19 | 20 | [project.entry-points.sqlite_utils] 21 | sqlite_vss = "sqlite_utils_sqlite_vss" 22 | -------------------------------------------------------------------------------- /bindings/sqlite-utils/pyproject.toml.tmpl: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "sqlite-utils-sqlite-vss" 3 | version = "${VERSION}" 4 | description = "TODO" 5 | readme = "README.md" 6 | authors = [{name = "Alex Garcia"}] 7 | license = {text = "Apache-2.0"} 8 | classifiers = [] 9 | 10 | dependencies = [ 11 | "sqlite-utils", 12 | "sqlite-vss" 13 | ] 14 | 15 | [project.urls] 16 | Homepage = "https://github.com/asg017/sqlite-vss" 17 | Changelog = "https://github.com/asg017/sqlite-vss/releases" 18 | Issues = "https://github.com/asg017/sqlite-vss/issues" 19 | 20 | [project.entry-points.sqlite_utils] 21 | sqlite_vss = "sqlite_utils_sqlite_vss" 22 | -------------------------------------------------------------------------------- /bindings/deno/test.ts: -------------------------------------------------------------------------------- 1 | import * as sqlite_vss from "./mod.ts"; 2 | import meta from "./deno.json" assert { type: "json" }; 3 | 4 | import { assertEquals } from "https://deno.land/std@0.177.0/testing/asserts.ts"; 5 | import { Database } from "https://deno.land/x/sqlite3@0.8.0/mod.ts"; 6 | 7 | Deno.test("x/sqlite3", (t) => { 8 | const db = new Database(":memory:"); 9 | 10 | db.enableLoadExtension = true; 11 | sqlite_vss.load(db); 12 | 13 | const [version] = db.prepare("select vss_version()").value<[string]>()!; 14 | 15 | assertEquals(version[0], "v"); 16 | assertEquals(version.substring(1), meta.version); 17 | 18 | db.close(); 19 | }); 20 | -------------------------------------------------------------------------------- /examples/mcu-wiki/README.md: -------------------------------------------------------------------------------- 1 | 2 | WIP 3 | 4 | 1. `sqlite3 test.db '.read build.sql'` 5 | 2. `deno run -A --unstable extract_plots.ts test.db` 6 | 3. `python3 add_embeddings.py test.db` 7 | 4. `sqlite3 test.db '.read add_vss.sql'` 8 | 9 | 10 | ```sql 11 | with matches as ( 12 | select rowid 13 | from vss_plot_sentences 14 | where vss_search(contents_embedding, vss_search_params(json(st_encode('who killed thanos?')), 30)) 15 | ), 16 | final as ( 17 | select 18 | plot_sentences.rowid, 19 | plot_sentences.contents 20 | from matches 21 | left join plot_sentences on plot_sentences.rowid = matches.rowid 22 | ) 23 | select * from final; 24 | ``` -------------------------------------------------------------------------------- /bindings/datasette/README.md: -------------------------------------------------------------------------------- 1 | # The `datasette-sqlite-vss` Datasette Plugin 2 | 3 | `datasette-sqlite-vss` is a [Datasette plugin](https://docs.datasette.io/en/stable/plugins.html) that loads the [`sqlite-vss`](https://github.com/asg017/sqlite-vss) extension in Datasette instances. 4 | ``` 5 | datasette install datasette-sqlite-vss 6 | ``` 7 | 8 | See [`docs.md`](../../docs.md) for a full API reference for the TODO SQL functions. 9 | 10 | Alternatively, when publishing Datasette instances, you can use the `--install` option to install the plugin. 11 | 12 | ``` 13 | datasette publish cloudrun data.db --service=my-service --install=datasette-sqlite-vss 14 | 15 | ``` 16 | -------------------------------------------------------------------------------- /bindings/python/sqlite_vss/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sqlite3 3 | 4 | def vector_loadable_path(): 5 | loadable_path = os.path.join(os.path.dirname(__file__), "vector0") 6 | return os.path.normpath(loadable_path) 7 | 8 | def vss_loadable_path(): 9 | loadable_path = os.path.join(os.path.dirname(__file__), "vss0") 10 | return os.path.normpath(loadable_path) 11 | 12 | def load_vector(conn: sqlite3.Connection) -> None: 13 | conn.load_extension(vector_loadable_path()) 14 | 15 | def load_vss(conn: sqlite3.Connection) -> None: 16 | conn.load_extension(vss_loadable_path()) 17 | 18 | def load(conn: sqlite3.Connection) -> None: 19 | load_vector(conn) 20 | load_vss(conn) 21 | -------------------------------------------------------------------------------- /bindings/node/sqlite-vss-linux-x64/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # sqlite-vss-linux-x64 4 | 5 | A `sqlite-vss` platform-specific package for `linux-x64`. 6 | 7 | When `sqlite-vss` is installed and the host computer has a `linux` operating system with `x64` architecture, then this package is downloaded with the pre-compiled SQLite extension bundled under `lib/vss0.so`. At runtime, the `sqlite-vss` package will resolve to this platform-specific package for use with [`better-sqlite3`](https://github.com/WiseLibs/better-sqlite3)' or [`node-sqlite3`](https://github.com/TryGhost/node-sqlite3). 8 | 9 | See the `sqlite-vss` package for more details. -------------------------------------------------------------------------------- /bindings/node/sqlite-vss-darwin-x64/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # sqlite-vss-darwin-x64 4 | 5 | A `sqlite-vss` platform-specific package for `darwin-x64`. 6 | 7 | When `sqlite-vss` is installed and the host computer has a `darwin` operating system with `x64` architecture, then this package is downloaded with the pre-compiled SQLite extension bundled under `lib/vss0.dylib`. At runtime, the `sqlite-vss` package will resolve to this platform-specific package for use with [`better-sqlite3`](https://github.com/WiseLibs/better-sqlite3)' or [`node-sqlite3`](https://github.com/TryGhost/node-sqlite3). 8 | 9 | See the `sqlite-vss` package for more details. -------------------------------------------------------------------------------- /bindings/node/sqlite-vss-darwin-arm64/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # sqlite-vss-darwin-arm64 4 | 5 | A `sqlite-vss` platform-specific package for `darwin-arm64`. 6 | 7 | When `sqlite-vss` is installed and the host computer has a `darwin` operating system with `arm64` architecture, then this package is downloaded with the pre-compiled SQLite extension bundled under `lib/vss0.dylib`. At runtime, the `sqlite-vss` package will resolve to this platform-specific package for use with [`better-sqlite3`](https://github.com/WiseLibs/better-sqlite3)' or [`node-sqlite3`](https://github.com/TryGhost/node-sqlite3). 8 | 9 | See the `sqlite-vss` package for more details. -------------------------------------------------------------------------------- /scripts/publish_release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail xtrace 4 | 5 | if [[ -n $(git status --porcelain | grep -v VERSION | grep -v vendor/faiss) ]]; then 6 | echo "❌ There are other un-staged changes to the repository besides VERSION" 7 | exit 1 8 | fi 9 | 10 | VERSION="$(cat VERSION)" 11 | 12 | echo "Publishing version v$VERSION..." 13 | 14 | make version 15 | git add --all 16 | git commit -m "v$VERSION" 17 | git tag v$VERSION 18 | git push origin main v$VERSION 19 | 20 | if grep -qE "alpha|beta" VERSION; then 21 | gh release create v$VERSION --title=v$VERSION --prerelease --notes="" 22 | else 23 | gh release create v$VERSION --title=v$VERSION 24 | fi 25 | 26 | 27 | echo "✅ Published! version v$VERSION" 28 | -------------------------------------------------------------------------------- /bindings/go/README.md: -------------------------------------------------------------------------------- 1 | # Golang bindings for `sqlite-vss` 2 | 3 | [![Go Reference](https://pkg.go.dev/badge/github.com/asg017/sqlite-vss/bindings/go.svg)](https://pkg.go.dev/github.com/asg017/sqlite-vss/bindings/go) 4 | 5 | ## Linking `sqlite-vss` libraries 6 | 7 | Option #1: 8 | 9 | ```go 10 | package main 11 | 12 | import ( 13 | "database/sql" 14 | 15 | _ "github.com/asg017/sqlite-vss/bindings/go" 16 | _ "github.com/mattn/go-sqlite3" 17 | ) 18 | 19 | // #cgo LDFLAGS: -Lpath/to/sqlite-vss-libs/ -Wl,-undefined,dynamic_lookup 20 | import "C" 21 | 22 | func main() { 23 | /* */ 24 | } 25 | 26 | ``` 27 | 28 | Option #2: 29 | 30 | ```bash 31 | CGO_LDFLAGS="-Lpath/to/sqlite-vss-libs/ -Wl,-undefined,dynamic_lookup" go build . 32 | ``` 33 | -------------------------------------------------------------------------------- /bindings/node/sqlite-vss/src/index.d.ts: -------------------------------------------------------------------------------- 1 | declare module 'sqlite-vss' { 2 | export interface Database { 3 | // after https://deno.land/x/sqlite3@0.8.0/mod.ts?s=Database#method_loadExtension_0 4 | loadExtension(file: string, entrypoint?: string | undefined): void; 5 | } 6 | 7 | /** 8 | * Loads the sqlite-vss extension on the given sqlite3 database. 9 | */ 10 | export function loadVector(database: Database): void; 11 | /** 12 | * Loads the sqlite-vss extension on the given sqlite3 database. 13 | */ 14 | export function loadVss(database: Database): void; 15 | 16 | /** 17 | * Loads the sqlite-vss extension on the given sqlite3 database. 18 | */ 19 | export function load(database: Database): void; 20 | } 21 | -------------------------------------------------------------------------------- /examples/headlines/build/import_articles.sql: -------------------------------------------------------------------------------- 1 | .load ./lines0 2 | 3 | create table articles( 4 | headline text, 5 | headline_embedding fvector, 6 | description text, 7 | description_embedding fvector, 8 | link text, 9 | category text, 10 | authors text, 11 | date 12 | ); 13 | 14 | insert into articles(headline, description, link, category, authors, date) 15 | select 16 | line ->> '$.headline' as headline, 17 | line ->> '$.short_description' as description, 18 | line ->> '$.link' as link, 19 | line ->> '$.category' as category, 20 | line ->> '$.authors' as authors, 21 | line ->> '$.date' as date 22 | from lines_read('News_Category_Dataset_v3.json'); -------------------------------------------------------------------------------- /examples/simonw_blog/build.sql: -------------------------------------------------------------------------------- 1 | .bail on 2 | .load ../../../sqlite-vector/build_release/vector0 3 | .load ../../build_release/vss0 4 | .timer on 5 | .mode box 6 | .header on 7 | 8 | attach "simonwillisonblog.db" as simonwillisonblog; 9 | 10 | create table blog_entry as 11 | select 12 | entries.*, 13 | embeddings.embedding as body_embedding 14 | from simonwillisonblog.blog_entry as entries 15 | left join simonwillisonblog.blog_entry_embeddings as embeddings 16 | on embeddings.id = entries.id 17 | where body_embedding is not null; 18 | 19 | create virtual table vss_blog_entry using vss0( 20 | body_embedding(1536) 21 | ); 22 | 23 | insert into vss_blog_entry(rowid, body_embedding) 24 | select id, body_embedding from blog_entry; -------------------------------------------------------------------------------- /bindings/go/vss/sqlite-vss.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef _SQLITE_VSS_H 3 | #define _SQLITE_VSS_H 4 | 5 | #include "sqlite3ext.h" 6 | #define SQLITE_VSS_VERSION "v0.1.2" 7 | 8 | #define SQLITE_VSS_VERSION_MAJOR @sqlite - vss_VERSION_MAJOR @ 9 | #define SQLITE_VSS_VERSION_MINOR @sqlite - vss_VERSION_MINOR @ 10 | #define SQLITE_VSS_VERSION_PATCH @sqlite - vss_VERSION_PATCH @ 11 | #define SQLITE_VSS_VERSION_TWEAK @sqlite - vss_VERSION_TWEAK @ 12 | 13 | #ifdef __cplusplus 14 | extern "C" { 15 | #endif 16 | 17 | int sqlite3_vss_init(sqlite3 *db, 18 | char **pzErrMsg, 19 | const sqlite3_api_routines *pApi); 20 | 21 | #ifdef __cplusplus 22 | } /* end of the 'extern "C"' block */ 23 | #endif 24 | 25 | #endif /* ifndef _SQLITE_VSS_H */ 26 | -------------------------------------------------------------------------------- /src/sqlite-vss.h.in: -------------------------------------------------------------------------------- 1 | 2 | #ifndef _SQLITE_VSS_H 3 | #define _SQLITE_VSS_H 4 | 5 | #include "sqlite3ext.h" 6 | #define SQLITE_VSS_VERSION "@SQLITE_VSS_VERSION@" 7 | 8 | #define SQLITE_VSS_VERSION_MAJOR @sqlite - vss_VERSION_MAJOR @ 9 | #define SQLITE_VSS_VERSION_MINOR @sqlite - vss_VERSION_MINOR @ 10 | #define SQLITE_VSS_VERSION_PATCH @sqlite - vss_VERSION_PATCH @ 11 | #define SQLITE_VSS_VERSION_TWEAK @sqlite - vss_VERSION_TWEAK @ 12 | 13 | #ifdef __cplusplus 14 | extern "C" { 15 | #endif 16 | 17 | int sqlite3_vss_init(sqlite3 *db, 18 | char **pzErrMsg, 19 | const sqlite3_api_routines *pApi); 20 | 21 | #ifdef __cplusplus 22 | } /* end of the 'extern "C"' block */ 23 | #endif 24 | 25 | #endif /* ifndef _SQLITE_VSS_H */ 26 | -------------------------------------------------------------------------------- /bindings/node/platform-package.README.md.tmpl: -------------------------------------------------------------------------------- 1 | 2 | 3 | # ${PACKAGE_NAME} 4 | 5 | A `${PACKAGE_NAME_BASE}` platform-specific package for `${PLATFORM_OS}-${PLATFORM_ARCH}`. 6 | 7 | When `${PACKAGE_NAME_BASE}` is installed and the host computer has a `${PLATFORM_OS}` operating system with `${PLATFORM_ARCH}` architecture, then this package is downloaded with the pre-compiled SQLite extension bundled under `lib/${EXTENSION_NAME}.${EXTENSION_SUFFIX}`. At runtime, the `${PACKAGE_NAME_BASE}` package will resolve to this platform-specific package for use with [`better-sqlite3`](https://github.com/WiseLibs/better-sqlite3)' or [`node-sqlite3`](https://github.com/TryGhost/node-sqlite3). 8 | 9 | See the `${PACKAGE_NAME_BASE}` package for more details. -------------------------------------------------------------------------------- /bindings/datasette/tests/test_sqlite_vector.py: -------------------------------------------------------------------------------- 1 | from datasette.app import Datasette 2 | import pytest 3 | 4 | 5 | @pytest.mark.asyncio 6 | async def test_plugin_is_installed(): 7 | datasette = Datasette(memory=True) 8 | response = await datasette.client.get("/-/plugins.json") 9 | assert response.status_code == 200 10 | installed_plugins = {p["name"] for p in response.json()} 11 | assert "datasette-sqlite-vector" in installed_plugins 12 | 13 | @pytest.mark.asyncio 14 | async def test_sqlite_vector_functions(): 15 | datasette = Datasette(memory=True) 16 | response = await datasette.client.get("/_memory.json?sql=select+vector_version(),vector()") 17 | assert response.status_code == 200 18 | vector_version, vector = response.json()["rows"][0] 19 | assert vector_version[0] == "v" 20 | assert len(vector) == 26 -------------------------------------------------------------------------------- /examples/mcu-wiki/build/extract_plots.ts: -------------------------------------------------------------------------------- 1 | import { Database } from "https://deno.land/x/sqlite3@0.7.2/mod.ts"; 2 | import wtf from 'npm:wtf_wikipedia'; 3 | 4 | const db = new Database(Deno.args[0]); 5 | 6 | interface Row { 7 | rowid: number; 8 | article: string; 9 | title: string; 10 | }; 11 | for(const r of db.prepare("select rowid, title, article from pages where title not like '%:%'")) { 12 | const row:Row = r as Row; 13 | const doc = wtf(row.article); 14 | for(const section of doc.sections()) { 15 | if(section.title() === 'Plot') { 16 | for(const [i, sentence] of (section.sentences() as any[]).entries()) { 17 | db.exec(` 18 | insert into plot_sentences(page, sentence, contents) 19 | values (?, ?, ?) 20 | `, row.rowid, i, sentence.text()) 21 | } 22 | 23 | } 24 | } 25 | } 26 | 27 | 28 | db.close(); -------------------------------------------------------------------------------- /site/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | # https://vitepress.dev/reference/default-theme-home-page 3 | layout: home 4 | 5 | hero: 6 | name: "sqlite-vss" 7 | text: "A SQLite Extension for Vector Search" 8 | tagline: "Local-first vector databases" 9 | actions: 10 | - theme: brand 11 | text: Getting Started 12 | link: /getting-started 13 | - theme: alt 14 | text: API Reference 15 | link: /api-reference 16 | #image: 17 | # src: /hero.png 18 | # alt: lol 19 | features: 20 | - title: Easy installation 21 | details: Pre-compiled extensions for MacOS and Linux. Packages available for Python, Node.js, Ruby, Deno, Go, and Rust 22 | - title: Embeddable 23 | details: Like SQLite, no external server, no network access, zero configuration 24 | - title: Based on Faiss 25 | details: Reliable, fast, and configurable vector search 26 | --- 27 | -------------------------------------------------------------------------------- /src/sqlite-vector.h.in: -------------------------------------------------------------------------------- 1 | #ifndef _SQLITE_VECTOR_H 2 | #define _SQLITE_VECTOR_H 3 | 4 | #include "sqlite3ext.h" 5 | 6 | #ifdef __cplusplus 7 | #include 8 | #include 9 | #include 10 | struct VectorFloat { 11 | int64_t size; 12 | float *data; 13 | }; 14 | 15 | struct vector0_api { 16 | 17 | int iVersion; 18 | std::unique_ptr> (*xValueAsVector)(sqlite3_value *value); 19 | void (*xResultVector)(sqlite3_context *context, std::vector *); 20 | }; 21 | 22 | #endif /* end of C++ specific APIs*/ 23 | 24 | #ifdef __cplusplus 25 | extern "C" { 26 | #endif 27 | 28 | int sqlite3_vector_init(sqlite3 *db, char **pzErrMsg, 29 | const sqlite3_api_routines *pApi); 30 | 31 | #ifdef __cplusplus 32 | } /* end of the 'extern "C"' block */ 33 | #endif 34 | 35 | #endif /* ifndef _SQLITE_VECTOR_H */ 36 | -------------------------------------------------------------------------------- /bindings/go/vector/vector.go: -------------------------------------------------------------------------------- 1 | // utilities 2 | package vector 3 | 4 | // #cgo LDFLAGS: -lsqlite_vector0 5 | // #cgo CFLAGS: -DSQLITE_CORE 6 | // #include 7 | // #include "sqlite-vector.h" 8 | import "C" 9 | 10 | // Once called, every future new SQLite3 connection created in this process 11 | // will have the sqlite-vector extension loaded. It will persist until Cancel() is 12 | // called. 13 | // 14 | // Calls sqlite3_auto_extension() under the hood. 15 | func Auto() { 16 | C.sqlite3_auto_extension( (*[0]byte) ((C.sqlite3_vector_init)) ); 17 | } 18 | 19 | // "Cancels" any previous calls to Auto(). Any new SQLite3 connections created 20 | // will not have the sqlite-vector extension loaded. 21 | // 22 | // Calls sqlite3_cancel_auto_extension() under the hood. 23 | func Cancel() { 24 | C.sqlite3_cancel_auto_extension( (*[0]byte) (C.sqlite3_vector_init) ); 25 | } 26 | -------------------------------------------------------------------------------- /bindings/go/vector/sqlite-vector.h: -------------------------------------------------------------------------------- 1 | #ifndef _SQLITE_VECTOR_H 2 | #define _SQLITE_VECTOR_H 3 | 4 | #include "sqlite3ext.h" 5 | 6 | #ifdef __cplusplus 7 | #include 8 | #include 9 | #include 10 | struct VectorFloat { 11 | int64_t size; 12 | float *data; 13 | }; 14 | 15 | struct vector0_api { 16 | 17 | int iVersion; 18 | std::unique_ptr> (*xValueAsVector)(sqlite3_value *value); 19 | void (*xResultVector)(sqlite3_context *context, std::vector *); 20 | }; 21 | 22 | #endif /* end of C++ specific APIs*/ 23 | 24 | #ifdef __cplusplus 25 | extern "C" { 26 | #endif 27 | 28 | int sqlite3_vector_init(sqlite3 *db, char **pzErrMsg, 29 | const sqlite3_api_routines *pApi); 30 | 31 | #ifdef __cplusplus 32 | } /* end of the 'extern "C"' block */ 33 | #endif 34 | 35 | #endif /* ifndef _SQLITE_VECTOR_H */ 36 | -------------------------------------------------------------------------------- /examples/headlines/fly.toml: -------------------------------------------------------------------------------- 1 | # fly.toml file generated for sqlite-vss-example on 2023-01-31T06:05:08Z 2 | 3 | app = "sqlite-vss-example" 4 | kill_signal = "SIGINT" 5 | kill_timeout = 5 6 | processes = [] 7 | 8 | [mounts] 9 | source="db" 10 | destination="/data" 11 | 12 | [env] 13 | 14 | [experimental] 15 | auto_rollback = true 16 | 17 | [[services]] 18 | http_checks = [] 19 | internal_port = 8001 20 | processes = ["app"] 21 | protocol = "tcp" 22 | script_checks = [] 23 | [services.concurrency] 24 | hard_limit = 25 25 | soft_limit = 20 26 | type = "connections" 27 | 28 | [[services.ports]] 29 | force_https = true 30 | handlers = ["http"] 31 | port = 80 32 | 33 | [[services.ports]] 34 | handlers = ["tls", "http"] 35 | port = 443 36 | 37 | [[services.tcp_checks]] 38 | grace_period = "1s" 39 | interval = "15s" 40 | restart_limit = 0 41 | timeout = "2s" 42 | -------------------------------------------------------------------------------- /examples/mcu-wiki/build/build.sql: -------------------------------------------------------------------------------- 1 | .bail on 2 | .load ./xml0 3 | 4 | .timer on 5 | 6 | create table pages( 7 | id text primary key, 8 | title text, 9 | updated_at datetime, 10 | article text 11 | ); 12 | 13 | CREATE TABLE plot_sentences( 14 | page integer, 15 | sentence integer, 16 | contents text, 17 | contents_embedding blob, 18 | foreign key (page) references pages(rowid) 19 | ); 20 | insert into pages 21 | select 22 | xml_extract(node, './/mediawiki:id/text()') as id, 23 | xml_extract(node, './/mediawiki:title/text()') as title, 24 | 25 | xml_extract(node, './/mediawiki:timestamp/text()') as updated_at, 26 | xml_extract(node, './/mediawiki:text/text()') as article 27 | from xml_each( 28 | readfile('marvelcinematicuniverse_pages_current.xml'), 29 | '//mediawiki:page', 30 | json_object( 31 | 'mediawiki', 'http://www.mediawiki.org/xml/export-0.11/' 32 | ) 33 | ); 34 | -------------------------------------------------------------------------------- /bindings/rust/Cargo.toml: -------------------------------------------------------------------------------- 1 | # autogenerated, don't edit by hand. 2 | 3 | [package] 4 | name = "sqlite-vss" 5 | version = "0.1.2" 6 | edition = "2021" 7 | authors = ["Alex Garcia "] 8 | description = "FFI bindings to the sqlite-vss SQLite extension" 9 | homepage = "https://github.com/asg017/sqlite-vss" 10 | repository = "https://github.com/asg017/sqlite-vss" 11 | keywords = ["sqlite", "sqlite-extension"] 12 | license = "MIT/Apache-2.0" 13 | 14 | [dependencies] 15 | 16 | [build-dependencies] 17 | anyhow = "1.0.71" 18 | ureq = { version = "2.6.2", optional = true } 19 | flate2 = { version = "1.0.26", optional = true } 20 | tar = { version = "0.4.38", optional = true } 21 | zip = { version = "0.6.6", optional = true } 22 | 23 | [dev-dependencies] 24 | rusqlite = "0.29.0" 25 | 26 | [features] 27 | default = ["vector", "vss"] 28 | download-libs = ["ureq", "flate2", "tar", "zip"] 29 | vector = [] 30 | vss = [] 31 | -------------------------------------------------------------------------------- /bindings/rust/Cargo.toml.tmpl: -------------------------------------------------------------------------------- 1 | # autogenerated, don't edit by hand. 2 | 3 | [package] 4 | name = "sqlite-vss" 5 | version = "${VERSION}" 6 | edition = "2021" 7 | authors = ["Alex Garcia "] 8 | description = "FFI bindings to the sqlite-vss SQLite extension" 9 | homepage = "https://github.com/asg017/sqlite-vss" 10 | repository = "https://github.com/asg017/sqlite-vss" 11 | keywords = ["sqlite", "sqlite-extension"] 12 | license = "MIT/Apache-2.0" 13 | 14 | [dependencies] 15 | 16 | [build-dependencies] 17 | anyhow = "1.0.71" 18 | ureq = { version = "2.6.2", optional = true } 19 | flate2 = { version = "1.0.26", optional = true } 20 | tar = { version = "0.4.38", optional = true } 21 | zip = { version = "0.6.6", optional = true } 22 | 23 | [dev-dependencies] 24 | rusqlite = "0.29.0" 25 | 26 | [features] 27 | default = ["vector", "vss"] 28 | download-libs = ["ureq", "flate2", "tar", "zip"] 29 | vector = [] 30 | vss = [] 31 | -------------------------------------------------------------------------------- /bindings/node/sqlite-vss/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "//": "Autogenerated by the npm_generate_platform_packages.sh script, do not edit by hand", 3 | "name": "sqlite-vss", 4 | "version": "0.1.2", 5 | "description": "", 6 | "repository": { 7 | "type": "git", 8 | "url": "https://github.com/asg017/sqlite-vss.git", 9 | "directory": "npm/sqlite-vss" 10 | }, 11 | "author": "Alex Garcia ", 12 | "license": "(MIT OR Apache-2.0)", 13 | "type": "module", 14 | "main": "src/index.js", 15 | "typings": "src/index.d.ts", 16 | "scripts": { 17 | "test": "node test.js" 18 | }, 19 | "files": [ 20 | "src/*", 21 | "*.dylib", 22 | "*.so", 23 | "*.dll" 24 | ], 25 | "optionalDependencies": { 26 | "sqlite-vss-darwin-x64": "0.1.2", 27 | "sqlite-vss-darwin-arm64": "0.1.2", 28 | "sqlite-vss-linux-x64": "0.1.2" 29 | }, 30 | "devDependencies": { 31 | "better-sqlite3": "^8.1.0", 32 | "sqlite3": "^5.1.4" 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /.github/workflows/site.yaml: -------------------------------------------------------------------------------- 1 | name: Deploy Site 2 | on: 3 | workflow_dispatch: {} 4 | push: 5 | branches: 6 | - main 7 | paths: 8 | - site/** 9 | - VERSION 10 | jobs: 11 | deploy: 12 | runs-on: ubuntu-latest 13 | permissions: 14 | pages: write 15 | id-token: write 16 | environment: 17 | name: github-pages 18 | url: ${{ steps.deployment.outputs.page_url }} 19 | steps: 20 | - uses: actions/checkout@v3 21 | with: 22 | fetch-depth: 0 23 | - uses: actions/setup-node@v3 24 | with: 25 | node-version: 16 26 | cache: npm 27 | cache-dependency-path: site/package-lock.json 28 | - run: npm ci 29 | working-directory: site/ 30 | - run: make site-build 31 | - uses: actions/configure-pages@v2 32 | - uses: actions/upload-pages-artifact@v1 33 | with: 34 | path: site/.vitepress/dist 35 | - id: deployment 36 | uses: actions/deploy-pages@v1 37 | -------------------------------------------------------------------------------- /scripts/elixir_generate_checksum.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # ./scripts/elixir_generate_checksum.sh "$(cat ~/tmp/checksums.txt)" 4 | 5 | CHECKSUMS_TXT="$1" 6 | 7 | 8 | generate () { 9 | export PLATFORM=$1 10 | export TARGET_ENV_NAME=$2 11 | export TARGET_ENV_SHA=$3 12 | result="$(echo "$CHECKSUMS_TXT" | grep "loadable-$PLATFORM")" 13 | 14 | NAME=$(echo "$result" | sed -E 's/.* (.*)/\1/') 15 | CHECKSUM=$(echo "$result" | sed -E 's/([^ ]*) .*/\1/') 16 | 17 | export "$TARGET_ENV_NAME"="$NAME" 18 | export "$TARGET_ENV_SHA"="$CHECKSUM" 19 | } 20 | 21 | 22 | generate linux-x86_64 LOADABLE_ASSET_LINUX_X86_64_NAME LOADABLE_ASSET_LINUX_X86_64_SHA256 23 | generate macos-x86_64 LOADABLE_ASSET_MACOS_X86_64_NAME LOADABLE_ASSET_MACOS_X86_64_SHA256 24 | generate macos-aarch64 LOADABLE_ASSET_MACOS_AARCH64_NAME LOADABLE_ASSET_MACOS_AARCH64_SHA256 25 | 26 | 27 | envsubst < bindings/elixir/sqlite-vss-checksum.exs.tmpl > bindings/elixir/sqlite-vss-checksum.exs 28 | echo "✅ generated bindings/elixir/sqlite-vss-checksum.exs" 29 | -------------------------------------------------------------------------------- /bindings/datasette/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | version = {} 4 | with open("datasette_sqlite_vss/version.py") as fp: 5 | exec(fp.read(), version) 6 | 7 | VERSION = version['__version__'] 8 | 9 | setup( 10 | name="datasette-sqlite-vss", 11 | description="", 12 | long_description="", 13 | long_description_content_type="text/markdown", 14 | author="Alex Garcia", 15 | url="https://github.com/asg017/sqlite-vss", 16 | project_urls={ 17 | "Issues": "https://github.com/asg017/sqlite-vss/issues", 18 | "CI": "https://github.com/asg017/sqlite-vss/actions", 19 | "Changelog": "https://github.com/asg017/sqlite-vss/releases", 20 | }, 21 | license="MIT License, Apache License, Version 2.0", 22 | version=VERSION, 23 | packages=["datasette_sqlite_vss"], 24 | entry_points={"datasette": ["sqlite_vss = datasette_sqlite_vss"]}, 25 | install_requires=["datasette", "sqlite-vss"], 26 | extras_require={"test": ["pytest"]}, 27 | python_requires=">=3.7", 28 | ) -------------------------------------------------------------------------------- /examples/headlines/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM datasetteproject/datasette 2 | 3 | RUN apt-get update && apt-get install -y curl 4 | 5 | 6 | RUN apt-get install -y libgomp1 libatlas-base-dev liblapack-dev 7 | 8 | RUN apt-get install -y cmake git pkg-config build-essential 9 | RUN pip install --upgrade pip 10 | RUN pip install sentence-transformers 11 | RUN python3 -c 'from sentence_transformers import SentenceTransformer; model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2");' 12 | ADD plugins/ /app/plugins 13 | 14 | # SQLite v3.41.2 15 | WORKDIR /sqlite 16 | ADD https://www.sqlite.org/2023/sqlite-autoconf-3410200.tar.gz . 17 | RUN tar -xvzf sqlite-autoconf-3410200.tar.gz 18 | WORKDIR /sqlite/sqlite-autoconf-3410200 19 | RUN ./configure 20 | RUN make 21 | ENV LD_LIBRARY_PATH=/sqlite/sqlite-autoconf-3410200/.libs 22 | 23 | # TODO outdated, use the datasette-sqlite-vss plugin instead 24 | 25 | RUN datasette install datasette-sqlite-vss 26 | 27 | CMD datasette /data/headlines.db \ 28 | -p 8001 -h 0.0.0.0 --cors \ 29 | --plugins-dir=/app/plugins 30 | -------------------------------------------------------------------------------- /examples/sift/build.sql: -------------------------------------------------------------------------------- 1 | .bail on 2 | .load ./vector0 3 | .load ./vss0 4 | .timer on 5 | .header on 6 | .mode box 7 | 8 | 9 | create virtual table vss_siftsmall using vss_index(vector(128) with "Flat,IDMap2"); 10 | 11 | insert into vss_siftsmall(rowid, vector) 12 | select 13 | rowid, vector 14 | from vector_fvecs_each(readfile('data/siftsmall/siftsmall_base.fvecs')); 15 | 16 | select 17 | rowid, 18 | distance 19 | from vss_siftsmall 20 | where vss_search( 21 | vector, 22 | vss_search_params((select vector from vector_fvecs_each(readfile('data/siftsmall/siftsmall_query.fvecs')) limit 1), 5) 23 | ); 24 | 25 | 26 | 27 | .exit 28 | select 29 | rowid, dimensions, vector_to_json(vector) 30 | from vector_fvecs_each(readfile('data/siftsmall/siftsmall_base.fvecs')) 31 | limit 10; 32 | 33 | select count(*) from vector_fvecs_each(readfile('data/siftsmall/siftsmall_base.fvecs')); 34 | select count(*) from vector_fvecs_each(readfile('data/sift/sift_base.fvecs')); 35 | --select count(*) from vector_fvecs_each(readfile('data/sift/sift_base.fvecs')); -------------------------------------------------------------------------------- /bindings/node/sqlite-vss/package.json.tmpl: -------------------------------------------------------------------------------- 1 | { 2 | "//": "Autogenerated by the npm_generate_platform_packages.sh script, do not edit by hand", 3 | "name": "${PACKAGE_NAME_BASE}", 4 | "version": "${VERSION}", 5 | "description": "", 6 | "repository": { 7 | "type": "git", 8 | "url": "https://github.com/asg017/${PACKAGE_NAME_BASE}.git", 9 | "directory": "npm/${PACKAGE_NAME_BASE}" 10 | }, 11 | "author": "Alex Garcia ", 12 | "license": "(MIT OR Apache-2.0)", 13 | "type": "module", 14 | "main": "src/index.js", 15 | "typings": "src/index.d.ts", 16 | "scripts": { 17 | "test": "node test.js" 18 | }, 19 | "files": [ 20 | "src/*", 21 | "*.dylib", 22 | "*.so", 23 | "*.dll" 24 | ], 25 | "optionalDependencies": { 26 | "${PACKAGE_NAME_BASE}-darwin-x64": "${VERSION}", 27 | "${PACKAGE_NAME_BASE}-darwin-arm64": "${VERSION}", 28 | "${PACKAGE_NAME_BASE}-linux-x64": "${VERSION}" 29 | }, 30 | "devDependencies": { 31 | "better-sqlite3": "^8.1.0", 32 | "sqlite3": "^5.1.4" 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2023 Alexander Garcia 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /examples/headlines/build/add_vector_search.sql: -------------------------------------------------------------------------------- 1 | .bail on 2 | .load ./vector0 3 | .load ../../build/vss0 4 | 5 | .timer on 6 | .mode box 7 | .header on 8 | 9 | create virtual table vss_articles using vss0( 10 | headline_embedding(384), 11 | description_embedding(384), 12 | ); 13 | 14 | -- 8s 15 | insert into vss_articles(rowid, headline_embedding, description_embedding) 16 | select 17 | rowid, 18 | headline_embedding, 19 | description_embedding 20 | from articles; 21 | 22 | 23 | create virtual table vss_ivf_articles using vss0( 24 | headline_embedding(384) factory="IVF4096,Flat,IDMap2", 25 | description_embedding(384) factory="IVF4096,Flat,IDMap2" 26 | ); 27 | 28 | -- 44m16s ! 29 | insert into vss_ivf_articles(operation, headline_embedding, description_embedding) 30 | select 31 | 'training', 32 | headline_embedding, 33 | description_embedding 34 | from articles; 35 | 36 | -- 4m31s 37 | insert into vss_ivf_articles(rowid, headline_embedding, description_embedding) 38 | select 39 | rowid, 40 | headline_embedding, 41 | description_embedding 42 | from articles; 43 | -------------------------------------------------------------------------------- /bindings/go/vss/vss.go: -------------------------------------------------------------------------------- 1 | // yoyo 2 | package vss 3 | 4 | // #cgo LDFLAGS: -lsqlite_vss0 -lfaiss_avx2 5 | // #cgo linux,amd64 LDFLAGS: -lm -lgomp -latlas -lblas -llapack 6 | // #cgo CFLAGS: -DSQLITE_CORE 7 | // #include 8 | // #include "sqlite-vss.h" 9 | // 10 | import "C" 11 | 12 | // Once called, every future new SQLite3 connection created in this process 13 | // will have the sqlite-vss extension loaded. It will persist until [Cancel] is 14 | // called. 15 | // 16 | // Calls [sqlite3_auto_extension()] under the hood. 17 | // 18 | // [sqlite3_auto_extension()]: https://www.sqlite.org/c3ref/auto_extension.html 19 | func Auto() { 20 | C.sqlite3_auto_extension( (*[0]byte) ((C.sqlite3_vss_init)) ); 21 | } 22 | 23 | // "Cancels" any previous calls to [Auto]. Any new SQLite3 connections created 24 | // will not have the sqlite-vss extension loaded. 25 | // 26 | // Calls [sqlite3_cancel_auto_extension()] under the hood. 27 | // [sqlite3_cancel_auto_extension()]: https://www.sqlite.org/c3ref/cancel_auto_extension.html 28 | func Cancel() { 29 | C.sqlite3_cancel_auto_extension( (*[0]byte) (C.sqlite3_vss_init) ); 30 | } 31 | -------------------------------------------------------------------------------- /examples/mcu-wiki/build/add_embeddings.py: -------------------------------------------------------------------------------- 1 | import sqlite3 2 | import sys 3 | import json 4 | 5 | db = sqlite3.connect(sys.argv[1]) 6 | db.load_extension("./vector0") 7 | db.load_extension("./vss0") 8 | 9 | 10 | from sentence_transformers import SentenceTransformer 11 | model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2") 12 | 13 | BATCH_SIZE = 256 14 | total_count = db.execute("select count(*) from plot_sentences where contents_embedding is null").fetchone()[0] 15 | 16 | while True: 17 | batch = db.execute("select rowid, contents from plot_sentences where contents_embedding is null limit ?", [BATCH_SIZE]).fetchall() 18 | 19 | if len(batch) == 0: 20 | break 21 | 22 | rowids = list(map(lambda x: x[0], batch)) 23 | contents = list(map(lambda x: x[1], batch)) 24 | 25 | print(f"[{rowids[-1]/total_count*100:.2f}] encoding [{rowids[0]}:{rowids[-1]}]") 26 | embeddings = model.encode(contents) 27 | 28 | for rowid, embedding in zip(rowids, embeddings): 29 | db.execute("update plot_sentences set contents_embedding = vector_to_blob(vector_from_json(?)) where rowid = ?", [json.dumps(embedding.tolist()), rowid]) 30 | 31 | db.commit() 32 | 33 | -------------------------------------------------------------------------------- /scripts/npm_generate_platform_packages.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | export PACKAGE_NAME_BASE="sqlite-vss" 6 | export EXTENSION_NAME="vss0" 7 | export VERSION=$(cat VERSION) 8 | 9 | generate () { 10 | export PLATFORM_OS=$1 11 | export PLATFORM_ARCH=$2 12 | export PACKAGE_NAME=$PACKAGE_NAME_BASE-$PLATFORM_OS-$PLATFORM_ARCH 13 | 14 | if [ "$PLATFORM_OS" == "windows" ]; then 15 | export EXTENSION_SUFFIX="dll" 16 | elif [ "$PLATFORM_OS" == "darwin" ]; then 17 | export EXTENSION_SUFFIX="dylib" 18 | else 19 | export EXTENSION_SUFFIX="so" 20 | fi 21 | 22 | 23 | mkdir -p bindings/node/$PACKAGE_NAME/lib 24 | 25 | envsubst < bindings/node/platform-package.package.json.tmpl > bindings/node/$PACKAGE_NAME/package.json 26 | envsubst < bindings/node/platform-package.README.md.tmpl > bindings/node/$PACKAGE_NAME/README.md 27 | 28 | touch bindings/node/$PACKAGE_NAME/lib/.gitkeep 29 | 30 | echo "✅ generated bindings/node/$PACKAGE_NAME" 31 | } 32 | 33 | envsubst < bindings/node/$PACKAGE_NAME_BASE/package.json.tmpl > bindings/node/$PACKAGE_NAME_BASE/package.json 34 | echo "✅ generated bindings/node/$PACKAGE_NAME_BASE" 35 | 36 | generate darwin x64 37 | generate darwin arm64 38 | generate linux x64 39 | -------------------------------------------------------------------------------- /bindings/rust/src/lib.rs: -------------------------------------------------------------------------------- 1 | #[cfg(feature = "vector")] 2 | #[link(name = "sqlite_vector0")] 3 | extern "C" { 4 | pub fn sqlite3_vector_init(); 5 | } 6 | 7 | #[cfg(feature = "vss")] 8 | #[link(name = "sqlite_vss0")] 9 | extern "C" { 10 | pub fn sqlite3_vss_init(); 11 | } 12 | 13 | #[cfg(test)] 14 | mod tests { 15 | use super::*; 16 | use rusqlite::{ffi::sqlite3_auto_extension, Connection}; 17 | 18 | #[test] 19 | fn test_rusqlite_auto_extension() { 20 | unsafe { 21 | sqlite3_auto_extension(Some(sqlite3_vector_init)); 22 | sqlite3_auto_extension(Some(sqlite3_vss_init)); 23 | } 24 | 25 | let conn = Connection::open_in_memory().unwrap(); 26 | 27 | let result: String = conn 28 | .query_row("select vss_version()", [], |row| row.get(0)) 29 | .unwrap(); 30 | 31 | assert_eq!(result, format!("v{}", env!("CARGO_PKG_VERSION"))); 32 | 33 | let result: String = conn 34 | .query_row( 35 | "select vector_to_json(?)", 36 | [[0x00, 0x00, 0x28, 0x42]], 37 | |row| row.get(0), 38 | ) 39 | .unwrap(); 40 | 41 | assert_eq!(result, "[42.0]"); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /site/using/ruby.md: -------------------------------------------------------------------------------- 1 | # `sqlite-vss` with Ruby 2 | 3 | ![Gem](https://img.shields.io/gem/v/sqlite-vss?color=red&logo=rubygems&logoColor=white) 4 | 5 | ::: warning 6 | The Ruby bindings for `sqlite-vss` are still in beta and are subject to change. If you come across problems, [please comment on this Ruby tracking issue](https://github.com/asg017/sqlite-vss/issues/52). 7 | ::: 8 | 9 | Ruby developers can use `sqlite-vss` with the [`sqlite-vss` Gem](https://rubygems.org/gems/sqlite-vss). 10 | 11 | ```bash 12 | gem install sqlite-vss 13 | ``` 14 | 15 | You can then use `SqliteVss.load()` to load `sqlite-vss` SQL functions in a given SQLite connection. 16 | 17 | ```ruby 18 | require 'sqlite3' 19 | require 'sqlite_vss' 20 | 21 | db = SQLite3::Database.new(':memory:') 22 | db.enable_load_extension(true) 23 | SqliteVss.load(db) 24 | db.enable_load_extension(false) 25 | 26 | result = db.execute('SELECT vss_version()') 27 | puts result.first.first 28 | 29 | ``` 30 | 31 | Checkout [the API Reference](./api-reference) for all available SQL functions. 32 | 33 | Also see _[Making SQLite extensions gem install-able](https://observablehq.com/@asg017/making-sqlite-extension-gem-installable) (June 2023)_ for more information on how gem install'able SQLite extensions work. 34 | -------------------------------------------------------------------------------- /site/using/datasette.md: -------------------------------------------------------------------------------- 1 | # `sqlite-vss` with Datasette 2 | 3 | [![Datasette](https://img.shields.io/pypi/v/datasette-sqlite-vss.svg?color=B6B6D9&label=Datasette+plugin&logoColor=white&logo=python)](https://datasette.io/plugins/datasette-sqlite-vss) 4 | 5 | `sqlite-vss` is also distributed as a [Datasette plugin](https://docs.datasette.io/en/stable/plugins.html) with [`datasette-sqlite-vss`](https://pypi.org/project/datasette-sqlite-vss/), which can be installed like so: 6 | 7 | ```bash 8 | datasette install datasette-sqlite-vss 9 | ``` 10 | 11 | `datasette-sqlite-vss` is just a PyPi package and can also be installed with `pip`. However, `datasette install` ensures that the plugin will be downloaded in the same Python environment as your Datasette installation, and will automatically include `sqlite-vss` in future Datasette instances. 12 | 13 | If you're using the [`datasette publish`](https://docs.datasette.io/en/stable/publish.html) command, you can [use the `--install` flag](https://docs.datasette.io/en/stable/plugins.html#deploying-plugins-using-datasette-publish) to include `datasette-sqlite-vss` in your Datasette projects. 14 | 15 | ```bash 16 | datasette publish cloudrun data.db \ 17 | --service=my-service \ 18 | --install=datasette-sqlite-vss 19 | ``` 20 | -------------------------------------------------------------------------------- /examples/headlines/README.md: -------------------------------------------------------------------------------- 1 | 2 | 1. `sqlite3 full.db '.read import_articles.sql'` 3 | 2. `python3 add_embeddings.py full.db` 4 | 3. `sqlite3 full.db '.read add_vector_search.sql'` 5 | 6 | ```sql 7 | with similar_matches as ( 8 | select rowid 9 | from vss_articles 10 | where vss_search( 11 | headline_embedding, 12 | vss_search_params(json(st_encode('charlie brown')), 20) 13 | ) 14 | ), final as ( 15 | select 16 | articles.* 17 | from similar_matches 18 | left join articles on articles.rowid = similar_matches.rowid 19 | ) 20 | select * from final; 21 | ``` 22 | 23 | ``` 24 | datasette full.db --host 0.0.0.0 --load-extension=../../build/vss0.so --load-extension=./vector0.so --plugins-dir=./plugins 25 | ``` 26 | 27 | ``` 28 | docker build -t x . 29 | 30 | docker run -p 8001:8001 -v $PWD/test.db:/mnt/test.db --rm -it x 31 | 32 | docker run -p 8001:8001 --rm -it \ 33 | -v $PWD/build.py:/build.py \ 34 | -v $PWD/test.db:/test.db \ 35 | -v $PWD/News_Category_Dataset_v3.json:/News_Category_Dataset_v3.json \ 36 | x python3 /build.py /test.db /News_Category_Dataset_v3.json 37 | ``` 38 | 39 | 40 | 41 | datasette test.db -p 8001 -h 0.0.0.0 \ 42 | --plugins-dir=plugins \ 43 | --load-extension ./vss0 \ 44 | --load-extension ./vector0 45 | 46 | 47 | ``` 48 | 49 | ``` -------------------------------------------------------------------------------- /tests/test-python.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import sqlite3 3 | import sqlite_vss 4 | 5 | class TestSqliteVectorPython(unittest.TestCase): 6 | def test_path(self): 7 | self.assertEqual(type(sqlite_vss.vss_loadable_path()), str) 8 | self.assertEqual(type(sqlite_vss.vector_loadable_path()), str) 9 | 10 | def test_loads(self): 11 | db = sqlite3.connect(':memory:') 12 | db.enable_load_extension(True) 13 | sqlite_vss.load(db) 14 | 15 | version, = db.execute('select vector_version()').fetchone() 16 | self.assertEqual(version[0], "v") 17 | 18 | version, = db.execute('select vss_version()').fetchone() 19 | self.assertEqual(version[0], "v") 20 | 21 | # test individual load functions 22 | 23 | db2 = sqlite3.connect(':memory:') 24 | db2.enable_load_extension(True) 25 | 26 | sqlite_vss.load_vector(db2) 27 | version, = db.execute('select vector_version()').fetchone() 28 | self.assertEqual(version[0], "v") 29 | 30 | with self.assertRaisesRegex(sqlite3.OperationalError, "no such function: vss_version"): 31 | db2.execute('select vss_version()').fetchone() 32 | 33 | sqlite_vss.load_vss(db2) 34 | version, = db.execute('select vss_version()').fetchone() 35 | self.assertEqual(version[0], "v") 36 | 37 | if __name__ == '__main__': 38 | unittest.main() -------------------------------------------------------------------------------- /bindings/ruby/sqlite_vss.gemspec: -------------------------------------------------------------------------------- 1 | 2 | lib = File.expand_path("../lib", __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | require "version" 5 | 6 | Gem::Specification.new do |spec| 7 | spec.name = "sqlite-vss" 8 | spec.version = SqliteVss::VERSION 9 | spec.authors = ["Alex Garcia"] 10 | spec.email = ["alexsebastian.garcia@gmail.com"] 11 | 12 | spec.summary = "a" 13 | spec.description = "b" 14 | spec.homepage = "https://github.com/asg017/sqlite-vss" 15 | spec.license = "MIT" 16 | 17 | # The --platform flag would work in most cases, but on a GH action 18 | # linux runner, it would set platform to "ruby" and not "x86-linux". 19 | # Setting this to Gem::Platform::CURRENT 20 | spec.platform = ENV['PLATFORM'] 21 | 22 | if spec.respond_to?(:metadata) 23 | 24 | spec.metadata["homepage_uri"] = spec.homepage 25 | spec.metadata["source_code_uri"] = spec.homepage 26 | spec.metadata["changelog_uri"] = spec.homepage 27 | else 28 | raise "RubyGems 2.0 or newer is required to protect against " \ 29 | "public gem pushes." 30 | end 31 | 32 | spec.files = Dir["lib/*.rb"] + Dir.glob('lib/*.{so,dylib,dll}') 33 | 34 | spec.require_paths = ["lib"] 35 | 36 | spec.add_development_dependency "bundler", "~> 1.17" 37 | spec.add_development_dependency "rake", "~> 10.0" 38 | end 39 | -------------------------------------------------------------------------------- /site/using/go.md: -------------------------------------------------------------------------------- 1 | # `sqlite-vss` with Go 2 | 3 | [![Go Reference](https://pkg.go.dev/badge/github.com/asg017/sqlite-vss/bindings/go.svg)](https://pkg.go.dev/github.com/asg017/sqlite-vss/bindings/go) 4 | 5 | ::: warning 6 | The Go bindings for `sqlite-vss` are still in beta and are subject to change. If you come across problems, [please comment on the Go tracking issue](https://github.com/asg017/sqlite-vss/issues/49). 7 | ::: 8 | 9 | ## Installing `sqlite-vss` into Go Projects 10 | 11 | The official `sqlite-vss` Go bindings can be installed like so: 12 | 13 | ``` 14 | go get -u github.com/asg017/sqlite-vss/bindings/go 15 | ``` 16 | 17 | You are required to provide pre-compiled static library files of `sqlite-vss`. You can do this by compiling `sqlite-vss` yourself, or [from a Github Release](https://github.com/asg017/sqlite-vss/releases). 18 | 19 | Once you have a directory with the pre-compiled static library files, you 20 | 21 | ## Working with Vectors in Go 22 | 23 | If your vectors in Go are represented as a slice of floats, you can insert them into a `vss0` table as a JSON string with [`json.Marshal`](https://pkg.go.dev/encoding/json#Marshal): 24 | 25 | ```go 26 | embedding := [3]float32{0.1, 0.2, 0.3} 27 | embeddingJson, err := json.Marshal(embedding) 28 | if err != nil { 29 | log.Fatal(err) 30 | } 31 | _, err := tx.Exec("INSERT INTO vss_demo(a) VALUES (?)", string(embeddingJson)) 32 | if err != nil { 33 | log.Fatal(err) 34 | } 35 | ``` 36 | -------------------------------------------------------------------------------- /bindings/node/sqlite-vss/test.js: -------------------------------------------------------------------------------- 1 | import test from "node:test"; 2 | import * as assert from "node:assert"; 3 | 4 | import * as sqliteVss from "./src/index.js"; 5 | import { basename, extname, isAbsolute } from "node:path"; 6 | 7 | import Database from "better-sqlite3"; 8 | import sqlite3 from "sqlite3"; 9 | 10 | test("loadable paths", (t) => { 11 | const vectorLoadablePath = sqliteVss.getVectorLoadablePath(); 12 | assert.strictEqual(isAbsolute(vectorLoadablePath), true); 13 | assert.strictEqual( 14 | basename(vectorLoadablePath, extname(vectorLoadablePath)), 15 | "vector0" 16 | ); 17 | 18 | const vssLoadablePath = sqliteVss.getVssLoadablePath(); 19 | assert.strictEqual(isAbsolute(vssLoadablePath), true); 20 | assert.strictEqual( 21 | basename(vssLoadablePath, extname(vssLoadablePath)), 22 | "vss0" 23 | ); 24 | }); 25 | 26 | test("better-sqlite3", (t) => { 27 | const db = new Database(":memory:"); 28 | sqliteVss.load(db); 29 | const version = db.prepare("select vss_version()").pluck().get(); 30 | assert.strictEqual(version[0], "v"); 31 | }); 32 | 33 | test("sqlite3", async (t) => { 34 | const db = new sqlite3.Database(":memory:"); 35 | sqliteVss.load(db); 36 | const version = await new Promise((resolve, reject) => { 37 | db.get("select vss_version()", (err, row) => { 38 | if (err) return reject(err); 39 | resolve(row["vss_version()"]); 40 | }); 41 | }); 42 | assert.strictEqual(version[0], "v"); 43 | }); 44 | -------------------------------------------------------------------------------- /examples/rust/demo.rs: -------------------------------------------------------------------------------- 1 | use rusqlite::{ffi::sqlite3_auto_extension, Connection, Result}; 2 | use sqlite_vss::{sqlite3_vector_init, sqlite3_vss_init}; 3 | 4 | fn main() -> Result<()> { 5 | unsafe { 6 | sqlite3_auto_extension(Some(sqlite3_vector_init)); 7 | sqlite3_auto_extension(Some(sqlite3_vss_init)); 8 | } 9 | 10 | let db = Connection::open_in_memory()?; 11 | 12 | let (version, vector): (String, String) = db.query_row( 13 | "SELECT vss_version(), vector_to_json(?)", 14 | [[0x00, 0x00, 0x28, 0x42]], 15 | |row| Ok((row.get(0)?, row.get(1)?)), 16 | )?; 17 | println!("version={version} vector={vector}"); 18 | 19 | db.execute_batch( 20 | r" 21 | CREATE VIRTUAL TABLE vss_demo USING vss0(a(2)); 22 | INSERT INTO vss_demo(rowid, a) 23 | VALUES 24 | (1, '[1.0, 2.0]'), 25 | (2, '[2.0, 2.0]'), 26 | (3, '[3.0, 2.0]') 27 | ", 28 | )?; 29 | 30 | let result: Vec<(i64, f32)> = db 31 | .prepare( 32 | r" 33 | SELECT 34 | rowid, 35 | distance 36 | FROM vss_demo 37 | WHERE vss_search(a, '[1.0, 2.0]') 38 | LIMIT 3 39 | ", 40 | )? 41 | .query_map([], |r| Ok((r.get(0)?, r.get(1)?)))? 42 | .collect::, _>>()?; 43 | 44 | for (rowid, distance) in result { 45 | println!("rowid={rowid}, distance={distance}"); 46 | } 47 | 48 | println!("✅ demo.rs ran successfully. \n"); 49 | Ok(()) 50 | } 51 | -------------------------------------------------------------------------------- /bindings/deno/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # `x/sqlite_vss` Deno Module 4 | 5 | [![Tags](https://img.shields.io/github/release/asg017/sqlite-vss)](https://github.com/asg017/sqlite-vss/releases) 6 | [![Doc](https://doc.deno.land/badge.svg)](https://doc.deno.land/https/deno.land/x/sqlite-vss@0.1.2/mod.ts) 7 | 8 | The [`sqlite-vss`](https://github.com/asg017/sqlite-vss) SQLite extension is available to Deno developers with the [`x/sqlite_vss`](https://deno.land/x/sqlite_vss) Deno module. It works with [`x/sqlite3`](https://deno.land/x/sqlite3), the fastest and native Deno SQLite3 module. 9 | 10 | ```js 11 | import { Database } from "https://deno.land/x/sqlite3@0.8.0/mod.ts"; 12 | import * as sqlite_vss from "https://deno.land/x/sqlite_vss@v0.1.2/mod.ts"; 13 | 14 | const db = new Database(":memory:"); 15 | 16 | db.enableLoadExtension = true; 17 | sqlite_vss.load(db); 18 | 19 | const [version] = db 20 | .prepare("select vss_version()") 21 | .value<[string]>()!; 22 | 23 | console.log(version); 24 | 25 | ``` 26 | 27 | Like `x/sqlite3`, `x/sqlite_vss` requires network and filesystem permissions to download and cache the pre-compiled SQLite extension for your machine. Though `x/sqlite3` already requires `--allow-ffi` and `--unstable`, so you might as well use `--allow-all`/`-A`. 28 | 29 | ```bash 30 | deno run -A --unstable 31 | ``` 32 | 33 | `x/sqlite_vss` does not work with [`x/sqlite`](https://deno.land/x/sqlite@v3.7.0), which is a WASM-based Deno SQLite module that does not support loading extensions. 34 | -------------------------------------------------------------------------------- /scripts/rename-wheels.py: -------------------------------------------------------------------------------- 1 | # This file is a small utility that rename all .whl files in a given directory 2 | # and "generalizes" them. The wheels made by python/sqlite_ulid contain the 3 | # pre-compiled sqlite extension, but those aren't bound by a specfic Python 4 | # runtime or version, that other wheels might be. So, this file will rename 5 | # those wheels to be "generalized", like replacing "c37-cp37" to "py3-none". 6 | import sys 7 | import os 8 | from pathlib import Path 9 | 10 | wheel_dir = sys.argv[1] 11 | 12 | is_macos_arm_build = '--is-macos-arm' in sys.argv 13 | 14 | for filename in os.listdir(wheel_dir): 15 | filename = Path(wheel_dir, filename) 16 | if not filename.suffix == '.whl': 17 | continue 18 | new_filename = (filename.name 19 | .replace('cp37-cp37', 'py3-none') 20 | .replace('cp38-cp38', 'py3-none') 21 | .replace('cp39-cp39', 'py3-none') 22 | .replace('cp310-cp310', 'py3-none') 23 | .replace('cp311-cp311', 'py3-none') 24 | .replace('linux_x86_64', 'manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux1_x86_64') 25 | 26 | 27 | ) 28 | if is_macos_arm_build: 29 | new_filename = (new_filename 30 | .replace('macosx_12_0_universal2', 'macosx_11_0_arm64') 31 | .replace('macosx_13_0_universal2', 'macosx_11_0_arm64') 32 | .replace('macosx_13_0_arm64', 'macosx_11_0_arm64') 33 | ) 34 | else: 35 | new_filename = (new_filename 36 | .replace('macosx_12_0_universal2', 'macosx_10_6_x86_64') 37 | .replace('macosx_12_0_x86_64', 'macosx_10_6_x86_64') 38 | ) 39 | 40 | os.rename(filename, Path(wheel_dir, new_filename)) 41 | -------------------------------------------------------------------------------- /bindings/deno/README.md.tmpl: -------------------------------------------------------------------------------- 1 | 2 | 3 | # `x/sqlite_vss` Deno Module 4 | 5 | [![Tags](https://img.shields.io/github/release/asg017/sqlite-vss)](https://github.com/asg017/sqlite-vss/releases) 6 | [![Doc](https://doc.deno.land/badge.svg)](https://doc.deno.land/https/deno.land/x/sqlite-vss@${VERSION}/mod.ts) 7 | 8 | The [`sqlite-vss`](https://github.com/asg017/sqlite-vss) SQLite extension is available to Deno developers with the [`x/sqlite_vss`](https://deno.land/x/sqlite_vss) Deno module. It works with [`x/sqlite3`](https://deno.land/x/sqlite3), the fastest and native Deno SQLite3 module. 9 | 10 | ```js 11 | import { Database } from "https://deno.land/x/sqlite3@0.8.0/mod.ts"; 12 | import * as sqlite_vss from "https://deno.land/x/sqlite_vss@v${VERSION}/mod.ts"; 13 | 14 | const db = new Database(":memory:"); 15 | 16 | db.enableLoadExtension = true; 17 | sqlite_vss.load(db); 18 | 19 | const [version] = db 20 | .prepare("select vss_version()") 21 | .value<[string]>()!; 22 | 23 | console.log(version); 24 | 25 | ``` 26 | 27 | Like `x/sqlite3`, `x/sqlite_vss` requires network and filesystem permissions to download and cache the pre-compiled SQLite extension for your machine. Though `x/sqlite3` already requires `--allow-ffi` and `--unstable`, so you might as well use `--allow-all`/`-A`. 28 | 29 | ```bash 30 | deno run -A --unstable 31 | ``` 32 | 33 | `x/sqlite_vss` does not work with [`x/sqlite`](https://deno.land/x/sqlite@v3.7.0), which is a WASM-based Deno SQLite module that does not support loading extensions. 34 | -------------------------------------------------------------------------------- /bindings/elixir/mix.exs: -------------------------------------------------------------------------------- 1 | defmodule SqliteVss.MixProject do 2 | use Mix.Project 3 | 4 | @source_url "https://github.com/asg017/sqlite-vss/tree/main/bindings/elixir" 5 | @version File.read!(Path.expand("./VERSION", __DIR__)) |> String.trim() 6 | 7 | def project do 8 | [ 9 | app: :sqlite_vss, 10 | version: @version, 11 | elixir: "~> 1.14", 12 | start_permanent: Mix.env() == :prod, 13 | deps: deps(), 14 | name: "sqlite_vss", 15 | source_url: @source_url, 16 | homepage_url: @source_url, 17 | docs: [ 18 | main: "SqliteVss", 19 | extras: ["README.md"], 20 | source_ref: "v" <> @version 21 | ], 22 | description: description(), 23 | package: package() 24 | ] 25 | end 26 | 27 | def application do 28 | [ 29 | extra_applications: [:logger, inets: :optional, ssl: :optional], 30 | mod: {SqliteVss, []}, 31 | env: [default: []] 32 | ] 33 | end 34 | 35 | defp deps do 36 | [ 37 | {:ex_doc, "~> 0.14", only: :dev, runtime: false}, 38 | {:ecto_sqlite3, ">= 0.0.0"}, 39 | {:castore, ">= 0.0.0"}, 40 | {:hex_core, "~> 0.10.0"} 41 | ] 42 | end 43 | 44 | defp description() do 45 | "sqlite-vss please." 46 | end 47 | 48 | defp package do 49 | [ 50 | files: [ 51 | "VERSION", 52 | "lib", 53 | "mix.exs", 54 | "README.md", 55 | "sqlite-vss-checksum.exs" 56 | ], 57 | links: %{"GitHub" => @source_url}, 58 | maintainers: ["Alex Garcia", "Tommy Rodriguez"], 59 | licenses: ["MIT"], 60 | ] 61 | end 62 | end 63 | -------------------------------------------------------------------------------- /bindings/python/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, Extension 2 | import os 3 | import platform 4 | 5 | version = {} 6 | with open("sqlite_vss/version.py") as fp: 7 | exec(fp.read(), version) 8 | 9 | VERSION = version['__version__'] 10 | 11 | system = platform.system() 12 | machine = platform.machine() 13 | 14 | print(system, machine) 15 | 16 | if system == 'Darwin': 17 | if machine not in ['x86_64', 'arm64']: 18 | raise Exception("unsupported platform") 19 | elif system == 'Linux': 20 | if machine not in ['x86_64']: 21 | raise Exception("unsupported platform") 22 | #elif system == 'Windows': 23 | else: 24 | raise Exception("unsupported platform") 25 | 26 | setup( 27 | name="sqlite-vss", 28 | description="", 29 | long_description="", 30 | long_description_content_type="text/markdown", 31 | author="Alex Garcia", 32 | url="https://github.com/asg017/sqlite-vss", 33 | project_urls={ 34 | "Issues": "https://github.com/asg017/sqlite-vss/issues", 35 | "CI": "https://github.com/asg017/sqlite-vss/actions", 36 | "Changelog": "https://github.com/asg017/sqlite-vss/releases", 37 | }, 38 | license="MIT License, Apache License, Version 2.0", 39 | version=VERSION, 40 | packages=["sqlite_vss"], 41 | package_data={"sqlite_vss": ['*.so', '*.dylib', '*.dll']}, 42 | install_requires=[], 43 | # Adding an Extension makes `pip wheel` believe that this isn't a 44 | # pure-python package. The noop.c was added since the windows build 45 | # didn't seem to respect optional=True 46 | ext_modules=[Extension("noop", ["noop.c"], optional=True)], 47 | extras_require={"test": ["pytest"]}, 48 | python_requires=">=3.7", 49 | ) -------------------------------------------------------------------------------- /examples/go/demo.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "log" 7 | 8 | _ "github.com/asg017/sqlite-vss/bindings/go" 9 | _ "github.com/mattn/go-sqlite3" 10 | ) 11 | 12 | // #cgo linux,amd64 LDFLAGS: -Wl,-undefined,dynamic_lookup -lstdc++ 13 | // #cgo darwin,amd64 LDFLAGS: -Wl,-undefined,dynamic_lookup -lomp 14 | // #cgo darwin,arm64 LDFLAGS: -Wl,-undefined,dynamic_lookup -lomp 15 | import "C" 16 | 17 | func main() { 18 | db, err := sql.Open("sqlite3", ":memory:") 19 | if err != nil { 20 | log.Fatal(err) 21 | } 22 | defer db.Close() 23 | 24 | var version, vector string 25 | err = db.QueryRow("SELECT vss_version(), vector_to_json(?)", []byte{0x00, 0x00, 0x28, 0x42}).Scan(&version, &vector) 26 | if err != nil { 27 | log.Fatal(err) 28 | } 29 | 30 | fmt.Printf("version=%s vector=%s\n", version, vector) 31 | 32 | _, err = db.Exec(` 33 | CREATE VIRTUAL TABLE vss_demo USING vss0(a(2)); 34 | INSERT INTO vss_demo(rowid, a) 35 | VALUES 36 | (1, '[1.0, 2.0]'), 37 | (2, '[2.0, 2.0]'), 38 | (3, '[3.0, 2.0]') 39 | `) 40 | if err != nil { 41 | log.Fatal(err) 42 | } 43 | 44 | rows, err := db.Query(` 45 | SELECT 46 | rowid, 47 | distance 48 | FROM vss_demo 49 | WHERE vss_search(a, '[1.0, 2.0]') 50 | LIMIT 3 51 | `) 52 | if err != nil { 53 | log.Fatal(err) 54 | } 55 | 56 | for rows.Next() { 57 | var rowid int64 58 | var distance float32 59 | err = rows.Scan(&rowid, &distance) 60 | if err != nil { 61 | log.Fatal(err) 62 | } 63 | fmt.Printf("rowid=%d, distance=%f\n", rowid, distance) 64 | } 65 | err = rows.Err() 66 | if err != nil { 67 | log.Fatal(err) 68 | } 69 | 70 | fmt.Println("✅ demo.go ran successfully. \n"); 71 | } 72 | -------------------------------------------------------------------------------- /scripts/openmp_static.patch: -------------------------------------------------------------------------------- 1 | diff --git a/faiss/CMakeLists.txt b/faiss/CMakeLists.txt 2 | index 16eb9e9c..230adee2 100644 3 | --- a/faiss/CMakeLists.txt 4 | +++ b/faiss/CMakeLists.txt 5 | @@ -251,8 +251,30 @@ target_compile_definitions(faiss PRIVATE FINTEGER=int) 6 | target_compile_definitions(faiss_avx2 PRIVATE FINTEGER=int) 7 | 8 | find_package(OpenMP REQUIRED) 9 | -target_link_libraries(faiss PRIVATE OpenMP::OpenMP_CXX) 10 | -target_link_libraries(faiss_avx2 PRIVATE OpenMP::OpenMP_CXX) 11 | + 12 | +# PATCH: statically link OpenMP into vss0 (for MacOS only) 13 | +# On Macs, OpenMP_CXX_LIBRARIES always ends in .dylib for some reason, 14 | +# to target_link_libraries will dynamically link openmp. This isn't ideal 15 | +# for pre-compiled SQLite extensions, since users will have to 16 | +# "brew install libomp", which doesn't always work. So this patch is included 17 | +# in MacOS builds for easier installations 18 | + 19 | +# same as OpenMP_CXX_LIBRARIES but with ".a" instead of ".dylib" 20 | +set(_OPENMP_ARCHIVE_PATH "${OpenMP_CXX_LIBRARIES}") 21 | +string(REPLACE ".dylib" ".a" _OPENMP_ARCHIVE_PATH ${_OPENMP_ARCHIVE_PATH}) 22 | +message("_OPENMP_ARCHIVE_PATH: ${_OPENMP_ARCHIVE_PATH}") 23 | + 24 | +# link to that .a file 25 | +target_link_libraries(faiss PRIVATE ${_OPENMP_ARCHIVE_PATH}) 26 | +target_link_libraries(faiss_avx2 PRIVATE ${_OPENMP_ARCHIVE_PATH}) 27 | + 28 | +# include openmp headers as well 29 | +target_include_directories(faiss PUBLIC ${OpenMP_CXX_INCLUDE_DIRS}) 30 | +target_include_directories(faiss_avx2 PUBLIC ${OpenMP_CXX_INCLUDE_DIRS}) 31 | + 32 | +# The original linking configurations 33 | +#target_link_libraries(faiss PRIVATE OpenMP::OpenMP_CXX) 34 | +#target_link_libraries(faiss_avx2 PRIVATE OpenMP::OpenMP_CXX) 35 | 36 | find_package(MKL) 37 | if(MKL_FOUND) 38 | -------------------------------------------------------------------------------- /examples/go-cybertron/go.mod: -------------------------------------------------------------------------------- 1 | module demo-cybertron 2 | 3 | go 1.20 4 | 5 | require ( 6 | github.com/asg017/sqlite-vss/bindings/go v0.0.0-20230530231750-2d0dd01d3dfa 7 | github.com/mattn/go-sqlite3 v1.14.17 8 | github.com/nlpodyssey/cybertron v0.1.2 9 | github.com/rs/zerolog v1.27.0 10 | ) 11 | 12 | require ( 13 | github.com/cespare/xxhash v1.1.0 // indirect 14 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 15 | github.com/dgraph-io/badger/v3 v3.2103.2 // indirect 16 | github.com/dgraph-io/ristretto v0.1.0 // indirect 17 | github.com/dlclark/regexp2 v1.7.0 // indirect 18 | github.com/dustin/go-humanize v1.0.0 // indirect 19 | github.com/gogo/protobuf v1.3.2 // indirect 20 | github.com/golang/glog v1.1.1 // indirect 21 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 22 | github.com/golang/protobuf v1.5.3 // indirect 23 | github.com/golang/snappy v0.0.4 // indirect 24 | github.com/google/flatbuffers v2.0.6+incompatible // indirect 25 | github.com/google/go-cmp v0.5.9 // indirect 26 | github.com/klauspost/compress v1.16.5 // indirect 27 | github.com/mattn/go-colorable v0.1.12 // indirect 28 | github.com/mattn/go-isatty v0.0.14 // indirect 29 | github.com/nlpodyssey/gopickle v0.1.0 // indirect 30 | github.com/nlpodyssey/gotokenizers v0.2.0 // indirect 31 | github.com/nlpodyssey/spago v1.0.1 // indirect 32 | github.com/nlpodyssey/spago/embeddings/store/diskstore v0.0.0-20220801114813-013d65be77fb // indirect 33 | github.com/pkg/errors v0.9.1 // indirect 34 | github.com/stretchr/testify v1.8.4 // indirect 35 | go.opencensus.io v0.24.0 // indirect 36 | golang.org/x/net v0.10.0 // indirect 37 | golang.org/x/sync v0.2.0 // indirect 38 | golang.org/x/sys v0.8.0 // indirect 39 | golang.org/x/text v0.9.0 // indirect 40 | google.golang.org/protobuf v1.30.0 // indirect 41 | ) 42 | -------------------------------------------------------------------------------- /bindings/elixir/README.md: -------------------------------------------------------------------------------- 1 | # SqliteVss Hex Package 2 | sqlite_vss is distributed on hex for Elixir developers. 3 | 4 | ## Installation 5 | 6 | If [available in Hex](https://hex.pm/docs/publish), the package can be installed 7 | by adding `sqlite_vss` to your list of dependencies in `mix.exs`: 8 | 9 | ```elixir 10 | def deps do 11 | [ 12 | {:sqlite_vss, "~> 0.0.0"} 13 | ] 14 | end 15 | ``` 16 | 17 | Now you can install sqlite_vss by running: 18 | 19 | `$ mix sqlite_vss.install` 20 | 21 | Documentation can be generated with [ExDoc](https://github.com/elixir-lang/ex_doc) 22 | and published on [HexDocs](https://hexdocs.pm). Once published, the docs can 23 | be found at . 24 | 25 | The `sqlite-vss` package is meant to be used with Exqlite like the following: 26 | 27 | ``` 28 | Mix.install([ 29 | {:sqlite_vss, path: "../"}, 30 | {:exqlite, "~> 0.13.0"} 31 | ], verbose: true) 32 | 33 | Mix.Task.run("sqlite_vss.install") 34 | 35 | alias Exqlite.Basic 36 | 37 | {:ok, conn} = Basic.open("example.db") 38 | 39 | :ok = Exqlite.Basic.enable_load_extension(conn) 40 | Exqlite.Basic.load_extension(conn, SqliteVss.loadable_path_vector0()) 41 | Exqlite.Basic.load_extension(conn, SqliteVss.loadable_path_vss0()) 42 | 43 | {:ok, [[version]], [_]} = Basic.exec(conn, "select vss_version()") |> Basic.rows() 44 | 45 | IO.puts("version: #{version}") 46 | ``` 47 | 48 | To load the extension files for an Ecto Repo add the following to your runtime.exs config. 49 | 50 | ``` 51 | # global 52 | config :exqlite, load_extensions: [SqliteVss.loadable_path_vector0(), SqliteVss.loadable_path_vss0()] 53 | 54 | # per connection in a Phoenix app 55 | config :my_app, MyApp.Repo, 56 | load_extensions: [SqliteVss.loadable_path_vector0(), SqliteVss.loadable_path_vss0()] 57 | ``` 58 | 59 | ## Running the demo 60 | 61 | ``` 62 | cd demo 63 | 64 | elixir demo.exs 65 | ``` 66 | -------------------------------------------------------------------------------- /.github/workflows/upload-deno-assets.js: -------------------------------------------------------------------------------- 1 | const fs = require("fs").promises; 2 | const { createHash } = require("node:crypto"); 3 | 4 | module.exports = async ({ github, context }) => { 5 | const VERSION = process.env.GITHUB_REF_NAME; 6 | const { owner, repo } = context.repo; 7 | 8 | const compiled_extensions = [ 9 | { 10 | path: `${process.env["ARTIFACT-MACOS-X86_64-EXTENSION"]}/vector0.dylib`, 11 | name: `sqlite-vss-${VERSION}-deno-darwin-x86_64.vector0.dylib`, 12 | }, 13 | { 14 | path: `${process.env["ARTIFACT-MACOS-X86_64-EXTENSION"]}/vss0.dylib`, 15 | name: `sqlite-vss-${VERSION}-deno-darwin-x86_64.vss0.dylib`, 16 | }, 17 | { 18 | path: `${process.env["ARTIFACT-MACOS-AARCH64-EXTENSION"]}/vector0.dylib`, 19 | name: `sqlite-vss-${VERSION}-deno-darwin-aarch64.vector0.dylib`, 20 | }, 21 | { 22 | path: `${process.env["ARTIFACT-MACOS-AARCH64-EXTENSION"]}/vss0.dylib`, 23 | name: `sqlite-vss-${VERSION}-deno-darwin-aarch64.vss0.dylib`, 24 | }, 25 | { 26 | path: `${process.env["ARTIFACT-LINUX-X86_64-EXTENSION"]}/vector0.so`, 27 | name: `sqlite-vss-${VERSION}-deno-linux-x86_64.vector0.so`, 28 | }, 29 | { 30 | path: `${process.env["ARTIFACT-LINUX-X86_64-EXTENSION"]}/vss0.so`, 31 | name: `sqlite-vss-${VERSION}-deno-linux-x86_64.vss0.so`, 32 | }, 33 | ]; 34 | 35 | const release = await github.rest.repos.getReleaseByTag({ 36 | owner, 37 | repo, 38 | tag: VERSION, 39 | }); 40 | const release_id = release.data.id; 41 | const outputAssetChecksums = []; 42 | 43 | await Promise.all( 44 | compiled_extensions.map(async ({ name, path }) => { 45 | const data = await fs.readFile(path); 46 | const checksum = createHash("sha256").update(data).digest("hex"); 47 | outputAssetChecksums.push({ name, checksum }); 48 | return github.rest.repos.uploadReleaseAsset({ 49 | owner, 50 | repo, 51 | release_id, 52 | name, 53 | data, 54 | }); 55 | }) 56 | ); 57 | return outputAssetChecksums.map((d) => `${d.checksum} ${d.name}`).join("\n"); 58 | }; -------------------------------------------------------------------------------- /site/using/nodejs.md: -------------------------------------------------------------------------------- 1 | # `sqlite-vss` with Node.js 2 | 3 | [![npm](https://img.shields.io/npm/v/sqlite-vss.svg?color=green&logo=nodedotjs&logoColor=white)](https://www.npmjs.com/package/sqlite-vss) 4 | 5 | Node.js developers can use `sqlite-vss` with the [`sqlite-vss` NPM package](https://www.npmjs.com/package/sqlite-vss). It can be installed with: 6 | 7 | ```bash 8 | npm install sqlite-vss 9 | ``` 10 | 11 | Once installed, the `sqlite-vss` package can be used with SQLite clients like [`better-sqlite3`](https://github.com/WiseLibs/better-sqlite3) and [`node-sqlite3`](https://github.com/TryGhost/node-sqlite3). 12 | 13 | ```js 14 | import Database from "better-sqlite3"; 15 | import * as sqlite_vss from "sqlite-vss"; 16 | 17 | const db = new Database(":memory:"); 18 | sqlite_vss.load(db); 19 | 20 | const version = db.prepare("select vss_version()").pluck().get(); 21 | console.log(version); 22 | ``` 23 | 24 | Checkout [the API Reference](./api-reference) for all available SQL functions. 25 | 26 | Also see _[Making SQLite extensions npm install-able](https://observablehq.com/@asg017/making-sqlite-extensions-npm-installable-and-deno) (March 2023)_ for more information on how npm install'able SQLite extensions work. 27 | 28 | ## Working with Vectors in Node.js 29 | 30 | ### Vectors as JSON 31 | 32 | If your vectors in Node.js are represented as an array of floats, you can insert them into a `vss0` table as a JSON string with `JSON.stringify()`. 33 | 34 | ```js 35 | const embedding = [0.1, 0.2, 0.3]; 36 | const stmt = db.prepare("INSERT INTO vss_demo VALUES (?)"); 37 | stmt.run(JSON.stringify(embedding)); 38 | ``` 39 | 40 | ### Vectors as Bytes 41 | 42 | Alternatively, if your vectors in Node.js are represented as a [Float32Array](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Float32Array), use the [`.buffer`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/TypedArray/buffer) accessor to insert the underlying ArrayBuffer. 43 | 44 | ```js 45 | const embedding = new Float32Array([0.1, 0.2, 0.3]); 46 | const stmt = db.prepare("INSERT INTO vss_demo VALUES (?)"); 47 | stmt.run(embedding.buffer); 48 | ``` 49 | -------------------------------------------------------------------------------- /NOTES: -------------------------------------------------------------------------------- 1 | ## TODO 2 | 3 | - [ ] `sqlite-vector` 4 | - [ ] light benchmarks 5 | - [ ] INSERT respect transactions 6 | - `pTable->vectors_to_add` 7 | - `xCommit`/`xRollback` 8 | - [ ] DELETE and UPDATE support 9 | - [ ] xUpdate DELETE, with `pTable->ids_to_data` and `xCommit`/`xRollback` support 10 | - [ ] update, call remove_id, add_id? 11 | 12 | 13 | 14 | - [ ] clustering? 15 | - [ ] [Distances](https://faiss.ai/cpp_api/file/distances_8h.html) 16 | - [ ] [extra distances](https://faiss.ai/cpp_api/file/extra__distances_8h.html) 17 | - [ ] binary index 18 | - [ ] hamming distance utils 19 | - [ ] vtab option to store index on disk instead (mmaped) 20 | - [ ] GPU? 21 | 22 | ```sql 23 | create table articles( 24 | headline text, 25 | body text 26 | ); 27 | create virtual table article_vectors using vss( 28 | headline(384) using "Flat,IDMap", 29 | body(384) using "IVF10,PQ4", 30 | ); 31 | 32 | with similar_headlines as ( 33 | select 34 | id, 35 | distance 36 | from article_vectors 37 | where vss_search(headline, vss_params('query', :query, 'k', 50)); 38 | ) 39 | select 40 | articles.headline 41 | from similar_headlines 42 | left join articles on articles.rowid = similar_headlines.id 43 | 44 | ``` 45 | 46 | https://github.com/matsui528/faiss_tips 47 | 48 | ``` 49 | cmake -B build; make -C build 50 | 51 | cmake -DCMAKE_BUILD_TYPE=Release -B build_release; make -C build_release 52 | ``` 53 | 54 | ```bash 55 | cd build/ 56 | cmake .. -DFAISS_ENABLE_GPU=OFF -DFAISS_ENABLE_PYTHON=OFF 57 | make 58 | ``` 59 | 60 | ``` 61 | PYO3_PYTHON=/Users/alex/projects/research-sqlite-vector/venv/bin/python LIBDIR=/usr/local/opt/python@3.8/Frameworks/Python.framework/Versions/3.8/lib cargo build 62 | 63 | PYTHONPATH=/Users/alex/projects/research-sqlite-vector/venv/lib/python3.8/site-packages/ sqlite3x :memory: '.read test.sql' 64 | ``` 65 | 66 | ## Embeddings generating 67 | 68 | `sqlite-vss` is a **Bring Your Own Vectors** database. 69 | 70 | ### Option 1: Application Defined Functions 71 | 72 | ### Option 2: With `sqlite-http` 73 | 74 | ### Option 3: With `sqlite-openai` or `sqlite-huggingface-inference` 75 | 76 | ### Option 4: With `sqlite-py` 77 | 78 | ### Option 5: With `sqlite-bert` (WIP) 79 | -------------------------------------------------------------------------------- /bindings/node/sqlite-vss/src/index.js: -------------------------------------------------------------------------------- 1 | import { join } from "node:path"; 2 | import { fileURLToPath } from "node:url"; 3 | import { arch, platform } from "node:process"; 4 | import { statSync } from "node:fs"; 5 | 6 | const supportedPlatforms = [ 7 | ["darwin", "x64"], 8 | ["darwin", "arm64"], 9 | ["linux", "x64"], 10 | ]; 11 | 12 | function validPlatform(platform, arch) { 13 | return ( 14 | supportedPlatforms.find(([p, a]) => platform == p && arch === a) !== null 15 | ); 16 | } 17 | function extensionSuffix(platform) { 18 | if (platform === "win32") return "dll"; 19 | if (platform === "darwin") return "dylib"; 20 | return "so"; 21 | } 22 | function platformPackageName(platform, arch) { 23 | const os = platform === "win32" ? "windows" : platform; 24 | return `sqlite-vss-${os}-${arch}`; 25 | } 26 | 27 | function loadablePathResolver(name) { 28 | if (!validPlatform(platform, arch)) { 29 | throw new Error( 30 | `Unsupported platform for sqlite-vss, on a ${platform}-${arch} machine, but not in supported platforms (${supportedPlatforms 31 | .map(([p, a]) => `${p}-${a}`) 32 | .join(",")}). Consult the sqlite-vss NPM package README for details. ` 33 | ); 34 | } 35 | const packageName = platformPackageName(platform, arch); 36 | const loadablePath = join( 37 | fileURLToPath(new URL(".", import.meta.url)), 38 | "..", 39 | "..", 40 | packageName, 41 | "lib", 42 | `${name}.${extensionSuffix(platform)}` 43 | ); 44 | if (!statSync(loadablePath, { throwIfNoEntry: false })) { 45 | throw new Error( 46 | `Loadble extension for sqlite-vss not found. Was the ${packageName} package installed? Avoid using the --no-optional flag, as the optional dependencies for sqlite-vss are required.` 47 | ); 48 | } 49 | 50 | return loadablePath; 51 | } 52 | 53 | export function getVectorLoadablePath() { 54 | return loadablePathResolver("vector0"); 55 | } 56 | 57 | export function getVssLoadablePath() { 58 | return loadablePathResolver("vss0"); 59 | } 60 | 61 | export function loadVector(db) { 62 | db.loadExtension(getVectorLoadablePath()); 63 | } 64 | export function loadVss(db) { 65 | db.loadExtension(getVssLoadablePath()); 66 | } 67 | export function load(db) { 68 | loadVector(db); 69 | loadVss(db); 70 | } 71 | -------------------------------------------------------------------------------- /site/compare.md: -------------------------------------------------------------------------------- 1 | # Comparisons with... 2 | 3 | `sqlite-vss` 4 | 5 | In general, `sqlite-vss` has the following unique features: 6 | 7 | - **Embeddable**: No separate process, server, or configuration is needed, `sqlite-vss` runs in the same process as your application, just like SQLite. 8 | - **Pure SQL**: There is no separate DSL or special API for inserting or querying vectors, it's all SQL. 9 | - **Configurable**: You can specify different [Faiss factory strings](https://github.com/facebookresearch/faiss/wiki/The-index-factory) to customize how your vectors are indexed. 10 | - **Easy Distribution and Installation**: `sqlite-vss` can be installed through several different package managers and languages, including Python, Node.js, Ruby, Deno, Go, rust, and more. 11 | 12 | But it also has the following disadvantages: 13 | 14 | - **Doesn't scale to millions of users**: A SQLite database can get you very far, but will eventually hit a limit with enough users or with extremely write-heavy applications. 15 | - **Depends on Faiss**: Faiss makes it tricky to compile yourself and hard to use on certain platforms. 16 | - **Young in development**: [`sqlite-vss`'s disadvantages listed on the README](https://github.com/asg017/sqlite-vss#disadvantages) go over several implementation-specific problems you may face using `sqlite-vss`, mostly because `sqlite-vss` is young and has 1 main developer. Consider [sponsoring my work](https://github.com/sponsors/asg017) to have these problems fixed! 17 | 18 | This page goes over other vector databases and vector storage techniques that exist, and how `sqlite-vss` differs. 19 | 20 | ## Vector Databases (Pinecone, Milvus, Qdrant, etc.) { #vector-dbs } 21 | 22 | In general, "real" vector databases like [Pinecone](https://www.pinecone.io/), [Milvus](https://milvus.io/), [Qdrant](https://qdrant.tech/), and dozens of others will likely "scale" better than `sqlite-vss`. These databases perform writes faster and more efficiently than `sqlite-vss`, can handle a large number of users at the same time, and can perform metadata-filtering on queries. 23 | 24 | On the other hand, `sqlite-vss` doesn't require an entire new server to maintain, is open source, and is most likely "fast enough" for many use-cases. 25 | 26 | 27 | ## JSON or Pickle 28 | 29 | ## Faiss 30 | 31 | ## `datasette-faiss` 32 | 33 | ## Chroma 34 | 35 | ## pgvector 36 | 37 | ## txtai 38 | -------------------------------------------------------------------------------- /site/using/deno.md: -------------------------------------------------------------------------------- 1 | 5 | 6 | # `sqlite-vss` with Deno 7 | 8 | [![deno.land/x release](https://img.shields.io/github/v/release/asg017/sqlite-vss?color=fef8d2&include_prereleases&label=deno.land%2Fx&logo=deno)](https://deno.land/x/sqlite_vss) 9 | 10 | `sqlite-vss` is available to Deno developers with the [`x/sqlite_vss`](https://deno.land/x/sqlite_vss) Deno module. It works with [`x/sqlite3`](https://deno.land/x/sqlite3), the native Deno SQLite module. 11 | 12 | ::: code-group 13 | 14 | ```ts-vue [main.ts] 15 | import { Database } from "https://deno.land/x/sqlite3@0.8.0/mod.ts"; 16 | import * as sqlite_vss from "https://deno.land/x/sqlite_vss@v{{ VERSION }}/mod.ts"; 17 | 18 | const db = new Database(":memory:"); 19 | db.enableLoadExtension = true; 20 | sqlite_vss.load(db); 21 | db.enableLoadExtension = false; 22 | 23 | const [version] = db 24 | .prepare("select vss_version()") 25 | .value<[string]>()!; 26 | 27 | console.log(version); 28 | ``` 29 | 30 | ::: 31 | 32 | ```bash-vue 33 | deno run -A --unstable main.ts 34 | ``` 35 | 36 | Checkout [the API Reference](./api-reference) for all available SQL functions. 37 | 38 | Also see _[Putting SQLite extensions on `deno.land/x`](https://observablehq.com/@asg017/making-sqlite-extensions-npm-installable-and-deno) (March 2023)_ for more information on how Deno SQLite extensions work. 39 | 40 | ## Working with Vectors in Deno 41 | 42 | ### Vectors as JSON 43 | 44 | If your vectors in Deno are represented as an array of floats, you can insert them into a `vss0` table as a JSON string with `JSON.stringify()`. 45 | 46 | ```js 47 | const embedding = [0.1, 0.2, 0.3]; 48 | const stmt = db.prepare("INSERT INTO vss_demo VALUES (?)"); 49 | stmt.run(JSON.stringify(embedding)); 50 | ``` 51 | 52 | ### Vectors as Bytes 53 | 54 | Alternatively, if your vectors in Node.js are represented as a [Float32Array](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Float32Array), use the [`.buffer`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/TypedArray/buffer) accessor to insert the underlying ArrayBuffer. 55 | 56 | ```js 57 | const embedding = new Float32Array([0.1, 0.2, 0.3]); 58 | const stmt = db.prepare("INSERT INTO vss_demo VALUES (?)"); 59 | stmt.run(embedding.buffer); 60 | ``` 61 | -------------------------------------------------------------------------------- /bindings/elixir/test/sqlite_vss_test.exs: -------------------------------------------------------------------------------- 1 | defmodule SqliteVssTest do 2 | use ExUnit.Case 3 | doctest SqliteVss 4 | 5 | describe "current_target/1" do 6 | test "aarch64 in an Apple machine with Darwin-based OS" do 7 | target_system = %{arch: "aarch64", vendor: "apple", os: "darwin20.3.0"} 8 | 9 | config = %{ 10 | target_system: target_system, 11 | os_type: {:unix, :darwin} 12 | } 13 | 14 | assert {:ok, "macos-aarch64"} = SqliteVss.current_target(config) 15 | end 16 | 17 | test "x86_64 in an Apple machine with Darwin-based OS" do 18 | target_system = %{arch: "x86_64", vendor: "apple", os: "darwin20.3.0"} 19 | 20 | config = %{ 21 | target_system: target_system, 22 | os_type: {:unix, :darwin} 23 | } 24 | 25 | assert {:ok, "macos-x86_64"} = SqliteVss.current_target(config) 26 | end 27 | 28 | test "x86_64 in a PC running RedHat Linux" do 29 | target_system = %{arch: "x86_64", vendor: "redhat", os: "linux", abi: "gnu"} 30 | 31 | config = %{ 32 | target_system: target_system, 33 | os_type: {:unix, :linux} 34 | } 35 | 36 | assert {:ok, "linux-x86_64"} = SqliteVss.current_target(config) 37 | end 38 | 39 | test "aarch64 in a PC running Linux" do 40 | target_system = %{arch: "aarch64", vendor: "pc", os: "linux", abi: "gnu"} 41 | 42 | config = %{ 43 | target_system: target_system, 44 | os_type: {:unix, :linux} 45 | } 46 | 47 | assert {:ok, "linux-aarch64"} = SqliteVss.current_target(config) 48 | end 49 | 50 | test "target not available" do 51 | config = %{ 52 | target_system: %{arch: "i686", vendor: "unknown", os: "linux", abi: "gnu"}, 53 | nif_version: "2.14", 54 | os_type: {:unix, :linux} 55 | } 56 | 57 | error_message = 58 | """ 59 | precompiled artifact is not available for this target: \"gnu-i686-linux-unknown\". 60 | The available targets are: 61 | - linux-x86_64 62 | - macos-aarch64 63 | - macos-x86_64 64 | """ 65 | |> String.trim() 66 | 67 | assert {:error, ^error_message} = SqliteVss.current_target(config) 68 | end 69 | end 70 | 71 | test "loads the vss and vector path" do 72 | assert String.match?(SqliteVss.loadable_path_vss0(), ~r/vss0/) 73 | assert String.match?(SqliteVss.loadable_path_vector0(), ~r/vector0/) 74 | end 75 | end 76 | -------------------------------------------------------------------------------- /site/.vitepress/theme/style.css: -------------------------------------------------------------------------------- 1 | /** 2 | * Customize default theme styling by overriding CSS variables: 3 | * https://github.com/vuejs/vitepress/blob/main/src/client/theme-default/styles/vars.css 4 | */ 5 | 6 | /** 7 | * Colors 8 | * -------------------------------------------------------------------------- */ 9 | 10 | :root { 11 | --vp-c-brand: #646cff; 12 | --vp-c-brand-light: #747bff; 13 | --vp-c-brand-lighter: #9499ff; 14 | --vp-c-brand-lightest: #bcc0ff; 15 | --vp-c-brand-dark: #535bf2; 16 | --vp-c-brand-darker: #454ce1; 17 | --vp-c-brand-dimm: rgba(100, 108, 255, 0.08); 18 | } 19 | 20 | /** 21 | * Component: Button 22 | * -------------------------------------------------------------------------- */ 23 | 24 | :root { 25 | --vp-button-brand-border: var(--vp-c-brand-light); 26 | --vp-button-brand-text: var(--vp-c-white); 27 | --vp-button-brand-bg: var(--vp-c-brand); 28 | --vp-button-brand-hover-border: var(--vp-c-brand-light); 29 | --vp-button-brand-hover-text: var(--vp-c-white); 30 | --vp-button-brand-hover-bg: var(--vp-c-brand-light); 31 | --vp-button-brand-active-border: var(--vp-c-brand-light); 32 | --vp-button-brand-active-text: var(--vp-c-white); 33 | --vp-button-brand-active-bg: var(--vp-button-brand-bg); 34 | } 35 | 36 | /** 37 | * Component: Home 38 | * -------------------------------------------------------------------------- */ 39 | 40 | :root { 41 | --vp-home-hero-name-color: transparent; 42 | --vp-home-hero-name-background: -webkit-linear-gradient( 43 | 120deg, 44 | #bd34fe 30%, 45 | #41d1ff 46 | ); 47 | 48 | --vp-home-hero-image-background-image: linear-gradient( 49 | -45deg, 50 | #bd34fe 50%, 51 | #47caff 50% 52 | ); 53 | --vp-home-hero-image-filter: blur(40px); 54 | } 55 | 56 | @media (min-width: 640px) { 57 | :root { 58 | --vp-home-hero-image-filter: blur(56px); 59 | } 60 | } 61 | 62 | @media (min-width: 960px) { 63 | :root { 64 | --vp-home-hero-image-filter: blur(72px); 65 | } 66 | } 67 | 68 | /** 69 | * Component: Custom Block 70 | * -------------------------------------------------------------------------- */ 71 | 72 | :root { 73 | --vp-custom-block-tip-border: var(--vp-c-brand); 74 | --vp-custom-block-tip-text: var(--vp-c-brand-darker); 75 | --vp-custom-block-tip-bg: var(--vp-c-brand-dimm); 76 | } 77 | 78 | .dark { 79 | --vp-custom-block-tip-border: var(--vp-c-brand); 80 | --vp-custom-block-tip-text: var(--vp-c-brand-lightest); 81 | --vp-custom-block-tip-bg: var(--vp-c-brand-dimm); 82 | } 83 | 84 | /** 85 | * Component: Algolia 86 | * -------------------------------------------------------------------------- */ 87 | 88 | .DocSearch { 89 | --docsearch-primary-color: var(--vp-c-brand) !important; 90 | } 91 | 92 | .VPSidebarItem .level-1 { 93 | /*margin-left: 0.5rem;*/ 94 | } 95 | -------------------------------------------------------------------------------- /bindings/node/.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | lerna-debug.log* 8 | .pnpm-debug.log* 9 | 10 | # Diagnostic reports (https://nodejs.org/api/report.html) 11 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 12 | 13 | # Runtime data 14 | pids 15 | *.pid 16 | *.seed 17 | *.pid.lock 18 | 19 | # Directory for instrumented libs generated by jscoverage/JSCover 20 | lib-cov 21 | 22 | # Coverage directory used by tools like istanbul 23 | coverage 24 | *.lcov 25 | 26 | # nyc test coverage 27 | .nyc_output 28 | 29 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 30 | .grunt 31 | 32 | # Bower dependency directory (https://bower.io/) 33 | bower_components 34 | 35 | # node-waf configuration 36 | .lock-wscript 37 | 38 | # Compiled binary addons (https://nodejs.org/api/addons.html) 39 | build/Release 40 | 41 | # Dependency directories 42 | node_modules/ 43 | jspm_packages/ 44 | 45 | # Snowpack dependency directory (https://snowpack.dev/) 46 | web_modules/ 47 | 48 | # TypeScript cache 49 | *.tsbuildinfo 50 | 51 | # Optional npm cache directory 52 | .npm 53 | 54 | # Optional eslint cache 55 | .eslintcache 56 | 57 | # Optional stylelint cache 58 | .stylelintcache 59 | 60 | # Microbundle cache 61 | .rpt2_cache/ 62 | .rts2_cache_cjs/ 63 | .rts2_cache_es/ 64 | .rts2_cache_umd/ 65 | 66 | # Optional REPL history 67 | .node_repl_history 68 | 69 | # Output of 'npm pack' 70 | *.tgz 71 | 72 | # Yarn Integrity file 73 | .yarn-integrity 74 | 75 | # dotenv environment variable files 76 | .env 77 | .env.development.local 78 | .env.test.local 79 | .env.production.local 80 | .env.local 81 | 82 | # parcel-bundler cache (https://parceljs.org/) 83 | .cache 84 | .parcel-cache 85 | 86 | # Next.js build output 87 | .next 88 | out 89 | 90 | # Nuxt.js build / generate output 91 | .nuxt 92 | dist 93 | 94 | # Gatsby files 95 | .cache/ 96 | # Comment in the public line in if your project uses Gatsby and not Next.js 97 | # https://nextjs.org/blog/next-9-1#public-directory-support 98 | # public 99 | 100 | # vuepress build output 101 | .vuepress/dist 102 | 103 | # vuepress v2.x temp and cache directory 104 | .temp 105 | .cache 106 | 107 | # Docusaurus cache and generated files 108 | .docusaurus 109 | 110 | # Serverless directories 111 | .serverless/ 112 | 113 | # FuseBox cache 114 | .fusebox/ 115 | 116 | # DynamoDB Local files 117 | .dynamodb/ 118 | 119 | # TernJS port file 120 | .tern-port 121 | 122 | # Stores VSCode versions used for testing VSCode extensions 123 | .vscode-test 124 | 125 | # yarn v2 126 | .yarn/cache 127 | .yarn/unplugged 128 | .yarn/build-state.yml 129 | .yarn/install-state.gz 130 | .pnp.* 131 | 132 | # not needed for libraries, right? running into some issues in ci with this, so gonna yeet 133 | package-lock.json -------------------------------------------------------------------------------- /examples/headlines/build/add_embeddings.py: -------------------------------------------------------------------------------- 1 | import sqlite3 2 | import sys 3 | 4 | print(sys.argv) 5 | db_path = sys.argv[1] 6 | 7 | db = sqlite3.connect(db_path) 8 | 9 | db.enable_load_extension(True) 10 | db.load_extension("./vector0") 11 | db.enable_load_extension(False) 12 | 13 | 14 | from sentence_transformers import SentenceTransformer 15 | model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2") 16 | 17 | BATCH_SIZE = 256 18 | total_count = db.execute("select count(*) from articles where headline_embedding is null").fetchone()[0] 19 | 20 | # by default, the headline_embedding and description_embeddings are NULL. This script will fill them in 21 | # with the embeddings from the sentence-transformers model, based on the headline and description columns. 22 | 23 | 24 | def fill_headline_embeddings(): 25 | # headline_embedding is set to NULL by default. Loop through all rows and fill that column 26 | # until they are all non-null. 27 | while True: 28 | 29 | # batch BATCH_SIZE rows at a time. Makes model.encode faster and uses less memory than all-at-once 30 | batch = db.execute( 31 | """ 32 | select 33 | rowid, 34 | headline 35 | from articles 36 | where headline_embedding is null 37 | limit ? 38 | """, 39 | [BATCH_SIZE] 40 | ).fetchall() 41 | 42 | if len(batch) == 0: 43 | break 44 | 45 | rowids = list(map(lambda x: x[0], batch)) 46 | headlines = list(map(lambda x: x[1], batch)) 47 | 48 | print(f"[{rowids[-1]/total_count*100:.2f}] encoding [{rowids[0]}:{rowids[-1]}]") 49 | embeddings = model.encode(headlines) 50 | 51 | for rowid, embedding in zip(rowids, embeddings): 52 | db.execute( 53 | """ 54 | update articles 55 | set headline_embedding = ? 56 | where rowid = ? 57 | """, 58 | [embedding.tobytes(), rowid] 59 | ) 60 | 61 | db.commit() 62 | 63 | def fill_description_embedding(): 64 | # description_embedding is set to NULL by default. Loop through all rows and fill that column 65 | # until they are all non-null. 66 | while True: 67 | batch = db.execute( 68 | """ 69 | select 70 | rowid, 71 | description 72 | from articles 73 | where description_embedding is null 74 | limit ? 75 | """, 76 | [BATCH_SIZE] 77 | ).fetchall() 78 | 79 | if len(batch) == 0: 80 | break 81 | 82 | rowids = list(map(lambda x: x[0], batch)) 83 | descriptions = list(map(lambda x: x[1], batch)) 84 | 85 | print(f"[{rowids[-1]/total_count*100:.2f}] encoding [{rowids[0]}:{rowids[-1]}]") 86 | embeddings = model.encode(descriptions) 87 | 88 | for rowid, embedding in zip(rowids, embeddings): 89 | db.execute( 90 | """ 91 | update articles 92 | set description_embedding = ? 93 | where rowid = ? 94 | """, 95 | [embedding.tobytes(), rowid] 96 | ) 97 | 98 | db.commit() 99 | 100 | fill_headline_embeddings() 101 | fill_description_embedding() 102 | -------------------------------------------------------------------------------- /site/using/python.md: -------------------------------------------------------------------------------- 1 | # `sqlite-vss` with Python 2 | 3 | [![PyPI](https://img.shields.io/pypi/v/sqlite-vss.svg?color=blue&logo=python&logoColor=white)](https://pypi.org/project/sqlite-vss/) 4 | 5 | Python developers can use `sqlite-vss` with the [`sqlite-vss` PyPi package](https://pypi.org/project/sqlite-vss/). 6 | 7 | ```bash 8 | pip install sqlite-vss 9 | ``` 10 | 11 | In Python, the `sqlite_vss` module has a `.load()` function that will load `sqlite-vss` SQL functions into a given SQLite connection. 12 | 13 | ```python 14 | import sqlite3 15 | import sqlite_vss 16 | 17 | db = sqlite3.connect(':memory:') 18 | db.enable_load_extension(True) 19 | sqlite_vss.load(db) 20 | db.enable_load_extension(False) 21 | 22 | version, = db.execute('select vss_version()').fetchone() 23 | print(version) 24 | ``` 25 | 26 | Checkout [the API Reference](./api-reference) for all available SQL functions. 27 | 28 | Also see _[Making SQLite extensions pip install-able](https://observablehq.com/@asg017/making-sqlite-extensions-pip-install-able) (February 2023)_ for more information on how pip install'able SQLite extensions work. 29 | 30 | ## Working with Vectors in Python 31 | 32 | ### Vectors as JSON 33 | 34 | If your vectors in Python are represented as a list of floats, you can insert them into a `vss0` table as a JSON string with [`json.dumps()`](https://docs.python.org/3/library/json.html#json.dumps). 35 | 36 | ```python 37 | import json 38 | 39 | embedding = [0.1, 0.2, 0.3] 40 | db.execute("insert into vss_demo(a) values (?)", [json.dumps(embedding)]) 41 | ``` 42 | 43 | ### Vectors as Bytes 44 | 45 | You can also convert a list of floats into the compact "raw bytes" format with [`struct.pack()`](https://docs.python.org/3/library/struct.html#struct.pack). 46 | 47 | ```python 48 | import struct 49 | 50 | def serialize(vector: List[float]) -> bytes: 51 | """ serializes a list of floats into a compact "raw bytes" format """ 52 | return struct.pack('%sf' % len(vector), *vector) 53 | 54 | embedding = [0.1, 0.2, 0.3] 55 | db.execute('insert into vss_demo(a) values (?)', [serialize(embedding)]) 56 | ``` 57 | 58 | ### `numpy` Arrays 59 | 60 | If your embeddings are a numpy array, you can use [`.tobytes()`](https://numpy.org/doc/stable/reference/generated/numpy.ndarray.tobytes.html) to convert the array into the `sqlite-vss`-compatible "raw bytes" format. 61 | 62 | ```python 63 | import numpy as np 64 | 65 | embedding = np.array([0.1, 0.2, 0.3]) 66 | 67 | db.execute( 68 | "insert into vss_demo(a) values (?)", [embedding.astype(np.float32).tobytes()] 69 | ) 70 | ``` 71 | 72 | The `.astype(np.float32)` cast is important! Faiss and `sqlite-vss` works with 4-byte floats, not 8-byte doubles. 73 | 74 | ### Which format should I choose? 75 | 76 | If you're inserting into a `vss0` table, the vector will be converting into a compact Faiss format anyway, so inserting a vector as JSON or "raw bytes" has no effect on the storage size of your `vss0` virtual table. 77 | 78 | However, if you are also saving the vector in a regular SQLite column, then the "raw bytes" format will be much more compact than the JSON string representation. 79 | 80 | You may also find the "raw bytes" method to be slightly faster than `json.dumps()`, especially in `numpy`. 81 | -------------------------------------------------------------------------------- /bindings/python/README.md: -------------------------------------------------------------------------------- 1 | # The `sqlite-vss` Python package 2 | 3 | `sqlite-vss` is also distributed on PyPi as a Python package, for use in Python applications. It works well with the builtin [`sqlite3`](https://docs.python.org/3/library/sqlite3.html) Python module. 4 | 5 | ``` 6 | pip install sqlite-vss 7 | ``` 8 | 9 | ## Usage 10 | 11 | The `sqlite-vss` python package exports two functions: `vss_loadable_path()`, which returns the full path to the loadable extension, and `load(conn)`, which loads the `sqlite-vss` extension into the given [sqlite3 Connection object](https://docs.python.org/3/library/sqlite3.html#connection-objects). 12 | 13 | ```python 14 | import sqlite_vss 15 | print(sqlite_vss.vss_loadable_path()) 16 | # '/.../venv/lib/python3.9/site-packages/sqlite_vss/vss0' 17 | 18 | import sqlite3 19 | conn = sqlite3.connect(':memory:') 20 | conn.enable_load_extension(True) 21 | sqlite_vss.load(conn) 22 | 23 | print(conn.execute('select vss_version()').fetchone()[0]) 24 | # 'v0.1.0' 25 | ``` 26 | 27 | See [the full API Reference](#api-reference) for the Python API, and [`docs.md`](../../docs.md) for documentation on the `sqlite-vss` SQL API. 28 | 29 | See [`datasette-sqlite-vss`](../datasette_sqlite_vss/) for a Datasette plugin that is a light wrapper around the `sqlite-vss` Python package. 30 | 31 | ## Compatibility 32 | 33 | Currently the `sqlite-vss` Python package is only distributed on PyPi as pre-build wheels, it's not possible to install from the source distribution. This is because the underlying `sqlite-vss` extension requires a lot of build dependencies like `make`, `cc`, and `cargo`. 34 | 35 | If you get a `unsupported platform` error when pip installing `sqlite-vss`, you'll have to build the `sqlite-vss` manually and load in the dynamic library manually. 36 | 37 | ## API Reference 38 | 39 |

vss_loadable_path()

40 | 41 | Returns the full path to the locally-install `sqlite-vss` extension, without the filename. 42 | 43 | This can be directly passed to [`sqlite3.Connection.load_extension()`](https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection.load_extension), but the [`sqlite_vss.load()`](#load) function is preferred. 44 | 45 | ```python 46 | import sqlite_vss 47 | print(sqlite_vss.vss_loadable_path()) 48 | # '/.../venv/lib/python3.9/site-packages/sqlite_vss/vss0' 49 | ``` 50 | 51 | > Note: this extension path doesn't include the file extension (`.dylib`, `.so`, `.dll`). This is because [SQLite will infer the correct extension](https://www.sqlite.org/loadext.html#loading_an_extension). 52 | 53 |

load(connection)

54 | 55 | Loads the `sqlite-vss` extension on the given [`sqlite3.Connection`](https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection) object, calling [`Connection.load_extension()`](https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection.load_extension). 56 | 57 | ```python 58 | import sqlite_vss 59 | import sqlite3 60 | conn = sqlite3.connect(':memory:') 61 | 62 | conn.enable_load_extension(True) 63 | sqlite_vss.load(conn) 64 | conn.enable_load_extension(False) 65 | 66 | conn.execute('select vss_version(), vss()').fetchone() 67 | # ('v0.1.0', '01gr7gwc5aq22ycea6j8kxq4s9') 68 | TODO 69 | ``` 70 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.17 FATAL_ERROR) 2 | project(sqlite-vss VERSION $ENV{SQLITE_VSS_CMAKE_VERSION}) 3 | project(sqlite-vss-static VERSION $ENV{SQLITE_VSS_CMAKE_VERSION}) 4 | project(sqlite-vector VERSION $ENV{SQLITE_VSS_CMAKE_VERSION}) 5 | project(sqlite-vector-static VERSION $ENV{SQLITE_VSS_CMAKE_VERSION}) 6 | 7 | if(PROJECT_VERSION_TWEAK) 8 | set(SQLITE_VSS_VERSION "v${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH}-alpha.${PROJECT_VERSION_TWEAK}") 9 | else() 10 | set(SQLITE_VSS_VERSION "v${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH}") 11 | endif() 12 | configure_file(src/sqlite-vss.h.in sqlite-vss.h) 13 | configure_file(src/sqlite-vector.h.in sqlite-vector.h) 14 | 15 | set(CMAKE_CXX_STANDARD 17) 16 | set(CMAKE_CXX_STANDARD_REQUIRED ON) 17 | 18 | option(FAISS_ENABLE_GPU "" OFF) 19 | option(FAISS_ENABLE_PYTHON "" OFF) 20 | option(BUILD_TESTING "" OFF) 21 | add_subdirectory(./vendor/faiss) 22 | 23 | # vendor in SQLite amalgammation 24 | include_directories(vendor/sqlite) 25 | link_directories(BEFORE vendor/sqlite) 26 | 27 | # Adding nlohmann_json for json parsing 28 | set(JSON_BuildTests OFF CACHE INTERNAL "") 29 | add_subdirectory(vendor/json) 30 | 31 | # ================================== sqlite-vector ================================== # 32 | add_library(sqlite-vector SHARED src/sqlite-vector.cpp) 33 | target_link_libraries(sqlite-vector sqlite3) 34 | target_link_libraries(sqlite-vector nlohmann_json::nlohmann_json) 35 | target_include_directories(sqlite-vector PUBLIC "${PROJECT_BINARY_DIR}") 36 | 37 | set_target_properties(sqlite-vector PROPERTIES PREFIX "") 38 | set_target_properties(sqlite-vector PROPERTIES OUTPUT_NAME "vector0") 39 | 40 | # ============================== sqlite-vector-static ============================== # 41 | add_library(sqlite-vector-static STATIC src/sqlite-vector.cpp) 42 | target_link_libraries(sqlite-vector-static sqlite3) 43 | target_link_libraries(sqlite-vector-static nlohmann_json::nlohmann_json) 44 | target_include_directories(sqlite-vector-static PUBLIC "${PROJECT_BINARY_DIR}") 45 | set_target_properties(sqlite-vector-static PROPERTIES OUTPUT_NAME "sqlite_vector0") 46 | target_compile_definitions(sqlite-vector-static PUBLIC SQLITE_CORE) 47 | 48 | 49 | # ================================== sqlite-vss ================================== # 50 | add_library(sqlite-vss SHARED src/sqlite-vss.cpp) 51 | target_link_libraries(sqlite-vss sqlite3) 52 | target_link_libraries(sqlite-vss faiss_avx2) 53 | target_include_directories(sqlite-vss PUBLIC "${PROJECT_BINARY_DIR}") 54 | 55 | set_target_properties(sqlite-vss PROPERTIES PREFIX "") 56 | set_target_properties(sqlite-vss PROPERTIES OUTPUT_NAME "vss0") 57 | 58 | # ============================== sqlite-vss-static =============================== # 59 | add_library(sqlite-vss-static STATIC src/sqlite-vss.cpp) 60 | target_link_libraries(sqlite-vss-static PRIVATE sqlite3) 61 | target_link_libraries(sqlite-vss-static PUBLIC faiss_avx2) 62 | target_link_options(sqlite-vss-static PRIVATE "-Wl,-all_load") 63 | target_include_directories(sqlite-vss-static PUBLIC "${PROJECT_BINARY_DIR}") 64 | set_target_properties(sqlite-vss-static PROPERTIES OUTPUT_NAME "sqlite_vss0") 65 | target_compile_definitions(sqlite-vss-static PRIVATE SQLITE_CORE) 66 | 67 | 68 | -------------------------------------------------------------------------------- /examples/c/demo.c: -------------------------------------------------------------------------------- 1 | #include "sqlite3.h" 2 | #include "sqlite-vector.h" 3 | #include "sqlite-vss.h" 4 | #include 5 | #include 6 | 7 | int main(int argc, char *argv[]) { 8 | int rc = SQLITE_OK; 9 | sqlite3 *db; 10 | sqlite3_stmt *stmt; 11 | char* error_message; 12 | 13 | // Call the sqlite-vector and sqlite-vss entrypoints on every new SQLite database. 14 | rc = sqlite3_auto_extension((void (*)())sqlite3_vector_init); 15 | if (rc != SQLITE_OK) { 16 | fprintf(stderr, "❌ demo.c could not load sqlite3_vector_init: %s\n", sqlite3_errmsg(db)); 17 | sqlite3_close(db); 18 | return 1; 19 | } 20 | rc = sqlite3_auto_extension((void (*)())sqlite3_vss_init); 21 | if (rc != SQLITE_OK) { 22 | fprintf(stderr, "❌ demo.c could not load sqlite3_vss_init: %s\n", sqlite3_errmsg(db)); 23 | sqlite3_close(db); 24 | return 1; 25 | } 26 | 27 | // this database connection will now have all sqlite-vss SQL functions available 28 | rc = sqlite3_open(":memory:", &db); 29 | 30 | if (rc != SQLITE_OK) { 31 | fprintf(stderr, "❌ demo.c cannot open database: %s\n", sqlite3_errmsg(db)); 32 | sqlite3_close(db); 33 | return 1; 34 | } 35 | 36 | rc = sqlite3_prepare_v2(db, "SELECT vss_version(), vector_to_json(?)", -1, &stmt, NULL); 37 | if(rc != SQLITE_OK) { 38 | fprintf(stderr, "❌ demo.c%s\n", error_message); 39 | sqlite3_free(error_message); 40 | sqlite3_close(db); 41 | return 1; 42 | } 43 | unsigned char blob[] = {0x00, 0x0, 0x28, 0x42}; 44 | sqlite3_bind_blob(stmt, 1, blob, sizeof(blob), SQLITE_STATIC); 45 | if (SQLITE_ROW != sqlite3_step(stmt)) { 46 | fprintf(stderr, "❌ demo.c%s\n", sqlite3_errmsg(db)); 47 | sqlite3_close(db); 48 | return 1; 49 | } 50 | printf("version=%s vector=%s\n", sqlite3_column_text(stmt, 0), sqlite3_column_text(stmt, 1)); 51 | sqlite3_finalize(stmt); 52 | 53 | rc = sqlite3_exec(db, 54 | "CREATE VIRTUAL TABLE vss_demo USING vss0(a(2)); \ 55 | INSERT INTO vss_demo(rowid, a) \ 56 | VALUES \ 57 | (1, '[1.0, 2.0]'), \ 58 | (2, '[2.0, 2.0]'), \ 59 | (3, '[3.0, 2.0]') \ 60 | " 61 | , NULL, NULL, &error_message); 62 | 63 | if (rc != SQLITE_OK) { 64 | fprintf(stderr, "❌%s\n", error_message); 65 | sqlite3_free(error_message); 66 | sqlite3_close(db); 67 | return 1; 68 | } 69 | 70 | rc = sqlite3_prepare_v2(db, "\ 71 | SELECT \ 72 | rowid, \ 73 | distance \ 74 | FROM vss_demo \ 75 | WHERE vss_search(a, '[1.0, 2.0]') \ 76 | LIMIT 10;\ 77 | ", -1, &stmt, NULL); 78 | if(rc != SQLITE_OK) { 79 | fprintf(stderr, "❌ demo.c %s\n", sqlite3_errmsg(db)); 80 | sqlite3_free(error_message); 81 | sqlite3_close(db); 82 | return 1; 83 | } 84 | while (SQLITE_ROW == (rc = sqlite3_step(stmt))) { 85 | long rowid = sqlite3_column_int64(stmt, 0); 86 | double distance = sqlite3_column_double(stmt, 1); 87 | printf("rowid=%ld distance=%f\n", rowid, distance); 88 | } 89 | if(rc != SQLITE_DONE) { 90 | fprintf(stderr, "❌ demo.c %s\n", sqlite3_errmsg(db)); 91 | sqlite3_free(error_message); 92 | sqlite3_close(db); 93 | sqlite3_finalize(stmt); 94 | return 1; 95 | } 96 | sqlite3_finalize(stmt); 97 | 98 | printf("✅ demo.c ran successfully. \n"); 99 | sqlite3_close(db); 100 | return 0; 101 | } 102 | 103 | -------------------------------------------------------------------------------- /bindings/deno/mod.ts: -------------------------------------------------------------------------------- 1 | import { download } from "https://deno.land/x/plug@1.0.1/mod.ts"; 2 | import meta from "./deno.json" assert { type: "json" }; 3 | 4 | const BASE = `${meta.github}/releases/download/v${meta.version}`; 5 | 6 | // Similar to https://github.com/denodrivers/sqlite3/blob/f7529897720631c2341b713f0d78d4d668593ea9/src/ffi.ts#L561 7 | let vssPath: string; 8 | let vectorPath: string; 9 | try { 10 | const customVectorPath = Deno.env.get("DENO_SQLITE_VECTOR_PATH"); 11 | if (customVectorPath) vectorPath = customVectorPath; 12 | else { 13 | vectorPath = await download({ 14 | url: { 15 | darwin: { 16 | aarch64: `${BASE}/sqlite-vss-v${meta.version}-deno-darwin-aarch64.vector0.dylib`, 17 | x86_64: `${BASE}/sqlite-vss-v${meta.version}-deno-darwin-x86_64.vector0.dylib`, 18 | }, 19 | linux: { 20 | x86_64: `${BASE}/sqlite-vss-v${meta.version}-deno-linux-x86_64.vector0.so`, 21 | }, 22 | }, 23 | suffixes: { 24 | darwin: "", 25 | linux: "", 26 | windows: "", 27 | }, 28 | }); 29 | } 30 | 31 | const customVssPath = Deno.env.get("DENO_SQLITE_VSS_PATH"); 32 | if (customVssPath) vssPath = customVssPath; 33 | else { 34 | vssPath = await download({ 35 | url: { 36 | darwin: { 37 | aarch64: `${BASE}/sqlite-vss-v${meta.version}-deno-darwin-aarch64.vss0.dylib`, 38 | x86_64: `${BASE}/sqlite-vss-v${meta.version}-deno-darwin-x86_64.vss0.dylib`, 39 | }, 40 | linux: { 41 | x86_64: `${BASE}/sqlite-vss-v${meta.version}-deno-linux-x86_64.vss0.so`, 42 | }, 43 | }, 44 | suffixes: { 45 | darwin: "", 46 | linux: "", 47 | windows: "", 48 | }, 49 | }); 50 | } 51 | } catch (e) { 52 | if (e instanceof Deno.errors.PermissionDenied) { 53 | throw e; 54 | } 55 | 56 | const error = new Error("Failed to load sqlite-vss extension"); 57 | error.cause = e; 58 | 59 | throw error; 60 | } 61 | 62 | /** 63 | * Returns the full path to the compiled sqlite-vss extension. 64 | * Caution: this will not be named "vss0.dylib|so|dll", since plug will 65 | * replace the name with a hash. 66 | */ 67 | export function getVssLoadablePath(): string { 68 | return vssPath; 69 | } 70 | 71 | /** 72 | * Returns the full path to the compiled sqlite-vector extension. 73 | * Caution: this will not be named "vss0.dylib|so|dll", since plug will 74 | * replace the name with a hash. 75 | */ 76 | export function getVectorLoadablePath(): string { 77 | return vectorPath; 78 | } 79 | 80 | /** 81 | * Entrypoint name for the sqlite-vss extensions. 82 | */ 83 | export const vssEntrypoint = "sqlite3_vss_init"; 84 | export const vectorEntrypoint = "sqlite3_vector_init"; 85 | 86 | interface Db { 87 | // after https://deno.land/x/sqlite3@0.8.0/mod.ts?s=Database#method_loadExtension_0 88 | loadExtension(file: string, entrypoint?: string | undefined): void; 89 | } 90 | 91 | /** 92 | * Loads the sqlite-vss extension on the given sqlite3 database. 93 | */ 94 | export function loadVector(db: Db): void { 95 | db.loadExtension(vectorPath, vectorEntrypoint); 96 | } 97 | /** 98 | * Loads the sqlite-vss extension on the given sqlite3 database. 99 | */ 100 | export function loadVss(db: Db): void { 101 | db.loadExtension(vssPath, vssEntrypoint); 102 | } 103 | 104 | /** 105 | * Loads the sqlite-vss extension on the given sqlite3 database. 106 | */ 107 | export function load(db: Db): void { 108 | loadVector(db); 109 | loadVss(db); 110 | } 111 | -------------------------------------------------------------------------------- /bindings/datasette/.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/#use-with-ide 110 | .pdm.toml 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | 131 | # Spyder project settings 132 | .spyderproject 133 | .spyproject 134 | 135 | # Rope project settings 136 | .ropeproject 137 | 138 | # mkdocs documentation 139 | /site 140 | 141 | # mypy 142 | .mypy_cache/ 143 | .dmypy.json 144 | dmypy.json 145 | 146 | # Pyre type checker 147 | .pyre/ 148 | 149 | # pytype static type analyzer 150 | .pytype/ 151 | 152 | # Cython debug symbols 153 | cython_debug/ 154 | 155 | # PyCharm 156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 158 | # and can be added to the global gitignore or merged into this file. For a more nuclear 159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 160 | #.idea/ -------------------------------------------------------------------------------- /bindings/python/.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/#use-with-ide 110 | .pdm.toml 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | 131 | # Spyder project settings 132 | .spyderproject 133 | .spyproject 134 | 135 | # Rope project settings 136 | .ropeproject 137 | 138 | # mkdocs documentation 139 | /site 140 | 141 | # mypy 142 | .mypy_cache/ 143 | .dmypy.json 144 | dmypy.json 145 | 146 | # Pyre type checker 147 | .pyre/ 148 | 149 | # pytype static type analyzer 150 | .pytype/ 151 | 152 | # Cython debug symbols 153 | cython_debug/ 154 | 155 | # PyCharm 156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 158 | # and can be added to the global gitignore or merged into this file. For a more nuclear 159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 160 | #.idea/ -------------------------------------------------------------------------------- /bindings/sqlite-utils/.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/#use-with-ide 110 | .pdm.toml 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | 131 | # Spyder project settings 132 | .spyderproject 133 | .spyproject 134 | 135 | # Rope project settings 136 | .ropeproject 137 | 138 | # mkdocs documentation 139 | /site 140 | 141 | # mypy 142 | .mypy_cache/ 143 | .dmypy.json 144 | dmypy.json 145 | 146 | # Pyre type checker 147 | .pyre/ 148 | 149 | # pytype static type analyzer 150 | .pytype/ 151 | 152 | # Cython debug symbols 153 | cython_debug/ 154 | 155 | # PyCharm 156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 158 | # and can be added to the global gitignore or merged into this file. For a more nuclear 159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 160 | #.idea/ 161 | -------------------------------------------------------------------------------- /site/getting-started.md: -------------------------------------------------------------------------------- 1 | 5 | 6 | # Getting Started 7 | 8 | ## Concepts 9 | 10 | ## Installing 11 | 12 | You have several options to include `sqlite-vss` into your projects, including PyPi packages for Python, NPM packages for Node.js, Gems for Ruby, and more. 13 | 14 | ::: code-group 15 | 16 | ```bash [Python] 17 | pip install sqlite-vss 18 | ``` 19 | 20 | ```bash [Node.js] 21 | npm install sqlite-vss 22 | ``` 23 | 24 | ```js-vue [Deno] 25 | import * as sqlite_vss from "https://deno.land/x/sqlite_vss@v{{VERSION}}/mod.ts"; 26 | ``` 27 | 28 | ```bash [Ruby] 29 | gem install sqlite-vss 30 | ``` 31 | 32 | ```elixir-vue [Elixir] 33 | {:sqlite_vss, "~> {{ VERSION }}"} 34 | ``` 35 | 36 | ```bash [Rust] 37 | cargo add sqlite-vss 38 | ``` 39 | 40 | ```bash [Go] 41 | go get -u github.com/asg017/sqlite-vss/bindings/go 42 | ``` 43 | 44 | ```bash [Datasette] 45 | datasette install datasette-sqlite-vss 46 | ``` 47 | 48 | ```bash [sqlite-utils] 49 | sqlite-utils install sqlite-utils-sqlite-vss 50 | ``` 51 | 52 | ::: 53 | 54 | Alternatively, you can download pre-compiled loadable extensions from the [`sqlite-vss` Github Releases](https://github.com/asg017/sqlite-vss/releases/latest). 55 | 56 | ## Basic Example: 2-Dimensional Vectors 57 | 58 | This example will go over the basics of `sqlite-vss`: How to create a `vss0` virtual table, how to populate it with your own vectors data, and how to query those vectors for similarity. 59 | 60 | Say we have a list of 2-dimensional vectors with the following values: 61 | 62 | | ID | Vector | 63 | | --- | -------------- | 64 | | 1 | `[1.0, 3.0]` | 65 | | 2 | `[3.0, 1.0]` | 66 | | 3 | `[-2.0, -2.0]` | 67 | | 4 | `[-4.0, 1.0]` | 68 | 69 | Visually, these vectors would look like this: 70 | 71 |

72 | 73 | Let's store these in a `vss0` virtual table! Let's create a new virtual table called `vss_demo`, with a single vector column called `a` for these four vectors 74 | 75 | ```sqlite 76 | create virtual table vss_demo using vss0( 77 | a(2) 78 | ); 79 | ``` 80 | 81 | Notice the `2` declaration in the `a` column definition. This is a required argument that tells `sqlite-vss` how many dimensions our vectors have. Also note that prefixing the virtual table name with `vss_` is a convention but not required. 82 | 83 | Let's insert our vectors! 84 | 85 | ```sqlite 86 | insert into vss_demo(rowid, a) 87 | select 88 | value ->> 0 as rowid, 89 | value ->> 1 as a 90 | from json_each(' 91 | [ 92 | [ 1, [1.0, 3.0] ], 93 | [ 2, [3.0, 1.0] ], 94 | [ 3, [-2.0, -2.0] ], 95 | [ 4, [-4.0, 1.0] ] 96 | ] 97 | '); 98 | ``` 99 | 100 | Here we are using [SQLite's builting JSON support](https://www.sqlite.org/json1.html) to define our vector data in a JSON string. `sqlite-vss` supports vectors in a few different formats, which you [can learn more about here](api-reference#inserting-data). 101 | 102 |

103 | 104 | ```sqlite 105 | 106 | select 107 | rowid, 108 | distance 109 | from vss_lookup 110 | where vss_search(a, json('[2.0, 2.0]')) 111 | limit 3; 112 | ``` 113 | 114 | ``` 115 | ┌───────┬──────────┐ 116 | │ rowid │ distance │ 117 | ├───────┼──────────┤ 118 | │ 1 │ 2.0 │ 119 | │ 2 │ 2.0 │ 120 | │ 3 │ 32.0 │ 121 | └───────┴──────────┘ 122 | ``` 123 | 124 |

125 | 126 | ```sqlite 127 | 128 | select 129 | rowid, 130 | distance 131 | from vss_lookup 132 | where vss_search(a, json('[-2.0, -2.0]')) 133 | limit 3; 134 | 135 | ``` 136 | 137 | ``` 138 | ┌───────┬──────────┐ 139 | │ rowid │ distance │ 140 | ├───────┼──────────┤ 141 | │ 3 │ 0.0 │ 142 | │ 4 │ 13.0 │ 143 | │ 1 │ 34.0 │ 144 | └───────┴──────────┘ 145 | ``` 146 | 147 | ## Next Steps 148 | -------------------------------------------------------------------------------- /bindings/go/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/asg017/sqlite-vss/bindings/go 2 | 3 | go 1.20 4 | 5 | require ( 6 | github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect 7 | github.com/Microsoft/go-winio v0.6.1 // indirect 8 | github.com/bufbuild/buf v1.20.0 // indirect 9 | github.com/bufbuild/connect-go v1.7.0 // indirect 10 | github.com/bufbuild/protocompile v0.5.1 // indirect 11 | github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect 12 | github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect 13 | github.com/docker/cli v24.0.2+incompatible // indirect 14 | github.com/docker/distribution v2.8.2+incompatible // indirect 15 | github.com/docker/docker v24.0.2+incompatible // indirect 16 | github.com/docker/docker-credential-helpers v0.7.0 // indirect 17 | github.com/docker/go-connections v0.4.0 // indirect 18 | github.com/docker/go-units v0.5.0 // indirect 19 | github.com/felixge/fgprof v0.9.3 // indirect 20 | github.com/go-chi/chi/v5 v5.0.8 // indirect 21 | github.com/go-logr/logr v1.2.4 // indirect 22 | github.com/go-logr/stdr v1.2.2 // indirect 23 | github.com/gofrs/flock v0.8.1 // indirect 24 | github.com/gofrs/uuid v4.4.0+incompatible // indirect 25 | github.com/gofrs/uuid/v5 v5.0.0 // indirect 26 | github.com/gogo/protobuf v1.3.2 // indirect 27 | github.com/golang/glog v1.1.1 // indirect 28 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 29 | github.com/golang/protobuf v1.5.3 // indirect 30 | github.com/google/go-containerregistry v0.15.2 // indirect 31 | github.com/google/pprof v0.0.0-20230510103437-eeec1cb781c3 // indirect 32 | github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 // indirect 33 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 34 | github.com/jdxcode/netrc v0.0.0-20221124155335-4616370d1a84 // indirect 35 | github.com/jhump/protocompile v0.0.0-20221021153901-4f6f732835e8 // indirect 36 | github.com/jhump/protoreflect v1.15.1 // indirect 37 | github.com/klauspost/compress v1.16.5 // indirect 38 | github.com/klauspost/pgzip v1.2.6 // indirect 39 | github.com/mitchellh/go-homedir v1.1.0 // indirect 40 | github.com/moby/term v0.5.0 // indirect 41 | github.com/morikuni/aec v1.0.0 // indirect 42 | github.com/nlpodyssey/cybertron v0.1.2 // indirect 43 | github.com/opencontainers/go-digest v1.0.0 // indirect 44 | github.com/opencontainers/image-spec v1.1.0-rc3 // indirect 45 | github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect 46 | github.com/pkg/errors v0.9.1 // indirect 47 | github.com/pkg/profile v1.7.0 // indirect 48 | github.com/rs/cors v1.9.0 // indirect 49 | github.com/russross/blackfriday/v2 v2.1.0 // indirect 50 | github.com/sirupsen/logrus v1.9.2 // indirect 51 | github.com/spf13/cobra v1.7.0 // indirect 52 | github.com/spf13/pflag v1.0.5 // indirect 53 | github.com/tetratelabs/wazero v1.1.0 // indirect 54 | github.com/vbatts/tar-split v0.11.3 // indirect 55 | go.opencensus.io v0.24.0 // indirect 56 | go.opentelemetry.io/otel v1.16.0 // indirect 57 | go.opentelemetry.io/otel/metric v1.16.0 // indirect 58 | go.opentelemetry.io/otel/sdk v1.16.0 // indirect 59 | go.opentelemetry.io/otel/trace v1.16.0 // indirect 60 | go.uber.org/atomic v1.11.0 // indirect 61 | go.uber.org/multierr v1.11.0 // indirect 62 | go.uber.org/zap v1.24.0 // indirect 63 | golang.org/x/crypto v0.9.0 // indirect 64 | golang.org/x/mod v0.10.0 // indirect 65 | golang.org/x/net v0.10.0 // indirect 66 | golang.org/x/sync v0.2.0 // indirect 67 | golang.org/x/sys v0.8.0 // indirect 68 | golang.org/x/term v0.8.0 // indirect 69 | golang.org/x/text v0.9.0 // indirect 70 | golang.org/x/tools v0.9.1 // indirect 71 | google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect 72 | google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect 73 | google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect 74 | google.golang.org/grpc v1.55.0 // indirect 75 | google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0 // indirect 76 | google.golang.org/protobuf v1.30.0 // indirect 77 | gopkg.in/yaml.v2 v2.4.0 // indirect 78 | gopkg.in/yaml.v3 v3.0.1 // indirect 79 | ) 80 | -------------------------------------------------------------------------------- /.github/workflows/upload.js: -------------------------------------------------------------------------------- 1 | const fs = require("fs").promises; 2 | const crypto = require("crypto"); 3 | const zlib = require("zlib"); 4 | const tar = require("tar-fs"); 5 | const { basename } = require("path"); 6 | 7 | const PROJECT = "sqlite-vss"; 8 | const vss = { 9 | name: "vss0", 10 | description: "", 11 | platforms: [ 12 | { 13 | path: `${process.env["ARTIFACT-LINUX-X86_64-EXTENSION"]}/vss0.so`, 14 | os: "linux", 15 | cpu: "x86_64", 16 | }, 17 | { 18 | path: `${process.env["ARTIFACT-MACOS-X86_64-EXTENSION"]}/vss0.dylib`, 19 | os: "darwin", 20 | cpu: "x86_64", 21 | }, 22 | { 23 | path: `${process.env["ARTIFACT-MACOS-AARCH64-EXTENSION"]}/vss0.dylib`, 24 | os: "darwin", 25 | cpu: "aarch64", 26 | }, 27 | ], 28 | }; 29 | const vector = { 30 | name: "vector0", 31 | description: "", 32 | platforms: [ 33 | { 34 | path: `${process.env["ARTIFACT-LINUX-X86_64-EXTENSION"]}/vector0.so`, 35 | os: "linux", 36 | cpu: "x86_64", 37 | }, 38 | { 39 | path: `${process.env["ARTIFACT-MACOS-X86_64-EXTENSION"]}/vector0.dylib`, 40 | os: "darwin", 41 | cpu: "x86_64", 42 | }, 43 | { 44 | path: `${process.env["ARTIFACT-MACOS-AARCH64-EXTENSION"]}/vector0.dylib`, 45 | os: "darwin", 46 | cpu: "aarch64", 47 | }, 48 | ], 49 | }; 50 | 51 | const extensions = [vector, vss]; 52 | 53 | function targz(files) { 54 | return new Promise((resolve, reject) => { 55 | console.log("targz files: ", files[0].name, files[0]); 56 | 57 | const tarStream = tar.pack(); 58 | 59 | for (const file of files) { 60 | tarStream.entry({ name: file.name }, file.data); 61 | } 62 | 63 | tarStream.finalize(); 64 | 65 | const gzip = zlib.createGzip(); 66 | 67 | const chunks = []; 68 | tarStream 69 | .pipe(gzip) 70 | .on("data", (chunk) => { 71 | chunks.push(chunk); 72 | }) 73 | .on("end", () => { 74 | const buffer = Buffer.concat(chunks); 75 | resolve(buffer); 76 | }) 77 | .on("error", reject); 78 | }); 79 | } 80 | 81 | module.exports = async ({ github, context }) => { 82 | const { 83 | repo: { owner, repo }, 84 | sha, 85 | } = context; 86 | const VERSION = process.env.GITHUB_REF_NAME; 87 | 88 | const release = await github.rest.repos.getReleaseByTag({ 89 | owner, 90 | repo, 91 | tag: process.env.GITHUB_REF.replace("refs/tags/", ""), 92 | }); 93 | const release_id = release.data.id; 94 | 95 | async function uploadPlatform(extension_name, platform) { 96 | const { path, os, cpu } = platform; 97 | 98 | const artifact = basename(path); 99 | const contents = await fs.readFile(path); 100 | const tar = await targz([{ name: artifact, data: contents }]); 101 | 102 | const asset_name = `${PROJECT}-${VERSION}-${extension_name}-${os}-${cpu}.tar.gz`; 103 | const asset_md5 = crypto.createHash("md5").update(tar).digest("base64"); 104 | const asset_sha256 = crypto.createHash("sha256").update(tar).digest("hex"); 105 | 106 | await github.rest.repos.uploadReleaseAsset({ 107 | owner, 108 | repo, 109 | release_id, 110 | name: asset_name, 111 | data: tar, 112 | }); 113 | 114 | return { 115 | os, 116 | cpu, 117 | asset_name, 118 | asset_sha256, 119 | asset_md5, 120 | }; 121 | } 122 | 123 | const spm_json = { 124 | version: 0, 125 | extensions: Object.fromEntries( 126 | await Promise.all( 127 | extensions.map(async ({ name, description, platforms }) => [ 128 | name, 129 | { 130 | description, 131 | platforms: await Promise.all( 132 | platforms.map((platform) => uploadPlatform(name, platform)) 133 | ), 134 | }, 135 | ]) 136 | ) 137 | ), 138 | }; 139 | 140 | await github.rest.repos.uploadReleaseAsset({ 141 | owner, 142 | repo, 143 | release_id, 144 | name: "spm.json", 145 | data: JSON.stringify(spm_json), 146 | }); 147 | }; -------------------------------------------------------------------------------- /site/building-source.md: -------------------------------------------------------------------------------- 1 | # Building `sqlite-vss` from source 2 | 3 | If there isn't a prebuilt `sqlite-vss` build for your operating system or CPU architecture, you can try building `sqlite-vss` yourself. It'll require a C++ compiler, cmake, and possibly a few other libraries to build correctly. 4 | 5 | Below are the general steps to build `sqlite-vss`. Your operating system may require a few more libraries or setup instructions, so some tips are listed below under [Platform-specific compiling tips](#platform-specific-compiling-tips). 6 | 7 | ## Instructions 8 | 9 | To start, clone this repository and its submodules. 10 | 11 | ```bash 12 | git clone --recurse-submodules https://github.com/asg017/sqlite-vss.git 13 | cd sqlite-vss 14 | ``` 15 | 16 | Next, you'll need to build a vendored version of SQLite under `vendor/sqlite`. To retrieve the SQLite amalgammation, run the `./vendor/get_sqlite.sh` script: 17 | 18 | ```bash 19 | ./vendor/get_sqlite.sh 20 | ``` 21 | 22 | Then navigate to the newly built `vendor/sqlite` directory and build the SQLite library. 23 | 24 | ```bash 25 | cd vendor/sqlite 26 | ./configure && make 27 | ``` 28 | 29 | Now that all dependencies are downloaded and configured, you can build the `sqlite-vss` extension! Run either `make loadable` to build a loadable SQLite extension. 30 | 31 | ```bash 32 | # Build a debug version of `sqlite-vss`. 33 | # Faster to compile, slower at runtime 34 | make loadable 35 | ``` 36 | 37 | After it finishes, you'll find debug version of `vector0` and `vss0` inside `dist/debug`, with file extensions `.dylib`, `.so`, or `.dll`, depending on your operating system. 38 | 39 | To compile static library files of `sqlite-vss` and `sqlite-vector`, run `make static`. 40 | 41 | ```bash 42 | # Build a debug static archive files of `sqlite-vss`, ".a" and ".h" files 43 | make static 44 | ``` 45 | 46 | After it completes, you'll find `.a` and `.h` files inside `dist/debug`, which can be used to statically link `sqlite-vss` into other projects. The [Rust bindings](./rust) and [Go bindings](./go) uses this approach. 47 | 48 | ``` 49 | -I./dist/debug -lsqlite_vector0 -lsqlite_vss0 -llfaiss_avx2 50 | ``` 51 | 52 | For better runtime performance and smaller binaries, consider instead `make loadable-release` and `make static-release`. 53 | 54 | ```bash 55 | 56 | # Build a release version of `sqlite-vss`. 57 | # Slower to compile, but faster at runtime 58 | make loadable-release 59 | 60 | make static-release 61 | ``` 62 | 63 | You'll file the optimized loadable and static files under `dist/release`. 64 | 65 | ## Platform-specific compiling tips 66 | 67 | ### MacOS (x86_64) 68 | 69 | On Macs, you may need to install and use `llvm` and `libomp` for compilation. It can be install with brew: 70 | 71 | ```bash 72 | brew install llvm libomp 73 | ``` 74 | 75 | Additionally, if you see other cryptic compiling errors, you may need to explicitly state to use the `llvm` compilers, with flags like so: 76 | 77 | ```bash 78 | export CC=/usr/local/opt/llvm/bin/clang 79 | export CXX=/usr/local/opt/llvm/bin/clang++ 80 | export LDFLAGS="-L/usr/local/opt/llvm/lib" 81 | export CPPFLAGS="-I/usr/local/opt/llvm/include" 82 | ``` 83 | 84 | If you come across any problems, please file an issue! 85 | 86 | ### MacOS (M1/M2 ARM) 87 | 88 | For Mac M1/M2 computers, you'll need `llvm` and `libomp`: 89 | 90 | ```bash 91 | brew install llvm libomp 92 | ``` 93 | 94 | You will also likely need the following compiler flags: 95 | 96 | ```bash 97 | export CC="/opt/homebrew/opt/llvm/bin/clang" 98 | export CXX="/opt/homebrew/opt/llvm/bin/clang++" 99 | export LDFLAGS="-L/opt/homebrew/opt/libomp/lib" 100 | export CPPFLAGS="-I/opt/homebrew/opt/libomp/include" 101 | ``` 102 | 103 | Note that these are _slightly_ different than the ones above, as `LDFLAGS`/`CPPFLAGS` are referencing libomp instead. 104 | 105 | ### Linux (x86_64) 106 | 107 | You most likely will need to install the following libraries before compiling: 108 | 109 | ```bash 110 | sudo apt-get update 111 | sudo apt-get install libgomp1 libatlas-base-dev liblapack-dev libsqlite3-dev 112 | ``` 113 | 114 | Explainations for these packages: 115 | 116 | - `libgomp1`: OpenMP implementation, for multi-threading. Required by Faiss 117 | - `libatlas-base-dev`: Provides a BLAS implementation. Required by Faiss 118 | - `liblapack-dev`: Provides an LAPACK implementation. Required by Faiss 119 | - `libsqlite3-dev`: Provides SQLite headers/static files for dev purposes. 120 | -------------------------------------------------------------------------------- /examples/go-cybertron/demo-cybertron.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The NLP Odyssey Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package main 6 | 7 | import ( 8 | "context" 9 | "database/sql" 10 | "encoding/json" 11 | "log" 12 | "os" 13 | 14 | "github.com/nlpodyssey/cybertron/pkg/models/bert" 15 | "github.com/nlpodyssey/cybertron/pkg/tasks" 16 | "github.com/nlpodyssey/cybertron/pkg/tasks/textencoding" 17 | "github.com/rs/zerolog" 18 | 19 | _ "github.com/asg017/sqlite-vss/bindings/go" 20 | sqlite "github.com/mattn/go-sqlite3" 21 | ) 22 | 23 | // #cgo linux,amd64 LDFLAGS: -L../../dist/debug -Wl,-undefined,dynamic_lookup -lstdc++ 24 | // #cgo darwin,amd64 LDFLAGS: -L../../dist/debug -Wl,-undefined,dynamic_lookup -lomp 25 | import "C" 26 | 27 | func main() { 28 | zerolog.SetGlobalLevel(zerolog.DebugLevel) 29 | 30 | // load the sentence-transformers model and define a new SQL scalar function 31 | // "st_encode" that creates a JSON representation of the embedding of a given text 32 | modelsDir := os.Getenv("CYBERTRON_MODELS_DIR") 33 | modelName := "sentence-transformers/all-MiniLM-L6-v2" 34 | m, err := tasks.Load[textencoding.Interface](&tasks.Config{ModelsDir: modelsDir, ModelName: modelName}) 35 | if err != nil { 36 | log.Fatal(err) 37 | } 38 | defer tasks.Finalize(m) 39 | st_encode := func(text string) string { 40 | result, err := m.Encode(context.Background(), text, int(bert.MeanPooling)) 41 | if err != nil { 42 | log.Fatal(err) 43 | } 44 | vec := result.Vector.Data().F32() 45 | vecBytes, err := json.Marshal(vec) 46 | if err != nil { 47 | panic(err) 48 | } 49 | return string(vecBytes) 50 | } 51 | 52 | sql.Register("sqlite3_custom", &sqlite.SQLiteDriver{ 53 | ConnectHook: func(conn *sqlite.SQLiteConn) error { 54 | if err := conn.RegisterFunc("st_encode", st_encode, true); err != nil { 55 | return err 56 | } 57 | return nil 58 | }, 59 | }) 60 | 61 | db, err := sql.Open("sqlite3_custom", "tmp.db") 62 | if err != nil { 63 | log.Fatal(err) 64 | } 65 | defer db.Close() 66 | 67 | // initialize fruits and vss_fruits tables 68 | _, err = db.Exec("CREATE TABLE IF NOT EXISTS fruits (id INTEGER PRIMARY KEY AUTOINCREMENT, name text)") 69 | if err != nil { 70 | log.Fatal(err) 71 | } 72 | // 384 == number of dimensions that the model outputs 73 | _, err = db.Exec("CREATE VIRTUAL TABLE IF NOT EXISTS vss_fruits USING vss0(embedding(384))") 74 | if err != nil { 75 | log.Fatal(err) 76 | } 77 | 78 | tx, err := db.Begin() 79 | if err != nil { 80 | log.Fatal(err) 81 | } 82 | 83 | // insert these fruits into the fruits table, then generate 84 | // embeddings for each fruit and store those vectors in vss_fruits 85 | fruits := []string{"apples", "bananas", "broccoli"} 86 | for _, fruit := range fruits { 87 | f, err := tx.Exec("INSERT INTO fruits (name) VALUES (?)", fruit) 88 | if err != nil { 89 | log.Fatal(err) 90 | } 91 | id, err := f.LastInsertId() 92 | if err != nil { 93 | log.Fatal(err) 94 | } 95 | 96 | _, err = tx.Exec("INSERT INTO vss_fruits(rowid, embedding) values (?, st_encode(?))", id, fruit) 97 | if err != nil { 98 | panic(err) 99 | } 100 | } 101 | tx.Commit() 102 | 103 | rows, err := db.Query(` 104 | with similar_matches as ( 105 | select rowid, vss_distance_linf(embedding, st_encode(?1)) distance 106 | from vss_fruits 107 | where vss_search(embedding, st_encode(?1)) 108 | limit 20 109 | ), final as ( 110 | select 111 | fruits.rowid, 112 | fruits.name, 113 | similar_matches.distance 114 | from similar_matches 115 | left join fruits on fruits.rowid = similar_matches.rowid 116 | group by fruits.name 117 | ) 118 | select rowid, name, distance from final order by distance`, 119 | "red") 120 | if err != nil { 121 | log.Fatal("vss_search query error:", err) 122 | } 123 | defer rows.Close() 124 | 125 | for rows.Next() { 126 | var rowid int64 127 | var name string 128 | var distance float64 129 | if err := rows.Scan(&rowid, &name, &distance); err != nil { 130 | log.Fatal(err) 131 | } 132 | log.Printf("rowid=%d name=%s distance=%f\n", rowid, name, distance) 133 | } 134 | if err := rows.Err(); err != nil { 135 | log.Fatal(err) 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /bindings/node/sqlite-vss/README.md: -------------------------------------------------------------------------------- 1 | # `sqlite-vss` NPM Package 2 | 3 | `sqlite-vss` is distributed on `npm` for Node.js developers. To install on [supported platforms](#supported-platforms), simply run: 4 | 5 | ``` 6 | npm install sqlite-vss 7 | ``` 8 | 9 | The `sqlite-vss` package is meant to be used with Node SQLite clients like [`better-sqlite3`](https://github.com/WiseLibs/better-sqlite3) and [`node-sqlite3`](https://github.com/TryGhost/node-sqlite3). 10 | 11 | ```js 12 | // using sqlite-vss with better-sqlite3 13 | import Database from "better-sqlite3"; 14 | import * as sqlite_vss from "sqlite-vss"; 15 | 16 | const db = new Database(":memory:"); 17 | 18 | sqlite_vss.load(db); 19 | 20 | const version = db.prepare("select vss_version()").pluck().get(); 21 | console.log(version); // "v0.2.0" 22 | ``` 23 | 24 | ```js 25 | // using sqlite-vss with node-sqlite3 26 | import sqlite3 from "sqlite3"; 27 | import * as sqlite_vss from "sqlite-vss"; 28 | 29 | const db = new sqlite3.Database(":memory:"); 30 | 31 | db.loadExtension(sqlite_vss.getLoadablePath()); 32 | 33 | db.get("select vss_version()", (err, row) => { 34 | console.log(row); // {vss_version(): "v0.2.0"} 35 | }); 36 | ``` 37 | 38 | See [the full API Reference](#api-reference) for the Node API, and [`docs.md`](../../docs.md) for documentation on the `sqlite-vss` SQL API. 39 | 40 | ## Supported Platforms 41 | 42 | Since the underlying `vss0` SQLite extension is pre-compiled, the `sqlite-vss` NPM package only works on a few "platforms" (operating systems + CPU architectures). These platforms include: 43 | 44 | - `darwin-x64` (MacOS x86_64) 45 | - `linux-x64` (Linux x86_64) 46 | 47 | To see which platform your machine is, check the [`process.arch`](https://nodejs.org/api/process.html#processarch) and [`process.platform`](https://nodejs.org/api/process.html#processplatform) values like so: 48 | 49 | ```bash 50 | $ node -e 'console.log([process.platform, process.arch])' 51 | [ 'darwin', 'x64' ] 52 | ``` 53 | 54 | When the `sqlite-vss` NPM package is installed, the correct pre-compiled extension for your operating system and CPU architecture will be downloaded from the [optional dependencies](https://docs.npmjs.com/cli/v9/configuring-npm/package-json#optionaldependencies), with platform-specific packages like `sqlite-vss-darwin-x64`. This will be automatically, there's no need to directly install those packages. 55 | 56 | More platforms may be supported in the future. Consider [supporting my work](https://github.com/sponsors/asg017/) if you'd like to see more operating systems and CPU architectures supported in `sqlite-vss`. 57 | 58 | ## API Reference 59 | 60 | # getLoadablePath [<>](https://github.com/asg017/sqlite-vss/blob/main/npm/sqlite-vss/src/index.js "Source") 61 | 62 | Returns the full path to where the `sqlite-vss` _should_ be installed, based on the `sqlite-vss`'s `package.json` optional dependencies and the host's operating system and architecture. 63 | 64 | This path can be directly passed into [`better-sqlite3`](https://github.com/WiseLibs/better-sqlite3)'s [`.loadExtension()`](https://github.com/WiseLibs/better-sqlite3/blob/master/docs/api.md#loadextensionpath-entrypoint---this). 65 | 66 | ```js 67 | import Database from "better-sqlite3"; 68 | import * as sqlite_vss from "sqlite-vss"; 69 | 70 | const db = new Database(":memory:"); 71 | db.loadExtension(sqlite_vss.getLoadablePath()); 72 | ``` 73 | 74 | It can also be used in [`node-sqlite3`](https://github.com/TryGhost/node-sqlite3)'s [`.loadExtension()`](https://github.com/TryGhost/node-sqlite3/wiki/API#loadextensionpath--callback). 75 | 76 | ```js 77 | import sqlite3 from "sqlite3"; 78 | import * as sqlite_vss from "sqlite-vss"; 79 | 80 | const db = new sqlite3.Database(":memory:"); 81 | db.loadExtension(sqlite_vss.getLoadablePath()); 82 | ``` 83 | 84 | This function throws an `Error` in two different cases. The first case is when `sqlite-vss` is installed and run on an [unsupported platform](#supported-platforms). The second case is when the platform-specific optional dependency is not installed. If you reach this, ensure you aren't using `--no-optional` flag, and [file an issue](https://github.com/asg017/sqlite-vss/issues/new) if you are stuck. 85 | 86 | The `db.loadExtension()` function may also throw an Error if the compiled extension is incompatible with your SQLite connection for any reason, including missing system packages, outdated glib versions, or other misconfigurations. If you reach this, please [file an issue](https://github.com/asg017/sqlite-vss/issues/new). 87 | -------------------------------------------------------------------------------- /bindings/rust/build.rs: -------------------------------------------------------------------------------- 1 | use std::path::PathBuf; 2 | 3 | #[cfg(feature = "download-libs")] 4 | use flate2::read::GzDecoder; 5 | #[cfg(feature = "download-libs")] 6 | use std::io::BufReader; 7 | #[cfg(feature = "download-libs")] 8 | use tar::Archive; 9 | #[cfg(feature = "download-libs")] 10 | use ureq::get; 11 | #[cfg(feature = "download-libs")] 12 | use zip::read::ZipArchive; 13 | 14 | #[cfg(feature = "download-libs")] 15 | enum Platform { 16 | MacosX86_64, 17 | MacosAarch64, 18 | LinuxX86_64, 19 | } 20 | 21 | #[cfg(not(feature = "download-libs"))] 22 | fn download_static_for_platform( 23 | _os: &str, 24 | _arch: &str, 25 | _version: String, 26 | _output_directory: &PathBuf, 27 | ) { 28 | } 29 | #[cfg(feature = "download-libs")] 30 | fn download_static_for_platform(os: &str, arch: &str, version: String, output_directory: &PathBuf) { 31 | let platform = match (os, arch) { 32 | ("linux", "x86_64") => Platform::LinuxX86_64, 33 | ("macos", "x86_64") => Platform::MacosX86_64, 34 | ("macos", "aarch64") => Platform::MacosAarch64, 35 | _ => panic!("Unsupported platform: {os} {arch}"), 36 | }; 37 | 38 | let base = "https://github.com/asg017/sqlite-vss/releases/download"; 39 | let url = match platform { 40 | Platform::MacosX86_64 => { 41 | format!("{base}/{version}/sqlite-vss-{version}-static-macos-x86_64.tar.gz") 42 | } 43 | 44 | Platform::MacosAarch64 => { 45 | format!("{base}/{version}/sqlite-vss-{version}-static-macos-aarch64.tar.gz") 46 | } 47 | Platform::LinuxX86_64 => { 48 | format!("{base}/{version}/sqlite-vss-{version}-static-linux-x86_64.tar.gz") 49 | } 50 | }; 51 | 52 | println!("{url}"); 53 | let response = get(url.as_str()).call().expect("Failed to download file"); 54 | println!("{}", response.get_url()); 55 | let mut reader = response.into_reader(); 56 | 57 | if url.ends_with(".zip") { 58 | let mut buf = Vec::new(); 59 | reader.read_to_end(&mut buf).unwrap(); 60 | let mut archive = 61 | ZipArchive::new(std::io::Cursor::new(buf)).expect("Failed to open zip archive"); 62 | archive 63 | .extract(output_directory) 64 | .expect("failed to extract .zip file"); 65 | } else { 66 | let buf_reader = BufReader::new(reader); 67 | let decoder = GzDecoder::new(buf_reader); 68 | let mut archive = Archive::new(decoder); 69 | archive 70 | .unpack(output_directory) 71 | .expect("Failed to extract tar.gz file"); 72 | } 73 | } 74 | fn main() { 75 | let version = format!("v{}", env!("CARGO_PKG_VERSION")); 76 | let os = std::env::var("CARGO_CFG_TARGET_OS").expect("CARGO_CFG_TARGET_OS not found"); 77 | let arch = std::env::var("CARGO_CFG_TARGET_ARCH").expect("CARGO_CFG_TARGET_ARCH not found"); 78 | 79 | let output_directory = if cfg!(feature = "download-libs") { 80 | let output_directory = std::path::Path::new(std::env::var("OUT_DIR").unwrap().as_str()) 81 | .join(format!("sqlite-vss-v{version}-{os}-{arch}")); 82 | if !output_directory.exists() { 83 | download_static_for_platform(os.as_str(), arch.as_str(), version, &output_directory); 84 | } 85 | output_directory 86 | } else { 87 | std::env::var("LIB_SQLITE_VSS").expect("The LIB_SQLITE_VSS environment variable needs to be defined if the download-libs feature is not enabled").into() 88 | }; 89 | 90 | println!("cargo:rerun-if-env-changed=LIB_SQLITE_VSS"); 91 | println!( 92 | "cargo:rustc-link-search=native={}", 93 | output_directory.to_string_lossy() 94 | ); 95 | println!("cargo:rustc-link-lib=static=faiss_avx2"); 96 | println!("cargo:rustc-link-lib=static=sqlite_vector0"); 97 | println!("cargo:rustc-link-lib=static=sqlite_vss0"); 98 | 99 | if cfg!(target_os = "macos") { 100 | println!("cargo:rustc-link-arg=-Wl,-undefined,dynamic_lookup"); 101 | } 102 | else if cfg!(target_os = "linux") { 103 | // TODO different builds of faiss/sqlite-vss may require other libs 104 | println!("cargo:rustc-link-lib=dylib=gomp"); 105 | println!("cargo:rustc-link-lib=dylib=atlas"); 106 | println!("cargo:rustc-link-lib=dylib=blas"); 107 | println!("cargo:rustc-link-lib=dylib=lapack"); 108 | println!("cargo:rustc-link-lib=dylib=m"); 109 | println!("cargo:rustc-link-lib=dylib=stdc++"); 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /site/.vitepress/config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig, DefaultTheme, HeadConfig } from "vitepress"; 2 | import { readFileSync } from "node:fs"; 3 | import { join, dirname } from "node:path"; 4 | import { fileURLToPath } from "node:url"; 5 | 6 | const PROJECT = "sqlite-vss"; 7 | const description = 8 | "A SQLite extension for efficient vector search, based on Faiss!"; 9 | 10 | const VERSION = readFileSync( 11 | join(dirname(fileURLToPath(import.meta.url)), "..", "..", "VERSION"), 12 | "utf8" 13 | ); 14 | 15 | function head(): HeadConfig[] { 16 | return [ 17 | [ 18 | "link", 19 | { 20 | rel: "shortcut icon", 21 | type: "image/svg+xml", 22 | href: "favicon.svg", 23 | }, 24 | ], 25 | [ 26 | "script", 27 | { 28 | defer: "", 29 | "data-domain": "alexgarcia.xyz/sqlite-vss", 30 | src: "https://plausible.io/js/script.js", 31 | }, 32 | ], 33 | ]; 34 | } 35 | 36 | const guides = { 37 | text: "Guides", 38 | items: [ 39 | //{ text: "OpenAI's Embedding API", link: "/openai-python" }, 40 | //{ text: "openai-to-sqlite", link: "/openai-to-sqlite" }, 41 | //{ text: "Sentence Transformers", link: "/sentence-transformers-python",}, 42 | //{ text: "HuggingFace Inference API", link: "/huggingface",}, 43 | //{ text: "Cohere's Embedding API", link: "/cohere" }, 44 | ], 45 | }; 46 | 47 | function nav(): DefaultTheme.NavItem[] { 48 | return [ 49 | { text: "Home", link: "/" }, 50 | //guides, 51 | { text: "API Reference", link: "/api-reference" }, 52 | { text: "♥ Sponsor", link: "https://github.com/sponsors/asg017" }, 53 | { 54 | text: `v${VERSION}`, 55 | items: [ 56 | { 57 | text: "Github Release", 58 | link: `https://github.com/asg017/${PROJECT}/releases/${VERSION}`, 59 | }, 60 | { 61 | text: "Bindings", 62 | items: [ 63 | { 64 | text: "Python: PyPi package", 65 | link: `https://pypi.org/project/${PROJECT}`, 66 | }, 67 | { 68 | text: "Node.js: NPM package", 69 | link: `https://www.npmjs.com/package/${PROJECT}`, 70 | }, 71 | { 72 | text: "Deno: deno.land/x module", 73 | link: `https://deno.land/x/${PROJECT.replace("-", "_")}`, 74 | }, 75 | { 76 | text: "Ruby: Ruby gem", 77 | link: `https://rubygems.org/gems/${PROJECT}`, 78 | }, 79 | { 80 | text: "Elixir: Hex package", 81 | link: `https://crates.io/crates/${PROJECT.replace("-", "_")}`, 82 | }, 83 | { 84 | text: "Rust: Cargo crate", 85 | link: `https://crates.io/crates/${PROJECT}`, 86 | }, 87 | { 88 | text: "Golang: Go module", 89 | link: `https://pkg.go.dev/github.com/asg017/${PROJECT}/bindings/go`, 90 | }, 91 | { 92 | text: "Datasette: Plugin", 93 | link: `https://datasette.io/plugins/datasette-${PROJECT}`, 94 | }, 95 | { 96 | text: "sqlite-utils: Plugin", 97 | link: `https://datasette.io/plugins/datasette-${PROJECT}`, 98 | }, 99 | ], 100 | }, 101 | ], 102 | }, 103 | ]; 104 | } 105 | 106 | function sidebar(): DefaultTheme.Sidebar { 107 | return [ 108 | { 109 | text: "Getting Started", 110 | collapsed: false, 111 | items: [ 112 | { 113 | text: "Introduction", 114 | link: "/getting-started#introduction", 115 | }, 116 | { text: "Installing", link: "/getting-started#installing" }, 117 | { text: "Basic Example", link: "/getting-started#basic-example" }, 118 | { text: "Next Steps", link: "/getting-started#next-steps" }, 119 | ], 120 | }, 121 | { 122 | text: "Using with...", 123 | collapsed: true, 124 | items: [ 125 | { text: "Python", link: "/python" }, 126 | { text: "Datasette", link: "/datasette" }, 127 | { text: "Node.js", link: "/nodejs" }, 128 | { text: "Deno", link: "/deno" }, 129 | { text: "Ruby", link: "/ruby" }, 130 | { text: "Elixir", link: "/elixir" }, 131 | { text: "Rust", link: "/rust" }, 132 | { text: "Go", link: "/go" }, 133 | { text: "Loadable Extension", link: "/loadable" }, 134 | { text: "Turso", link: "/turso" }, 135 | ], 136 | }, 137 | //guides, 138 | { 139 | text: "Comparisons with...", 140 | link: "/compare", 141 | }, 142 | { 143 | text: "Documentation", 144 | items: [ 145 | { text: "Building from Source", link: "/building-source" }, 146 | { text: "API Reference", link: "/api-reference" }, 147 | ], 148 | }, 149 | { 150 | text: "See also", 151 | items: [ 152 | { 153 | text: "sqlite-ecosystem", 154 | link: "https://github.com/asg017/sqlite-ecosystem", 155 | }, 156 | { text: "sqlite-http", link: "https://github.com/asg017/sqlite-http" }, 157 | { text: "sqlite-xsv", link: "https://github.com/asg017/sqlite-xsv" }, 158 | ], 159 | }, 160 | ]; 161 | } 162 | export default defineConfig({ 163 | title: PROJECT, 164 | description, 165 | lastUpdated: true, 166 | head: head(), 167 | base: "/sqlite-vss/", 168 | themeConfig: { 169 | nav: nav(), 170 | sidebar: sidebar(), 171 | footer: { 172 | message: "MIT License", 173 | copyright: "Copyright © 2023 Alex Garcia", 174 | }, 175 | outline: "deep", 176 | search: { 177 | provider: "local", 178 | }, 179 | socialLinks: [ 180 | { icon: "github", link: `https://github.com/asg017/${PROJECT}` }, 181 | { icon: "discord", link: "https://discord.gg/F9BZdpABMN" }, 182 | ], 183 | editLink: { 184 | pattern: `https://github.com/asg017/${PROJECT}/edit/main/site/:path`, 185 | }, 186 | }, 187 | rewrites: { 188 | "using/:pkg.md": ":pkg.md", 189 | "guides/:pkg.md": ":pkg.md", 190 | }, 191 | markdown: { 192 | languages: [ 193 | { 194 | id: "sqlite", 195 | scopeName: "source.sqlite", 196 | grammar: JSON.parse(readFileSync("sql.tmLanguage.json", "utf8")), 197 | path: "sqlite", 198 | aliases: ["sqlite"], 199 | }, 200 | ], 201 | }, 202 | }); 203 | -------------------------------------------------------------------------------- /bindings/deno/deno.lock: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2", 3 | "remote": { 4 | "https://deno.land/std@0.152.0/_util/assert.ts": "e94f2eb37cebd7f199952e242c77654e43333c1ac4c5c700e929ea3aa5489f74", 5 | "https://deno.land/std@0.152.0/_util/os.ts": "3b4c6e27febd119d36a416d7a97bd3b0251b77c88942c8f16ee5953ea13e2e49", 6 | "https://deno.land/std@0.152.0/path/_constants.ts": "df1db3ffa6dd6d1252cc9617e5d72165cd2483df90e93833e13580687b6083c3", 7 | "https://deno.land/std@0.152.0/path/_interface.ts": "ee3b431a336b80cf445441109d089b70d87d5e248f4f90ff906820889ecf8d09", 8 | "https://deno.land/std@0.152.0/path/_util.ts": "c1e9686d0164e29f7d880b2158971d805b6e0efc3110d0b3e24e4b8af2190d2b", 9 | "https://deno.land/std@0.152.0/path/common.ts": "bee563630abd2d97f99d83c96c2fa0cca7cee103e8cb4e7699ec4d5db7bd2633", 10 | "https://deno.land/std@0.152.0/path/glob.ts": "cb5255638de1048973c3e69e420c77dc04f75755524cb3b2e160fe9277d939ee", 11 | "https://deno.land/std@0.152.0/path/mod.ts": "56fec03ad0ebd61b6ab39ddb9b0ddb4c4a5c9f2f4f632e09dd37ec9ebfd722ac", 12 | "https://deno.land/std@0.152.0/path/posix.ts": "c1f7afe274290ea0b51da07ee205653b2964bd74909a82deb07b69a6cc383aaa", 13 | "https://deno.land/std@0.152.0/path/separator.ts": "fe1816cb765a8068afb3e8f13ad272351c85cbc739af56dacfc7d93d710fe0f9", 14 | "https://deno.land/std@0.152.0/path/win32.ts": "bd7549042e37879c68ff2f8576a25950abbfca1d696d41d82c7bca0b7e6f452c", 15 | "https://deno.land/std@0.176.0/_util/asserts.ts": "178dfc49a464aee693a7e285567b3d0b555dc805ff490505a8aae34f9cfb1462", 16 | "https://deno.land/std@0.176.0/_util/os.ts": "d932f56d41e4f6a6093d56044e29ce637f8dcc43c5a90af43504a889cf1775e3", 17 | "https://deno.land/std@0.176.0/encoding/hex.ts": "50f8c95b52eae24395d3dfcb5ec1ced37c5fe7610ef6fffdcc8b0fdc38e3b32f", 18 | "https://deno.land/std@0.176.0/fmt/colors.ts": "938c5d44d889fb82eff6c358bea8baa7e85950a16c9f6dae3ec3a7a729164471", 19 | "https://deno.land/std@0.176.0/fs/_util.ts": "65381f341af1ff7f40198cee15c20f59951ac26e51ddc651c5293e24f9ce6f32", 20 | "https://deno.land/std@0.176.0/fs/copy.ts": "14214efd94fc3aa6db1e4af2b4b9578e50f7362b7f3725d5a14ad259a5df26c8", 21 | "https://deno.land/std@0.176.0/fs/empty_dir.ts": "c3d2da4c7352fab1cf144a1ecfef58090769e8af633678e0f3fabaef98594688", 22 | "https://deno.land/std@0.176.0/fs/ensure_dir.ts": "724209875497a6b4628dfb256116e5651c4f7816741368d6c44aab2531a1e603", 23 | "https://deno.land/std@0.176.0/fs/ensure_file.ts": "c38602670bfaf259d86ca824a94e6cb9e5eb73757fefa4ebf43a90dd017d53d9", 24 | "https://deno.land/std@0.176.0/fs/ensure_link.ts": "c0f5b2f0ec094ed52b9128eccb1ee23362a617457aa0f699b145d4883f5b2fb4", 25 | "https://deno.land/std@0.176.0/fs/ensure_symlink.ts": "2955cc8332aeca9bdfefd05d8d3976b94e282b0f353392a71684808ed2ffdd41", 26 | "https://deno.land/std@0.176.0/fs/eol.ts": "f1f2eb348a750c34500741987b21d65607f352cf7205f48f4319d417fff42842", 27 | "https://deno.land/std@0.176.0/fs/exists.ts": "b8c8a457b71e9d7f29b9d2f87aad8dba2739cbe637e8926d6ba6e92567875f8e", 28 | "https://deno.land/std@0.176.0/fs/expand_glob.ts": "45d17e89796a24bd6002e4354eda67b4301bb8ba67d2cac8453cdabccf1d9ab0", 29 | "https://deno.land/std@0.176.0/fs/mod.ts": "bc3d0acd488cc7b42627044caf47d72019846d459279544e1934418955ba4898", 30 | "https://deno.land/std@0.176.0/fs/move.ts": "4cb47f880e3f0582c55e71c9f8b1e5e8cfaacb5e84f7390781dd563b7298ec19", 31 | "https://deno.land/std@0.176.0/fs/walk.ts": "ea95ffa6500c1eda6b365be488c056edc7c883a1db41ef46ec3bf057b1c0fe32", 32 | "https://deno.land/std@0.176.0/path/_constants.ts": "e49961f6f4f48039c0dfed3c3f93e963ca3d92791c9d478ac5b43183413136e0", 33 | "https://deno.land/std@0.176.0/path/_interface.ts": "6471159dfbbc357e03882c2266d21ef9afdb1e4aa771b0545e90db58a0ba314b", 34 | "https://deno.land/std@0.176.0/path/_util.ts": "d7abb1e0dea065f427b89156e28cdeb32b045870acdf865833ba808a73b576d0", 35 | "https://deno.land/std@0.176.0/path/common.ts": "ee7505ab01fd22de3963b64e46cff31f40de34f9f8de1fff6a1bd2fe79380000", 36 | "https://deno.land/std@0.176.0/path/glob.ts": "d479e0a695621c94d3fd7fe7abd4f9499caf32a8de13f25073451c6ef420a4e1", 37 | "https://deno.land/std@0.176.0/path/mod.ts": "4b83694ac500d7d31b0cdafc927080a53dc0c3027eb2895790fb155082b0d232", 38 | "https://deno.land/std@0.176.0/path/posix.ts": "8b7c67ac338714b30c816079303d0285dd24af6b284f7ad63da5b27372a2c94d", 39 | "https://deno.land/std@0.176.0/path/separator.ts": "0fb679739d0d1d7bf45b68dacfb4ec7563597a902edbaf3c59b50d5bcadd93b1", 40 | "https://deno.land/std@0.176.0/path/win32.ts": "d186344e5583bcbf8b18af416d13d82b35a317116e6460a5a3953508c3de5bba", 41 | "https://deno.land/std@0.177.0/fmt/colors.ts": "938c5d44d889fb82eff6c358bea8baa7e85950a16c9f6dae3ec3a7a729164471", 42 | "https://deno.land/std@0.177.0/testing/_diff.ts": "1a3c044aedf77647d6cac86b798c6417603361b66b54c53331b312caeb447aea", 43 | "https://deno.land/std@0.177.0/testing/_format.ts": "a69126e8a469009adf4cf2a50af889aca364c349797e63174884a52ff75cf4c7", 44 | "https://deno.land/std@0.177.0/testing/asserts.ts": "984ab0bfb3faeed92ffaa3a6b06536c66811185328c5dd146257c702c41b01ab", 45 | "https://deno.land/x/plug@1.0.1/deps.ts": "35ea2acd5e3e11846817a429b7ef4bec47b80f2d988f5d63797147134cbd35c2", 46 | "https://deno.land/x/plug@1.0.1/download.ts": "8d6a023ade0806a0653b48cd5f6f8b15fcfaa1dbf2aa1f4bc90fc5732d27b144", 47 | "https://deno.land/x/plug@1.0.1/mod.ts": "5dec80ee7a3a325be45c03439558531bce7707ac118f4376cebbd6740ff24bfb", 48 | "https://deno.land/x/plug@1.0.1/types.ts": "d8eb738fc6ed883e6abf77093442c2f0b71af9090f15c7613621d4039e410ee1", 49 | "https://deno.land/x/plug@1.0.1/util.ts": "5ba8127b9adc36e070b9e22971fb8106869eea1741f452a87b4861e574f13481", 50 | "https://deno.land/x/sqlite3@0.8.0/deno.json": "61fbd0665a1b48f5e0f1773371d49b776f559cd6c3747a4e674adc3eb423686c", 51 | "https://deno.land/x/sqlite3@0.8.0/deps.ts": "722c865b9cef27b4cde0bb1ac9ebb08e94c43ad090a7313cea576658ff1e3bb0", 52 | "https://deno.land/x/sqlite3@0.8.0/mod.ts": "d41b8b30e1b20b537ef4d78cae98d90f6bd65c727b64aa1a18bffbb28f7d6ec3", 53 | "https://deno.land/x/sqlite3@0.8.0/src/blob.ts": "a956fc0cf4a8c7a21dc3fcb71a07ef773bcd08b5fd72e8ace89b1bfbd031bf06", 54 | "https://deno.land/x/sqlite3@0.8.0/src/constants.ts": "85fd27aa6e199093f25f5f437052e16fd0e0870b96ca9b24a98e04ddc8b7d006", 55 | "https://deno.land/x/sqlite3@0.8.0/src/database.ts": "c68c7fdfa7548000ea7e194360cdce86b81b667aab6b0778ea7ed9b74a37b7cb", 56 | "https://deno.land/x/sqlite3@0.8.0/src/ffi.ts": "d5d5e3b4524cf0b980d23e57b08f75dc0debc7af2e93ff8b00cdc5af52637b31", 57 | "https://deno.land/x/sqlite3@0.8.0/src/statement.ts": "0b2b3c8b5564ad3c35f2c7d57607e5755c38b23b835e9a46aa1002e68e6ec3a2", 58 | "https://deno.land/x/sqlite3@0.8.0/src/util.ts": "9627ebecc7a5eb250d2df9386a456a9a9ed7842a20fe32be1ee6b7c663c77bd3" 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /bindings/elixir/lib/sqlite_vss.ex: -------------------------------------------------------------------------------- 1 | defmodule SqliteVss do 2 | @moduledoc """ 3 | `SqliteVss` module which installs the vector0 and vss0 extensions. 4 | """ 5 | use Application 6 | require Logger 7 | 8 | @doc false 9 | def start(_, _) do 10 | Supervisor.start_link([], strategy: :one_for_one) 11 | end 12 | 13 | @doc """ 14 | Returns the configured sqlite_vss version. 15 | """ 16 | def current_version do 17 | config = :hex_core.default_config() 18 | 19 | {:ok, _} = Application.ensure_all_started(:inets) 20 | {:ok, _} = Application.ensure_all_started(:ssl) 21 | 22 | updated_config = 23 | Map.put(config, :http_adapter, {:hex_http_httpc, %{http_options: http_options()}}) 24 | 25 | case :hex_api_package.get(updated_config, "sqlite_vss") do 26 | {:ok, {200, _headers, response}} -> 27 | response 28 | |> Map.get("releases") 29 | |> Enum.map(& &1["version"]) 30 | |> List.first() 31 | 32 | {:ok, {404, _, _}} -> 33 | # TODO: remove this fallback once sqlite_vss is published 34 | "0.1.1-alpha.8" 35 | 36 | {:error, reason} -> 37 | reason 38 | end 39 | end 40 | 41 | @doc """ 42 | Installs sqlite_vss with `current_version/0`. 43 | """ 44 | def install(base_url \\ default_base_url()) do 45 | url = get_url(base_url) 46 | 47 | tar_binary = fetch_body!(url) 48 | 49 | bin_path = bin_path() 50 | 51 | with :ok <- 52 | :erl_tar.extract({:binary, tar_binary}, [ 53 | :compressed, 54 | cwd: to_charlist(bin_path) 55 | ]) do 56 | Logger.debug("Copying artifact from release and extracting to #{bin_path}") 57 | :ok 58 | end 59 | end 60 | 61 | @doc """ 62 | Returns the path to the executable. 63 | 64 | The executable may not be available if it was not yet installed. 65 | """ 66 | def bin_path do 67 | {:ok, current_target} = current_target() 68 | name = "sqlite-vss-#{current_target}" 69 | 70 | Application.get_env(:sqlite_vss, :path) || 71 | if Code.ensure_loaded?(Mix.Project) do 72 | Path.join(Path.dirname(Mix.Project.build_path()), name) 73 | else 74 | Path.expand("_build/#{name}") 75 | end 76 | end 77 | 78 | def filename(target) do 79 | "sqlite-vss-v#{current_version()}-loadable-#{target}.tar.gz" 80 | end 81 | 82 | @doc """ 83 | The default URL to install SqliteVss from. 84 | """ 85 | def default_base_url do 86 | "https://github.com/asg017/sqlite-vss/releases/download/v$version/sqlite-vss-v$version-loadable-$target.tar.gz" 87 | end 88 | 89 | def loadable_path_vector0() do 90 | SqliteVss.bin_path() <> "/vector0" 91 | end 92 | 93 | def loadable_path_vss0() do 94 | SqliteVss.bin_path() <> "/vss0" 95 | end 96 | 97 | defp target_config do 98 | current_system_arch = system_arch() 99 | 100 | %{ 101 | os_type: :os.type(), 102 | target_system: current_system_arch, 103 | word_size: :erlang.system_info(:wordsize) 104 | } 105 | end 106 | 107 | def current_target!(target_config \\ target_config()) do 108 | current_target(target_config) 109 | |> elem(1) 110 | end 111 | 112 | def current_target(target_config \\ target_config()) do 113 | %{os_type: os_type, target_system: target_system} = target_config 114 | arch_str = target_system |> Map.values() |> Enum.join("-") 115 | 116 | case os_type do 117 | {:win32, _} -> 118 | {:error, "sqlite-vss is not available for architecture: #{arch_str}"} 119 | 120 | {:unix, osname} -> 121 | osname = if osname == :darwin, do: "macos", else: osname 122 | 123 | case target_config.target_system.arch do 124 | "x86_64" -> 125 | {:ok, "#{osname}-x86_64"} 126 | 127 | "aarch64" -> 128 | {:ok, "#{osname}-aarch64"} 129 | 130 | _ -> 131 | {:error, 132 | "precompiled artifact is not available for this target: \"#{arch_str}\".\nThe available targets are:\n - linux-x86_64\n - macos-aarch64\n - macos-x86_64"} 133 | end 134 | end 135 | end 136 | 137 | defp http_options(scheme) do 138 | http_options() 139 | |> maybe_add_proxy_auth(scheme) 140 | end 141 | 142 | defp http_options do 143 | # https://erlef.github.io/security-wg/secure_coding_and_deployment_hardening/inets 144 | cacertfile = cacertfile() |> String.to_charlist() 145 | 146 | [ 147 | ssl: [ 148 | verify: :verify_peer, 149 | cacertfile: cacertfile, 150 | depth: 2, 151 | customize_hostname_check: [ 152 | match_fun: :public_key.pkix_verify_hostname_match_fun(:https) 153 | ] 154 | ] 155 | ] 156 | end 157 | 158 | defp fetch_body!(url) do 159 | scheme = URI.parse(url).scheme 160 | url = String.to_charlist(url) 161 | Logger.debug("Downloading sqlite-vss from #{url}") 162 | 163 | {:ok, _} = Application.ensure_all_started(:inets) 164 | {:ok, _} = Application.ensure_all_started(:ssl) 165 | 166 | if proxy = proxy_for_scheme(scheme) do 167 | %{host: host, port: port} = URI.parse(proxy) 168 | Logger.debug("Using #{String.upcase(scheme)}_PROXY: #{proxy}") 169 | set_option = if "https" == scheme, do: :https_proxy, else: :proxy 170 | :httpc.set_options([{set_option, {{String.to_charlist(host), port}, []}}]) 171 | end 172 | 173 | options = [body_format: :binary] 174 | 175 | case :httpc.request(:get, {url, []}, http_options(scheme), options) do 176 | {:ok, {{_, 200, _}, _headers, body}} -> 177 | body 178 | 179 | other -> 180 | raise """ 181 | couldn't fetch #{url}: #{inspect(other)} 182 | 183 | You may also install the "sqlite_vss" executable manually, \ 184 | see the docs: https://hexdocs.pm/sqlite_vss 185 | """ 186 | end 187 | end 188 | 189 | defp proxy_for_scheme("http") do 190 | System.get_env("HTTP_PROXY") || System.get_env("http_proxy") 191 | end 192 | 193 | defp proxy_for_scheme("https") do 194 | System.get_env("HTTPS_PROXY") || System.get_env("https_proxy") 195 | end 196 | 197 | defp maybe_add_proxy_auth(http_options, scheme) do 198 | case proxy_auth(scheme) do 199 | nil -> http_options 200 | auth -> [{:proxy_auth, auth} | http_options] 201 | end 202 | end 203 | 204 | defp proxy_auth(scheme) do 205 | with proxy when is_binary(proxy) <- proxy_for_scheme(scheme), 206 | %{userinfo: userinfo} when is_binary(userinfo) <- URI.parse(proxy), 207 | [username, password] <- String.split(userinfo, ":") do 208 | {String.to_charlist(username), String.to_charlist(password)} 209 | else 210 | _ -> nil 211 | end 212 | end 213 | 214 | defp cacertfile() do 215 | Application.get_env(:sqlite_vss, :cacerts_path) || CAStore.file_path() 216 | end 217 | 218 | defp get_url(base_url, target \\ current_target!()) do 219 | base_url 220 | |> String.replace("$version", current_version()) 221 | |> String.replace("$target", target) 222 | end 223 | 224 | # Returns a map with `:arch`, `:vendor`, `:os` and maybe `:abi`. 225 | defp system_arch do 226 | base = 227 | :erlang.system_info(:system_architecture) 228 | |> List.to_string() 229 | |> String.split("-") 230 | 231 | triple_keys = 232 | case length(base) do 233 | 4 -> 234 | [:arch, :vendor, :os, :abi] 235 | 236 | 3 -> 237 | [:arch, :vendor, :os] 238 | 239 | _ -> 240 | # It's too complicated to find out, and we won't support this for now. 241 | [] 242 | end 243 | 244 | triple_keys 245 | |> Enum.zip(base) 246 | |> Enum.into(%{}) 247 | end 248 | end 249 | -------------------------------------------------------------------------------- /bindings/elixir/mix.lock: -------------------------------------------------------------------------------- 1 | %{ 2 | "bypass": {:hex, :bypass, "2.1.0", "909782781bf8e20ee86a9cabde36b259d44af8b9f38756173e8f5e2e1fabb9b1", [:mix], [{:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:ranch, "~> 1.3", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "d9b5df8fa5b7a6efa08384e9bbecfe4ce61c77d28a4282f79e02f1ef78d96b80"}, 3 | "castore": {:hex, :castore, "1.0.2", "0c6292ecf3e3f20b7c88408f00096337c4bfd99bd46cc2fe63413ddbe45b3573", [:mix], [], "hexpm", "40b2dd2836199203df8500e4a270f10fc006cc95adc8a319e148dc3077391d96"}, 4 | "cc_precompiler": {:hex, :cc_precompiler, "0.1.7", "77de20ac77f0e53f20ca82c563520af0237c301a1ec3ab3bc598e8a96c7ee5d9", [:mix], [{:elixir_make, "~> 0.7.3", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "2768b28bf3c2b4f788c995576b39b8cb5d47eb788526d93bd52206c1d8bf4b75"}, 5 | "cowboy": {:hex, :cowboy, "2.10.0", "ff9ffeff91dae4ae270dd975642997afe2a1179d94b1887863e43f681a203e26", [:make, :rebar3], [{:cowlib, "2.12.1", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "3afdccb7183cc6f143cb14d3cf51fa00e53db9ec80cdcd525482f5e99bc41d6b"}, 6 | "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"}, 7 | "cowlib": {:hex, :cowlib, "2.12.1", "a9fa9a625f1d2025fe6b462cb865881329b5caff8f1854d1cbc9f9533f00e1e1", [:make, :rebar3], [], "hexpm", "163b73f6367a7341b33c794c4e88e7dbfe6498ac42dcd69ef44c5bc5507c8db0"}, 8 | "db_connection": {:hex, :db_connection, "2.5.0", "bb6d4f30d35ded97b29fe80d8bd6f928a1912ca1ff110831edcd238a1973652c", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c92d5ba26cd69ead1ff7582dbb860adeedfff39774105a4f1c92cbb654b55aa2"}, 9 | "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, 10 | "earmark_parser": {:hex, :earmark_parser, "1.4.32", "fa739a0ecfa34493de19426681b23f6814573faee95dfd4b4aafe15a7b5b32c6", [:mix], [], "hexpm", "b8b0dd77d60373e77a3d7e8afa598f325e49e8663a51bcc2b88ef41838cca755"}, 11 | "ecto": {:hex, :ecto, "3.10.1", "c6757101880e90acc6125b095853176a02da8f1afe056f91f1f90b80c9389822", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "d2ac4255f1601bdf7ac74c0ed971102c6829dc158719b94bd30041bbad77f87a"}, 12 | "ecto_sql": {:hex, :ecto_sql, "3.10.1", "6ea6b3036a0b0ca94c2a02613fd9f742614b5cfe494c41af2e6571bb034dd94c", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.10.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.0 or ~> 0.17.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f6a25bdbbd695f12c8171eaff0851fa4c8e72eec1e98c7364402dda9ce11c56b"}, 13 | "ecto_sqlite3": {:hex, :ecto_sqlite3, "0.10.3", "82ce316a8727f1daec397a9932b1a20130ea1ac33c3257b78eded1d3f45ae9b3", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:ecto, "~> 3.10", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:exqlite, "~> 0.9", [hex: :exqlite, repo: "hexpm", optional: false]}], "hexpm", "b4fa32d09f5e5c05d3401ade3dd4416e3c7072d5117c150cb4adeea72760fb93"}, 14 | "elixir_make": {:hex, :elixir_make, "0.7.6", "67716309dc5d43e16b5abbd00c01b8df6a0c2ab54a8f595468035a50189f9169", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}], "hexpm", "5a0569756b0f7873a77687800c164cca6dfc03a09418e6fcf853d78991f49940"}, 15 | "ex_doc": {:hex, :ex_doc, "0.29.4", "6257ecbb20c7396b1fe5accd55b7b0d23f44b6aa18017b415cb4c2b91d997729", [:mix], [{:earmark_parser, "~> 1.4.31", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "2c6699a737ae46cb61e4ed012af931b57b699643b24dabe2400a8168414bc4f5"}, 16 | "exqlite": {:hex, :exqlite, "0.13.12", "29996a0773869f98b6764b27fe55ab48744d03baabbbca8c1b990a3f15448a30", [:make, :mix], [{:cc_precompiler, "~> 0.1", [hex: :cc_precompiler, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.7", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "0fdc3ec52e432d45379265d0aad3e0e8524656cdf45f19a9d6dc45e3ce048cd7"}, 17 | "hex_core": {:hex, :hex_core, "0.10.0", "6e739a159b0141fa6c3c60c92b73aa6dec5b7909647a9b9ecea9da6709b75709", [:rebar3], [], "hexpm", "1c229aeb2df3a7ffc0c00fa4fc1721995058b2c617f083cf617e29258b1d9f57"}, 18 | "makeup": {:hex, :makeup, "1.1.0", "6b67c8bc2882a6b6a445859952a602afc1a41c2e08379ca057c0f525366fc3ca", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "0a45ed501f4a8897f580eabf99a2e5234ea3e75a4373c8a52824f6e873be57a6"}, 19 | "makeup_elixir": {:hex, :makeup_elixir, "0.16.1", "cc9e3ca312f1cfeccc572b37a09980287e243648108384b97ff2b76e505c3555", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "e127a341ad1b209bd80f7bd1620a15693a9908ed780c3b763bccf7d200c767c6"}, 20 | "makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"}, 21 | "mime": {:hex, :mime, "2.0.3", "3676436d3d1f7b81b5a2d2bd8405f412c677558c81b1c92be58c00562bb59095", [:mix], [], "hexpm", "27a30bf0db44d25eecba73755acf4068cbfe26a4372f9eb3e4ea3a45956bff6b"}, 22 | "nimble_parsec": {:hex, :nimble_parsec, "1.3.1", "2c54013ecf170e249e9291ed0a62e5832f70a476c61da16f6aac6dca0189f2af", [:mix], [], "hexpm", "2682e3c0b2eb58d90c6375fc0cc30bc7be06f365bf72608804fb9cffa5e1b167"}, 23 | "plug": {:hex, :plug, "1.14.2", "cff7d4ec45b4ae176a227acd94a7ab536d9b37b942c8e8fa6dfc0fff98ff4d80", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "842fc50187e13cf4ac3b253d47d9474ed6c296a8732752835ce4a86acdf68d13"}, 24 | "plug_cowboy": {:hex, :plug_cowboy, "2.6.1", "9a3bbfceeb65eff5f39dab529e5cd79137ac36e913c02067dba3963a26efe9b2", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "de36e1a21f451a18b790f37765db198075c25875c64834bcc82d90b309eb6613"}, 25 | "plug_crypto": {:hex, :plug_crypto, "1.2.5", "918772575e48e81e455818229bf719d4ab4181fcbf7f85b68a35620f78d89ced", [:mix], [], "hexpm", "26549a1d6345e2172eb1c233866756ae44a9609bd33ee6f99147ab3fd87fd842"}, 26 | "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, 27 | "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"}, 28 | } 29 | -------------------------------------------------------------------------------- /.github/workflows/test.yaml: -------------------------------------------------------------------------------- 1 | name: "test build" 2 | on: 3 | push: 4 | branches: 5 | - main 6 | permissions: 7 | contents: read 8 | env: 9 | ARTIFACT-LINUX-X86_64-EXTENSION: sqlite-vss-linux-x86_64 10 | ARTIFACT-MACOS-X86_64-EXTENSION: sqlite-vss-macos-x86_64 11 | ARTIFACT-MACOS-AARCH64-EXTENSION: sqlite-vss-macos-aarch64 12 | ARTIFACT-WINDOWS-X86_64-EXTENSION: sqlite-vss-windows-x86_64 13 | ARTIFACT-LINUX-X86_64-WHEELS: sqlite-vss-linux-x86_64-wheels 14 | ARTIFACT-MACOS-X86_64-WHEELS: sqlite-vss-macos-x86_64-wheels 15 | ARTIFACT-MACOS-AARCH64-WHEELS: sqlite-vss-macos-aarch64-wheels 16 | jobs: 17 | build-linux-x86_64-extension: 18 | runs-on: ubuntu-20.04 19 | steps: 20 | - uses: actions/checkout@v3 21 | with: 22 | submodules: "recursive" 23 | - name: Cache sqlite build 24 | id: cache-sqlite-build 25 | uses: actions/cache@v3 26 | with: 27 | path: vendor/sqlite 28 | key: ${{ runner.os }}-${{ hashFiles('vendor/get_sqlite.sh') }} 29 | - if: steps.cache-sqlite-build.outputs.cache-hit != 'true' 30 | run: ./vendor/get_sqlite.sh 31 | - if: steps.cache-sqlite-build.outputs.cache-hit != 'true' 32 | working-directory: vendor/sqlite 33 | run: ./configure && make 34 | 35 | # TODO how cache this? 36 | - run: sudo apt-get update -y && sudo apt-get install -y cmake libgomp1 37 | - run: make loadable static 38 | - uses: actions/upload-artifact@v3 39 | with: 40 | name: ${{ env.ARTIFACT-LINUX-X86_64-EXTENSION }} 41 | path: dist/debug/* 42 | build-linux-x86_64-python: 43 | runs-on: ubuntu-latest 44 | needs: [build-linux-x86_64-extension] 45 | steps: 46 | - uses: actions/checkout@v3 47 | - uses: actions/download-artifact@v3 48 | with: 49 | name: ${{ env.ARTIFACT-LINUX-X86_64-EXTENSION }} 50 | path: dist/debug/ 51 | - uses: actions/setup-python@v3 52 | - run: pip install wheel 53 | - run: make python 54 | - run: make datasette 55 | - uses: actions/upload-artifact@v3 56 | with: 57 | name: ${{ env.ARTIFACT-LINUX-X86_64-WHEELS }} 58 | path: dist/debug/wheels/*.whl 59 | test-linux-x86_64: 60 | runs-on: ubuntu-latest 61 | needs: [build-linux-x86_64-extension, build-linux-x86_64-python] 62 | env: 63 | DENO_DIR: deno_cache 64 | steps: 65 | - uses: actions/checkout@v3 66 | - uses: actions/download-artifact@v3 67 | with: 68 | name: ${{ env.ARTIFACT-LINUX-X86_64-EXTENSION }} 69 | path: dist/debug/ 70 | - uses: actions/download-artifact@v3 71 | with: 72 | name: ${{ env.ARTIFACT-LINUX-X86_64-WHEELS }} 73 | path: dist/debug/ 74 | - run: pip install --find-links dist/debug/ sqlite_vss 75 | - run: make test-loadable 76 | - run: make test-python 77 | # for test-npm 78 | - uses: actions/setup-node@v3 79 | with: 80 | cache: npm 81 | cache-dependency-path: bindings/node/sqlite-vss/package.json 82 | - run: npm install 83 | working-directory: bindings/node/sqlite-vss 84 | - run: cp dist/debug/vss0.so bindings/node/sqlite-vss-linux-x64/lib 85 | - run: cp dist/debug/vector0.so bindings/node/sqlite-vss-linux-x64/lib 86 | - run: make test-npm 87 | # for test-deno 88 | - uses: denoland/setup-deno@v1 89 | with: 90 | deno-version: v1.30 91 | - uses: actions/cache@v3 92 | with: 93 | path: ${{ env.DENO_DIR }} 94 | key: ${{ runner.os }}-${{ hashFiles('bindings/deno/deno.lock') }} 95 | - run: make test-deno 96 | env: 97 | DENO_SQLITE_VECTOR_PATH: ${{ github.workspace }}/dist/debug/vector0 98 | DENO_SQLITE_VSS_PATH: ${{ github.workspace }}/dist/debug/vss0 99 | build-macos-x86_64-extension: 100 | runs-on: macos-11 101 | steps: 102 | - uses: actions/checkout@v3 103 | with: 104 | submodules: "recursive" 105 | - name: Cache sqlite build 106 | id: cache-sqlite-build 107 | uses: actions/cache@v3 108 | with: 109 | path: vendor/sqlite 110 | key: ${{ runner.os }}-${{ hashFiles('vendor/get_sqlite.sh') }} 111 | - if: steps.cache-sqlite-build.outputs.cache-hit != 'true' 112 | run: ./vendor/get_sqlite.sh 113 | - if: steps.cache-sqlite-build.outputs.cache-hit != 'true' 114 | working-directory: vendor/sqlite 115 | run: ./configure && make 116 | - run: brew install llvm libomp 117 | - name: Cache cmake build 118 | id: cache-cmake-build 119 | uses: actions/cache@v3 120 | with: 121 | path: build 122 | key: ${{ runner.os }}-build 123 | - run: make patch-openmp 124 | - run: find /usr/local/opt/libomp/* 125 | - run: make loadable static 126 | env: 127 | # `brew info libomp` gives the correct one, with .a file for static openmp builds 128 | CC: /usr/local/opt/llvm/bin/clang 129 | CXX: /usr/local/opt/llvm/bin/clang++ 130 | LDFLAGS: "-L/usr/local/opt/libomp/lib" 131 | CPPFLAGS: "-I/usr/local/opt/libomp/include" 132 | - uses: actions/upload-artifact@v3 133 | with: 134 | name: ${{ env.ARTIFACT-MACOS-X86_64-EXTENSION }} 135 | path: dist/debug/* 136 | build-macos-x86_64-python: 137 | runs-on: macos-latest 138 | needs: [build-macos-x86_64-extension] 139 | steps: 140 | - uses: actions/checkout@v3 141 | - uses: actions/download-artifact@v3 142 | with: 143 | name: ${{ env.ARTIFACT-MACOS-X86_64-EXTENSION }} 144 | path: dist/debug/ 145 | - uses: actions/setup-python@v3 146 | - run: pip install wheel 147 | - run: make python 148 | - run: make datasette 149 | - uses: actions/upload-artifact@v3 150 | with: 151 | name: ${{ env.ARTIFACT-MACOS-X86_64-WHEELS }} 152 | path: dist/debug/wheels/*.whl 153 | test-macos-x86_64: 154 | runs-on: macos-latest 155 | needs: [build-macos-x86_64-extension, build-macos-x86_64-python] 156 | # can't get automated tests on macos to work yet :( 157 | if: false 158 | steps: 159 | - uses: actions/checkout@v3 160 | - uses: actions/download-artifact@v3 161 | with: 162 | name: ${{ env.ARTIFACT-MACOS-X86_64-EXTENSION }} 163 | path: dist/debug/ 164 | - uses: actions/download-artifact@v3 165 | with: 166 | name: ${{ env.ARTIFACT-MACOS-X86_64-WHEELS }} 167 | path: dist/debug/wheels 168 | #- run: brew install llvm 169 | - run: /usr/local/opt/python@3/libexec/bin/pip install --find-links dist/debug/wheels sqlite_vss 170 | - run: find dist 171 | #- run: make test-loadable python=/usr/local/opt/python@3/libexec/bin/python 172 | #- run: make test-python python=/usr/local/opt/python@3/libexec/bin/python 173 | build-macos-aarch64-extension: 174 | runs-on: [self-hosted, mm1] 175 | steps: 176 | - uses: actions/checkout@v3 177 | with: 178 | submodules: "recursive" 179 | - name: Cache sqlite build 180 | id: cache-sqlite-build 181 | uses: actions/cache@v3 182 | with: 183 | path: vendor/sqlite 184 | key: ${{ runner.os }}-${{ hashFiles('vendor/get_sqlite.sh') }} 185 | - if: steps.cache-sqlite-build.outputs.cache-hit != 'true' 186 | run: ./vendor/get_sqlite.sh 187 | - if: steps.cache-sqlite-build.outputs.cache-hit != 'true' 188 | working-directory: vendor/sqlite 189 | run: ./configure && make 190 | - run: make patch-openmp 191 | - run: make loadable static 192 | env: 193 | # `brew info libomp` gives the correct one, with .a file for static openmp builds 194 | CC: /opt/homebrew/opt/llvm/bin/clang 195 | CXX: /opt/homebrew/opt/llvm/bin/clang++ 196 | LDFLAGS: "-L/opt/homebrew/opt/libomp/lib" 197 | CPPFLAGS: "-I/opt/homebrew/opt/libomp/include" 198 | - uses: actions/upload-artifact@v3 199 | with: 200 | name: ${{ env.ARTIFACT-MACOS-AARCH64-EXTENSION }} 201 | path: dist/debug/* 202 | build-macos-aarch64-python: 203 | runs-on: [self-hosted, mm1] 204 | needs: [build-macos-aarch64-extension] 205 | steps: 206 | - uses: actions/checkout@v3 207 | - uses: actions/download-artifact@v3 208 | with: 209 | name: ${{ env.ARTIFACT-MACOS-AARCH64-EXTENSION }} 210 | path: dist/debug/ 211 | - run: pip3 install wheel 212 | - run: make python IS_MACOS_ARM=1 213 | - run: make datasette 214 | - uses: actions/upload-artifact@v3 215 | with: 216 | name: ${{ env.ARTIFACT-MACOS-AARCH64-WHEELS }} 217 | path: dist/debug/wheels/*.whl 218 | build-windows-x86_64-extension: 219 | runs-on: windows-2019 220 | if: false 221 | steps: 222 | - uses: actions/checkout@v3 223 | with: 224 | submodules: "recursive" 225 | - name: Cache sqlite build 226 | id: cache-sqlite-build 227 | uses: actions/cache@v3 228 | with: 229 | path: vendor/sqlite 230 | key: ${{ runner.os }}-${{ hashFiles('vendor/get_sqlite.sh') }} 231 | - if: steps.cache-sqlite-build.outputs.cache-hit != 'true' 232 | run: choco install wget --no-progress && ./vendor/get_sqlite.sh 233 | shell: bash 234 | - if: steps.cache-sqlite-build.outputs.cache-hit != 'true' 235 | working-directory: vendor\sqlite 236 | run: ./configure && make 237 | - run: make loadable static 238 | - uses: actions/upload-artifact@v3 239 | with: 240 | name: ${{ env.ARTIFACT-WINDOWS-X86_64-EXTENSION }} 241 | path: dist/debug/* 242 | --------------------------------------------------------------------------------