├── .github └── workflows │ └── build.yml ├── .gitignore ├── .gitmodules ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Cargo.lock ├── Cargo.toml ├── LICENSE.txt ├── README-zh.md ├── README.md ├── VERSION ├── cozo-bin ├── Cargo.toml ├── README-zh.md ├── README.md └── src │ ├── client.rs │ ├── index.html │ ├── main.rs │ ├── repl.rs │ ├── security.txt │ └── server.rs ├── cozo-core-examples ├── Cargo.toml └── src │ └── bin │ ├── run.rs │ ├── run_ast.rs │ └── run_parse_ast.rs ├── cozo-core ├── Cargo.toml ├── README-zh.md ├── README.md ├── benches │ ├── pokec.rs │ ├── time_travel.rs │ └── wiki_pagerank.rs ├── src │ ├── cozoscript.pest │ ├── data │ │ ├── aggr.rs │ │ ├── expr.rs │ │ ├── functions.rs │ │ ├── json.rs │ │ ├── memcmp.rs │ │ ├── mod.rs │ │ ├── program.rs │ │ ├── relation.rs │ │ ├── symb.rs │ │ ├── tests │ │ │ ├── aggrs.rs │ │ │ ├── exprs.rs │ │ │ ├── functions.rs │ │ │ ├── json.rs │ │ │ ├── memcmp.rs │ │ │ ├── mod.rs │ │ │ ├── validity.rs │ │ │ └── values.rs │ │ ├── tuple.rs │ │ └── value.rs │ ├── fixed_rule │ │ ├── algos │ │ │ ├── all_pairs_shortest_path.rs │ │ │ ├── astar.rs │ │ │ ├── bfs.rs │ │ │ ├── degree_centrality.rs │ │ │ ├── dfs.rs │ │ │ ├── kruskal.rs │ │ │ ├── label_propagation.rs │ │ │ ├── louvain.rs │ │ │ ├── mod.rs │ │ │ ├── pagerank.rs │ │ │ ├── prim.rs │ │ │ ├── random_walk.rs │ │ │ ├── shortest_path_bfs.rs │ │ │ ├── shortest_path_dijkstra.rs │ │ │ ├── strongly_connected_components.rs │ │ │ ├── top_sort.rs │ │ │ ├── triangles.rs │ │ │ └── yen.rs │ │ ├── mod.rs │ │ └── utilities │ │ │ ├── constant.rs │ │ │ ├── csv.rs │ │ │ ├── jlines.rs │ │ │ ├── mod.rs │ │ │ └── reorder_sort.rs │ ├── fts │ │ ├── README.md │ │ ├── ast.rs │ │ ├── cangjie │ │ │ ├── mod.rs │ │ │ ├── options.rs │ │ │ ├── stream.rs │ │ │ └── tokenizer.rs │ │ ├── indexing.rs │ │ ├── mod.rs │ │ └── tokenizer │ │ │ ├── alphanum_only.rs │ │ │ ├── ascii_folding_filter.rs │ │ │ ├── empty_tokenizer.rs │ │ │ ├── lower_caser.rs │ │ │ ├── mod.rs │ │ │ ├── ngram_tokenizer.rs │ │ │ ├── raw_tokenizer.rs │ │ │ ├── remove_long.rs │ │ │ ├── simple_tokenizer.rs │ │ │ ├── split_compound_words.rs │ │ │ ├── stemmer.rs │ │ │ ├── stop_word_filter │ │ │ ├── gen_stopwords.py │ │ │ ├── mod.rs │ │ │ └── stopwords.rs │ │ │ ├── tokenized_string.rs │ │ │ ├── tokenizer_impl.rs │ │ │ └── whitespace_tokenizer.rs │ ├── lib.rs │ ├── parse │ │ ├── expr.rs │ │ ├── fts.rs │ │ ├── imperative.rs │ │ ├── mod.rs │ │ ├── query.rs │ │ ├── schema.rs │ │ └── sys.rs │ ├── query │ │ ├── compile.rs │ │ ├── eval.rs │ │ ├── graph.rs │ │ ├── logical.rs │ │ ├── magic.rs │ │ ├── mod.rs │ │ ├── ra.rs │ │ ├── reorder.rs │ │ ├── sort.rs │ │ ├── stored.rs │ │ └── stratify.rs │ ├── runtime │ │ ├── callback.rs │ │ ├── db.rs │ │ ├── hnsw.rs │ │ ├── imperative.rs │ │ ├── minhash_lsh.rs │ │ ├── mod.rs │ │ ├── relation.rs │ │ ├── temp_store.rs │ │ ├── tests.rs │ │ └── transact.rs │ ├── storage │ │ ├── mem.rs │ │ ├── mod.rs │ │ ├── newrocks.rs │ │ ├── rocks.rs │ │ ├── sled.rs │ │ ├── sqlite.rs │ │ ├── temp.rs │ │ └── tikv.rs │ └── utils.rs └── tests │ ├── air-routes-latest-edges.csv │ ├── air-routes-latest-nodes.csv │ ├── air-routes.json │ └── air_routes.rs ├── cozo-lib-c ├── .gitignore ├── Cargo.toml ├── README-zh.md ├── README.md ├── build.rs ├── comiple-ios.sh ├── cozo_c.h ├── example.c └── src │ └── lib.rs ├── cozo-lib-java ├── .gitignore ├── Cargo.toml ├── CozoJavaBridge.java ├── README-zh.md ├── README.md ├── org_cozodb_CozoJavaBridge.h └── src │ └── lib.rs ├── cozo-lib-nodejs ├── .gitignore ├── Cargo.toml ├── README-zh.md ├── README.md ├── example.js ├── index.d.ts ├── index.js ├── package.json ├── src │ └── lib.rs ├── test_build.sh └── yarn.lock ├── cozo-lib-python ├── .github_disabled │ └── workflows │ │ └── CI.yml ├── .gitignore ├── Cargo.toml ├── README-zh.md ├── README.md ├── build.sh ├── pyproject.toml └── src │ └── lib.rs ├── cozo-lib-swift ├── .gitignore ├── Cargo.toml ├── CozoDB.swift ├── CozoSwiftBridge.podspec ├── Package.swift ├── README-zh.md ├── README.md ├── build-rust.sh ├── build.rs ├── publish_pod.sh └── src │ └── lib.rs ├── cozo-lib-wasm ├── .appveyor.yml ├── .gitignore ├── .travis.yml ├── Cargo.toml ├── README-zh.md ├── README.md ├── build.sh ├── src │ ├── lib.rs │ └── utils.rs └── tests │ └── web.rs ├── cozorocks ├── .gitignore ├── CMakeLists.txt ├── Cargo.toml ├── README-zh.md ├── README.md ├── bridge │ ├── bridge.h │ ├── common.h │ ├── db.cpp │ ├── db.h │ ├── iter.h │ ├── opts.h │ ├── slice.h │ ├── status.cpp │ ├── status.h │ ├── tx.cpp │ └── tx.h ├── build.rs ├── build_version.cc ├── rocksdb_lib_sources.txt └── src │ ├── bridge │ ├── db.rs │ ├── iter.rs │ ├── mod.rs │ └── tx.rs │ └── lib.rs ├── scripts ├── build-docs.sh ├── build-release-linux.sh ├── build-release-mac.sh ├── build-release-windows.ps1 ├── compress.sh └── update-images.sh ├── signatures └── version1 │ └── cla.json └── static └── logo_c.png /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: 4 | push: 5 | branches: [ "main" ] 6 | pull_request: 7 | branches: [ "main" ] 8 | 9 | env: 10 | CARGO_TERM_COLOR: always 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - uses: actions/checkout@v3 19 | with: 20 | submodules: recursive 21 | - name: Build 22 | run: cargo build -p cozo --release --verbose 23 | - name: Run tests 24 | run: cargo test -p cozo --release --verbose 25 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | ### Rust template 3 | # Generated by Cargo 4 | # will have compiled files and executables 5 | debug/ 6 | target/ 7 | 8 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries 9 | # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html 10 | 11 | # These are backup files generated by rustfmt 12 | **/*.rs.bk 13 | 14 | _path* 15 | *.db### CMake template 16 | CMakeLists.txt.user 17 | CMakeCache.txt 18 | CMakeFiles 19 | CMakeScripts 20 | Testing 21 | cmake_install.cmake 22 | install_manifest.txt 23 | compile_commands.json 24 | CTestTestfile.cmake 25 | _deps 26 | _test* 27 | *.dll 28 | *.db 29 | .DS_Store 30 | release.zip 31 | .idea 32 | .fleet 33 | .vscode 34 | release/ 35 | release* 36 | Cross.toml 37 | /tools 38 | *.cozo_auth 39 | .cozo_repl_history 40 | /venv/ 41 | flamegraph*.svg -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "cozorocks/rocksdb"] 2 | path = cozorocks/rocksdb 3 | url = https://github.com/facebook/rocksdb.git 4 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## Contributing 2 | 3 | Thanks for your interest in contributing to the Cozo project! 4 | As Cozo is just getting started, we aim to keep everything simple. 5 | 6 | Contributions to code or other materials 7 | should be done via [pull requests](https://github.com/cozodb/cozo/pulls). 8 | You will be automatically prompted to sign a [Contributor Agreement (CLA)](https://github.com/cozodb/cozo-docs/blob/main/CLA.md). 9 | 10 | As the project matures, ideally before the end of 2023, we will change the CLA requirement to a DCO requirement: see [here](https://opensource.com/article/18/3/cla-vs-dco-whats-difference) 11 | if you are not familiar with the difference. 12 | The reason that we are asking for CLA now is that we want to retain the option to make changes that would become 13 | very difficult if copyright is distributed among the contributors, for example if we decide (again) 14 | that a different (FOSS) license would suit the project better. If you are uncomfortable with any form of CLA, 15 | you can wait until then. 16 | 17 | For code contributions other than simple bug fixes, please 18 | [discuss](https://github.com/cozodb/cozo/discussions) it 19 | first before opening a pull request, 20 | otherwise it is unlikely to be accepted. -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | resolver = "2" 3 | members = [ 4 | "cozo-core", 5 | "cozorocks", 6 | "cozo-bin", 7 | "cozo-lib-c", 8 | "cozo-lib-java", 9 | "cozo-lib-wasm", 10 | "cozo-lib-swift", 11 | "cozo-lib-python", 12 | "cozo-lib-nodejs", 13 | "cozo-core-examples", 14 | ] 15 | 16 | [profile.bench] 17 | lto = true 18 | debug = true 19 | 20 | [profile.dev.package."cozorocks"] 21 | opt-level = 3 22 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 0.7.6 -------------------------------------------------------------------------------- /cozo-bin/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "cozo-bin" 3 | version = "0.7.6" 4 | edition = "2021" 5 | license = "MPL-2.0" 6 | description = "Standalone Cozo database" 7 | authors = ["Ziyang Hu"] 8 | homepage = "https://www.cozodb.org" 9 | repository = "https://github.com/cozodb/cozo" 10 | documentation = "https://docs.cozodb.org" 11 | 12 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 13 | 14 | [features] 15 | #! # Features 16 | 17 | ## Enables the `minimal`, `requests` and `graph-algo` features 18 | compact = ["minimal", "requests", "graph-algo"] 19 | ## Enables the `storage-sqlite` and `graph-algo` features 20 | mobile = ["storage-sqlite", "graph-algo"] 21 | ## Enables the `minimal`, `requests` and `graph-algo` features in single threaded mode 22 | compact-single-threaded = ["minimal", "requests", "graph-algo"] 23 | ## Enables the `storage-sqlite` feature 24 | minimal = ["storage-sqlite"] 25 | ## Enables the [Sqlite](https://www.sqlite.org/index.html) backend, also allows backup and restore with Sqlite data files. 26 | storage-sqlite = ["cozo/storage-sqlite"] 27 | ## Enables the [RocksDB](http://rocksdb.org/) backend 28 | storage-rocksdb = ["cozo/storage-rocksdb"] 29 | ## Enables the graph algorithms 30 | graph-algo = ["cozo/graph-algo"] 31 | ## Allows the utilities to make web requests to fetch data 32 | requests = ["cozo/requests"] 33 | ## Uses jemalloc as the global allocator, can make a difference in performance 34 | jemalloc = ["cozo/jemalloc"] 35 | ## Enables io-uring option for the RocksDB storage 36 | io-uring = ["cozo/io-uring"] 37 | ## Enables the [Sled](https://github.com/spacejam/sled) backend 38 | storage-sled = ["cozo/storage-sled"] 39 | ## Enables the [TiKV](https://tikv.org/) client backend 40 | storage-tikv = ["cozo/storage-tikv"] 41 | 42 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 43 | 44 | [dependencies] 45 | cozo = { version = "0.7.6", path = "../cozo-core", default-features = false } 46 | clap = { version = "4.5.4", features = ["derive"] } 47 | env_logger = "0.11.3" 48 | log = "0.4.21" 49 | rand = "0.8.5" 50 | serde_derive = "1.0.199" 51 | serde = { version = "1.0.199" } 52 | chrono = "0.4.38" 53 | serde_json = "1.0.116" 54 | prettytable = "0.10.0" 55 | rustyline = "14.0.0" 56 | minreq = { version = "2.11.2", features = ["https-rustls"] } 57 | miette = { version = "5.10.0", features = ["fancy"] } 58 | ctrlc = "3.4.4" 59 | axum = "0.7.5" 60 | axum-macros = "0.4.1" 61 | itertools = "0.12.1" 62 | tokio = { version = "1.37.0", features = ["full"] } 63 | async-stream = "0.3.5" 64 | futures = "0.3.30" 65 | crossbeam = "0.8.4" 66 | eventsource-client = "0.12.2" 67 | tower-http = { version = "0.5.2", features = ["full"] } 68 | rayon = "1.10.0" 69 | -------------------------------------------------------------------------------- /cozo-bin/README-zh.md: -------------------------------------------------------------------------------- 1 | # Cozo（独立程序） 2 | 3 | [![server](https://img.shields.io/github/v/release/cozodb/cozo)](https://github.com/cozodb/cozo/releases) 4 | 5 | 本文叙述的是如何安装设置 Cozo 的独立程序本身。有关如何使用 6 | CozoDB（CozoScript）的信息，见 [文档](https://docs.cozodb.org/zh_CN/latest/index.html) 。 7 | 8 | ## 下载 9 | 10 | 独立服务的程序可以从 [GitHub 发布页](https://github.com/cozodb/cozo/releases) 11 | 或 [Gitee 发布页](https://gitee.com/cozodb/cozo/releases) 下载，其中名为 `cozo-*` 的是独立服务程序，名为 `cozo_all-*` 12 | 的独立程序同时支持更多地存储引擎，比如 [TiKV](https://tikv.org/)。 13 | 14 | ## 启动服务程序 15 | 16 | 在终端中执行： 17 | 18 | ```bash 19 | ./cozo server 20 | ``` 21 | 22 | 如此执行命令会使用纯内存的非持久化存储引擎。执行 `./cozo server -h` 可查看如何启用其它引擎，以及其它参数。 23 | 24 | 若要终止程序，按下 `CTRL-C` 按键，或向进程发送 `SIGTERM` （比如通过 `kill` 命令）。 25 | 26 | ## 命令行界面 27 | 28 | `./cozo repl` 可开启命令行界面（REPL），同时不会启动 web 服务。其它选择存储引擎的参数可一同使用。 29 | 30 | 在界面中可以使用以下特殊命令： 31 | 32 | * `%set <键> <值>`：设置在查询中可用的参数值。 33 | * `%unset <键>`：删除已设置的参数值。 34 | * `%clear`：清空所有已设置的参数。 35 | * `%params`：显示当前所有参数。 36 | * `%run <文件>`: 运行 `<文件>` 中包含的查询。 37 | * `%import <文件或 URL>`：将文件或 URL 里的 JSON 数据导入至数据库。 38 | * `%save <文件>`：下一个成功查询的结果将会以 JSON 格式存储在指定的文件中。如果文件参数未给出，则清除上次的文件设置。 39 | * `%backup <文件>`：备份全部数据至指定的文件。 40 | * `%restore <文件>`：将指定的备份文件中的数据加载到当前数据库中。当前数据库必须为空。 41 | 42 | ## 查询 API 43 | 44 | 查询通过向 API 发送 POST 请求来完成。默认的请求地址是 `http://127.0.0.1:9070/text-query` 。请求必须包含 JSON 格式的正文，具体内容如下： 45 | 46 | ```json 47 | { 48 | "script": "", 49 | "params": {} 50 | } 51 | ``` 52 | 53 | `params` 给出了查询文本中可用的变量。例如，当 `params` 为 `{"num": 1}` 时，查询文本中可以以 `$num` 来代替常量 `1` 54 | 。请善用此功能，而不是手动拼接查询字符串。 55 | 56 | HTTP API 返回的结果永远是 JSON 格式的。如果请求成功，则返回结果的 `"ok"` 字段将为 `true`，且 `"rows"` 57 | 字段将含有查询结果的行，而 `"headers"` 将含有表头。如果查询报错，则 `"ok"` 字段将为 `false`，而错误信息会在 `"message"` 58 | 字段中，同时 `"display"` 字段会包含格式化好的友好的错误提示。 59 | 60 | > Cozo 的设计，基于其在一个受信任的环境中运行，且其所有用户也是由受信任的这种假设。因此 Cozo 没有内置认证与复杂的安全机制。如果你需要远程访问 61 | > Cozo 服务，你必须自己设置防火墙、加密和代理等，用来保护服务器上资源的安全。 62 | > 63 | > 由于总是会有用户不小心将服务接口暴露于外网，Cozo 有一个补救措施：如果允许了非回传地址访问 Cozo，则必须在所有请求中以 HTTP 64 | > 文件头 `x-cozo-auth` 的形式附上访问令牌。访问令牌的内容在启动服务的终端中有提示。注意这仅仅是一个补救措施，并不是特别可靠的安全机制，是为了尽量防止一些不由于小心而造成严重后果的悲剧。 65 | > 66 | > 在有些客户端里，部分 API 加入 HTTP 文件头可能会比较困难或者根本不可能。这时也可以在查询参数 `auth` 中传入令牌。 67 | 68 | ## 所有 API 69 | 70 | * `POST /text-query`，见上。 71 | * `GET /export/{relations: String}`，导出指定表中的数据，其中 `relations` 是以逗号分割的表名。 72 | * `PUT /import`，向数据库导入数据。所导入的数据应以在正文中以 `application/json` MIME 类型传入，具体格式与 `/export` 73 | 返回值中的 `data` 字段相同。 74 | * `POST /backup`，备份数据库，需要传入 JSON 正文 `{"path": <路径>}`。 75 | * `POST /import-from-backup`，将备份中指定存储表中的数据插入当前数据库中同名存储表。需要传入 JSON 76 | 正文 `{"path": <路径>, "relations": <表名数组>}`. 77 | * `GET /`，用浏览器打开这个地址，然后打开浏览器的调试工具，就可以使用一个简陋的 JS 客户端。 78 | 79 | > 注意 `import` 与 `import-from-backup` 接口不会激活任何触发器。 80 | 81 | 82 | 以下为试验性的 API： 83 | 84 | * `GET(SSE) /changes/{relation: String}` 85 | 获取某个存储表的更新，基于 [SSE](https://developer.mozilla.org/zh-CN/docs/Web/API/Server-sent_events/Using_server-sent_events). 86 | 87 | ## 编译 88 | 89 | 编译 `cozo` 需要 [Rust 工具链](https://rustup.rs)。运行 90 | 91 | ```bash 92 | cargo build --release -p cozo-bin -F compact -F storage-rocksdb 93 | ``` 94 | -------------------------------------------------------------------------------- /cozo-bin/src/client.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023, The Cozo Project Authors. 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 5 | * If a copy of the MPL was not distributed with this file, 6 | * You can obtain one at https://mozilla.org/MPL/2.0/. 7 | */ 8 | 9 | -------------------------------------------------------------------------------- /cozo-bin/src/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Cozo database 6 | 7 | 8 |

Cozo API is running.

9 | 51 | 52 | -------------------------------------------------------------------------------- /cozo-bin/src/main.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022, The Cozo Project Authors. 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 5 | * If a copy of the MPL was not distributed with this file, 6 | * You can obtain one at https://mozilla.org/MPL/2.0/. 7 | */ 8 | 9 | extern crate core; 10 | 11 | use std::process::exit; 12 | 13 | use clap::{Parser, Subcommand}; 14 | use env_logger::Env; 15 | 16 | use crate::repl::{repl_main, ReplArgs}; 17 | use crate::server::{server_main, ServerArgs}; 18 | 19 | mod client; 20 | mod repl; 21 | mod server; 22 | 23 | #[derive(Parser)] 24 | #[command(author, version, about, long_about = None)] 25 | #[command(propagate_version = true)] 26 | struct AppArgs { 27 | #[command(subcommand)] 28 | command: Commands, 29 | } 30 | 31 | #[derive(Subcommand)] 32 | enum Commands { 33 | Server(ServerArgs), 34 | Repl(ReplArgs), 35 | } 36 | 37 | fn main() { 38 | match AppArgs::parse().command { 39 | Commands::Server(args) => { 40 | env_logger::Builder::from_env(Env::default().default_filter_or("info")).init(); 41 | tokio::runtime::Builder::new_multi_thread() 42 | .enable_all() 43 | .build() 44 | .unwrap() 45 | .block_on(server_main(args)) 46 | } 47 | Commands::Repl(args) => { 48 | if let Err(e) = repl_main(args) { 49 | eprintln!("{e}"); 50 | exit(-1); 51 | } 52 | } 53 | }; 54 | 55 | // if args.repl { 56 | 57 | // } else { 58 | 59 | // server_main(args, db) 60 | // } 61 | } 62 | 63 | // fn server_main(args: Server, db: DbInstance) { 64 | // 65 | // let addr = if Ipv6Addr::from_str(&args.bind).is_ok() { 66 | // format!("[{}]:{}", args.bind, args.port) 67 | // } else { 68 | // format!("{}:{}", args.bind, args.port) 69 | // }; 70 | // println!( 71 | // "Database ({} backend) web API running at http://{}", 72 | // args.engine, addr 73 | // ); 74 | // println!("The auth file is at {conf_path}"); 75 | // rouille::start_server(addr, move |request| { 76 | // let now = chrono::Utc::now().format("%Y-%m-%d %H:%M:%S%.6f"); 77 | // let log_ok = |req: &Request, _resp: &Response, elap: std::time::Duration| { 78 | // info!("{} {} {} {:?}", now, req.method(), req.raw_url(), elap); 79 | // }; 80 | // let log_err = |req: &Request, elap: std::time::Duration| { 81 | // error!( 82 | // "{} Handler panicked: {} {} {:?}", 83 | // now, 84 | // req.method(), 85 | // req.raw_url(), 86 | // elap 87 | // ); 88 | // }; 89 | // }); 90 | // } 91 | -------------------------------------------------------------------------------- /cozo-bin/src/security.txt: -------------------------------------------------------------------------------- 1 | 2 | ==================================================================================== 3 | !! SECURITY NOTICE, PLEASE READ !! 4 | ==================================================================================== 5 | You instructed Cozo to bind to a non-default address. 6 | 7 | Cozo is designed to be accessed by trusted clients in a trusted network. 8 | As a last defense against unauthorized access when everything else fails, 9 | any requests now require the HTTP request header `x-cozo-auth` to be set, 10 | or the query parameter `auth=` be set to the auth token. 11 | The auth token is found in a file indicated below. 12 | 13 | This is required even if the request comes from localhost. 14 | 15 | This is not a sufficient protection against attacks, and you must set up 16 | proper authentication schemes, encryptions, etc. by firewalls and/or proxies. 17 | ==================================================================================== 18 | -------------------------------------------------------------------------------- /cozo-core-examples/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "cozo-core-examples" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | cozo = { version = "0.7.6", path = "../cozo-core", default-features = false, features = [ 8 | "rayon", 9 | ] } 10 | -------------------------------------------------------------------------------- /cozo-core-examples/src/bin/run.rs: -------------------------------------------------------------------------------- 1 | use cozo::{DbInstance, ScriptMutability}; 2 | 3 | fn main() { 4 | let db = DbInstance::new("mem", "", Default::default()).unwrap(); 5 | let script = "?[a] := a in [1, 2, 3]"; 6 | let result = db 7 | .run_script(script, Default::default(), ScriptMutability::Immutable) 8 | .unwrap(); 9 | println!("{:?}", result); 10 | } 11 | -------------------------------------------------------------------------------- /cozo-core-examples/src/bin/run_ast.rs: -------------------------------------------------------------------------------- 1 | use std::collections::BTreeMap; 2 | 3 | use cozo::{ 4 | data::{ 5 | functions::current_validity, 6 | program::{InputAtom, InputInlineRule, InputInlineRulesOrFixed, InputProgram, Unification}, 7 | symb::PROG_ENTRY, 8 | }, 9 | parse::{CozoScript, ImperativeStmt, ImperativeStmtClause}, 10 | DataValue, DbInstance, Num, ScriptMutability, Symbol, 11 | }; 12 | 13 | fn main() { 14 | let db = DbInstance::new("mem", "", Default::default()).unwrap(); 15 | let sym_a = Symbol::new("a", Default::default()); 16 | let script = CozoScript::Imperative(vec![ImperativeStmt::Program { 17 | prog: ImperativeStmtClause { 18 | prog: InputProgram { 19 | prog: { 20 | let mut p = BTreeMap::new(); 21 | p.insert( 22 | Symbol::new(PROG_ENTRY, Default::default()), 23 | InputInlineRulesOrFixed::Rules { 24 | rules: vec![InputInlineRule { 25 | head: vec![sym_a.clone()], 26 | aggr: vec![None], 27 | body: vec![InputAtom::Unification { 28 | inner: Unification { 29 | binding: sym_a, 30 | expr: cozo::Expr::Const { 31 | val: DataValue::List(vec![ 32 | DataValue::Num(Num::Int(1)), 33 | DataValue::Num(Num::Int(2)), 34 | DataValue::Num(Num::Int(3)), 35 | ]), 36 | span: Default::default(), 37 | }, 38 | one_many_unif: true, 39 | span: Default::default(), 40 | }, 41 | }], 42 | span: Default::default(), 43 | }], 44 | }, 45 | ); 46 | p 47 | }, 48 | out_opts: Default::default(), 49 | disable_magic_rewrite: false, 50 | }, 51 | store_as: None, 52 | }, 53 | }]); 54 | let result = db 55 | .run_script_ast(script, current_validity(), ScriptMutability::Immutable) 56 | .unwrap(); 57 | println!("{:?}", result); 58 | } 59 | -------------------------------------------------------------------------------- /cozo-core-examples/src/bin/run_parse_ast.rs: -------------------------------------------------------------------------------- 1 | use cozo::{data::functions::current_validity, parse::parse_script, DbInstance, ScriptMutability}; 2 | 3 | fn main() { 4 | let db = DbInstance::new("mem", "", Default::default()).unwrap(); 5 | let script = "?[a] := a in [1, 2, 3]"; 6 | let cur_vld = current_validity(); 7 | let script_ast = 8 | parse_script(script, &Default::default(), &db.get_fixed_rules(), cur_vld).unwrap(); 9 | println!("AST: {:?}", script_ast); 10 | let result = db 11 | .run_script_ast(script_ast, cur_vld, ScriptMutability::Immutable) 12 | .unwrap(); 13 | println!("Result: {:?}", result); 14 | } 15 | -------------------------------------------------------------------------------- /cozo-core/README-zh.md: -------------------------------------------------------------------------------- 1 | # Cozo-core 2 | 3 | [![Crates.io](https://img.shields.io/crates/v/cozo)](https://crates.io/crates/cozo) 4 | 5 | Cozo 数据库核心部分的实现。 -------------------------------------------------------------------------------- /cozo-core/README.md: -------------------------------------------------------------------------------- 1 | # Cozo-core 2 | 3 | [![Crates.io](https://img.shields.io/crates/v/cozo)](https://crates.io/crates/cozo) 4 | 5 | This crate contains the implementation proper of CozoDB. -------------------------------------------------------------------------------- /cozo-core/benches/wiki_pagerank.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022, The Cozo Project Authors. 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 5 | * If a copy of the MPL was not distributed with this file, 6 | * You can obtain one at https://mozilla.org/MPL/2.0/. 7 | */ 8 | 9 | #![feature(test)] 10 | 11 | extern crate test; 12 | 13 | use std::collections::BTreeMap; 14 | use std::fs::File; 15 | use std::io::BufRead; 16 | use std::path::PathBuf; 17 | use std::time::Instant; 18 | use std::{env, io}; 19 | use test::Bencher; 20 | 21 | use lazy_static::{initialize, lazy_static}; 22 | 23 | use cozo::{DbInstance, NamedRows, DataValue}; 24 | 25 | lazy_static! { 26 | static ref TEST_DB: DbInstance = { 27 | let data_dir = PathBuf::from(env::var("COZO_BENCH_WIKI_DIR").unwrap()); 28 | 29 | let db = DbInstance::new("mem", "", "").unwrap(); 30 | let mut file_path = data_dir.clone(); 31 | file_path.push("wikipedia-articles.el"); 32 | 33 | // dbg!(&db_kind); 34 | // dbg!(&data_dir); 35 | // dbg!(&file_path); 36 | // dbg!(&data_size); 37 | // dbg!(&n_threads); 38 | 39 | db.run_script(":create article {fr: Int, to: Int}", 40 | Default::default(), 41 | ).unwrap(); 42 | 43 | let file = File::open(&file_path).unwrap(); 44 | let mut articles = vec![]; 45 | 46 | let import_time = Instant::now(); 47 | for line in io::BufReader::new(file).lines() { 48 | let line = line.unwrap(); 49 | if line.len() < 2 { 50 | continue 51 | } 52 | let mut splits = line.split_whitespace(); 53 | let fr = splits.next().unwrap(); 54 | let to = splits.next().unwrap(); 55 | articles.push(vec![DataValue::from(fr.parse::().unwrap()), DataValue::from(to.parse::().unwrap())]) 56 | } 57 | db.import_relations(BTreeMap::from([("article".to_string(), NamedRows { 58 | headers: vec![ 59 | "fr".to_string(), 60 | "to".to_string(), 61 | ], 62 | rows: articles, 63 | next: None, 64 | })])).unwrap(); 65 | dbg!(import_time.elapsed()); 66 | db 67 | }; 68 | } 69 | 70 | #[bench] 71 | fn wikipedia_pagerank(b: &mut Bencher) { 72 | initialize(&TEST_DB); 73 | b.iter(|| { 74 | TEST_DB 75 | .run_script("?[id, rank] <~ PageRank(*article[])", Default::default()) 76 | .unwrap() 77 | }); 78 | } 79 | 80 | #[bench] 81 | fn wikipedia_louvain(b: &mut Bencher) { 82 | initialize(&TEST_DB); 83 | b.iter(|| { 84 | let start = Instant::now(); 85 | TEST_DB 86 | .run_script( 87 | "?[grp, idx] <~ CommunityDetectionLouvain(*article[])", 88 | Default::default(), 89 | ) 90 | .unwrap(); 91 | dbg!(start.elapsed()); 92 | }) 93 | } 94 | -------------------------------------------------------------------------------- /cozo-core/src/data/json.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022, The Cozo Project Authors. 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 5 | * If a copy of the MPL was not distributed with this file, 6 | * You can obtain one at https://mozilla.org/MPL/2.0/. 7 | */ 8 | 9 | use base64::engine::general_purpose::STANDARD; 10 | use base64::Engine; 11 | use serde_json::json; 12 | pub(crate) use serde_json::Value as JsonValue; 13 | 14 | use crate::data::value::{DataValue, Num, Vector}; 15 | use crate::JsonData; 16 | 17 | impl From for DataValue { 18 | fn from(v: JsonValue) -> Self { 19 | match v { 20 | JsonValue::Null => DataValue::Null, 21 | JsonValue::Bool(b) => DataValue::Bool(b), 22 | JsonValue::Number(n) => match n.as_i64() { 23 | Some(i) => DataValue::from(i), 24 | None => match n.as_f64() { 25 | Some(f) => DataValue::from(f), 26 | None => DataValue::from(n.to_string()), 27 | }, 28 | }, 29 | JsonValue::String(s) => DataValue::from(s), 30 | JsonValue::Array(arr) => DataValue::List(arr.iter().map(DataValue::from).collect()), 31 | JsonValue::Object(d) => DataValue::Json(JsonData(JsonValue::Object(d))), 32 | } 33 | } 34 | } 35 | 36 | impl<'a> From<&'a JsonValue> for DataValue { 37 | fn from(v: &'a JsonValue) -> Self { 38 | match v { 39 | JsonValue::Null => DataValue::Null, 40 | JsonValue::Bool(b) => DataValue::Bool(*b), 41 | JsonValue::Number(n) => match n.as_i64() { 42 | Some(i) => DataValue::from(i), 43 | None => match n.as_f64() { 44 | Some(f) => DataValue::from(f), 45 | None => DataValue::from(n.to_string()), 46 | }, 47 | }, 48 | JsonValue::String(s) => DataValue::Str(s.into()), 49 | JsonValue::Array(arr) => DataValue::List(arr.iter().map(DataValue::from).collect()), 50 | JsonValue::Object(d) => DataValue::Json(JsonData(JsonValue::Object(d.clone()))), 51 | } 52 | } 53 | } 54 | 55 | impl From for JsonValue { 56 | fn from(v: DataValue) -> Self { 57 | match v { 58 | DataValue::Null => JsonValue::Null, 59 | DataValue::Bool(b) => JsonValue::Bool(b), 60 | DataValue::Num(Num::Int(i)) => JsonValue::Number(i.into()), 61 | DataValue::Num(Num::Float(f)) => { 62 | if f.is_finite() { 63 | json!(f) 64 | } else if f.is_nan() { 65 | json!(()) 66 | } else if f.is_infinite() { 67 | if f.is_sign_negative() { 68 | json!("NEGATIVE_INFINITY") 69 | } else { 70 | json!("INFINITY") 71 | } 72 | } else { 73 | unreachable!() 74 | } 75 | } 76 | DataValue::Str(t) => JsonValue::String(t.into()), 77 | DataValue::Bytes(bytes) => JsonValue::String(STANDARD.encode(bytes)), 78 | DataValue::List(l) => { 79 | JsonValue::Array(l.iter().map(|v| JsonValue::from(v.clone())).collect()) 80 | } 81 | DataValue::Bot => panic!("found bottom"), 82 | DataValue::Set(l) => { 83 | JsonValue::Array(l.iter().map(|v| JsonValue::from(v.clone())).collect()) 84 | } 85 | DataValue::Regex(r) => { 86 | json!(r.0.as_str()) 87 | } 88 | DataValue::Uuid(u) => { 89 | json!(u.0) 90 | } 91 | DataValue::Vec(arr) => match arr { 92 | Vector::F32(a) => json!(a.as_slice().unwrap()), 93 | Vector::F64(a) => json!(a.as_slice().unwrap()), 94 | }, 95 | DataValue::Validity(v) => { 96 | json!([v.timestamp.0, v.is_assert]) 97 | } 98 | DataValue::Json(j) => j.0, 99 | } 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /cozo-core/src/data/mod.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022, The Cozo Project Authors. 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 5 | * If a copy of the MPL was not distributed with this file, 6 | * You can obtain one at https://mozilla.org/MPL/2.0/. 7 | */ 8 | 9 | pub(crate) mod aggr; 10 | pub(crate) mod expr; 11 | pub mod functions; 12 | pub(crate) mod json; 13 | pub(crate) mod memcmp; 14 | pub mod program; 15 | pub(crate) mod relation; 16 | pub mod symb; 17 | pub(crate) mod tuple; 18 | pub(crate) mod value; 19 | 20 | #[cfg(test)] 21 | mod tests; 22 | -------------------------------------------------------------------------------- /cozo-core/src/data/symb.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022, The Cozo Project Authors. 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 5 | * If a copy of the MPL was not distributed with this file, 6 | * You can obtain one at https://mozilla.org/MPL/2.0/. 7 | */ 8 | 9 | use std::cmp::Ordering; 10 | use std::fmt::{Debug, Display, Formatter}; 11 | use std::hash::{Hash, Hasher}; 12 | use std::ops::Deref; 13 | 14 | use miette::{bail, Diagnostic, Result}; 15 | use serde_derive::{Deserialize, Serialize}; 16 | use smartstring::{LazyCompact, SmartString}; 17 | use thiserror::Error; 18 | 19 | use crate::parse::SourceSpan; 20 | 21 | /// Names with associated source span 22 | #[derive(Clone, Deserialize, Serialize)] 23 | pub struct Symbol { 24 | pub(crate) name: SmartString, 25 | #[serde(skip)] 26 | pub(crate) span: SourceSpan, 27 | } 28 | 29 | impl Deref for Symbol { 30 | type Target = str; 31 | 32 | fn deref(&self) -> &Self::Target { 33 | &self.name 34 | } 35 | } 36 | 37 | impl Hash for Symbol { 38 | fn hash(&self, state: &mut H) { 39 | self.name.hash(state) 40 | } 41 | } 42 | 43 | impl PartialEq for Symbol { 44 | fn eq(&self, other: &Self) -> bool { 45 | self.name == other.name 46 | } 47 | } 48 | 49 | impl Eq for Symbol {} 50 | 51 | impl PartialOrd for Symbol { 52 | fn partial_cmp(&self, other: &Self) -> Option { 53 | Some(self.cmp(other)) 54 | } 55 | } 56 | 57 | impl Ord for Symbol { 58 | fn cmp(&self, other: &Self) -> Ordering { 59 | self.name.cmp(&other.name) 60 | } 61 | } 62 | 63 | impl Display for Symbol { 64 | fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { 65 | write!(f, "{}", self.name) 66 | } 67 | } 68 | 69 | impl Debug for Symbol { 70 | fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { 71 | write!(f, "{}", self.name) 72 | } 73 | } 74 | 75 | impl Symbol { 76 | pub fn new(name: impl Into>, span: SourceSpan) -> Self { 77 | Self { 78 | name: name.into(), 79 | span, 80 | } 81 | } 82 | pub(crate) fn is_temp_store_name(&self) -> bool { 83 | self.name.starts_with('_') 84 | } 85 | pub(crate) fn is_prog_entry(&self) -> bool { 86 | self.name == "?" 87 | } 88 | pub(crate) fn is_ignored_symbol(&self) -> bool { 89 | self.name == "_" 90 | } 91 | pub(crate) fn is_generated_ignored_symbol(&self) -> bool { 92 | self.name.starts_with('~') 93 | } 94 | pub(crate) fn ensure_valid_field(&self) -> Result<()> { 95 | if self.name.contains('(') || self.name.contains(')') { 96 | #[derive(Debug, Error, Diagnostic)] 97 | #[error("The symbol {0} is not valid as a field")] 98 | #[diagnostic(code(parser::symbol_invalid_as_field))] 99 | struct SymbolInvalidAsField(String, #[label] SourceSpan); 100 | 101 | bail!(SymbolInvalidAsField(self.name.to_string(), self.span)) 102 | } 103 | Ok(()) 104 | } 105 | } 106 | 107 | pub const PROG_ENTRY: &str = "?"; 108 | -------------------------------------------------------------------------------- /cozo-core/src/data/tests/exprs.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022, The Cozo Project Authors. 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 5 | * If a copy of the MPL was not distributed with this file, 6 | * You can obtain one at https://mozilla.org/MPL/2.0/. 7 | */ 8 | 9 | use crate::{DataValue, DbInstance}; 10 | 11 | #[test] 12 | fn expression_eval() { 13 | let db = DbInstance::default(); 14 | 15 | let res = db 16 | .run_default( 17 | r#" 18 | ?[a] := a = if(2 + 3 > 1 * 99999, 190291021 + 14341234212 / 2121) 19 | "#, 20 | ) 21 | .unwrap(); 22 | assert_eq!(res.rows[0][0], DataValue::Null); 23 | 24 | let res = db 25 | .run_default( 26 | r#" 27 | ?[a] := a = if(2 + 3 > 1, true, false) 28 | "#, 29 | ) 30 | .unwrap(); 31 | assert_eq!(res.rows[0][0].get_bool().unwrap(), true); 32 | } 33 | -------------------------------------------------------------------------------- /cozo-core/src/data/tests/json.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022, The Cozo Project Authors. 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 5 | * If a copy of the MPL was not distributed with this file, 6 | * You can obtain one at https://mozilla.org/MPL/2.0/. 7 | * 8 | */ 9 | 10 | use serde_json::json; 11 | 12 | use crate::data::json::JsonValue; 13 | use crate::data::value::DataValue; 14 | 15 | #[test] 16 | fn bad_values() { 17 | println!("{}", json!(f64::INFINITY)); 18 | println!("{}", JsonValue::from(DataValue::from(f64::INFINITY))); 19 | println!("{}", JsonValue::from(DataValue::from(f64::NEG_INFINITY))); 20 | println!("{}", JsonValue::from(DataValue::from(f64::NAN))); 21 | } 22 | -------------------------------------------------------------------------------- /cozo-core/src/data/tests/mod.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022, The Cozo Project Authors. 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 5 | * If a copy of the MPL was not distributed with this file, 6 | * You can obtain one at https://mozilla.org/MPL/2.0/. 7 | */ 8 | 9 | mod aggrs; 10 | mod exprs; 11 | mod functions; 12 | mod json; 13 | mod memcmp; 14 | mod validity; 15 | mod values; 16 | -------------------------------------------------------------------------------- /cozo-core/src/data/tests/validity.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022, The Cozo Project Authors. 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 5 | * If a copy of the MPL was not distributed with this file, 6 | * You can obtain one at https://mozilla.org/MPL/2.0/. 7 | * 8 | */ 9 | 10 | use crate::data::value::DataValue; 11 | use crate::DbInstance; 12 | use serde_json::json; 13 | use std::env; 14 | 15 | #[test] 16 | fn test_validity() { 17 | let path = "_test_validity"; 18 | let _ = std::fs::remove_file(path); 19 | let _ = std::fs::remove_dir_all(path); 20 | let db_kind = env::var("COZO_TEST_DB_ENGINE").unwrap_or("mem".to_string()); 21 | println!("Using {} engine", db_kind); 22 | let db = DbInstance::default(); 23 | 24 | db.run_default(":create vld {a, v: Validity => d}").unwrap(); 25 | 26 | assert!(db 27 | .run_default( 28 | r#" 29 | ?[a, v, d] <- [[1, [9223372036854775807, true], null]] 30 | :put vld {a, v => d} 31 | "#, 32 | ) 33 | .is_err()); 34 | 35 | assert!(db 36 | .run_default( 37 | r#" 38 | ?[a, v, d] <- [[1, [-9223372036854775808, true], null]] 39 | :put vld {a, v => d} 40 | "#, 41 | ) 42 | .is_err()); 43 | 44 | db.run_default( 45 | r#" 46 | ?[a, v, d] <- [[1, [0, true], 0]] 47 | :put vld {a, v => d} 48 | "#, 49 | ) 50 | .unwrap(); 51 | 52 | let res = db 53 | .run_default( 54 | r#" 55 | ?[a, v, d] := *vld{a, v, d @ "NOW"} 56 | "#, 57 | ) 58 | .unwrap() 59 | .rows; 60 | assert_eq!(res.len(), 1); 61 | 62 | let res = db 63 | .run_default( 64 | r#" 65 | ?[a, v, d] := *vld{a, v, d} 66 | "#, 67 | ) 68 | .unwrap() 69 | .rows; 70 | assert_eq!(res.len(), 1); 71 | 72 | db.run_default( 73 | r#" 74 | ?[a, v, d] <- [[1, [1, false], 1]] 75 | :put vld {a, v => d} 76 | "#, 77 | ) 78 | .unwrap(); 79 | 80 | let res = db 81 | .run_default( 82 | r#" 83 | ?[a, v, d] := *vld{a, v, d @ "NOW"} 84 | "#, 85 | ) 86 | .unwrap() 87 | .rows; 88 | assert_eq!(res.len(), 0); 89 | 90 | let res = db 91 | .run_default( 92 | r#" 93 | ?[a, v, d] := *vld{a, v, d} 94 | "#, 95 | ) 96 | .unwrap() 97 | .rows; 98 | assert_eq!(res.len(), 2); 99 | 100 | db.run_default( 101 | r#" 102 | ?[a, v, d] <- [[1, "ASSERT", 2]] 103 | :put vld {a, v => d} 104 | "#, 105 | ) 106 | .unwrap(); 107 | 108 | let res = db 109 | .run_default( 110 | r#" 111 | ?[a, v, d] := *vld{a, v, d @ "NOW"} 112 | "#, 113 | ) 114 | .unwrap() 115 | .rows; 116 | assert_eq!(res.len(), 1); 117 | assert_eq!(res[0][2].get_int().unwrap(), 2); 118 | 119 | let res = db 120 | .run_default( 121 | r#" 122 | ?[a, v, d] := *vld{a, v, d} 123 | "#, 124 | ) 125 | .unwrap() 126 | .rows; 127 | assert_eq!(res.len(), 3); 128 | 129 | db.run_default( 130 | r#" 131 | ?[a, v, d] <- [[1, "RETRACT", 3]] 132 | :put vld {a, v => d} 133 | "#, 134 | ) 135 | .unwrap(); 136 | 137 | let res = db 138 | .run_default( 139 | r#" 140 | ?[a, v, d] := *vld{a, v, d @ "NOW"} 141 | "#, 142 | ) 143 | .unwrap() 144 | .rows; 145 | assert_eq!(res.len(), 0); 146 | 147 | let res = db 148 | .run_default( 149 | r#" 150 | ?[a, v, d] := *vld{a, v, d} 151 | "#, 152 | ) 153 | .unwrap() 154 | .rows; 155 | assert_eq!(res.len(), 4); 156 | db.run_default( 157 | r#" 158 | ?[a, v, d] <- [[1, [9223372036854775806, true], null]] 159 | :put vld {a, v => d} 160 | "#, 161 | ) 162 | .unwrap(); 163 | 164 | let res = db 165 | .run_default( 166 | r#" 167 | ?[a, v, d] := *vld{a, v, d @ "NOW"} 168 | "#, 169 | ) 170 | .unwrap() 171 | .rows; 172 | assert_eq!(res.len(), 0); 173 | 174 | let res = db 175 | .run_default( 176 | r#" 177 | ?[a, v, d] := *vld{a, v, d @ "END"} 178 | "#, 179 | ) 180 | .unwrap() 181 | .rows; 182 | assert_eq!(res.len(), 1); 183 | assert_eq!(res[0][2], DataValue::Null); 184 | 185 | let res = db 186 | .run_default( 187 | r#" 188 | ?[a, v, d] := *vld{a, v, d} 189 | "#, 190 | ) 191 | .unwrap() 192 | .rows; 193 | assert_eq!(res.len(), 5); 194 | 195 | println!("{}", json!(res)); 196 | } 197 | -------------------------------------------------------------------------------- /cozo-core/src/data/tests/values.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022, The Cozo Project Authors. 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 5 | * If a copy of the MPL was not distributed with this file, 6 | * You can obtain one at https://mozilla.org/MPL/2.0/. 7 | * 8 | */ 9 | 10 | use std::collections::{BTreeMap, HashMap}; 11 | use std::mem::size_of; 12 | 13 | use crate::data::symb::Symbol; 14 | use crate::data::value::DataValue; 15 | 16 | #[test] 17 | fn show_size() { 18 | dbg!(size_of::()); 19 | dbg!(size_of::()); 20 | dbg!(size_of::()); 21 | dbg!(size_of::>()); 22 | dbg!(size_of::>()); 23 | } 24 | 25 | #[test] 26 | fn utf8() { 27 | let c = char::from_u32(0x10FFFF).unwrap(); 28 | let mut s = String::new(); 29 | s.push(c); 30 | println!("{}", s); 31 | println!( 32 | "{:b} {:b} {:b} {:b}", 33 | s.as_bytes()[0], 34 | s.as_bytes()[1], 35 | s.as_bytes()[2], 36 | s.as_bytes()[3] 37 | ); 38 | dbg!(s); 39 | } 40 | 41 | #[test] 42 | fn display_datavalues() { 43 | println!("{}", DataValue::Null); 44 | println!("{}", DataValue::from(true)); 45 | println!("{}", DataValue::from(-1)); 46 | println!("{}", DataValue::from(-1121212121.331212121)); 47 | println!("{}", DataValue::from(f64::NAN)); 48 | println!("{}", DataValue::from(f64::NEG_INFINITY)); 49 | println!("{}", DataValue::from(vec![10, 20])); 50 | println!("{}", DataValue::from(vec!["hello", "你好"])); 51 | println!( 52 | "{}", 53 | DataValue::List(vec![ 54 | DataValue::from(false), 55 | DataValue::from(r###"abc"你"好'啊👌"###), 56 | DataValue::from(f64::NEG_INFINITY), 57 | ]) 58 | ); 59 | } 60 | -------------------------------------------------------------------------------- /cozo-core/src/data/tuple.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022, The Cozo Project Authors. 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 5 | * If a copy of the MPL was not distributed with this file, 6 | * You can obtain one at https://mozilla.org/MPL/2.0/. 7 | */ 8 | 9 | use crate::data::functions::TERMINAL_VALIDITY; 10 | use miette::Result; 11 | use std::cmp::Reverse; 12 | 13 | use crate::data::memcmp::MemCmpEncoder; 14 | use crate::data::value::{DataValue, Validity, ValidityTs}; 15 | use crate::runtime::relation::RelationId; 16 | 17 | pub type Tuple = Vec; 18 | 19 | pub(crate) type TupleIter<'a> = Box> + 'a>; 20 | 21 | pub(crate) trait TupleT { 22 | fn encode_as_key(&self, prefix: RelationId) -> Vec; 23 | } 24 | 25 | impl TupleT for T 26 | where 27 | T: AsRef<[DataValue]>, 28 | { 29 | fn encode_as_key(&self, prefix: RelationId) -> Vec { 30 | let len = self.as_ref().len(); 31 | let mut ret = Vec::with_capacity(4 + 4 * len + 10 * len); 32 | let prefix_bytes = prefix.0.to_be_bytes(); 33 | ret.extend(prefix_bytes); 34 | for val in self.as_ref().iter() { 35 | ret.encode_datavalue(val); 36 | } 37 | ret 38 | } 39 | } 40 | 41 | pub fn decode_tuple_from_key(key: &[u8], size_hint: usize) -> Tuple { 42 | let mut remaining = &key[ENCODED_KEY_MIN_LEN..]; 43 | let mut ret = Vec::with_capacity(size_hint); 44 | while !remaining.is_empty() { 45 | let (val, next) = DataValue::decode_from_key(remaining); 46 | ret.push(val); 47 | remaining = next; 48 | } 49 | ret 50 | } 51 | 52 | const DEFAULT_SIZE_HINT: usize = 16; 53 | 54 | /// Check if the tuple key passed in should be a valid return for a validity query. 55 | /// 56 | /// Returns two elements, the first element contains `Some(tuple)` if the key should be included 57 | /// in the return set and `None` otherwise, 58 | /// the second element gives the next binary key for the seek to be used as an inclusive 59 | /// lower bound. 60 | pub fn check_key_for_validity(key: &[u8], valid_at: ValidityTs, size_hint: Option) -> (Option, Vec) { 61 | let mut decoded = decode_tuple_from_key(key, size_hint.unwrap_or(DEFAULT_SIZE_HINT)); 62 | let rel_id = RelationId::raw_decode(key); 63 | let vld = match decoded.last().unwrap() { 64 | DataValue::Validity(vld) => vld, 65 | _ => unreachable!(), 66 | }; 67 | if vld.timestamp < valid_at { 68 | *decoded.last_mut().unwrap() = DataValue::Validity(Validity { 69 | timestamp: valid_at, 70 | is_assert: Reverse(true), 71 | }); 72 | let nxt_seek = decoded.encode_as_key(rel_id); 73 | (None, nxt_seek) 74 | } else if !vld.is_assert.0 { 75 | *decoded.last_mut().unwrap() = DataValue::Validity(TERMINAL_VALIDITY); 76 | let nxt_seek = decoded.encode_as_key(rel_id); 77 | (None, nxt_seek) 78 | } else { 79 | let ret = decoded.clone(); 80 | *decoded.last_mut().unwrap() = DataValue::Validity(TERMINAL_VALIDITY); 81 | let nxt_seek = decoded.encode_as_key(rel_id); 82 | (Some(ret), nxt_seek) 83 | } 84 | } 85 | 86 | pub(crate) const ENCODED_KEY_MIN_LEN: usize = 8; 87 | -------------------------------------------------------------------------------- /cozo-core/src/fixed_rule/algos/degree_centrality.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022, The Cozo Project Authors. 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 5 | * If a copy of the MPL was not distributed with this file, 6 | * You can obtain one at https://mozilla.org/MPL/2.0/. 7 | */ 8 | 9 | use std::collections::BTreeMap; 10 | 11 | use miette::Result; 12 | use smartstring::{LazyCompact, SmartString}; 13 | 14 | use crate::data::expr::Expr; 15 | use crate::data::symb::Symbol; 16 | use crate::data::value::DataValue; 17 | use crate::fixed_rule::{FixedRule, FixedRulePayload}; 18 | use crate::parse::SourceSpan; 19 | use crate::runtime::db::Poison; 20 | use crate::runtime::temp_store::RegularTempStore; 21 | 22 | pub(crate) struct DegreeCentrality; 23 | 24 | impl FixedRule for DegreeCentrality { 25 | fn run( 26 | &self, 27 | payload: FixedRulePayload<'_, '_>, 28 | out: &mut RegularTempStore, 29 | poison: Poison, 30 | ) -> Result<()> { 31 | let it = payload.get_input(0)?.ensure_min_len(2)?.iter()?; 32 | let mut counter: BTreeMap = BTreeMap::new(); 33 | for tuple in it { 34 | let tuple = tuple?; 35 | let from = tuple[0].clone(); 36 | let (from_total, from_out, _) = counter.entry(from).or_default(); 37 | *from_total += 1; 38 | *from_out += 1; 39 | 40 | let to = tuple[1].clone(); 41 | let (to_total, _, to_in) = counter.entry(to).or_default(); 42 | *to_total += 1; 43 | *to_in += 1; 44 | poison.check()?; 45 | } 46 | if let Ok(nodes) = payload.get_input(1) { 47 | for tuple in nodes.iter()? { 48 | let tuple = tuple?; 49 | let id = &tuple[0]; 50 | if !counter.contains_key(id) { 51 | counter.insert(id.clone(), (0, 0, 0)); 52 | } 53 | poison.check()?; 54 | } 55 | } 56 | for (k, (total_d, out_d, in_d)) in counter.into_iter() { 57 | let tuple = vec![ 58 | k, 59 | DataValue::from(total_d as i64), 60 | DataValue::from(out_d as i64), 61 | DataValue::from(in_d as i64), 62 | ]; 63 | out.put(tuple); 64 | } 65 | Ok(()) 66 | } 67 | 68 | fn arity( 69 | &self, 70 | _options: &BTreeMap, Expr>, 71 | _rule_head: &[Symbol], 72 | _span: SourceSpan, 73 | ) -> Result { 74 | Ok(4) 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /cozo-core/src/fixed_rule/algos/kruskal.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022, The Cozo Project Authors. 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 5 | * If a copy of the MPL was not distributed with this file, 6 | * You can obtain one at https://mozilla.org/MPL/2.0/. 7 | */ 8 | 9 | use graph::prelude::{DirectedCsrGraph, DirectedNeighborsWithValues, Graph}; 10 | use std::cmp::Reverse; 11 | use std::collections::BTreeMap; 12 | 13 | use itertools::Itertools; 14 | use miette::Result; 15 | use ordered_float::OrderedFloat; 16 | use priority_queue::PriorityQueue; 17 | use smartstring::{LazyCompact, SmartString}; 18 | 19 | use crate::data::expr::Expr; 20 | use crate::data::symb::Symbol; 21 | use crate::data::value::DataValue; 22 | use crate::fixed_rule::{FixedRule, FixedRulePayload}; 23 | use crate::parse::SourceSpan; 24 | use crate::runtime::db::Poison; 25 | use crate::runtime::temp_store::RegularTempStore; 26 | 27 | pub(crate) struct MinimumSpanningForestKruskal; 28 | 29 | impl FixedRule for MinimumSpanningForestKruskal { 30 | fn run( 31 | &self, 32 | payload: FixedRulePayload<'_, '_>, 33 | out: &mut RegularTempStore, 34 | poison: Poison, 35 | ) -> Result<()> { 36 | let edges = payload.get_input(0)?; 37 | let (graph, indices, _) = edges.as_directed_weighted_graph(true, true)?; 38 | if graph.node_count() == 0 { 39 | return Ok(()); 40 | } 41 | let msp = kruskal(&graph, poison)?; 42 | for (src, dst, cost) in msp { 43 | out.put(vec![ 44 | indices[src as usize].clone(), 45 | indices[dst as usize].clone(), 46 | DataValue::from(cost as f64), 47 | ]); 48 | } 49 | 50 | Ok(()) 51 | } 52 | 53 | fn arity( 54 | &self, 55 | _options: &BTreeMap, Expr>, 56 | _rule_head: &[Symbol], 57 | _span: SourceSpan, 58 | ) -> Result { 59 | Ok(3) 60 | } 61 | } 62 | 63 | fn kruskal(edges: &DirectedCsrGraph, poison: Poison) -> Result> { 64 | let mut pq = PriorityQueue::new(); 65 | let mut uf = UnionFind::new(edges.node_count()); 66 | let mut mst = Vec::with_capacity((edges.node_count() - 1) as usize); 67 | for from in 0..edges.node_count() { 68 | for target in edges.out_neighbors_with_values(from) { 69 | let to = target.target; 70 | let cost = target.value; 71 | pq.push((from, to), Reverse(OrderedFloat(cost))); 72 | } 73 | } 74 | while let Some(((from, to), Reverse(OrderedFloat(cost)))) = pq.pop() { 75 | if uf.connected(from, to) { 76 | continue; 77 | } 78 | uf.union(from, to); 79 | 80 | mst.push((from, to, cost)); 81 | if uf.szs[0] == edges.node_count() { 82 | break; 83 | } 84 | poison.check()?; 85 | } 86 | Ok(mst) 87 | } 88 | 89 | struct UnionFind { 90 | ids: Vec, 91 | szs: Vec, 92 | } 93 | 94 | impl UnionFind { 95 | fn new(n: u32) -> Self { 96 | Self { 97 | ids: (0..n).collect_vec(), 98 | szs: vec![1; n as usize], 99 | } 100 | } 101 | fn union(&mut self, p: u32, q: u32) { 102 | let root1 = self.find(p); 103 | let root2 = self.find(q); 104 | if root1 != root2 { 105 | if self.szs[root1 as usize] < self.szs[root2 as usize] { 106 | self.szs[root2 as usize] += self.szs[root1 as usize]; 107 | self.ids[root1 as usize] = root2; 108 | } else { 109 | self.szs[root1 as usize] += self.szs[root2 as usize]; 110 | self.ids[root2 as usize] = root1; 111 | } 112 | } 113 | } 114 | fn find(&mut self, mut p: u32) -> u32 { 115 | let mut root = p; 116 | while root != self.ids[root as usize] { 117 | root = self.ids[root as usize]; 118 | } 119 | while p != root { 120 | let next = self.ids[p as usize]; 121 | self.ids[p as usize] = root; 122 | p = next; 123 | } 124 | root 125 | } 126 | fn connected(&mut self, p: u32, q: u32) -> bool { 127 | self.find(p) == self.find(q) 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /cozo-core/src/fixed_rule/algos/label_propagation.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022, The Cozo Project Authors. 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 5 | * If a copy of the MPL was not distributed with this file, 6 | * You can obtain one at https://mozilla.org/MPL/2.0/. 7 | */ 8 | 9 | use std::collections::BTreeMap; 10 | 11 | use graph::prelude::{DirectedCsrGraph, DirectedNeighborsWithValues, Graph}; 12 | use itertools::Itertools; 13 | use miette::Result; 14 | use rand::prelude::*; 15 | use smartstring::{LazyCompact, SmartString}; 16 | 17 | use crate::data::expr::Expr; 18 | use crate::data::symb::Symbol; 19 | use crate::data::value::DataValue; 20 | use crate::fixed_rule::{FixedRule, FixedRulePayload}; 21 | use crate::parse::SourceSpan; 22 | use crate::runtime::db::Poison; 23 | use crate::runtime::temp_store::RegularTempStore; 24 | 25 | pub(crate) struct LabelPropagation; 26 | 27 | impl FixedRule for LabelPropagation { 28 | fn run( 29 | &self, 30 | payload: FixedRulePayload<'_, '_>, 31 | out: &mut RegularTempStore, 32 | poison: Poison, 33 | ) -> Result<()> { 34 | let edges = payload.get_input(0)?; 35 | let undirected = payload.bool_option("undirected", Some(false))?; 36 | let max_iter = payload.pos_integer_option("max_iter", Some(10))?; 37 | let (graph, indices, _inv_indices) = edges.as_directed_weighted_graph(undirected, true)?; 38 | let labels = label_propagation(&graph, max_iter, poison)?; 39 | for (idx, label) in labels.into_iter().enumerate() { 40 | let node = indices[idx].clone(); 41 | out.put(vec![DataValue::from(label as i64), node]); 42 | } 43 | Ok(()) 44 | } 45 | 46 | fn arity( 47 | &self, 48 | _options: &BTreeMap, Expr>, 49 | _rule_head: &[Symbol], 50 | _span: SourceSpan, 51 | ) -> Result { 52 | Ok(2) 53 | } 54 | } 55 | 56 | fn label_propagation( 57 | graph: &DirectedCsrGraph, 58 | max_iter: usize, 59 | poison: Poison, 60 | ) -> Result> { 61 | let n_nodes = graph.node_count(); 62 | let mut labels = (0..n_nodes).collect_vec(); 63 | let mut rng = thread_rng(); 64 | let mut iter_order = (0..n_nodes).collect_vec(); 65 | for _ in 0..max_iter { 66 | iter_order.shuffle(&mut rng); 67 | let mut changed = false; 68 | for node in &iter_order { 69 | let mut labels_for_node: BTreeMap = BTreeMap::new(); 70 | for edge in graph.out_neighbors_with_values(*node) { 71 | let label = labels[edge.target as usize]; 72 | *labels_for_node.entry(label).or_default() += edge.value; 73 | } 74 | if labels_for_node.is_empty() { 75 | continue; 76 | } 77 | let mut labels_by_score = labels_for_node.into_iter().collect_vec(); 78 | labels_by_score.sort_by(|a, b| a.1.total_cmp(&b.1).reverse()); 79 | let max_score = labels_by_score[0].1; 80 | let candidate_labels = labels_by_score 81 | .into_iter() 82 | .take_while(|(_, score)| *score == max_score) 83 | .map(|(l, _)| l) 84 | .collect_vec(); 85 | let new_label = *candidate_labels.choose(&mut rng).unwrap(); 86 | if new_label != labels[*node as usize] { 87 | changed = true; 88 | labels[*node as usize] = new_label; 89 | } 90 | poison.check()?; 91 | } 92 | if !changed { 93 | break; 94 | } 95 | } 96 | Ok(labels) 97 | } 98 | -------------------------------------------------------------------------------- /cozo-core/src/fixed_rule/algos/mod.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022, The Cozo Project Authors. 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 5 | * If a copy of the MPL was not distributed with this file, 6 | * You can obtain one at https://mozilla.org/MPL/2.0/. 7 | */ 8 | 9 | pub(crate) mod all_pairs_shortest_path; 10 | pub(crate) mod astar; 11 | pub(crate) mod bfs; 12 | pub(crate) mod degree_centrality; 13 | pub(crate) mod dfs; 14 | pub(crate) mod kruskal; 15 | pub(crate) mod label_propagation; 16 | pub(crate) mod louvain; 17 | pub(crate) mod pagerank; 18 | pub(crate) mod prim; 19 | pub(crate) mod random_walk; 20 | pub(crate) mod shortest_path_bfs; 21 | pub(crate) mod shortest_path_dijkstra; 22 | pub(crate) mod strongly_connected_components; 23 | pub(crate) mod top_sort; 24 | pub(crate) mod triangles; 25 | pub(crate) mod yen; 26 | 27 | pub(crate) use all_pairs_shortest_path::{BetweennessCentrality, ClosenessCentrality}; 28 | pub(crate) use astar::ShortestPathAStar; 29 | pub(crate) use bfs::Bfs; 30 | pub(crate) use degree_centrality::DegreeCentrality; 31 | pub(crate) use dfs::Dfs; 32 | pub(crate) use kruskal::MinimumSpanningForestKruskal; 33 | pub(crate) use label_propagation::LabelPropagation; 34 | pub(crate) use louvain::CommunityDetectionLouvain; 35 | pub(crate) use pagerank::PageRank; 36 | pub(crate) use prim::MinimumSpanningTreePrim; 37 | pub(crate) use random_walk::RandomWalk; 38 | pub(crate) use shortest_path_bfs::ShortestPathBFS; 39 | pub(crate) use shortest_path_dijkstra::ShortestPathDijkstra; 40 | pub(crate) use strongly_connected_components::StronglyConnectedComponent; 41 | pub(crate) use top_sort::TopSort; 42 | pub(crate) use triangles::ClusteringCoefficients; 43 | pub(crate) use yen::KShortestPathYen; 44 | -------------------------------------------------------------------------------- /cozo-core/src/fixed_rule/algos/pagerank.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022, The Cozo Project Authors. 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 5 | * If a copy of the MPL was not distributed with this file, 6 | * You can obtain one at https://mozilla.org/MPL/2.0/. 7 | */ 8 | 9 | use std::collections::BTreeMap; 10 | 11 | #[cfg(not(feature = "rayon"))] 12 | use approx::AbsDiffEq; 13 | use graph::prelude::{page_rank, PageRankConfig}; 14 | use miette::Result; 15 | use smartstring::{LazyCompact, SmartString}; 16 | 17 | use crate::data::expr::Expr; 18 | use crate::data::symb::Symbol; 19 | use crate::data::value::DataValue; 20 | use crate::fixed_rule::{FixedRule, FixedRulePayload}; 21 | use crate::parse::SourceSpan; 22 | use crate::runtime::db::Poison; 23 | use crate::runtime::temp_store::RegularTempStore; 24 | 25 | pub(crate) struct PageRank; 26 | 27 | impl FixedRule for PageRank { 28 | #[allow(unused_variables)] 29 | fn run( 30 | &self, 31 | payload: FixedRulePayload<'_, '_>, 32 | out: &mut RegularTempStore, 33 | poison: Poison, 34 | ) -> Result<()> { 35 | let edges = payload.get_input(0)?; 36 | let undirected = payload.bool_option("undirected", Some(false))?; 37 | let theta = payload.unit_interval_option("theta", Some(0.85))? as f32; 38 | let epsilon = payload.unit_interval_option("epsilon", Some(0.0001))? as f32; 39 | let iterations = payload.pos_integer_option("iterations", Some(10))?; 40 | 41 | let (graph, indices, _) = edges.as_directed_graph(undirected)?; 42 | 43 | if indices.is_empty() { 44 | return Ok(()); 45 | } 46 | 47 | let (ranks, _n_run, _) = page_rank( 48 | &graph, 49 | PageRankConfig::new(iterations, epsilon as f64, theta), 50 | ); 51 | 52 | for (idx, score) in ranks.iter().enumerate() { 53 | out.put(vec![indices[idx].clone(), DataValue::from(*score as f64)]); 54 | } 55 | Ok(()) 56 | } 57 | 58 | fn arity( 59 | &self, 60 | _options: &BTreeMap, Expr>, 61 | _rule_head: &[Symbol], 62 | _span: SourceSpan, 63 | ) -> Result { 64 | Ok(2) 65 | } 66 | } 67 | 68 | #[cfg(not(feature = "rayon"))] 69 | fn pagerank( 70 | edges: &[Vec], 71 | theta: f32, 72 | epsilon: f32, 73 | iterations: usize, 74 | poison: Poison, 75 | ) -> Result> { 76 | let init_val = (1. - theta) / edges.len() as f32; 77 | let mut g_mat = OMatrix::::repeat(edges.len(), edges.len(), init_val); 78 | let n = edges.len(); 79 | let empty_score = theta / n as f32; 80 | for (node, to_nodes) in edges.iter().enumerate() { 81 | let l = to_nodes.len(); 82 | if l == 0 { 83 | for to_node in 0..n { 84 | g_mat[(node, to_node)] = empty_score; 85 | } 86 | } else { 87 | let score = theta / n as f32; 88 | for to_node in to_nodes { 89 | g_mat[(node, *to_node)] = score; 90 | } 91 | } 92 | } 93 | let mut pi_vec = OMatrix::::repeat(edges.len(), 1.); 94 | let scale_target = (n as f32).sqrt(); 95 | let mut last_pi_vec = pi_vec.clone(); 96 | for _ in 0..iterations { 97 | std::mem::swap(&mut pi_vec, &mut last_pi_vec); 98 | pi_vec = g_mat.tr_mul(&last_pi_vec); 99 | pi_vec.normalize_mut(); 100 | let f = pi_vec.norm() / scale_target; 101 | pi_vec.unscale_mut(f); 102 | 103 | if pi_vec.abs_diff_eq(&last_pi_vec, epsilon) { 104 | break; 105 | } 106 | poison.check()?; 107 | } 108 | Ok(pi_vec) 109 | } 110 | -------------------------------------------------------------------------------- /cozo-core/src/fixed_rule/algos/prim.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022, The Cozo Project Authors. 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 5 | * If a copy of the MPL was not distributed with this file, 6 | * You can obtain one at https://mozilla.org/MPL/2.0/. 7 | */ 8 | 9 | use graph::prelude::{DirectedCsrGraph, DirectedNeighborsWithValues, Graph}; 10 | use std::cmp::Reverse; 11 | use std::collections::BTreeMap; 12 | 13 | use miette::Diagnostic; 14 | use miette::Result; 15 | use ordered_float::OrderedFloat; 16 | use priority_queue::PriorityQueue; 17 | use smartstring::{LazyCompact, SmartString}; 18 | use thiserror::Error; 19 | 20 | use crate::data::expr::Expr; 21 | use crate::data::symb::Symbol; 22 | use crate::data::value::DataValue; 23 | use crate::fixed_rule::{FixedRule, FixedRulePayload}; 24 | use crate::parse::SourceSpan; 25 | use crate::runtime::db::Poison; 26 | use crate::runtime::temp_store::RegularTempStore; 27 | 28 | pub(crate) struct MinimumSpanningTreePrim; 29 | 30 | impl FixedRule for MinimumSpanningTreePrim { 31 | fn run( 32 | &self, 33 | payload: FixedRulePayload<'_, '_>, 34 | out: &mut RegularTempStore, 35 | poison: Poison, 36 | ) -> Result<()> { 37 | let edges = payload.get_input(0)?; 38 | let (graph, indices, inv_indices) = edges.as_directed_weighted_graph(true, true)?; 39 | if graph.node_count() == 0 { 40 | return Ok(()); 41 | } 42 | let starting = match payload.get_input(1) { 43 | Err(_) => 0, 44 | Ok(rel) => { 45 | let tuple = rel.iter()?.next().ok_or_else(|| { 46 | #[derive(Debug, Error, Diagnostic)] 47 | #[error("The provided starting nodes relation is empty")] 48 | #[diagnostic(code(algo::empty_starting))] 49 | struct EmptyStarting(#[label] SourceSpan); 50 | 51 | EmptyStarting(rel.span()) 52 | })??; 53 | let dv = &tuple[0]; 54 | *inv_indices.get(dv).ok_or_else(|| { 55 | #[derive(Debug, Error, Diagnostic)] 56 | #[error("The requested starting node {0:?} is not found")] 57 | #[diagnostic(code(algo::starting_node_not_found))] 58 | struct StartingNodeNotFound(DataValue, #[label] SourceSpan); 59 | 60 | StartingNodeNotFound(dv.clone(), rel.span()) 61 | })? 62 | } 63 | }; 64 | let msp = prim(&graph, starting, poison)?; 65 | for (src, dst, cost) in msp { 66 | out.put(vec![ 67 | indices[src as usize].clone(), 68 | indices[dst as usize].clone(), 69 | DataValue::from(cost as f64), 70 | ]); 71 | } 72 | Ok(()) 73 | } 74 | 75 | fn arity( 76 | &self, 77 | _options: &BTreeMap, Expr>, 78 | _rule_head: &[Symbol], 79 | _span: SourceSpan, 80 | ) -> Result { 81 | Ok(3) 82 | } 83 | } 84 | 85 | fn prim( 86 | graph: &DirectedCsrGraph, 87 | starting: u32, 88 | poison: Poison, 89 | ) -> Result> { 90 | let mut visited = vec![false; graph.node_count() as usize]; 91 | let mut mst_edges = Vec::with_capacity((graph.node_count() - 1) as usize); 92 | let mut pq = PriorityQueue::new(); 93 | 94 | let mut relax_edges_at_node = |node: u32, pq: &mut PriorityQueue<_, _>| { 95 | visited[node as usize] = true; 96 | for target in graph.out_neighbors_with_values(node) { 97 | let to_node = target.target; 98 | let cost = target.value; 99 | if visited[to_node as usize] { 100 | continue; 101 | } 102 | pq.push_increase(to_node, (Reverse(OrderedFloat(cost)), node)); 103 | } 104 | }; 105 | 106 | relax_edges_at_node(starting, &mut pq); 107 | 108 | while let Some((to_node, (Reverse(OrderedFloat(cost)), from_node))) = pq.pop() { 109 | if mst_edges.len() == (graph.node_count() - 1) as usize { 110 | break; 111 | } 112 | mst_edges.push((from_node, to_node, cost)); 113 | relax_edges_at_node(to_node, &mut pq); 114 | poison.check()?; 115 | } 116 | 117 | Ok(mst_edges) 118 | } 119 | -------------------------------------------------------------------------------- /cozo-core/src/fixed_rule/algos/top_sort.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022, The Cozo Project Authors. 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 5 | * If a copy of the MPL was not distributed with this file, 6 | * You can obtain one at https://mozilla.org/MPL/2.0/. 7 | */ 8 | 9 | use graph::prelude::{DirectedCsrGraph, DirectedNeighbors, Graph}; 10 | use std::collections::BTreeMap; 11 | 12 | use miette::Result; 13 | use smartstring::{LazyCompact, SmartString}; 14 | 15 | use crate::data::expr::Expr; 16 | use crate::data::symb::Symbol; 17 | use crate::data::value::DataValue; 18 | use crate::fixed_rule::{FixedRule, FixedRulePayload}; 19 | use crate::parse::SourceSpan; 20 | use crate::runtime::db::Poison; 21 | use crate::runtime::temp_store::RegularTempStore; 22 | 23 | pub(crate) struct TopSort; 24 | 25 | impl FixedRule for TopSort { 26 | fn run( 27 | &self, 28 | payload: FixedRulePayload<'_, '_>, 29 | out: &mut RegularTempStore, 30 | poison: Poison, 31 | ) -> Result<()> { 32 | let edges = payload.get_input(0)?; 33 | 34 | let (graph, indices, _) = edges.as_directed_graph(false)?; 35 | 36 | let sorted = kahn_g(&graph, poison)?; 37 | 38 | for (idx, val_id) in sorted.iter().enumerate() { 39 | let val = indices.get(*val_id as usize).unwrap(); 40 | let tuple = vec![DataValue::from(idx as i64), val.clone()]; 41 | out.put(tuple); 42 | } 43 | 44 | Ok(()) 45 | } 46 | 47 | fn arity( 48 | &self, 49 | _options: &BTreeMap, Expr>, 50 | _rule_head: &[Symbol], 51 | _span: SourceSpan, 52 | ) -> Result { 53 | Ok(2) 54 | } 55 | } 56 | 57 | pub(crate) fn kahn_g(graph: &DirectedCsrGraph, poison: Poison) -> Result> { 58 | let graph_size = graph.node_count(); 59 | let mut in_degree = vec![0; graph_size as usize]; 60 | for tos in 0..graph_size { 61 | for to in graph.out_neighbors(tos) { 62 | in_degree[*to as usize] += 1; 63 | } 64 | } 65 | let mut sorted = Vec::with_capacity(graph_size as usize); 66 | let mut pending = vec![]; 67 | 68 | for (node, degree) in in_degree.iter().enumerate() { 69 | if *degree == 0 { 70 | pending.push(node as u32); 71 | } 72 | } 73 | 74 | while let Some(removed) = pending.pop() { 75 | sorted.push(removed); 76 | for nxt in graph.out_neighbors(removed) { 77 | in_degree[*nxt as usize] -= 1; 78 | if in_degree[*nxt as usize] == 0 { 79 | pending.push(*nxt); 80 | } 81 | } 82 | poison.check()?; 83 | } 84 | 85 | Ok(sorted) 86 | } 87 | -------------------------------------------------------------------------------- /cozo-core/src/fixed_rule/algos/triangles.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022, The Cozo Project Authors. 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 5 | * If a copy of the MPL was not distributed with this file, 6 | * You can obtain one at https://mozilla.org/MPL/2.0/. 7 | */ 8 | 9 | use std::collections::BTreeMap; 10 | 11 | use graph::prelude::{DirectedCsrGraph, DirectedNeighbors, Graph}; 12 | use itertools::Itertools; 13 | use miette::Result; 14 | #[cfg(feature = "rayon")] 15 | use rayon::prelude::*; 16 | use smartstring::{LazyCompact, SmartString}; 17 | 18 | use crate::data::expr::Expr; 19 | use crate::data::symb::Symbol; 20 | use crate::data::value::DataValue; 21 | use crate::fixed_rule::{FixedRule, FixedRulePayload}; 22 | use crate::parse::SourceSpan; 23 | use crate::runtime::db::Poison; 24 | use crate::runtime::temp_store::RegularTempStore; 25 | 26 | pub(crate) struct ClusteringCoefficients; 27 | 28 | impl FixedRule for ClusteringCoefficients { 29 | fn run( 30 | &self, 31 | payload: FixedRulePayload<'_, '_>, 32 | out: &mut RegularTempStore, 33 | poison: Poison, 34 | ) -> Result<()> { 35 | let edges = payload.get_input(0)?; 36 | let (graph, indices, _) = edges.as_directed_graph(true)?; 37 | let coefficients = clustering_coefficients(&graph, poison)?; 38 | for (idx, (cc, n_triangles, degree)) in coefficients.into_iter().enumerate() { 39 | out.put(vec![ 40 | indices[idx].clone(), 41 | DataValue::from(cc), 42 | DataValue::from(n_triangles as i64), 43 | DataValue::from(degree as i64), 44 | ]); 45 | } 46 | 47 | Ok(()) 48 | } 49 | 50 | fn arity( 51 | &self, 52 | _options: &BTreeMap, Expr>, 53 | _rule_head: &[Symbol], 54 | _span: SourceSpan, 55 | ) -> Result { 56 | Ok(4) 57 | } 58 | } 59 | 60 | fn clustering_coefficients( 61 | graph: &DirectedCsrGraph, 62 | poison: Poison, 63 | ) -> Result> { 64 | let node_size = graph.node_count(); 65 | 66 | (0..node_size) 67 | .into_par_iter() 68 | .map(|node_idx| -> Result<(f64, usize, usize)> { 69 | let edges = graph.out_neighbors(node_idx).collect_vec(); 70 | let degree = edges.len(); 71 | if degree < 2 { 72 | Ok((0., 0, degree)) 73 | } else { 74 | let n_triangles = edges 75 | .iter() 76 | .map(|e_src| { 77 | edges 78 | .iter() 79 | .filter(|e_dst| { 80 | if e_src <= e_dst { 81 | return false; 82 | } 83 | for nb in graph.out_neighbors(**e_src) { 84 | if nb == **e_dst { 85 | return true; 86 | } 87 | } 88 | false 89 | }) 90 | .count() 91 | }) 92 | .sum(); 93 | let cc = 2. * n_triangles as f64 / ((degree as f64) * ((degree as f64) - 1.)); 94 | poison.check()?; 95 | Ok((cc, n_triangles, degree)) 96 | } 97 | }) 98 | .collect::>() 99 | } 100 | -------------------------------------------------------------------------------- /cozo-core/src/fixed_rule/utilities/mod.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022, The Cozo Project Authors. 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 5 | * If a copy of the MPL was not distributed with this file, 6 | * You can obtain one at https://mozilla.org/MPL/2.0/. 7 | */ 8 | 9 | pub(crate) mod constant; 10 | pub(crate) mod csv; 11 | pub(crate) mod jlines; 12 | pub(crate) mod reorder_sort; 13 | 14 | pub(crate) use self::csv::CsvReader; 15 | pub(crate) use constant::Constant; 16 | pub(crate) use jlines::JsonReader; 17 | pub(crate) use reorder_sort::ReorderSort; 18 | -------------------------------------------------------------------------------- /cozo-core/src/fts/README.md: -------------------------------------------------------------------------------- 1 | 2 | Stop words: 3 | 4 | ``` 5 | https://raw.githubusercontent.com/stopwords-iso/stopwords-iso/master/python/stopwordsiso/stopwords-iso.json 6 | ``` -------------------------------------------------------------------------------- /cozo-core/src/fts/cangjie/mod.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Code under this module is adapted from the Cang-jie project 3 | * https://github.com/DCjanus/cang-jie 4 | * All code here are licensed under the MIT license, as in the original project. 5 | */ 6 | 7 | pub(crate) mod options; 8 | pub(crate) mod stream; 9 | pub(crate) mod tokenizer; 10 | -------------------------------------------------------------------------------- /cozo-core/src/fts/cangjie/options.rs: -------------------------------------------------------------------------------- 1 | /// Tokenizer Option 2 | #[derive(Debug, Clone)] 3 | pub(crate) enum TokenizerOption { 4 | /// Cut the input text, return all possible words 5 | All, 6 | /// Cut the input text 7 | Default { 8 | /// `hmm`: enable HMM or not 9 | hmm: bool, 10 | }, 11 | 12 | /// Cut the input text in search mode 13 | ForSearch { 14 | /// `hmm`: enable HMM or not 15 | hmm: bool, 16 | }, 17 | /// Cut the input text into UTF-8 characters 18 | Unicode, 19 | } 20 | -------------------------------------------------------------------------------- /cozo-core/src/fts/cangjie/stream.rs: -------------------------------------------------------------------------------- 1 | use crate::fts::tokenizer::Token; 2 | 3 | #[derive(Debug)] 4 | pub(crate) struct CangjieTokenStream<'a> { 5 | result: Vec<&'a str>, 6 | // Begin with 1 7 | index: usize, 8 | offset_from: usize, 9 | token: Token, 10 | } 11 | 12 | impl<'a> CangjieTokenStream<'a> { 13 | pub(crate) fn new(result: Vec<&'a str>) -> Self { 14 | CangjieTokenStream { 15 | result, 16 | index: 0, 17 | offset_from: 0, 18 | token: Token::default(), 19 | } 20 | } 21 | } 22 | 23 | impl<'a> crate::fts::tokenizer::TokenStream for CangjieTokenStream<'a> { 24 | fn advance(&mut self) -> bool { 25 | if self.index < self.result.len() { 26 | let current_word = self.result[self.index]; 27 | let offset_to = self.offset_from + current_word.len(); 28 | 29 | self.token = Token { 30 | offset_from: self.offset_from, 31 | offset_to, 32 | position: self.index, 33 | text: current_word.to_string(), 34 | position_length: self.result.len(), 35 | }; 36 | 37 | self.index += 1; 38 | self.offset_from = offset_to; 39 | true 40 | } else { 41 | false 42 | } 43 | } 44 | 45 | fn token(&self) -> &crate::fts::tokenizer::Token { 46 | &self.token 47 | } 48 | 49 | fn token_mut(&mut self) -> &mut crate::fts::tokenizer::Token { 50 | &mut self.token 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /cozo-core/src/fts/cangjie/tokenizer.rs: -------------------------------------------------------------------------------- 1 | use super::{options::TokenizerOption, stream::CangjieTokenStream}; 2 | use jieba_rs::Jieba; 3 | use log::trace; 4 | use std::sync::Arc; 5 | use crate::fts::tokenizer::BoxTokenStream; 6 | 7 | #[derive(Clone, Debug)] 8 | pub(crate) struct CangJieTokenizer { 9 | /// Separation algorithm provider 10 | pub(crate) worker: Arc, 11 | /// Separation config 12 | pub(crate) option: TokenizerOption, 13 | } 14 | 15 | impl Default for CangJieTokenizer { 16 | fn default() -> Self { 17 | CangJieTokenizer { 18 | worker: Arc::new(Jieba::empty()), 19 | option: TokenizerOption::Default { hmm: false }, 20 | } 21 | } 22 | } 23 | 24 | impl crate::fts::tokenizer::Tokenizer for CangJieTokenizer { 25 | /// Cut text into tokens 26 | fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> { 27 | let result = match self.option { 28 | TokenizerOption::All => self.worker.cut_all(text), 29 | TokenizerOption::Default { hmm: use_hmm } => self.worker.cut(text, use_hmm), 30 | TokenizerOption::ForSearch { hmm: use_hmm } => { 31 | self.worker.cut_for_search(text, use_hmm) 32 | } 33 | TokenizerOption::Unicode => { 34 | text.chars() 35 | .fold((0usize, vec![]), |(offset, mut result), the_char| { 36 | result.push(&text[offset..offset + the_char.len_utf8()]); 37 | (offset + the_char.len_utf8(), result) 38 | }) 39 | .1 40 | } 41 | }; 42 | trace!("{:?}->{:?}", text, result); 43 | BoxTokenStream::from(CangjieTokenStream::new(result)) 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /cozo-core/src/fts/tokenizer/alphanum_only.rs: -------------------------------------------------------------------------------- 1 | // # Example 2 | // ```rust 3 | // use tantivy::tokenizer::*; 4 | // 5 | // let tokenizer = TextAnalyzer::from(RawTokenizer) 6 | // .filter(AlphaNumOnlyFilter); 7 | // 8 | // let mut stream = tokenizer.token_stream("hello there"); 9 | // // is none because the raw filter emits one token that 10 | // // contains a space 11 | // assert!(stream.next().is_none()); 12 | // 13 | // let tokenizer = TextAnalyzer::from(SimpleTokenizer) 14 | // .filter(AlphaNumOnlyFilter); 15 | // 16 | // let mut stream = tokenizer.token_stream("hello there 💣"); 17 | // assert!(stream.next().is_some()); 18 | // assert!(stream.next().is_some()); 19 | // // the "emoji" is dropped because its not an alphanum 20 | // assert!(stream.next().is_none()); 21 | // ``` 22 | use super::{BoxTokenStream, Token, TokenFilter, TokenStream}; 23 | 24 | /// `TokenFilter` that removes all tokens that contain non 25 | /// ascii alphanumeric characters. 26 | #[derive(Clone)] 27 | pub(crate) struct AlphaNumOnlyFilter; 28 | 29 | pub(crate) struct AlphaNumOnlyFilterStream<'a> { 30 | tail: BoxTokenStream<'a>, 31 | } 32 | 33 | impl<'a> AlphaNumOnlyFilterStream<'a> { 34 | fn predicate(&self, token: &Token) -> bool { 35 | token.text.chars().all(|c| c.is_ascii_alphanumeric()) 36 | } 37 | } 38 | 39 | impl TokenFilter for AlphaNumOnlyFilter { 40 | fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> { 41 | BoxTokenStream::from(AlphaNumOnlyFilterStream { tail: token_stream }) 42 | } 43 | } 44 | 45 | impl<'a> TokenStream for AlphaNumOnlyFilterStream<'a> { 46 | fn advance(&mut self) -> bool { 47 | while self.tail.advance() { 48 | if self.predicate(self.tail.token()) { 49 | return true; 50 | } 51 | } 52 | 53 | false 54 | } 55 | 56 | fn token(&self) -> &Token { 57 | self.tail.token() 58 | } 59 | 60 | fn token_mut(&mut self) -> &mut Token { 61 | self.tail.token_mut() 62 | } 63 | } 64 | 65 | #[cfg(test)] 66 | mod tests { 67 | use crate::fts::tokenizer::tests::assert_token; 68 | use crate::fts::tokenizer::{AlphaNumOnlyFilter, SimpleTokenizer, TextAnalyzer, Token}; 69 | 70 | #[test] 71 | fn test_alphanum_only() { 72 | let tokens = token_stream_helper("I am a cat. 我輩は猫である。(1906)"); 73 | assert_eq!(tokens.len(), 5); 74 | assert_token(&tokens[0], 0, "I", 0, 1); 75 | assert_token(&tokens[1], 1, "am", 2, 4); 76 | assert_token(&tokens[2], 2, "a", 5, 6); 77 | assert_token(&tokens[3], 3, "cat", 7, 10); 78 | assert_token(&tokens[4], 5, "1906", 37, 41); 79 | } 80 | 81 | fn token_stream_helper(text: &str) -> Vec { 82 | let a = TextAnalyzer::from(SimpleTokenizer).filter(AlphaNumOnlyFilter); 83 | let mut token_stream = a.token_stream(text); 84 | let mut tokens: Vec = vec![]; 85 | let mut add_token = |token: &Token| { 86 | tokens.push(token.clone()); 87 | }; 88 | token_stream.process(&mut add_token); 89 | tokens 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /cozo-core/src/fts/tokenizer/empty_tokenizer.rs: -------------------------------------------------------------------------------- 1 | use crate::fts::tokenizer::{BoxTokenStream, Token, TokenStream, Tokenizer}; 2 | 3 | #[derive(Clone)] 4 | pub(crate) struct EmptyTokenizer; 5 | 6 | impl Tokenizer for EmptyTokenizer { 7 | fn token_stream<'a>(&self, _text: &'a str) -> BoxTokenStream<'a> { 8 | EmptyTokenStream::default().into() 9 | } 10 | } 11 | 12 | #[derive(Default)] 13 | struct EmptyTokenStream { 14 | token: Token, 15 | } 16 | 17 | impl TokenStream for EmptyTokenStream { 18 | fn advance(&mut self) -> bool { 19 | false 20 | } 21 | 22 | fn token(&self) -> &super::Token { 23 | &self.token 24 | } 25 | 26 | fn token_mut(&mut self) -> &mut super::Token { 27 | &mut self.token 28 | } 29 | } 30 | 31 | #[cfg(test)] 32 | mod tests { 33 | use crate::fts::tokenizer::Tokenizer; 34 | 35 | #[test] 36 | fn test_empty_tokenizer() { 37 | let tokenizer = super::EmptyTokenizer; 38 | let mut empty = tokenizer.token_stream("whatever string"); 39 | assert!(!empty.advance()); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /cozo-core/src/fts/tokenizer/lower_caser.rs: -------------------------------------------------------------------------------- 1 | use std::mem; 2 | 3 | use super::{Token, TokenFilter, TokenStream}; 4 | use crate::fts::tokenizer::BoxTokenStream; 5 | 6 | impl TokenFilter for LowerCaser { 7 | fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> { 8 | BoxTokenStream::from(LowerCaserTokenStream { 9 | tail: token_stream, 10 | buffer: String::with_capacity(100), 11 | }) 12 | } 13 | } 14 | 15 | /// Token filter that lowercase terms. 16 | #[derive(Clone)] 17 | pub(crate) struct LowerCaser; 18 | 19 | pub(crate) struct LowerCaserTokenStream<'a> { 20 | buffer: String, 21 | tail: BoxTokenStream<'a>, 22 | } 23 | 24 | // writes a lowercased version of text into output. 25 | fn to_lowercase_unicode(text: &str, output: &mut String) { 26 | output.clear(); 27 | for c in text.chars() { 28 | // Contrary to the std, we do not take care of sigma special case. 29 | // This will have an normalizationo effect, which is ok for search. 30 | output.extend(c.to_lowercase()); 31 | } 32 | } 33 | 34 | impl<'a> TokenStream for LowerCaserTokenStream<'a> { 35 | fn advance(&mut self) -> bool { 36 | if !self.tail.advance() { 37 | return false; 38 | } 39 | if self.token_mut().text.is_ascii() { 40 | // fast track for ascii. 41 | self.token_mut().text.make_ascii_lowercase(); 42 | } else { 43 | to_lowercase_unicode(&self.tail.token().text, &mut self.buffer); 44 | mem::swap(&mut self.tail.token_mut().text, &mut self.buffer); 45 | } 46 | true 47 | } 48 | 49 | fn token(&self) -> &Token { 50 | self.tail.token() 51 | } 52 | 53 | fn token_mut(&mut self) -> &mut Token { 54 | self.tail.token_mut() 55 | } 56 | } 57 | 58 | #[cfg(test)] 59 | mod tests { 60 | use crate::fts::tokenizer::tests::assert_token; 61 | use crate::fts::tokenizer::{LowerCaser, SimpleTokenizer, TextAnalyzer, Token}; 62 | 63 | #[test] 64 | fn test_to_lower_case() { 65 | let tokens = token_stream_helper("Tree"); 66 | assert_eq!(tokens.len(), 1); 67 | assert_token(&tokens[0], 0, "tree", 0, 4); 68 | 69 | let tokens = token_stream_helper("Русский текст"); 70 | assert_eq!(tokens.len(), 2); 71 | assert_token(&tokens[0], 0, "русский", 0, 14); 72 | assert_token(&tokens[1], 1, "текст", 15, 25); 73 | } 74 | 75 | fn token_stream_helper(text: &str) -> Vec { 76 | let mut token_stream = TextAnalyzer::from(SimpleTokenizer) 77 | .filter(LowerCaser) 78 | .token_stream(text); 79 | let mut tokens = vec![]; 80 | let mut add_token = |token: &Token| { 81 | tokens.push(token.clone()); 82 | }; 83 | token_stream.process(&mut add_token); 84 | tokens 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /cozo-core/src/fts/tokenizer/raw_tokenizer.rs: -------------------------------------------------------------------------------- 1 | use super::{Token, TokenStream, Tokenizer}; 2 | use crate::fts::tokenizer::BoxTokenStream; 3 | 4 | /// For each value of the field, emit a single unprocessed token. 5 | #[derive(Clone)] 6 | pub(crate) struct RawTokenizer; 7 | 8 | pub(crate) struct RawTokenStream { 9 | token: Token, 10 | has_token: bool, 11 | } 12 | 13 | impl Tokenizer for RawTokenizer { 14 | fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> { 15 | let token = Token { 16 | offset_from: 0, 17 | offset_to: text.len(), 18 | position: 0, 19 | text: text.to_string(), 20 | position_length: 1, 21 | }; 22 | RawTokenStream { 23 | token, 24 | has_token: true, 25 | } 26 | .into() 27 | } 28 | } 29 | 30 | impl TokenStream for RawTokenStream { 31 | fn advance(&mut self) -> bool { 32 | let result = self.has_token; 33 | self.has_token = false; 34 | result 35 | } 36 | 37 | fn token(&self) -> &Token { 38 | &self.token 39 | } 40 | 41 | fn token_mut(&mut self) -> &mut Token { 42 | &mut self.token 43 | } 44 | } 45 | 46 | #[cfg(test)] 47 | mod tests { 48 | use crate::fts::tokenizer::tests::assert_token; 49 | use crate::fts::tokenizer::{RawTokenizer, TextAnalyzer, Token}; 50 | 51 | #[test] 52 | fn test_raw_tokenizer() { 53 | let tokens = token_stream_helper("Hello, happy tax payer!"); 54 | assert_eq!(tokens.len(), 1); 55 | assert_token(&tokens[0], 0, "Hello, happy tax payer!", 0, 23); 56 | } 57 | 58 | fn token_stream_helper(text: &str) -> Vec { 59 | let a = TextAnalyzer::from(RawTokenizer); 60 | let mut token_stream = a.token_stream(text); 61 | let mut tokens: Vec = vec![]; 62 | let mut add_token = |token: &Token| { 63 | tokens.push(token.clone()); 64 | }; 65 | token_stream.process(&mut add_token); 66 | tokens 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /cozo-core/src/fts/tokenizer/remove_long.rs: -------------------------------------------------------------------------------- 1 | //! # Example 2 | //! ```text 3 | //! use tantivy::tokenizer::*; 4 | //! 5 | //! let tokenizer = TextAnalyzer::from(SimpleTokenizer) 6 | //! .filter(RemoveLongFilter::limit(5)); 7 | //! 8 | //! let mut stream = tokenizer.token_stream("toolong nice"); 9 | //! // because `toolong` is more than 5 characters, it is filtered 10 | //! // out of the token stream. 11 | //! assert_eq!(stream.next().unwrap().text, "nice"); 12 | //! assert!(stream.next().is_none()); 13 | //! ``` 14 | use super::{Token, TokenFilter, TokenStream}; 15 | use crate::fts::tokenizer::BoxTokenStream; 16 | 17 | /// `RemoveLongFilter` removes tokens that are longer 18 | /// than a given number of bytes (in UTF-8 representation). 19 | /// 20 | /// It is especially useful when indexing unconstrained content. 21 | /// e.g. Mail containing base-64 encoded pictures etc. 22 | #[derive(Clone)] 23 | pub(crate) struct RemoveLongFilter { 24 | length_limit: usize, 25 | } 26 | 27 | impl RemoveLongFilter { 28 | /// Creates a `RemoveLongFilter` given a limit in bytes of the UTF-8 representation. 29 | pub(crate) fn limit(length_limit: usize) -> RemoveLongFilter { 30 | RemoveLongFilter { length_limit } 31 | } 32 | } 33 | 34 | impl<'a> RemoveLongFilterStream<'a> { 35 | fn predicate(&self, token: &Token) -> bool { 36 | token.text.len() < self.token_length_limit 37 | } 38 | } 39 | 40 | impl TokenFilter for RemoveLongFilter { 41 | fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> { 42 | BoxTokenStream::from(RemoveLongFilterStream { 43 | token_length_limit: self.length_limit, 44 | tail: token_stream, 45 | }) 46 | } 47 | } 48 | 49 | pub(crate) struct RemoveLongFilterStream<'a> { 50 | token_length_limit: usize, 51 | tail: BoxTokenStream<'a>, 52 | } 53 | 54 | impl<'a> TokenStream for RemoveLongFilterStream<'a> { 55 | fn advance(&mut self) -> bool { 56 | while self.tail.advance() { 57 | if self.predicate(self.tail.token()) { 58 | return true; 59 | } 60 | } 61 | false 62 | } 63 | 64 | fn token(&self) -> &Token { 65 | self.tail.token() 66 | } 67 | 68 | fn token_mut(&mut self) -> &mut Token { 69 | self.tail.token_mut() 70 | } 71 | } 72 | 73 | #[cfg(test)] 74 | mod tests { 75 | use crate::fts::tokenizer::tests::assert_token; 76 | use crate::fts::tokenizer::{RemoveLongFilter, SimpleTokenizer, TextAnalyzer, Token}; 77 | 78 | #[test] 79 | fn test_remove_long() { 80 | let tokens = token_stream_helper("hello tantivy, happy searching!"); 81 | assert_eq!(tokens.len(), 2); 82 | assert_token(&tokens[0], 0, "hello", 0, 5); 83 | assert_token(&tokens[1], 2, "happy", 15, 20); 84 | } 85 | 86 | fn token_stream_helper(text: &str) -> Vec { 87 | let a = TextAnalyzer::from(SimpleTokenizer).filter(RemoveLongFilter::limit(6)); 88 | let mut token_stream = a.token_stream(text); 89 | let mut tokens: Vec = vec![]; 90 | let mut add_token = |token: &Token| { 91 | tokens.push(token.clone()); 92 | }; 93 | token_stream.process(&mut add_token); 94 | tokens 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /cozo-core/src/fts/tokenizer/simple_tokenizer.rs: -------------------------------------------------------------------------------- 1 | use std::str::CharIndices; 2 | 3 | use super::{BoxTokenStream, Token, TokenStream, Tokenizer}; 4 | 5 | /// Tokenize the text by splitting on whitespaces and punctuation. 6 | #[derive(Clone)] 7 | pub(crate) struct SimpleTokenizer; 8 | 9 | pub(crate) struct SimpleTokenStream<'a> { 10 | text: &'a str, 11 | chars: CharIndices<'a>, 12 | token: Token, 13 | } 14 | 15 | impl Tokenizer for SimpleTokenizer { 16 | fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> { 17 | BoxTokenStream::from(SimpleTokenStream { 18 | text, 19 | chars: text.char_indices(), 20 | token: Token::default(), 21 | }) 22 | } 23 | } 24 | 25 | impl<'a> SimpleTokenStream<'a> { 26 | // search for the end of the current token. 27 | fn search_token_end(&mut self) -> usize { 28 | (&mut self.chars) 29 | .filter(|(_, c)| !c.is_alphanumeric()) 30 | .map(|(offset, _)| offset) 31 | .next() 32 | .unwrap_or(self.text.len()) 33 | } 34 | } 35 | 36 | impl<'a> TokenStream for SimpleTokenStream<'a> { 37 | fn advance(&mut self) -> bool { 38 | self.token.text.clear(); 39 | self.token.position = self.token.position.wrapping_add(1); 40 | while let Some((offset_from, c)) = self.chars.next() { 41 | if c.is_alphanumeric() { 42 | let offset_to = self.search_token_end(); 43 | self.token.offset_from = offset_from; 44 | self.token.offset_to = offset_to; 45 | self.token.text.push_str(&self.text[offset_from..offset_to]); 46 | return true; 47 | } 48 | } 49 | false 50 | } 51 | 52 | fn token(&self) -> &Token { 53 | &self.token 54 | } 55 | 56 | fn token_mut(&mut self) -> &mut Token { 57 | &mut self.token 58 | } 59 | } 60 | 61 | #[cfg(test)] 62 | mod tests { 63 | use crate::fts::tokenizer::tests::assert_token; 64 | use crate::fts::tokenizer::{SimpleTokenizer, TextAnalyzer, Token}; 65 | 66 | #[test] 67 | fn test_simple_tokenizer() { 68 | let tokens = token_stream_helper("Hello, happy tax payer!"); 69 | assert_eq!(tokens.len(), 4); 70 | assert_token(&tokens[0], 0, "Hello", 0, 5); 71 | assert_token(&tokens[1], 1, "happy", 7, 12); 72 | assert_token(&tokens[2], 2, "tax", 13, 16); 73 | assert_token(&tokens[3], 3, "payer", 17, 22); 74 | } 75 | 76 | fn token_stream_helper(text: &str) -> Vec { 77 | let a = TextAnalyzer::from(SimpleTokenizer); 78 | let mut token_stream = a.token_stream(text); 79 | let mut tokens: Vec = vec![]; 80 | let mut add_token = |token: &Token| { 81 | tokens.push(token.clone()); 82 | }; 83 | token_stream.process(&mut add_token); 84 | tokens 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /cozo-core/src/fts/tokenizer/stemmer.rs: -------------------------------------------------------------------------------- 1 | use std::borrow::Cow; 2 | use std::mem; 3 | 4 | use rust_stemmers::{self, Algorithm}; 5 | 6 | use super::{Token, TokenFilter, TokenStream}; 7 | use crate::fts::tokenizer::BoxTokenStream; 8 | 9 | /// Available stemmer languages. 10 | #[derive(Debug, serde_derive::Serialize, serde_derive::Deserialize, Eq, PartialEq, Copy, Clone)] 11 | #[allow(missing_docs)] 12 | pub(crate) enum Language { 13 | Arabic, 14 | Danish, 15 | Dutch, 16 | English, 17 | Finnish, 18 | French, 19 | German, 20 | Greek, 21 | Hungarian, 22 | Italian, 23 | Norwegian, 24 | Portuguese, 25 | Romanian, 26 | Russian, 27 | Spanish, 28 | Swedish, 29 | Tamil, 30 | Turkish, 31 | } 32 | 33 | impl Language { 34 | fn algorithm(self) -> Algorithm { 35 | use self::Language::*; 36 | match self { 37 | Arabic => Algorithm::Arabic, 38 | Danish => Algorithm::Danish, 39 | Dutch => Algorithm::Dutch, 40 | English => Algorithm::English, 41 | Finnish => Algorithm::Finnish, 42 | French => Algorithm::French, 43 | German => Algorithm::German, 44 | Greek => Algorithm::Greek, 45 | Hungarian => Algorithm::Hungarian, 46 | Italian => Algorithm::Italian, 47 | Norwegian => Algorithm::Norwegian, 48 | Portuguese => Algorithm::Portuguese, 49 | Romanian => Algorithm::Romanian, 50 | Russian => Algorithm::Russian, 51 | Spanish => Algorithm::Spanish, 52 | Swedish => Algorithm::Swedish, 53 | Tamil => Algorithm::Tamil, 54 | Turkish => Algorithm::Turkish, 55 | } 56 | } 57 | } 58 | 59 | /// `Stemmer` token filter. Several languages are supported, see [`Language`] for the available 60 | /// languages. 61 | /// Tokens are expected to be lowercased beforehand. 62 | #[derive(Clone)] 63 | pub(crate) struct Stemmer { 64 | stemmer_algorithm: Algorithm, 65 | } 66 | 67 | impl Stemmer { 68 | /// Creates a new `Stemmer` [`TokenFilter`] for a given language algorithm. 69 | pub(crate) fn new(language: Language) -> Stemmer { 70 | Stemmer { 71 | stemmer_algorithm: language.algorithm(), 72 | } 73 | } 74 | } 75 | 76 | impl Default for Stemmer { 77 | /// Creates a new `Stemmer` [`TokenFilter`] for [`Language::English`]. 78 | fn default() -> Self { 79 | Stemmer::new(Language::English) 80 | } 81 | } 82 | 83 | impl TokenFilter for Stemmer { 84 | fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> { 85 | let inner_stemmer = rust_stemmers::Stemmer::create(self.stemmer_algorithm); 86 | BoxTokenStream::from(StemmerTokenStream { 87 | tail: token_stream, 88 | stemmer: inner_stemmer, 89 | buffer: String::new(), 90 | }) 91 | } 92 | } 93 | 94 | pub(crate) struct StemmerTokenStream<'a> { 95 | tail: BoxTokenStream<'a>, 96 | stemmer: rust_stemmers::Stemmer, 97 | buffer: String, 98 | } 99 | 100 | impl<'a> TokenStream for StemmerTokenStream<'a> { 101 | fn advance(&mut self) -> bool { 102 | if !self.tail.advance() { 103 | return false; 104 | } 105 | let token = self.tail.token_mut(); 106 | let stemmed_str = self.stemmer.stem(&token.text); 107 | match stemmed_str { 108 | Cow::Owned(stemmed_str) => token.text = stemmed_str, 109 | Cow::Borrowed(stemmed_str) => { 110 | self.buffer.clear(); 111 | self.buffer.push_str(stemmed_str); 112 | mem::swap(&mut token.text, &mut self.buffer); 113 | } 114 | } 115 | true 116 | } 117 | 118 | fn token(&self) -> &Token { 119 | self.tail.token() 120 | } 121 | 122 | fn token_mut(&mut self) -> &mut Token { 123 | self.tail.token_mut() 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /cozo-core/src/fts/tokenizer/stop_word_filter/gen_stopwords.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | resp = requests.get("https://raw.githubusercontent.com/stopwords-iso/stopwords-iso/master/stopwords-iso.json") 4 | resp.raise_for_status() 5 | data = resp.json() 6 | 7 | with requests.Session() as sess, open("stopwords.rs", "w") as mod: 8 | mod.write("/*\n") 9 | mod.write( 10 | "These stop word lists are from the stopwords-iso project (https://github.com/stopwords-iso/stopwords-iso/) " 11 | "which carries the MIT license." 12 | ) 13 | mod.write("\n*/\n\n") 14 | 15 | for lang, data in data.items(): 16 | 17 | mod.write(f"pub(crate) const {lang.upper()}: &[&str] = &[\n") 18 | 19 | for word in data: 20 | mod.write(f' r#"{word}"#,\n') 21 | 22 | mod.write("];\n\n") 23 | print(f'"{lang}" => stopwords::{lang.upper()},') 24 | -------------------------------------------------------------------------------- /cozo-core/src/fts/tokenizer/tokenized_string.rs: -------------------------------------------------------------------------------- 1 | use std::cmp::Ordering; 2 | 3 | use crate::fts::tokenizer::{Token, TokenStream}; 4 | 5 | /// Struct representing pre-tokenized text 6 | #[derive(Debug, Clone, serde_derive::Serialize, serde_derive::Deserialize, Eq, PartialEq)] 7 | pub(crate) struct PreTokenizedString { 8 | /// Original text 9 | pub(crate) text: String, 10 | /// Tokens derived from the text 11 | pub(crate) tokens: Vec, 12 | } 13 | 14 | impl Ord for PreTokenizedString { 15 | fn cmp(&self, other: &Self) -> Ordering { 16 | self.text.cmp(&other.text) 17 | } 18 | } 19 | 20 | impl PartialOrd for PreTokenizedString { 21 | fn partial_cmp(&self, other: &Self) -> Option { 22 | Some(self.cmp(other)) 23 | } 24 | } 25 | 26 | /// [`TokenStream`] implementation which wraps [`PreTokenizedString`] 27 | pub(crate) struct PreTokenizedStream { 28 | tokenized_string: PreTokenizedString, 29 | current_token: i64, 30 | } 31 | 32 | impl From for PreTokenizedStream { 33 | fn from(s: PreTokenizedString) -> PreTokenizedStream { 34 | PreTokenizedStream { 35 | tokenized_string: s, 36 | current_token: -1, 37 | } 38 | } 39 | } 40 | 41 | impl TokenStream for PreTokenizedStream { 42 | fn advance(&mut self) -> bool { 43 | self.current_token += 1; 44 | self.current_token < self.tokenized_string.tokens.len() as i64 45 | } 46 | 47 | fn token(&self) -> &Token { 48 | assert!( 49 | self.current_token >= 0, 50 | "TokenStream not initialized. You should call advance() at least once." 51 | ); 52 | &self.tokenized_string.tokens[self.current_token as usize] 53 | } 54 | 55 | fn token_mut(&mut self) -> &mut Token { 56 | assert!( 57 | self.current_token >= 0, 58 | "TokenStream not initialized. You should call advance() at least once." 59 | ); 60 | &mut self.tokenized_string.tokens[self.current_token as usize] 61 | } 62 | } 63 | 64 | #[cfg(test)] 65 | mod tests { 66 | 67 | use super::*; 68 | use crate::fts::tokenizer::Token; 69 | 70 | #[test] 71 | fn test_tokenized_stream() { 72 | let tok_text = PreTokenizedString { 73 | text: String::from("A a"), 74 | tokens: vec![ 75 | Token { 76 | offset_from: 0, 77 | offset_to: 1, 78 | position: 0, 79 | text: String::from("A"), 80 | position_length: 1, 81 | }, 82 | Token { 83 | offset_from: 2, 84 | offset_to: 3, 85 | position: 1, 86 | text: String::from("a"), 87 | position_length: 1, 88 | }, 89 | ], 90 | }; 91 | 92 | let mut token_stream = PreTokenizedStream::from(tok_text.clone()); 93 | 94 | for expected_token in tok_text.tokens { 95 | assert!(token_stream.advance()); 96 | assert_eq!(token_stream.token(), &expected_token); 97 | } 98 | assert!(!token_stream.advance()); 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /cozo-core/src/fts/tokenizer/whitespace_tokenizer.rs: -------------------------------------------------------------------------------- 1 | use std::str::CharIndices; 2 | 3 | use super::{BoxTokenStream, Token, TokenStream, Tokenizer}; 4 | 5 | /// Tokenize the text by splitting on whitespaces. 6 | #[derive(Clone)] 7 | pub(crate) struct WhitespaceTokenizer; 8 | 9 | pub(crate) struct WhitespaceTokenStream<'a> { 10 | text: &'a str, 11 | chars: CharIndices<'a>, 12 | token: Token, 13 | } 14 | 15 | impl Tokenizer for WhitespaceTokenizer { 16 | fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> { 17 | BoxTokenStream::from(WhitespaceTokenStream { 18 | text, 19 | chars: text.char_indices(), 20 | token: Token::default(), 21 | }) 22 | } 23 | } 24 | 25 | impl<'a> WhitespaceTokenStream<'a> { 26 | // search for the end of the current token. 27 | fn search_token_end(&mut self) -> usize { 28 | (&mut self.chars) 29 | .filter(|(_, c)| c.is_ascii_whitespace()) 30 | .map(|(offset, _)| offset) 31 | .next() 32 | .unwrap_or(self.text.len()) 33 | } 34 | } 35 | 36 | impl<'a> TokenStream for WhitespaceTokenStream<'a> { 37 | fn advance(&mut self) -> bool { 38 | self.token.text.clear(); 39 | self.token.position = self.token.position.wrapping_add(1); 40 | while let Some((offset_from, c)) = self.chars.next() { 41 | if !c.is_ascii_whitespace() { 42 | let offset_to = self.search_token_end(); 43 | self.token.offset_from = offset_from; 44 | self.token.offset_to = offset_to; 45 | self.token.text.push_str(&self.text[offset_from..offset_to]); 46 | return true; 47 | } 48 | } 49 | false 50 | } 51 | 52 | fn token(&self) -> &Token { 53 | &self.token 54 | } 55 | 56 | fn token_mut(&mut self) -> &mut Token { 57 | &mut self.token 58 | } 59 | } 60 | 61 | #[cfg(test)] 62 | mod tests { 63 | use crate::fts::tokenizer::tests::assert_token; 64 | use crate::fts::tokenizer::{TextAnalyzer, Token, WhitespaceTokenizer}; 65 | 66 | #[test] 67 | fn test_whitespace_tokenizer() { 68 | let tokens = token_stream_helper("Hello, happy tax payer!"); 69 | assert_eq!(tokens.len(), 4); 70 | assert_token(&tokens[0], 0, "Hello,", 0, 6); 71 | assert_token(&tokens[1], 1, "happy", 7, 12); 72 | assert_token(&tokens[2], 2, "tax", 13, 16); 73 | assert_token(&tokens[3], 3, "payer!", 17, 23); 74 | } 75 | 76 | fn token_stream_helper(text: &str) -> Vec { 77 | let a = TextAnalyzer::from(WhitespaceTokenizer); 78 | let mut token_stream = a.token_stream(text); 79 | let mut tokens: Vec = vec![]; 80 | let mut add_token = |token: &Token| { 81 | tokens.push(token.clone()); 82 | }; 83 | token_stream.process(&mut add_token); 84 | tokens 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /cozo-core/src/query/mod.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022, The Cozo Project Authors. 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 5 | * If a copy of the MPL was not distributed with this file, 6 | * You can obtain one at https://mozilla.org/MPL/2.0/. 7 | */ 8 | 9 | pub(crate) mod compile; 10 | pub(crate) mod eval; 11 | pub(crate) mod graph; 12 | pub(crate) mod logical; 13 | pub(crate) mod magic; 14 | pub(crate) mod ra; 15 | pub(crate) mod reorder; 16 | pub(crate) mod sort; 17 | pub(crate) mod stored; 18 | pub(crate) mod stratify; 19 | -------------------------------------------------------------------------------- /cozo-core/src/query/sort.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022, The Cozo Project Authors. 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 5 | * If a copy of the MPL was not distributed with this file, 6 | * You can obtain one at https://mozilla.org/MPL/2.0/. 7 | */ 8 | 9 | use std::cmp::Ordering; 10 | use std::collections::BTreeMap; 11 | 12 | use itertools::Itertools; 13 | use miette::Result; 14 | 15 | use crate::data::program::SortDir; 16 | use crate::data::symb::Symbol; 17 | use crate::data::tuple::Tuple; 18 | use crate::runtime::temp_store::EpochStore; 19 | use crate::runtime::transact::SessionTx; 20 | 21 | impl<'a> SessionTx<'a> { 22 | pub(crate) fn sort_and_collect( 23 | &mut self, 24 | original: EpochStore, 25 | sorters: &[(Symbol, SortDir)], 26 | head: &[Symbol], 27 | ) -> Result> { 28 | let head_indices: BTreeMap<_, _> = head.iter().enumerate().map(|(i, k)| (k, i)).collect(); 29 | let idx_sorters = sorters 30 | .iter() 31 | .map(|(k, dir)| (head_indices[k], *dir)) 32 | .collect_vec(); 33 | 34 | let mut all_data: Vec<_> = original.all_iter().map(|v| v.into_tuple()).collect_vec(); 35 | all_data.sort_by(|a, b| { 36 | for (idx, dir) in &idx_sorters { 37 | match a[*idx].cmp(&b[*idx]) { 38 | Ordering::Equal => {} 39 | o => { 40 | return match dir { 41 | SortDir::Asc => o, 42 | SortDir::Dsc => o.reverse(), 43 | } 44 | } 45 | } 46 | } 47 | Ordering::Equal 48 | }); 49 | 50 | Ok(all_data) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /cozo-core/src/runtime/callback.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022, The Cozo Project Authors. 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 5 | * If a copy of the MPL was not distributed with this file, 6 | * You can obtain one at https://mozilla.org/MPL/2.0/. 7 | */ 8 | 9 | use std::collections::{BTreeMap, BTreeSet}; 10 | use std::fmt::{Display, Formatter}; 11 | 12 | use crossbeam::channel::Sender; 13 | use smartstring::{LazyCompact, SmartString}; 14 | 15 | use crate::{Db, NamedRows, Storage}; 16 | 17 | /// Represents the kind of operation that triggered the callback 18 | #[derive(Copy, Clone, Debug, Eq, PartialEq)] 19 | pub enum CallbackOp { 20 | /// Triggered by Put operations 21 | Put, 22 | /// Triggered by Rm operations 23 | Rm, 24 | } 25 | 26 | impl Display for CallbackOp { 27 | fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { 28 | match self { 29 | CallbackOp::Put => f.write_str("Put"), 30 | CallbackOp::Rm => f.write_str("Rm"), 31 | } 32 | } 33 | } 34 | 35 | impl CallbackOp { 36 | /// Get the string representation 37 | pub fn as_str(&self) -> &'static str { 38 | match self { 39 | CallbackOp::Put => "Put", 40 | CallbackOp::Rm => "Rm", 41 | } 42 | } 43 | } 44 | 45 | #[allow(dead_code)] 46 | pub struct CallbackDeclaration { 47 | pub(crate) dependent: SmartString, 48 | pub(crate) sender: Sender<(CallbackOp, NamedRows, NamedRows)>, 49 | } 50 | 51 | pub(crate) type CallbackCollector = 52 | BTreeMap, Vec<(CallbackOp, NamedRows, NamedRows)>>; 53 | 54 | #[allow(dead_code)] 55 | pub(crate) type EventCallbackRegistry = ( 56 | BTreeMap, 57 | BTreeMap, BTreeSet>, 58 | ); 59 | 60 | impl<'s, S: Storage<'s>> Db { 61 | pub(crate) fn current_callback_targets(&self) -> BTreeSet> { 62 | #[cfg(not(target_arch = "wasm32"))] 63 | { 64 | self.event_callbacks 65 | .read() 66 | .unwrap() 67 | .1 68 | .keys() 69 | .cloned() 70 | .collect() 71 | } 72 | 73 | #[cfg(target_arch = "wasm32")] 74 | { 75 | Default::default() 76 | } 77 | } 78 | #[cfg(not(target_arch = "wasm32"))] 79 | pub(crate) fn send_callbacks(&'s self, collector: CallbackCollector) { 80 | let mut to_remove = vec![]; 81 | 82 | for (table, vals) in collector { 83 | for (op, new, old) in vals { 84 | let (cbs, cb_dir) = &*self.event_callbacks.read().unwrap(); 85 | if let Some(cb_ids) = cb_dir.get(&table) { 86 | let mut it = cb_ids.iter(); 87 | if let Some(fst) = it.next() { 88 | for cb_id in it { 89 | if let Some(cb) = cbs.get(cb_id) { 90 | if cb.sender.send((op, new.clone(), old.clone())).is_err() { 91 | to_remove.push(*cb_id) 92 | } 93 | } 94 | } 95 | 96 | if let Some(cb) = cbs.get(fst) { 97 | if cb.sender.send((op, new, old)).is_err() { 98 | to_remove.push(*fst) 99 | } 100 | } 101 | } 102 | } 103 | } 104 | } 105 | 106 | if !to_remove.is_empty() { 107 | let (cbs, cb_dir) = &mut *self.event_callbacks.write().unwrap(); 108 | for removing_id in &to_remove { 109 | if let Some(removed) = cbs.remove(removing_id) { 110 | if let Some(set) = cb_dir.get_mut(&removed.dependent) { 111 | set.remove(removing_id); 112 | } 113 | } 114 | } 115 | } 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /cozo-core/src/runtime/mod.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022, The Cozo Project Authors. 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 5 | * If a copy of the MPL was not distributed with this file, 6 | * You can obtain one at https://mozilla.org/MPL/2.0/. 7 | */ 8 | 9 | pub(crate) mod callback; 10 | pub(crate) mod db; 11 | pub(crate) mod imperative; 12 | pub(crate) mod relation; 13 | pub(crate) mod temp_store; 14 | pub(crate) mod transact; 15 | pub(crate) mod hnsw; 16 | pub(crate) mod minhash_lsh; 17 | #[cfg(test)] 18 | mod tests; 19 | -------------------------------------------------------------------------------- /cozo-core/src/storage/temp.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022, The Cozo Project Authors. 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 5 | * If a copy of the MPL was not distributed with this file, 6 | * You can obtain one at https://mozilla.org/MPL/2.0/. 7 | */ 8 | 9 | use std::collections::BTreeMap; 10 | use std::default::Default; 11 | 12 | use miette::Result; 13 | 14 | use crate::data::tuple::Tuple; 15 | use crate::data::value::ValidityTs; 16 | use crate::runtime::relation::decode_tuple_from_kv; 17 | use crate::storage::mem::SkipIterator; 18 | use crate::storage::{Storage, StoreTx}; 19 | 20 | #[derive(Default, Clone)] 21 | pub(crate) struct TempStorage; 22 | 23 | impl<'s> Storage<'s> for TempStorage { 24 | type Tx = TempTx; 25 | 26 | fn storage_kind(&self) -> &'static str { 27 | "temp" 28 | } 29 | 30 | fn transact(&'s self, _write: bool) -> Result { 31 | Ok(TempTx { 32 | store: Default::default(), 33 | }) 34 | } 35 | 36 | fn range_compact(&'s self, _lower: &[u8], _upper: &[u8]) -> Result<()> { 37 | panic!("range compact called on temp store") 38 | } 39 | 40 | fn batch_put<'a>( 41 | &'a self, 42 | _data: Box, Vec)>> + 'a>, 43 | ) -> Result<()> { 44 | panic!("batch put compact called on temp store") 45 | } 46 | } 47 | 48 | pub(crate) struct TempTx { 49 | store: BTreeMap, Vec>, 50 | } 51 | 52 | impl<'s> StoreTx<'s> for TempTx { 53 | fn get(&self, key: &[u8], _for_update: bool) -> Result>> { 54 | Ok(self.store.get(key).cloned()) 55 | } 56 | 57 | fn put(&mut self, key: &[u8], val: &[u8]) -> Result<()> { 58 | self.store.insert(key.to_vec(), val.to_vec()); 59 | Ok(()) 60 | } 61 | 62 | fn supports_par_put(&self) -> bool { 63 | false 64 | } 65 | 66 | fn del(&mut self, key: &[u8]) -> Result<()> { 67 | self.store.remove(key); 68 | Ok(()) 69 | } 70 | 71 | fn del_range_from_persisted(&mut self, _lower: &[u8], _upper: &[u8]) -> Result<()> { 72 | Ok(()) 73 | } 74 | 75 | fn exists(&self, key: &[u8], _for_update: bool) -> Result { 76 | Ok(self.store.contains_key(key)) 77 | } 78 | 79 | fn commit(&mut self) -> Result<()> { 80 | Ok(()) 81 | } 82 | 83 | fn range_scan_tuple<'a>( 84 | &'a self, 85 | lower: &[u8], 86 | upper: &[u8], 87 | ) -> Box> + 'a> 88 | where 89 | 's: 'a, 90 | { 91 | Box::new( 92 | self.store 93 | .range(lower.to_vec()..upper.to_vec()) 94 | .map(|(k, v)| Ok(decode_tuple_from_kv(k, v, None))), 95 | ) 96 | } 97 | 98 | fn range_skip_scan_tuple<'a>( 99 | &'a self, 100 | lower: &[u8], 101 | upper: &[u8], 102 | valid_at: ValidityTs, 103 | ) -> Box> + 'a> { 104 | Box::new( 105 | SkipIterator { 106 | inner: &self.store, 107 | upper: upper.to_vec(), 108 | valid_at, 109 | next_bound: lower.to_vec(), 110 | size_hint: None, 111 | } 112 | .map(Ok), 113 | ) 114 | } 115 | 116 | fn range_scan<'a>( 117 | &'a self, 118 | lower: &[u8], 119 | upper: &[u8], 120 | ) -> Box, Vec)>> + 'a> 121 | where 122 | 's: 'a, 123 | { 124 | Box::new( 125 | self.store 126 | .range(lower.to_vec()..upper.to_vec()) 127 | .map(|(k, v)| Ok((k.clone(), v.clone()))), 128 | ) 129 | } 130 | 131 | fn range_count<'a>(&'a self, lower: &[u8], upper: &[u8]) -> Result where 's: 'a { 132 | Ok(self.store.range(lower.to_vec()..upper.to_vec()).count()) 133 | } 134 | 135 | fn total_scan<'a>(&'a self) -> Box, Vec)>> + 'a> 136 | where 137 | 's: 'a, 138 | { 139 | Box::new(self.store.iter().map(|(k, v)| Ok((k.clone(), v.clone())))) 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /cozo-core/src/utils.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022, The Cozo Project Authors. 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 5 | * If a copy of the MPL was not distributed with this file, 6 | * You can obtain one at https://mozilla.org/MPL/2.0/. 7 | */ 8 | 9 | #[inline(always)] 10 | pub(crate) fn swap_option_result(d: Result, E>) -> Option> { 11 | match d { 12 | Ok(Some(s)) => Some(Ok(s)), 13 | Ok(None) => None, 14 | Err(e) => Some(Err(e)), 15 | } 16 | } 17 | 18 | #[derive(Default)] 19 | pub(crate) struct TempCollector serde::Deserialize<'a>> { 20 | // pub(crate) inner: Vec, 21 | pub(crate) inner: swapvec::SwapVec, 22 | } 23 | 24 | impl serde::Deserialize<'a>> TempCollector { 25 | pub(crate) fn push(&mut self, val: T) { 26 | self.inner.push(val).unwrap(); 27 | } 28 | pub(crate) fn into_iter(self) -> impl Iterator { 29 | self.inner.into_iter().map(|v| v.unwrap()) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /cozo-lib-c/.gitignore: -------------------------------------------------------------------------------- 1 | example -------------------------------------------------------------------------------- /cozo-lib-c/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "cozo_c" 3 | version = "0.7.6" 4 | edition = "2021" 5 | license = "MPL-2.0" 6 | homepage = "https://www.cozodb.org" 7 | repository = "https://github.com/cozodb/cozo" 8 | documentation = "https://docs.cozodb.org" 9 | description = "C bindings for CozoDB" 10 | 11 | [lib] 12 | crate-type = ["cdylib", "staticlib"] 13 | 14 | [features] 15 | #! # Features 16 | 17 | ## Enables the `minimal`, `requests` and `graph-algo` features 18 | compact = ["minimal", "requests", "graph-algo"] 19 | ## Enables the `storage-sqlite` and `graph-algo` features 20 | mobile = ["storage-sqlite", "graph-algo"] 21 | ## Enables the `minimal`, `requests` and `graph-algo` features in single threaded mode 22 | compact-single-threaded = ["minimal", "requests", "graph-algo"] 23 | ## Enables the `storage-sqlite` feature 24 | minimal = ["storage-sqlite"] 25 | ## Enables the [Sqlite](https://www.sqlite.org/index.html) backend, also allows backup and restore with Sqlite data files. 26 | storage-sqlite = ["cozo/storage-sqlite"] 27 | ## Enables the [RocksDB](http://rocksdb.org/) backend 28 | storage-rocksdb = ["cozo/storage-rocksdb"] 29 | ## Enables the graph algorithms 30 | graph-algo = ["cozo/graph-algo"] 31 | ## Allows the utilities to make web requests to fetch data 32 | requests = ["cozo/requests"] 33 | ## Uses jemalloc as the global allocator, can make a difference in performance 34 | jemalloc = ["cozo/jemalloc"] 35 | ## Enables io-uring option for the RocksDB storage 36 | io-uring = ["cozo/io-uring"] 37 | 38 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 39 | 40 | [dependencies] 41 | cozo = { version = "0.7.6", path = "../cozo-core", default_features = false } 42 | lazy_static = "1.4.0" 43 | 44 | [build-dependencies] 45 | cbindgen = "0.26.0" 46 | -------------------------------------------------------------------------------- /cozo-lib-c/README-zh.md: -------------------------------------------------------------------------------- 1 | # Cozo C 语言库 2 | 3 | [![C](https://img.shields.io/github/v/release/cozodb/cozo)](https://github.com/cozodb/cozo/releases) 4 | 5 | 这里是 Cozo 的 C API 的源代码。 6 | 7 | 本文叙述的是如何安装设置 Cozo 的 C 语言库。有关如何使用 CozoDB（CozoScript）的信息，见 [文档](https://docs.cozodb.org/zh_CN/latest/index.html) 。 8 | 9 | 预编译的库可从 [GitHub 发布页面](https://github.com/cozodb/cozo/releases) 下载，其中 C 语言库以 `libcozo_c` 开头。 10 | 11 | C 语言的 API 在这个 [头文件](./cozo_c.h) 中。 12 | 13 | [这个程序](./example.c) 举例说明了如何调用此 API，程序可使用以下命令编译安装： 14 | 15 | ```bash 16 | gcc -L../target/release/ -lcozo_c example.c -o example && ./example 17 | ``` 18 | 19 | # 从源码编译 20 | 21 | 首先需要安装 [Rust 工具链](https://www.rust-lang.org/tools/install) ，然后： 22 | 23 | ```bash 24 | cargo build --release -p cozo_c -F compact -F storage-rocksdb 25 | ``` 26 | -------------------------------------------------------------------------------- /cozo-lib-c/README.md: -------------------------------------------------------------------------------- 1 | # Cozo C lib 2 | 3 | [![C](https://img.shields.io/github/v/release/cozodb/cozo)](https://github.com/cozodb/cozo/releases) 4 | 5 | This directory contains the source of the Cozo C API. 6 | 7 | This document describes how to set up the C library. 8 | To learn how to use CozoDB (CozoScript), read the [docs](https://docs.cozodb.org/en/latest/index.html). 9 | 10 | You can download pre-built libraries from the [release page](https://github.com/cozodb/cozo/releases), 11 | look for those starting with `libcozo_c`. 12 | 13 | The API is contained in this single [header file](./cozo_c.h). 14 | 15 | An example for using the API is [here](./example.c). 16 | 17 | To build and run the example: 18 | 19 | ```bash 20 | gcc -L../target/release/ -lcozo_c example.c -o example && ./example 21 | ``` 22 | 23 | # Building Cozo from source 24 | 25 | You need to install the [Rust toolchain](https://www.rust-lang.org/tools/install) on your system. Then: 26 | 27 | ```bash 28 | cargo build --release -p cozo_c -F compact -F storage-rocksdb 29 | ``` 30 | -------------------------------------------------------------------------------- /cozo-lib-c/build.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022, The Cozo Project Authors. 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 5 | * If a copy of the MPL was not distributed with this file, 6 | * You can obtain one at https://mozilla.org/MPL/2.0/. 7 | */ 8 | 9 | use cbindgen::{Config, Language}; 10 | use std::env; 11 | 12 | fn main() { 13 | let crate_dir = env::var("CARGO_MANIFEST_DIR").unwrap(); 14 | 15 | let mut config = Config::default(); 16 | config.cpp_compat = true; 17 | 18 | cbindgen::Builder::new() 19 | .with_config(config) 20 | .with_crate(crate_dir) 21 | .with_language(Language::C) 22 | .with_include_guard("COZO_C_H") 23 | .with_autogen_warning( 24 | "/* Warning, this file is autogenerated by cbindgen. Don't modify this manually. */", 25 | ) 26 | .with_header( 27 | r#"/* 28 | Copyright 2022, The Cozo Project Authors. 29 | 30 | This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 31 | If a copy of the MPL was not distributed with this file, 32 | You can obtain one at https://mozilla.org/MPL/2.0/. 33 | */"#, 34 | ) 35 | .with_documentation(true) 36 | .generate() 37 | .expect("Unable to generate bindings") 38 | .write_to_file("cozo_c.h"); 39 | } 40 | -------------------------------------------------------------------------------- /cozo-lib-c/comiple-ios.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ "$#" -ne 2 ] 4 | then 5 | echo "Usage (note: only call inside xcode!):" 6 | echo "compile-library.sh " 7 | exit 1 8 | fi 9 | 10 | # what to pass to cargo build -p, e.g. your_lib_ffi 11 | FFI_TARGET=$1 12 | # buildvariant from our xcconfigs 13 | BUILDVARIANT=$2 14 | 15 | RELFLAG= 16 | if [[ "$BUILDVARIANT" != "debug" ]]; then 17 | RELFLAG=--release 18 | fi 19 | 20 | set -euvx 21 | 22 | if [[ -n "${DEVELOPER_SDK_DIR:-}" ]]; then 23 | # Assume we're in Xcode, which means we're probably cross-compiling. 24 | # In this case, we need to add an extra library search path for build scripts and proc-macros, 25 | # which run on the host instead of the target. 26 | # (macOS Big Sur does not have linkable libraries in /usr/lib/.) 27 | export LIBRARY_PATH="${DEVELOPER_SDK_DIR}/MacOSX.sdk/usr/lib:${LIBRARY_PATH:-}" 28 | fi 29 | 30 | IS_SIMULATOR=${IS_SIMULATOR:=0} 31 | if [ "${LLVM_TARGET_TRIPLE_SUFFIX-}" = "-simulator" ]; then 32 | IS_SIMULATOR=1 33 | fi 34 | 35 | for arch in $ARCHS; do 36 | case "$arch" in 37 | x86_64) 38 | if [ $IS_SIMULATOR -eq 0 ]; then 39 | echo "Building for x86_64, but not a simulator build. What's going on?" >&2 40 | exit 2 41 | fi 42 | 43 | # Intel iOS simulator 44 | export CFLAGS_x86_64_apple_ios="-target x86_64-apple-ios" 45 | $HOME/.cargo/bin/cargo build -p $FFI_TARGET --lib $RELFLAG --target x86_64-apple-ios -F mobile 46 | ;; 47 | 48 | arm64) 49 | if [ $IS_SIMULATOR -eq 0 ]; then 50 | # Hardware iOS targets 51 | $HOME/.cargo/bin/cargo build -p $FFI_TARGET --lib $RELFLAG --target aarch64-apple-ios -F mobile 52 | else 53 | $HOME/.cargo/bin/cargo build -p $FFI_TARGET --lib $RELFLAG --target aarch64-apple-ios-sim -F mobile 54 | fi 55 | esac 56 | done -------------------------------------------------------------------------------- /cozo-lib-c/example.c: -------------------------------------------------------------------------------- 1 | // Copyright 2022, The Cozo Project Authors. 2 | // 3 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 4 | // If a copy of the MPL was not distributed with this file, 5 | // You can obtain one at https://mozilla.org/MPL/2.0/. 6 | 7 | #include 8 | #include 9 | #include 10 | #include "cozo_c.h" 11 | 12 | void run_query(int32_t db_id, const char *query) { 13 | const char *empty_params = "{}"; 14 | char *res; 15 | res = cozo_run_query(db_id, query, empty_params, false); 16 | printf("%s\n", res); 17 | cozo_free_str(res); 18 | } 19 | 20 | int main() { 21 | int32_t db_id; 22 | char *err = cozo_open_db("mem", "", "{}", &db_id); 23 | 24 | if (err) { 25 | printf("%s", err); 26 | cozo_free_str(err); 27 | return -1; 28 | } 29 | 30 | run_query(db_id, "?[] <- [[1, 2, 3]]"); 31 | 32 | cozo_close_db(db_id); 33 | 34 | return 0; 35 | } 36 | -------------------------------------------------------------------------------- /cozo-lib-java/.gitignore: -------------------------------------------------------------------------------- 1 | *.class -------------------------------------------------------------------------------- /cozo-lib-java/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "cozo_java" 3 | version = "0.7.6" 4 | edition = "2021" 5 | license = "MPL-2.0" 6 | homepage = "https://www.cozodb.org" 7 | repository = "https://github.com/cozodb/cozo" 8 | documentation = "https://docs.cozodb.org" 9 | description = "Java bindings for CozoDB" 10 | 11 | [lib] 12 | crate-type = ["cdylib"] 13 | 14 | [features] 15 | #! # Features 16 | 17 | ## Enables the `minimal`, `requests` and `graph-algo` features 18 | compact = ["minimal", "requests", "graph-algo"] 19 | ## Enables the `storage-sqlite` and `graph-algo` features 20 | mobile = ["storage-sqlite", "graph-algo"] 21 | ## Enables the `minimal`, `requests` and `graph-algo` features in single threaded mode 22 | compact-single-threaded = ["minimal", "requests", "graph-algo"] 23 | ## Enables the `storage-sqlite` feature 24 | minimal = ["storage-sqlite"] 25 | ## Enables the [Sqlite](https://www.sqlite.org/index.html) backend, also allows backup and restore with Sqlite data files. 26 | storage-sqlite = ["cozo/storage-sqlite"] 27 | ## Enables the [RocksDB](http://rocksdb.org/) backend 28 | storage-rocksdb = ["cozo/storage-rocksdb"] 29 | ## Enables the graph algorithms 30 | graph-algo = ["cozo/graph-algo"] 31 | ## Allows the utilities to make web requests to fetch data 32 | requests = ["cozo/requests"] 33 | ## Uses jemalloc as the global allocator, can make a difference in performance 34 | jemalloc = ["cozo/jemalloc"] 35 | ## Enables io-uring option for the RocksDB storage 36 | io-uring = ["cozo/io-uring"] 37 | 38 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 39 | 40 | [dependencies] 41 | jni = "0.21.1" 42 | # , features = ["compact"] 43 | cozo = { version = "0.7.6", path = "../cozo-core", default_features = false, features = ["compact"] } 44 | lazy_static = "1.4.0" 45 | -------------------------------------------------------------------------------- /cozo-lib-java/CozoJavaBridge.java: -------------------------------------------------------------------------------- 1 | package org.cozodb; 2 | 3 | public class CozoJavaBridge { 4 | private static native int openDb(String engine, String path, String options); 5 | private static native boolean closeDb(int id); 6 | private static native String runQuery(int id, String script, String params); 7 | private static native String exportRelations(int id, String rel); 8 | private static native String importRelations(int id, String data); 9 | private static native String backup(int id, String file); 10 | private static native String restore(int id, String file); 11 | private static native String importFromBackup(int id, String data); 12 | } -------------------------------------------------------------------------------- /cozo-lib-java/README-zh.md: -------------------------------------------------------------------------------- 1 | # Cozo Java 语言库 2 | 3 | 这里是 Cozo 的 JNI 接口库，可以在 Java/JVM 语言/安卓中使用。 4 | 5 | 一般情况下，这个库不是直接使用的。用于应当使用以下调用此库的库： 6 | 7 | * [Cozo Java](https://github.com/cozodb/cozo-lib-java) （[镜像](https://gitee.com/cozodb/cozo-lib-java)）：在 JVM Java 中使用 8 | * [Cozo Clojure](https://github.com/cozodb/cozo-clj) （[镜像](https://gitee.com/cozodb/cozo-clj)）：在 JVM Clojure 中使用 9 | * [Cozo Android](https://github.com/cozodb/cozo-lib-android) （[镜像](https://gitee.com/cozodb/cozo-lib-android)）：在安卓中使用 10 | 11 | 下面几个小节介绍在上面几个库不支持你的平台时如何从源码编译此库。 12 | 13 | ## 为 JDK 编译 14 | 15 | 首先安装 Rust 工具链，然后： 16 | ```bash 17 | cargo build --release -p cozo_java -F storage-rocksdb 18 | ``` 19 | 20 | ## 为安卓编译 21 | 22 | 为安卓编译较为复杂，以下仅做简要叙述。 23 | 24 | 首先，在编译时请不要使用 `-F storage-rocksdb` 选项，除非你有能力在 `build.rs` 中做出大量调整使得 [cozorocks](../cozorocks) 能够成功为安卓编译。 25 | 26 | 然后，在 Rust 工具链中添加安卓目标，设置好安卓 NDK 以及其编译路径、库路径等。手动搞定这些非常复杂，不过 [这里](https://github.com/cross-rs/cross) 有一些系统镜像可以省去你不少工作。 27 | 28 | 所有上面所述都设置好了之后，执行下面命令就可以编译安卓库了： 29 | 30 | ```bash 31 | for TARGET in aarch64-linux-android armv7-linux-androideabi i686-linux-android x86_64-linux-android; do 32 | cross build -p cozo_java --release --target=$TARGET 33 | done 34 | ``` 35 | 36 | 上面编译了多个架构的安卓库。如果只是想在常见的安卓手机平板上运行，`aarch64-linux-android` 一个目标其实就够了。 -------------------------------------------------------------------------------- /cozo-lib-java/README.md: -------------------------------------------------------------------------------- 1 | # Cozo-lib-java 2 | 3 | This crate provides the JNI bindings for using Cozo in Java/JVM languages/Android. 4 | 5 | You do not use this crate directly. Instead, use: 6 | 7 | * [Cozo Java](https://github.com/cozodb/cozo-lib-java) for Java or other JVM languages 8 | * [Cozo Clojure](https://github.com/cozodb/cozo-clj) for Clojure on JVM (you can also use the Java library, but this one is nicer) 9 | * [Cozo Android](https://github.com/cozodb/cozo-lib-android) for Android 10 | 11 | Keep reading only if the prebuilt binaries provided by these libraries do not suit your needs. 12 | 13 | ## Building for JDK 14 | 15 | With the Rust toolchain installed, 16 | ```bash 17 | cargo build --release -p cozo_java -F storage-rocksdb 18 | ``` 19 | 20 | ## Building for Android 21 | 22 | Building for Android is not easy, and we will be very sketchy. 23 | 24 | The first thing to note is that you should omit `-F storage-rocksdb` from the build command above, 25 | unless you are prepared to manually change lots of `build.rs` flags in 26 | [cozorocks](../cozorocks) to build the RocksDB dependency. 27 | 28 | Then, in addition to adding Android targets to the Rust toolchain, 29 | you also need to set up the Android NDK 30 | cross-compilation and libraries paths, etc. 31 | This is notoriously hard to get right, but fortunately 32 | you can just use the Docker image [here](https://github.com/cross-rs/cross) 33 | which has everything set up for you. 34 | 35 | When everything is set up correctly, the following command show complete without errors: 36 | 37 | ```bash 38 | for TARGET in aarch64-linux-android armv7-linux-androideabi i686-linux-android x86_64-linux-android; do 39 | cross build -p cozo_java --release --target=$TARGET 40 | done 41 | ``` 42 | 43 | For running on modern Android phones, the single target `aarch64-linux-android` is probably enough. -------------------------------------------------------------------------------- /cozo-lib-java/org_cozodb_CozoJavaBridge.h: -------------------------------------------------------------------------------- 1 | /* DO NOT EDIT THIS FILE - it is machine generated */ 2 | #include 3 | /* Header for class org_cozodb_CozoJavaBridge */ 4 | 5 | #ifndef _Included_org_cozodb_CozoJavaBridge 6 | #define _Included_org_cozodb_CozoJavaBridge 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | /* 11 | * Class: org_cozodb_CozoJavaBridge 12 | * Method: openDb 13 | * Signature: (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)I 14 | */ 15 | JNIEXPORT jint JNICALL Java_org_cozodb_CozoJavaBridge_openDb 16 | (JNIEnv *, jclass, jstring, jstring, jstring); 17 | 18 | /* 19 | * Class: org_cozodb_CozoJavaBridge 20 | * Method: closeDb 21 | * Signature: (I)Z 22 | */ 23 | JNIEXPORT jboolean JNICALL Java_org_cozodb_CozoJavaBridge_closeDb 24 | (JNIEnv *, jclass, jint); 25 | 26 | /* 27 | * Class: org_cozodb_CozoJavaBridge 28 | * Method: runQuery 29 | * Signature: (ILjava/lang/String;Ljava/lang/String;)Ljava/lang/String; 30 | */ 31 | JNIEXPORT jstring JNICALL Java_org_cozodb_CozoJavaBridge_runQuery 32 | (JNIEnv *, jclass, jint, jstring, jstring); 33 | 34 | /* 35 | * Class: org_cozodb_CozoJavaBridge 36 | * Method: exportRelations 37 | * Signature: (ILjava/lang/String;)Ljava/lang/String; 38 | */ 39 | JNIEXPORT jstring JNICALL Java_org_cozodb_CozoJavaBridge_exportRelations 40 | (JNIEnv *, jclass, jint, jstring); 41 | 42 | /* 43 | * Class: org_cozodb_CozoJavaBridge 44 | * Method: importRelations 45 | * Signature: (ILjava/lang/String;)Ljava/lang/String; 46 | */ 47 | JNIEXPORT jstring JNICALL Java_org_cozodb_CozoJavaBridge_importRelations 48 | (JNIEnv *, jclass, jint, jstring); 49 | 50 | /* 51 | * Class: org_cozodb_CozoJavaBridge 52 | * Method: backup 53 | * Signature: (ILjava/lang/String;)Ljava/lang/String; 54 | */ 55 | JNIEXPORT jstring JNICALL Java_org_cozodb_CozoJavaBridge_backup 56 | (JNIEnv *, jclass, jint, jstring); 57 | 58 | /* 59 | * Class: org_cozodb_CozoJavaBridge 60 | * Method: restore 61 | * Signature: (ILjava/lang/String;)Ljava/lang/String; 62 | */ 63 | JNIEXPORT jstring JNICALL Java_org_cozodb_CozoJavaBridge_restore 64 | (JNIEnv *, jclass, jint, jstring); 65 | 66 | /* 67 | * Class: org_cozodb_CozoJavaBridge 68 | * Method: importFromBackup 69 | * Signature: (ILjava/lang/String;)Ljava/lang/String; 70 | */ 71 | JNIEXPORT jstring JNICALL Java_org_cozodb_CozoJavaBridge_importFromBackup 72 | (JNIEnv *, jclass, jint, jstring); 73 | 74 | #ifdef __cplusplus 75 | } 76 | #endif 77 | #endif 78 | -------------------------------------------------------------------------------- /cozo-lib-nodejs/.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | native/index.node 3 | **/node_modules 4 | **/.DS_Store 5 | npm-debug.log* 6 | _test_db/ 7 | build/ 8 | native/ 9 | .idea/ 10 | -------------------------------------------------------------------------------- /cozo-lib-nodejs/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "cozo-node" 3 | version = "0.7.6" 4 | description = "Cozo database for NodeJS" 5 | authors = ["Ziyang Hu"] 6 | license = "MPL-2.0" 7 | homepage = "https://www.cozodb.org" 8 | repository = "https://github.com/cozodb/cozo" 9 | documentation = "https://docs.cozodb.org" 10 | edition = "2021" 11 | exclude = ["index.node"] 12 | 13 | [lib] 14 | crate-type = ["cdylib"] 15 | 16 | [features] 17 | #! # Features 18 | 19 | ## Enables the `minimal`, `requests` and `graph-algo` features 20 | compact = ["minimal", "requests", "graph-algo"] 21 | ## Enables the `storage-sqlite` and `graph-algo` features 22 | mobile = ["storage-sqlite", "graph-algo"] 23 | ## Enables the `minimal`, `requests` and `graph-algo` features in single threaded mode 24 | compact-single-threaded = ["minimal", "requests", "graph-algo"] 25 | ## Enables the `storage-sqlite` feature 26 | minimal = ["storage-sqlite"] 27 | ## Enables the [Sqlite](https://www.sqlite.org/index.html) backend, also allows backup and restore with Sqlite data files. 28 | storage-sqlite = ["cozo/storage-sqlite"] 29 | ## Enables the [RocksDB](http://rocksdb.org/) backend 30 | storage-rocksdb = ["cozo/storage-rocksdb"] 31 | ## Enables the graph algorithms 32 | graph-algo = ["cozo/graph-algo"] 33 | ## Allows the utilities to make web requests to fetch data 34 | requests = ["cozo/requests"] 35 | ## Uses jemalloc as the global allocator, can make a difference in performance 36 | jemalloc = ["cozo/jemalloc"] 37 | ## Enables io-uring option for the RocksDB storage 38 | io-uring = ["cozo/io-uring"] 39 | 40 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 41 | 42 | [dependencies] 43 | cozo = { version = "0.7.6", path = "../cozo-core", default-features = false } 44 | lazy_static = "1.4.0" 45 | crossbeam = "0.8.4" 46 | miette = "5.10.0" 47 | serde_json = "1.0.116" 48 | rayon = "1.10.0" 49 | 50 | [dependencies.neon] 51 | version = "0.10" 52 | default-features = false 53 | features = ["napi-6", "channel-api"] 54 | -------------------------------------------------------------------------------- /cozo-lib-nodejs/README-zh.md: -------------------------------------------------------------------------------- 1 | # Cozo NodeJS 库 2 | 3 | [![cozo-node](https://img.shields.io/npm/v/cozo-node)](https://www.npmjs.com/package/cozo-node) 4 | 5 | NodeJS 的嵌入式 [CozoDB](https://www.cozodb.org) 库。 6 | 7 | 本文叙述的是如何安装设置库本身。有关如何使用 CozoDB（CozoScript）的信息，见 [文档](https://docs.cozodb.org/zh_CN/latest/index.html) 。 8 | 9 | ## 安装 10 | 11 | ```bash 12 | npm install --save cozo-node 13 | ``` 14 | 15 | 安装过程中需要从 [GitHub 发布页](https://github.com/cozodb/cozo-lib-nodejs/releases/) 下载二进制文件。如果因为网络问题失败，可以使用此 [镜像](https://github.com/cozodb/cozo-lib-nodejs/releases/) ，使用方法为 16 | 17 | ```bash 18 | npm install --save cozo-node --cozo_node_prebuilt_binary_host_mirror=https://gitee.com/cozodb/cozo-lib-nodejs/releases/download/ 19 | ``` 20 | 21 | 注意：如果你用的是 Yarn 而不是 NPM，类似的命令 [可能不奏效](https://github.com/mapbox/node-pre-gyp/issues/514) 。 22 | 23 | 如果你的操作系统、平台不是常见的平台，可能会报错说找不到预编译库。这种情况下可以参见后面关于如何从源码编译的内容。 24 | 25 | ## 用法 26 | 27 | ```javascript 28 | const {CozoDb} = require('cozo-node') 29 | 30 | const db = new CozoDb() 31 | 32 | function printQuery(query, params) { 33 | db.run(query, params) 34 | .then(data => console.log(data)) 35 | .catch(err => console.error(err.display || err.message)) 36 | } 37 | 38 | printQuery("?[] <- [['hello', 'world!']]") 39 | printQuery("?[] <- [['hello', 'world', $name]]", {"name": "JavaScript"}) 40 | printQuery("?[a] <- [[1, 2]]") 41 | ``` 42 | 43 | ### 基本 API 44 | 45 | ```ts 46 | class CozoDb { 47 | /** 48 | * 构造函数 49 | * 50 | * @param engine: 默认为 'mem'，即纯内存的非持久化存储。其他值可以是 'sqlite'、'rocksdb' 等 51 | * @param path: 存储文件或文件夹的路径，默认为 'data.db'。在 'mem' 引擎下无用。 52 | * @param options: 默认为 {}，在 NodeJS 支持的引擎中无用。 53 | */ 54 | constructor(engine: string, path: string, options: object): CozoDb; 55 | 56 | /** 57 | * 关闭数据库，并释放其原生资源。如果不调用此方法而直接删除数据库的变量，则会造成原生资源泄漏。 58 | */ 59 | close(): void; 60 | 61 | /** 62 | * 执行查询文本 63 | * 64 | * @param script: 查询文本 65 | * @param params: 传入的参数，默认为 {} 66 | */ 67 | async run(script: string, params: object): object; 68 | 69 | /** 70 | * 导出存储表 71 | * 72 | * @param relations: 需要导出的存储表名称 73 | */ 74 | async exportRelations(relations: Array): object; 75 | 76 | /** 77 | * 导入数据至存储表 78 | * 79 | * 注意：以此方法导入数据不会激活存储表上任何的触发器。 80 | * 81 | * @param data: 导入的表以及数据，格式与 `exportRelations` 返回的相同 82 | */ 83 | async importRelations(data: object): object; 84 | 85 | /** 86 | * 备份数据库 87 | * 88 | * @param path: 备份文件路径 89 | */ 90 | async backup(path: string): object; 91 | 92 | /** 93 | * 从备份文件恢复数据至当前数据库。若当前数据库非空，则报错。 94 | * 95 | * @param path: 备份文件路径 96 | */ 97 | async restore(path: string): object; 98 | 99 | /** 100 | * 将备份文件中指定存储表里的数据插入当前数据库中同名表里。 101 | * 102 | * 注意：以此方法导入数据不会激活存储表上任何的触发器。 103 | * 104 | * @param path: 备份文件路径 105 | * @param rels: 需导入数据的表名 106 | */ 107 | async importRelationsFromBackup(path: string, rels: Array): object; 108 | } 109 | ``` 110 | 111 | 更多信息 [见此](https://docs.cozodb.org/zh_CN/latest/nonscript.html) 。 112 | 113 | ### 进阶 API 114 | 115 | Cozo 支持多语句事务、存储表更新时回调以及使用 NodeJS 实现自定义固定规则，与 [Python 库](https://github.com/cozodb/pycozo) （[国内镜像](https://gitee.com/cozodb/pycozo)）所支持的差不多。可参考此 [示例](./example.js)。 116 | 117 | ## 编译 118 | 119 | 编译 `cozo-node` 需要 [Rust 工具链](https://rustup.rs)。运行 120 | 121 | ```bash 122 | cargo build --release -p cozo-node -F compact -F storage-rocksdb 123 | ``` 124 | 125 | 完成后，动态链接库可以在 `../target/` 文件夹中找到（具体文件名根据平台与操作系统会有差异，一般来说 Linux 上 扩展名为 `.so`，Mac 上为 `.dylib`，Windows 上为 `.dll`）。 126 | 将找到的动态库拷贝为此目录下的 `native/6/cozo_node_prebuilt.node` 文件（中间目录若不存在，则需建立）。 127 | 128 | 如果一切操作正确，在此目录下执行下列命令则会正常返回： 129 | 130 | ```bash 131 | node example.js 132 | ``` -------------------------------------------------------------------------------- /cozo-lib-nodejs/example.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022, The Cozo Project Authors. 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 5 | * If a copy of the MPL was not distributed with this file, 6 | * You can obtain one at https://mozilla.org/MPL/2.0/. 7 | */ 8 | 9 | const {Buffer} = require('node:buffer') 10 | const {CozoDb} = require('.'); 11 | 12 | (async () => { 13 | const db = new CozoDb() 14 | try { 15 | const result = await db.run('?[a] <- [["hello"], ["world"], [$b]]', {b: Buffer.alloc(8, 255)}); 16 | console.log(result.rows) 17 | } catch (e) { 18 | console.error(e) 19 | } 20 | const cb_id = db.registerCallback('test', (op, new_rows, old_rows) => { 21 | console.log(`${op} ${JSON.stringify(new_rows)} ${JSON.stringify(old_rows)}`) 22 | }) 23 | 24 | await db.run(`?[a] <- [[1],[2],[3]] :create test {a}`); 25 | 26 | db.registerNamedRule('Pipipy', 1, async (inputs, options) => { 27 | console.log(`rule inputs: ${JSON.stringify(inputs)} ${JSON.stringify(options)}`) 28 | await sleep(1000); 29 | return inputs[0].map((row) => [row[0] * options.mul]) 30 | }) 31 | 32 | try { 33 | let r = await db.run(` 34 | rel[] <- [[1],[2]] 35 | 36 | ?[a] <~ Pipipy(rel[], mul: 3) 37 | `); 38 | console.log(r); 39 | } catch (e) { 40 | console.error(e.display); 41 | } 42 | 43 | console.log((await db.exportRelations(['test']))['test']['rows']) 44 | 45 | const tx = db.multiTransact(true); 46 | await tx.run(':create a {a}'); 47 | await tx.run('?[a] <- [[1]] :put a {a}'); 48 | try { 49 | await tx.run(':create a {a}') 50 | } catch (e) { 51 | } 52 | await tx.run('?[a] <- [[2]] :put a {a}') 53 | await tx.run('?[a] <- [[3]] :put a {a}') 54 | tx.commit() 55 | 56 | const res = await db.run('?[a] := *a[a]'); 57 | console.log(res); 58 | 59 | db.unregisterCallback(cb_id) 60 | db.unregisterNamedRule('Pipipy') 61 | })() 62 | 63 | function sleep(ms) { 64 | return new Promise((resolve, reject) => { 65 | setTimeout(() => { 66 | resolve() 67 | }, ms); 68 | }) 69 | } -------------------------------------------------------------------------------- /cozo-lib-nodejs/index.d.ts: -------------------------------------------------------------------------------- 1 | declare module "cozo-node" { 2 | export class CozoDb { 3 | /** 4 | * Constructor 5 | * 6 | * @param engine: defaults to 'mem', the in-memory non-persistent engine. 7 | * 'sqlite', 'rocksdb' and maybe others are available, 8 | * depending on compile time flags. 9 | * @param path: path to store the data on disk, defaults to 'data.db', 10 | * may not be applicable for some engines such as 'mem' 11 | * @param options: defaults to {}, ignored by all the engines in the published NodeJS artefact 12 | */ 13 | constructor(engine?: string, path?: string, options?: object); 14 | 15 | /** 16 | * You must call this method for any database you no longer want to use: 17 | * otherwise the native resources associated with it may linger for as 18 | * long as your program runs. Simply `delete` the variable is not enough. 19 | */ 20 | close(): void; 21 | 22 | /** 23 | * Runs a query 24 | * 25 | * @param script: the query 26 | * @param params: the parameters as key-value pairs, defaults to {} 27 | */ 28 | run(script: string, params?: Record): Promise; 29 | 30 | /** 31 | * Export several relations 32 | * 33 | * @param relations: names of relations to export, in an array. 34 | */ 35 | exportRelations(relations: Array): Promise; 36 | 37 | /** 38 | * Import several relations. 39 | * 40 | * Note that triggers are _not_ run for the relations, if any exists. 41 | * If you need to activate triggers, use queries with parameters. 42 | * 43 | * @param data: in the same form as returned by `exportRelations`. The relations 44 | * must already exist in the database. 45 | */ 46 | importRelations(data: object): Promise; 47 | 48 | /** 49 | * Backup database 50 | * 51 | * @param path: path to file to store the backup. 52 | */ 53 | backup(path: string): Promise; 54 | 55 | /** 56 | * Restore from a backup. Will fail if the current database already contains data. 57 | * 58 | * @param path: path to the backup file. 59 | */ 60 | restore(path: string): Promise; 61 | 62 | /** 63 | * Import several relations from a backup. The relations must already exist in the database. 64 | * 65 | * Note that triggers are _not_ run for the relations, if any exists. 66 | * If you need to activate triggers, use queries with parameters. 67 | * 68 | * @param path: path to the backup file. 69 | * @param rels: the relations to import. 70 | */ 71 | importRelationsFromBackup(path: string, rels: Array): Promise; 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /cozo-lib-nodejs/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cozo-node", 3 | "version": "0.7.6", 4 | "description": "Cozo database for NodeJS", 5 | "main": "index", 6 | "types": "index.d.ts", 7 | "files": [ 8 | "index.js", 9 | "index.d.ts", 10 | "LICENSE.txt" 11 | ], 12 | "binary": { 13 | "module_name": "cozo_node_prebuilt", 14 | "host": "https://github.com/cozodb/cozo-lib-nodejs/releases/download/", 15 | "remote_path": "{version}", 16 | "package_name": "{napi_build_version}-{platform}-{arch}.tar.gz", 17 | "module_path": "./native/{napi_build_version}", 18 | "pkg_path": ".", 19 | "napi_versions": [ 20 | 6 21 | ] 22 | }, 23 | "scripts": { 24 | "install": "node-pre-gyp install", 25 | "package": "node-pre-gyp package" 26 | }, 27 | "author": "Ziyang Hu", 28 | "license": "MIT", 29 | "devDependencies": { 30 | "cargo-cp-artifact": "^0.1" 31 | }, 32 | "repository": { 33 | "type": "git", 34 | "url": "git+https://github.com/cozodb/cozo-lib-nodejs.git" 35 | }, 36 | "keywords": [ 37 | "database", 38 | "datalog", 39 | "graph" 40 | ], 41 | "bugs": { 42 | "url": "https://github.com/cozodb/cozo/issues" 43 | }, 44 | "homepage": "https://www.cozodb.org", 45 | "dependencies": { 46 | "@mapbox/node-pre-gyp": "^1.0.10" 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /cozo-lib-nodejs/test_build.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022, The Cozo Project Authors. 3 | # 4 | # This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 5 | # If a copy of the MPL was not distributed with this file, 6 | # You can obtain one at https://mozilla.org/MPL/2.0/. 7 | # 8 | 9 | cargo build -p cozo-node -F compact -F storage-rocksdb 10 | mv ../target/debug/libcozo_node.dylib native/6/cozo_node_prebuilt.node 11 | node example.js -------------------------------------------------------------------------------- /cozo-lib-python/.github_disabled/workflows/CI.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | - master 8 | pull_request: 9 | 10 | jobs: 11 | linux: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v3 15 | - uses: messense/maturin-action@v1 16 | with: 17 | manylinux: auto 18 | command: build 19 | args: --release --sdist -o dist --find-interpreter 20 | - name: Upload wheels 21 | uses: actions/upload-artifact@v2 22 | with: 23 | name: wheels 24 | path: dist 25 | 26 | windows: 27 | runs-on: windows-latest 28 | steps: 29 | - uses: actions/checkout@v3 30 | - uses: messense/maturin-action@v1 31 | with: 32 | command: build 33 | args: --release -o dist --find-interpreter 34 | - name: Upload wheels 35 | uses: actions/upload-artifact@v2 36 | with: 37 | name: wheels 38 | path: dist 39 | 40 | macos: 41 | runs-on: macos-latest 42 | steps: 43 | - uses: actions/checkout@v3 44 | - uses: messense/maturin-action@v1 45 | with: 46 | command: build 47 | args: --release -o dist --universal2 --find-interpreter 48 | - name: Upload wheels 49 | uses: actions/upload-artifact@v2 50 | with: 51 | name: wheels 52 | path: dist 53 | 54 | release: 55 | name: Release 56 | runs-on: ubuntu-latest 57 | if: "startsWith(github.ref, 'refs/tags/')" 58 | needs: [ macos, windows, linux ] 59 | steps: 60 | - uses: actions/download-artifact@v2 61 | with: 62 | name: wheels 63 | - name: Publish to PyPI 64 | uses: messense/maturin-action@v1 65 | env: 66 | MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} 67 | with: 68 | command: upload 69 | args: --skip-existing * -------------------------------------------------------------------------------- /cozo-lib-python/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | 3 | # Byte-compiled / optimized / DLL files 4 | __pycache__/ 5 | .pytest_cache/ 6 | *.py[cod] 7 | 8 | # C extensions 9 | *.so 10 | 11 | # Distribution / packaging 12 | .Python 13 | .venv/ 14 | env/ 15 | bin/ 16 | build/ 17 | develop-eggs/ 18 | dist/ 19 | eggs/ 20 | lib/ 21 | lib64/ 22 | parts/ 23 | sdist/ 24 | var/ 25 | include/ 26 | man/ 27 | venv/ 28 | *.egg-info/ 29 | .installed.cfg 30 | *.egg 31 | 32 | # Installer logs 33 | pip-log.txt 34 | pip-delete-this-directory.txt 35 | pip-selfcheck.json 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .cache 42 | nosetests.xml 43 | coverage.xml 44 | 45 | # Translations 46 | *.mo 47 | 48 | # Mr Developer 49 | .mr.developer.cfg 50 | .project 51 | .pydevproject 52 | 53 | # Rope 54 | .ropeproject 55 | 56 | # Django stuff: 57 | *.log 58 | *.pot 59 | 60 | .DS_Store 61 | 62 | # Sphinx documentation 63 | docs/_build/ 64 | 65 | # PyCharm 66 | .idea/ 67 | 68 | # VSCode 69 | .vscode/ 70 | 71 | # Pyenv 72 | .python-version 73 | _test_db/ 74 | -------------------------------------------------------------------------------- /cozo-lib-python/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "cozo_py" 3 | version = "0.7.6" 4 | edition = "2021" 5 | description = "Cozo database for python" 6 | authors = ["Ziyang Hu"] 7 | license = "MPL-2.0" 8 | homepage = "https://www.cozodb.org" 9 | repository = "https://github.com/cozodb/cozo" 10 | documentation = "https://docs.cozodb.org" 11 | 12 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 13 | 14 | [lib] 15 | name = "cozo_embedded" 16 | crate-type = ["cdylib"] 17 | 18 | [features] 19 | #! # Features 20 | 21 | ## Enables the `minimal`, `requests` and `graph-algo` features 22 | compact = ["minimal", "requests", "graph-algo"] 23 | ## Enables the `storage-sqlite` and `graph-algo` features 24 | mobile = ["storage-sqlite", "graph-algo"] 25 | ## Enables the `minimal`, `requests` and `graph-algo` features in single threaded mode 26 | compact-single-threaded = ["minimal", "requests", "graph-algo"] 27 | ## Enables the `storage-sqlite` feature 28 | minimal = ["storage-sqlite"] 29 | ## Enables the [Sqlite](https://www.sqlite.org/index.html) backend, also allows backup and restore with Sqlite data files. 30 | storage-sqlite = ["cozo/storage-sqlite"] 31 | ## Enables the [RocksDB](http://rocksdb.org/) backend 32 | storage-rocksdb = ["cozo/storage-rocksdb"] 33 | ## Enables the graph algorithms 34 | graph-algo = ["cozo/graph-algo"] 35 | ## Allows the utilities to make web requests to fetch data 36 | requests = ["cozo/requests"] 37 | ## Uses jemalloc as the global allocator, can make a difference in performance 38 | jemalloc = ["cozo/jemalloc"] 39 | ## Enables io-uring option for the RocksDB storage 40 | io-uring = ["cozo/io-uring"] 41 | 42 | 43 | [dependencies] 44 | cozo = { version = "0.7.6", path = "../cozo-core", default-features = false } 45 | pyo3 = { version = "0.21.2", features = ["extension-module", "abi3", "abi3-py37"] } 46 | miette = "5.10.0" 47 | serde_json = "1.0.116" 48 | rayon = "1.10.0" 49 | -------------------------------------------------------------------------------- /cozo-lib-python/README-zh.md: -------------------------------------------------------------------------------- 1 | # Cozo Python 库 2 | 3 | [![pypi](https://img.shields.io/pypi/v/cozo_embedded)](https://pypi.org/project/cozo_embedded/) 4 | 5 | [CozoDB](https://www.cozodb.org) 的 Python 嵌入式库 `cozo_embedded` 的源代码。 6 | 7 | 一般来说你应该使用 [PyCozo](https://github.com/cozodb/pycozo) 库（ [Gitee 镜像](https://gitee.com/cozodb/pycozo) ），而不是直接使用此库。 8 | 9 | 编译此库需要安装 Rust 工具链以及 [maturin](https://github.com/PyO3/maturin) 。安装好后运行 10 | 11 | ```bash 12 | maturin build -F compact -F storage-rocksdb --release 13 | ``` 14 | 15 | 更多选项参见 maturin 的 [文档](https://www.maturin.rs/) 。 -------------------------------------------------------------------------------- /cozo-lib-python/README.md: -------------------------------------------------------------------------------- 1 | # cozo-lib-python 2 | 3 | [![pypi](https://img.shields.io/pypi/v/cozo_embedded)](https://pypi.org/project/cozo_embedded/) 4 | 5 | Native bindings for embedding [CozoDB](https://www.cozodb.org) in Python, providing the 6 | `cozo_embedded` package. 7 | 8 | You are not supposed to be using this package directly in your code. Use [PyCozo](https://github.com/cozodb/pycozo), 9 | which depends on this package. 10 | 11 | To build this package, you need to install the Rust toolchain 12 | as well as the [maturin](https://github.com/PyO3/maturin) python package. 13 | Then run 14 | 15 | ```bash 16 | maturin build -F compact -F storage-rocksdb --release 17 | ``` 18 | 19 | Refer maturin's docs for more information about how to [develop](https://www.maturin.rs/develop.html) 20 | and [build](https://www.maturin.rs/distribution.html) this package. -------------------------------------------------------------------------------- /cozo-lib-python/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | PYO3_NO_PYTHON=1 maturin build -F compact -F storage-rocksdb --release --strip -------------------------------------------------------------------------------- /cozo-lib-python/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["maturin>=0.13,<0.14"] 3 | build-backend = "maturin" 4 | 5 | [project] 6 | name = "cozo_embedded" 7 | requires-python = ">=3.7" 8 | classifiers = [ 9 | "Topic :: Database", 10 | "Programming Language :: Rust", 11 | "Programming Language :: Python :: Implementation :: CPython", 12 | "Programming Language :: Python :: Implementation :: PyPy", 13 | ] 14 | 15 | 16 | -------------------------------------------------------------------------------- /cozo-lib-swift/.gitignore: -------------------------------------------------------------------------------- 1 | generated 2 | CozoSwiftBridge 3 | .idea -------------------------------------------------------------------------------- /cozo-lib-swift/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "cozo-swift" 3 | version = "0.7.6" 4 | edition = "2021" 5 | description = "Cozo database for Swift" 6 | authors = ["Ziyang Hu"] 7 | license = "MPL-2.0" 8 | homepage = "https://www.cozodb.org" 9 | repository = "https://github.com/cozodb/cozo" 10 | documentation = "https://docs.cozodb.org" 11 | 12 | [lib] 13 | crate-type = ["staticlib"] 14 | 15 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 16 | 17 | [features] 18 | ## Enables the `minimal`, `requests` and `graph-algo` features 19 | compact = ["minimal", "requests", "graph-algo"] 20 | ## Enables the `storage-sqlite` and `graph-algo` features 21 | mobile = ["storage-sqlite", "graph-algo"] 22 | ## Enables the `minimal`, `requests` and `graph-algo` features in single threaded mode 23 | compact-single-threaded = ["minimal", "requests", "graph-algo"] 24 | ## Enables the `storage-sqlite` feature 25 | minimal = ["storage-sqlite"] 26 | ## Enables the [Sqlite](https://www.sqlite.org/index.html) backend, also allows backup and restore with Sqlite data files. 27 | storage-sqlite = ["cozo/storage-sqlite"] 28 | ## Enables the [RocksDB](http://rocksdb.org/) backend 29 | storage-rocksdb = ["cozo/storage-rocksdb"] 30 | ## Enables the graph algorithms 31 | graph-algo = ["cozo/graph-algo"] 32 | ## Allows the utilities to make web requests to fetch data 33 | requests = ["cozo/requests"] 34 | ## Uses jemalloc as the global allocator, can make a difference in performance 35 | jemalloc = ["cozo/jemalloc"] 36 | ## Enables io-uring option for the RocksDB storage 37 | io-uring = ["cozo/io-uring"] 38 | 39 | [build-dependencies] 40 | swift-bridge-build = "0.1.41" 41 | 42 | [dependencies] 43 | cozo = { version = "0.7.6", path = "../cozo-core", default-features = false } 44 | swift-bridge = "0.1.53" 45 | -------------------------------------------------------------------------------- /cozo-lib-swift/CozoSwiftBridge.podspec: -------------------------------------------------------------------------------- 1 | Pod::Spec.new do |spec| 2 | spec.name = "CozoSwiftBridge" 3 | spec.version = "0.7.6" 4 | spec.summary = "CozoDB for Swift" 5 | spec.description = "This library allows you to use CozoDB embedded in your Swift application" 6 | spec.homepage = "https://github.com/cozodb/cozo/" 7 | spec.license = "MPL-2.0" 8 | spec.author = { "Ziyang Hu" => "hu.ziyang@cantab.net" } 9 | spec.source = { :http => "https://github.com/cozodb/cozo/releases/download/v0.7.6/CozoSwiftBridge-0.7.6.tgz" } 10 | spec.source_files = "Sources/CozoSwiftBridge/*" 11 | spec.vendored_frameworks = "RustXcframework.xcframework" 12 | spec.requires_arc = true 13 | spec.swift_version = "5.0" 14 | spec.osx.deployment_target = "10.13" 15 | spec.ios.deployment_target = "11.0" 16 | spec.dependency "SwiftyJSON", "~> 4.0" 17 | end 18 | -------------------------------------------------------------------------------- /cozo-lib-swift/Package.swift: -------------------------------------------------------------------------------- 1 | // swift-tools-version:5.5.0 2 | import PackageDescription 3 | let package = Package( 4 | name: "CozoSwiftBridge", 5 | products: [ 6 | .library( 7 | name: "CozoSwiftBridge", 8 | targets: ["CozoSwiftBridge"]), 9 | ], 10 | dependencies: [ 11 | .package(url: "https://github.com/SwiftyJSON/SwiftyJSON.git", from: "4.0.0"), 12 | ], 13 | targets: [ 14 | .binaryTarget( 15 | name: "RustXcframework", 16 | path: "RustXcframework.xcframework" 17 | ), 18 | .target( 19 | name: "CozoSwiftBridge", 20 | dependencies: ["RustXcframework", "SwiftyJSON"]) 21 | ] 22 | ) 23 | -------------------------------------------------------------------------------- /cozo-lib-swift/README-zh.md: -------------------------------------------------------------------------------- 1 | # Cozo Swift 库（仅支持苹果硬件） 2 | 3 | [![pod](https://img.shields.io/cocoapods/v/CozoSwiftBridge)](https://github.com/cozodb/cozo/tree/main/cozo-lib-swift) 4 | 5 | 本文叙述的是如何安装设置库本身。有关如何使用 CozoDB（CozoScript）的信息，见 [文档](https://docs.cozodb.org/zh_CN/latest/index.html) 。 6 | 7 | 本库可在 macOS（苹果 ARM 或 Intel 芯片）以及 iOS（iPad、iPhone、模拟器）中使用。 8 | 9 | 预编译的二进制包里，持久化引擎中，只有 `storage-sqlite` 在此库中可用。如果需要其他引擎请从源码编译。 10 | 11 | ## 安装 12 | 13 | ### CocoaPods 14 | 15 | ```ruby 16 | target 'YourApp' do 17 | use_frameworks! 18 | 19 | pod 'CozoSwiftBridge', '~> 0.7.1' 20 | end 21 | ``` 22 | 23 | ### Swift Package Manager (SPM) 24 | 25 | 从 [GitHub 发布页面](https://github.com/cozodb/cozo/releases) 下载名为 `CozoSwiftBridge.tgz` 的包，然后手动导入至 XCode 中。详见 [英文文档](./README.md)。 26 | 27 | ## 基本使用 28 | 29 | ```swift 30 | import CozoSwiftBridge 31 | 32 | { 33 | let path = NSHomeDirectory() 34 | let file = path + "/cozo-data.db" 35 | let db = CozoDB("sqlite", file) 36 | let res = try! db.run("?[] <- [[1,2,3]]").toString() 37 | } 38 | ``` 39 | 40 | 上例中创建了一个 SQLite 引擎支持的数据库。如果要用纯内存引擎： 41 | 42 | ```swift 43 | let db = CozoDB() 44 | ``` 45 | 46 | ### API 47 | 48 | ``` 49 | public class CozoDB { 50 | public let db: DbInstance 51 | 52 | /** 53 | * 构造一个纯内存引擎的数据库 54 | */ 55 | public init(); 56 | 57 | /** 58 | * 构造一个数据库 59 | * 60 | * `kind`: 引擎类型，`mem` 或 `sqlite`。 61 | * `path`: 存储文件的路径，仅在 `sqlite` 引擎下有效。 62 | */ 63 | public init(kind: String, path: String) throws; 64 | 65 | /** 66 | * 执行查询文本 67 | * 68 | * `query`: 查询文本 69 | */ 70 | public func run(_ query: String) throws -> [NamedRow]; 71 | 72 | /** 73 | * 执行查询文本 74 | * 75 | * `query`: 查询文本 76 | * `params`: 文本中可用的参数 77 | */ 78 | public func run(_ query: String, params: JSON) throws -> [NamedRow]; 79 | 80 | /** 81 | * 导出纯出表至 JSON 82 | * 83 | * `relations`: 需导出的表名 84 | */ 85 | public func exportRelations(relations: [String]) throws -> JSON; 86 | 87 | /** 88 | * 导入数据至存储表中 89 | * 90 | * 注意此方法不会激活任何触发器。 91 | * 92 | * `data`: 导入内容，与 `exportRelations` 返回的格式相同 93 | */ 94 | public func importRelations(data: JSON) throws; 95 | 96 | /** 97 | * 备份数据库 98 | * 99 | * `path`: 备份路径 100 | */ 101 | public func backup(path: String) throws; 102 | 103 | /** 104 | * 将备份恢复到当前数据库 105 | * 106 | * `path`: 备份路径 107 | */ 108 | public func restore(path: String) throws; 109 | 110 | /** 111 | * 将备份中表里的数据插入当前数据库中选定的同名表中 112 | * 113 | * 注意此方法不会激活任何触发器。 114 | * 115 | * `path`: 备份路径 116 | * `relations`: 需要导入数据的表名 117 | */ 118 | public func importRelationsFromBackup(path: String, relations: [String]) throws; 119 | } 120 | ``` 121 | 122 | 更多信息 [见此](https://docs.cozodb.org/zh_CN/latest/nonscript.html) 。 123 | 124 | ## 编译 125 | 126 | 首先安装 [Rust 工具链](https://rustup.rs)。然后执行 [此批处理文件](build-rust.sh) 。建议执行时设置环境变量`CARGO_PROFILE_RELEASE_LTO=fat`：编译时间会变长，但是生成的库更快。 127 | 128 | 如果一切都没问题，则 `CozoSwiftBridge` 文件夹里会有编译好的文件。 129 | 130 | 如果想使用 RocksDB 引擎，则在批处理文件中，将以下两行 131 | ```bash 132 | cargo build -p cozo-swift -F compact --target x86_64-apple-darwin --release 133 | cargo build -p cozo-swift -F compact --target aarch64-apple-darwin --release 134 | ``` 135 | 改为 136 | ```bash 137 | cargo build -p cozo-swift -F compact -F storage-rocksdb --target x86_64-apple-darwin --release 138 | cargo build -p cozo-swift -F compact -F storage-rocksdb --target aarch64-apple-darwin --release 139 | ``` 140 | 注意，给 iOS 编译 RocksDB 不是一件简单的事情。 141 | 142 | 在使用生成的库时，需要在 XCode 中选择链接至 `libc++` 动态库。 -------------------------------------------------------------------------------- /cozo-lib-swift/build-rust.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | THISDIR=$(dirname $0) 5 | cd $THISDIR 6 | 7 | export SWIFT_BRIDGE_OUT_DIR="$(pwd)/generated" 8 | # Build the project for the desired platforms: 9 | cargo build -p cozo-swift -F compact --target x86_64-apple-darwin --release 10 | cargo build -p cozo-swift -F compact --target aarch64-apple-darwin --release 11 | mkdir -p ../target/universal-macos/release 12 | 13 | lipo \ 14 | ../target/aarch64-apple-darwin/release/libcozo_swift.a \ 15 | ../target/x86_64-apple-darwin/release/libcozo_swift.a -create -output \ 16 | ../target/universal-macos/release/libcozo_swift.a 17 | 18 | cargo build -p cozo-swift -F compact --target aarch64-apple-ios --release 19 | cargo build -p cozo-swift -F compact --target x86_64-apple-ios --release 20 | cargo build -p cozo-swift -F compact --target aarch64-apple-ios-sim --release 21 | 22 | mkdir -p ../target/universal-ios/release 23 | 24 | lipo \ 25 | ../target/aarch64-apple-ios-sim/release/libcozo_swift.a \ 26 | ../target/x86_64-apple-ios/release/libcozo_swift.a -create -output \ 27 | ../target/universal-ios/release/libcozo_swift.a 28 | 29 | swift-bridge-cli create-package \ 30 | --bridges-dir ./generated \ 31 | --out-dir CozoSwiftBridge \ 32 | --ios ../target/aarch64-apple-ios/release/libcozo_swift.a \ 33 | --simulator ../target/universal-ios/release/libcozo_swift.a \ 34 | --macos ../target/universal-macos/release/libcozo_swift.a \ 35 | --name CozoSwiftBridge 36 | 37 | cp CozoDB.swift CozoSwiftBridge/Sources/CozoSwiftBridge 38 | cp Package.swift CozoSwiftBridge/ 39 | cp CozoSwiftBridge.podspec CozoSwiftBridge/ 40 | 41 | VERSION=$(cat ../VERSION) 42 | 43 | cd CozoSwiftBridge 44 | 45 | tar cvzf ../../release/CozoSwiftBridge-$VERSION.tgz . 46 | 47 | cd .. -------------------------------------------------------------------------------- /cozo-lib-swift/build.rs: -------------------------------------------------------------------------------- 1 | use std::path::PathBuf; 2 | 3 | fn main() { 4 | let out_dir = PathBuf::from("./generated"); 5 | 6 | let bridges = vec!["src/lib.rs"]; 7 | for path in &bridges { 8 | println!("cargo:rerun-if-changed={path}"); 9 | } 10 | 11 | swift_bridge_build::parse_bridges(bridges) 12 | .write_all_concatenated(out_dir, env!("CARGO_PKG_NAME")); 13 | } 14 | -------------------------------------------------------------------------------- /cozo-lib-swift/publish_pod.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright 2022, The Cozo Project Authors. 4 | # 5 | # This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 6 | # If a copy of the MPL was not distributed with this file, 7 | # You can obtain one at https://mozilla.org/MPL/2.0/. 8 | # 9 | 10 | pod trunk push CozoSwiftBridge.podspec --allow-warnings --verbose -------------------------------------------------------------------------------- /cozo-lib-swift/src/lib.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022, The Cozo Project Authors. 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 5 | * If a copy of the MPL was not distributed with this file, 6 | * You can obtain one at https://mozilla.org/MPL/2.0/. 7 | */ 8 | 9 | use cozo::*; 10 | 11 | #[swift_bridge::bridge] 12 | mod ffi { 13 | extern "Rust" { 14 | type DbInstance; 15 | 16 | fn new_cozo_db(engine: &str, path: &str, options: &str) -> Option; 17 | 18 | fn run_script_str(&self, payload: &str, params: &str, immutable: bool) -> String; 19 | fn export_relations_str(&self, data: &str) -> String; 20 | fn import_relations_str(&self, data: &str) -> String; 21 | fn backup_db_str(&self, out_file: &str) -> String; 22 | fn restore_backup_str(&self, in_file: &str) -> String; 23 | fn import_from_backup_str(&self, data: &str) -> String; 24 | } 25 | } 26 | 27 | fn new_cozo_db(engine: &str, path: &str, options: &str) -> Option { 28 | let options = if options.is_empty() { "{}" } else { options }; 29 | match DbInstance::new_with_str(engine, path, options) { 30 | Ok(db) => Some(db), 31 | Err(err) => { 32 | eprintln!("{err}"); 33 | None 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /cozo-lib-wasm/.appveyor.yml: -------------------------------------------------------------------------------- 1 | install: 2 | - appveyor-retry appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe 3 | - if not defined RUSTFLAGS rustup-init.exe -y --default-host x86_64-pc-windows-msvc --default-toolchain nightly 4 | - set PATH=%PATH%;C:\Users\appveyor\.cargo\bin 5 | - rustc -V 6 | - cargo -V 7 | 8 | build: false 9 | 10 | test_script: 11 | - cargo test --locked 12 | -------------------------------------------------------------------------------- /cozo-lib-wasm/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | **/*.rs.bk 3 | Cargo.lock 4 | bin/ 5 | pkg/ 6 | wasm-pack.log 7 | -------------------------------------------------------------------------------- /cozo-lib-wasm/.travis.yml: -------------------------------------------------------------------------------- 1 | language: rust 2 | sudo: false 3 | 4 | cache: cargo 5 | 6 | matrix: 7 | include: 8 | 9 | # Builds with wasm-pack. 10 | - rust: beta 11 | env: RUST_BACKTRACE=1 12 | addons: 13 | firefox: latest 14 | chrome: stable 15 | before_script: 16 | - (test -x $HOME/.cargo/bin/cargo-install-update || cargo install cargo-update) 17 | - (test -x $HOME/.cargo/bin/cargo-generate || cargo install --vers "^0.2" cargo-generate) 18 | - cargo install-update -a 19 | - curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh -s -- -f 20 | script: 21 | - cargo generate --git . --name testing 22 | # Having a broken Cargo.toml (in that it has curlies in fields) anywhere 23 | # in any of our parent dirs is problematic. 24 | - mv Cargo.toml Cargo.toml.tmpl 25 | - cd testing 26 | - wasm-pack build 27 | - wasm-pack test --chrome --firefox --headless 28 | 29 | # Builds on nightly. 30 | - rust: nightly 31 | env: RUST_BACKTRACE=1 32 | before_script: 33 | - (test -x $HOME/.cargo/bin/cargo-install-update || cargo install cargo-update) 34 | - (test -x $HOME/.cargo/bin/cargo-generate || cargo install --vers "^0.2" cargo-generate) 35 | - cargo install-update -a 36 | - rustup target add wasm32-unknown-unknown 37 | script: 38 | - cargo generate --git . --name testing 39 | - mv Cargo.toml Cargo.toml.tmpl 40 | - cd testing 41 | - cargo check 42 | - cargo check --target wasm32-unknown-unknown 43 | - cargo check --no-default-features 44 | - cargo check --target wasm32-unknown-unknown --no-default-features 45 | - cargo check --no-default-features --features console_error_panic_hook 46 | - cargo check --target wasm32-unknown-unknown --no-default-features --features console_error_panic_hook 47 | - cargo check --no-default-features --features "console_error_panic_hook wee_alloc" 48 | - cargo check --target wasm32-unknown-unknown --no-default-features --features "console_error_panic_hook wee_alloc" 49 | 50 | # Builds on beta. 51 | - rust: beta 52 | env: RUST_BACKTRACE=1 53 | before_script: 54 | - (test -x $HOME/.cargo/bin/cargo-install-update || cargo install cargo-update) 55 | - (test -x $HOME/.cargo/bin/cargo-generate || cargo install --vers "^0.2" cargo-generate) 56 | - cargo install-update -a 57 | - rustup target add wasm32-unknown-unknown 58 | script: 59 | - cargo generate --git . --name testing 60 | - mv Cargo.toml Cargo.toml.tmpl 61 | - cd testing 62 | - cargo check 63 | - cargo check --target wasm32-unknown-unknown 64 | - cargo check --no-default-features 65 | - cargo check --target wasm32-unknown-unknown --no-default-features 66 | - cargo check --no-default-features --features console_error_panic_hook 67 | - cargo check --target wasm32-unknown-unknown --no-default-features --features console_error_panic_hook 68 | # Note: no enabling the `wee_alloc` feature here because it requires 69 | # nightly for now. 70 | -------------------------------------------------------------------------------- /cozo-lib-wasm/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "cozo-lib-wasm" 3 | version = "0.7.6" 4 | edition = "2021" 5 | description = "Cozo database for WASM" 6 | authors = ["Ziyang Hu"] 7 | license = "MPL-2.0" 8 | homepage = "https://www.cozodb.org" 9 | repository = "https://github.com/cozodb/cozo" 10 | documentation = "https://docs.cozodb.org" 11 | 12 | [lib] 13 | crate-type = ["cdylib", "rlib"] 14 | 15 | [features] 16 | default = ["console_error_panic_hook"] 17 | 18 | [dependencies] 19 | wasm-bindgen = "0.2.92" 20 | cozo = { version = "0.7.6", path = "../cozo-core", default-features = false, features = ["wasm"] } 21 | 22 | # The `console_error_panic_hook` crate provides better debugging of panics by 23 | # logging them with `console.error`. This is great for development, but requires 24 | # all the `std::fmt` and `std::panicking` infrastructure, so isn't great for 25 | # code size when deploying. 26 | console_error_panic_hook = { version = "0.1.6", optional = true } 27 | 28 | # `wee_alloc` is a tiny allocator for wasm that is only ~1K in code size 29 | # compared to the default allocator's ~10K. It is slower than the default 30 | # allocator, however. 31 | wee_alloc = { version = "0.4.5", optional = true } 32 | 33 | [dev-dependencies] 34 | wasm-bindgen-test = "0.3.13" 35 | 36 | #[profile.release] 37 | ## Tell `rustc` to optimize for small code size. 38 | #opt-level = "s" 39 | -------------------------------------------------------------------------------- /cozo-lib-wasm/README-zh.md: -------------------------------------------------------------------------------- 1 | # Cozo WASM 库（浏览器） 2 | 3 | 可以在浏览器中运行的 Cozo WASM 库。NodeJS 用户请使用 [原生库](../cozo-lib-nodejs) ：速度更快，功能也更多。 4 | 5 | 本文叙述的是如何安装设置库本身。有关如何使用 CozoDB（CozoScript）的信息，见 [文档](https://docs.cozodb.org/zh_CN/latest/index.html) 。 6 | 7 | 安装 8 | 9 | ``` 10 | npm install cozo-lib-wasm 11 | ``` 12 | 13 | 你也可以直接从 [发布页面](https://github.com/cozodb/cozo/releases) 下载 `cozo_wasm--wasm32-unknown-unknown.zip` 文件，然后直接在你的网页代码中引用：见 [此处](https://rustwasm.github.io/docs/wasm-bindgen/examples/without-a-bundler.html) 的 `index.html` 范例。 14 | 15 | ## 使用 16 | 17 | 参考 [此文件](wasm-react-demo/src/App.js)。简单地说： 18 | 19 | ```js 20 | import init, {CozoDb} from "cozo-lib-wasm"; 21 | ``` 22 | 23 | 然后： 24 | 25 | ```js 26 | let db; 27 | init().then(() => { 28 | db = CozoDb.new(); 29 | // db can only be used after the promise resolves 30 | }) 31 | ``` 32 | 33 | ## API 34 | 35 | ```ts 36 | export class CozoDb { 37 | free(): void; 38 | 39 | static new(): CozoDb; 40 | 41 | run(script: string, params: string): string; 42 | 43 | export_relations(data: string): string; 44 | 45 | // 注意：通过此接口载入数据不会激活触发器 46 | import_relations(data: string): string; 47 | } 48 | ``` 49 | 50 | 注意所有的 API 都是同步的。如果你的查询需要比较长的时间返回，浏览器的主线程会被阻塞。阻塞浏览器主线程不是好事，因此在这种情况下你可以考虑在 web worker 中运行 Cozo WASM 模块。不过预编译的 WASM 模块不支持在有些浏览器的 web worker 中运行：见 [此页面](https://developer.mozilla.org/en-US/docs/Web/API/Worker/Worker#browser_compatibility) 的 "Support for ECMAScript 51 | modules" 信息。 52 | 53 | ## 编译 54 | 55 | 编译需要 [Rust 工具链](https://rustup.rs/)，[NodeJS 与 npm](https://nodejs.org/)，再加上 [wasm-pack](https://github.com/rustwasm/wasm-pack)。 56 | 57 | 用以下命令来编译： 58 | 59 | ```bash 60 | wasm-pack build --target web --release 61 | ``` 62 | 63 | 建议编译时设置环境变量 `CARGO_PROFILE_RELEASE_LTO=fat` 使生成的库更快（以增加编译时间为代价）。 64 | 65 | 以上我们给出了参数 `--target web`：上面在浏览器中的使用例子只支持用此参数编译出的库。更多信息参见 [WASM 的文档](https://rustwasm.github.io/wasm-pack/book/commands/build.html#target)。 66 | 67 | 使用 `--target no-modules` 编译出的库可以在更多浏览器中的 web worker 运行，但是调用方式与上面给出的例子有区别，也更麻烦。详情见 [文档](https://rustwasm.github.io/wasm-bindgen/examples/wasm-in-web-worker.html) 。 -------------------------------------------------------------------------------- /cozo-lib-wasm/README.md: -------------------------------------------------------------------------------- 1 | # Cozo in web assembly 2 | 3 | This crate provides Cozo web assembly modules for browsers. 4 | If you are targeting NodeJS, use [this](../cozo-lib-nodejs) instead: 5 | native code is still _much_ faster than WASM. 6 | 7 | This document describes how to set up the Cozo WASM module for use. 8 | To learn how to use CozoDB (CozoScript), read the [docs](https://docs.cozodb.org/en/latest/index.html). 9 | 10 | ## Installation 11 | 12 | ``` 13 | npm install cozo-lib-wasm 14 | ``` 15 | 16 | Alternatively, you can download `cozo_wasm--wasm32-unknown-unknown.zip` 17 | from the [release page](https://github.com/cozodb/cozo/releases) and include 18 | the JS and WASM files directly in your project: see the `index.html` example 19 | [here](https://rustwasm.github.io/docs/wasm-bindgen/examples/without-a-bundler.html) for 20 | what is required in your code. 21 | 22 | ## Usage 23 | 24 | See the code [here](wasm-react-demo/src/App.js). Basically, you write 25 | 26 | ```js 27 | import init, {CozoDb} from "cozo-lib-wasm"; 28 | ``` 29 | 30 | and call 31 | 32 | ```js 33 | let db; 34 | init().then(() => { 35 | db = CozoDb.new(); 36 | // db can only be used after the promise resolves 37 | }) 38 | ``` 39 | 40 | ## API 41 | 42 | ```ts 43 | export class CozoDb { 44 | free(): void; 45 | 46 | static new(): CozoDb; 47 | 48 | run(script: string, params: string): string; 49 | 50 | export_relations(data: string): string; 51 | 52 | // Note that triggers are _not_ run for the relations, if any exists. 53 | // If you need to activate triggers, use queries with parameters. 54 | import_relations(data: string): string; 55 | } 56 | ``` 57 | 58 | Note that this API is synchronous. If your computation runs for a long time, 59 | **it will block the main thread**. If you know that some of your queries are going to be heavy, 60 | you should consider running Cozo in a web worker. However, the published module 61 | may not work across browsers in web workers (look for the row "Support for ECMAScript 62 | modules" [here](https://developer.mozilla.org/en-US/docs/Web/API/Worker/Worker#browser_compatibility)). 63 | 64 | The next section contains some pointers for how to alleviate this, but expect a lot of work. 65 | 66 | ## Compiling 67 | 68 | You will need to install [Rust](https://rustup.rs/), [NodeJS with npm](https://nodejs.org/), 69 | and [wasm-pack](https://github.com/rustwasm/wasm-pack) first. 70 | 71 | The published module was built with 72 | 73 | ```bash 74 | wasm-pack build --target web --release 75 | ``` 76 | 77 | and the environment variable `CARGO_PROFILE_RELEASE_LTO=fat`. 78 | 79 | The important option is `--target web`: the above usage instructions only work for this target. 80 | See the documentation [here](https://rustwasm.github.io/wasm-pack/book/commands/build.html#target). 81 | 82 | if you are interested in running Cozo in a web worker and expect it to run across browsers, 83 | you will need to use the `--target no-modules` option, and write a lot of gluing code. 84 | See [here](https://rustwasm.github.io/wasm-bindgen/examples/wasm-in-web-worker.html) for tips. -------------------------------------------------------------------------------- /cozo-lib-wasm/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | CARGO_PROFILE_RELEASE_LTO=fat wasm-pack build --target web --release -------------------------------------------------------------------------------- /cozo-lib-wasm/src/lib.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022, The Cozo Project Authors. 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 5 | * If a copy of the MPL was not distributed with this file, 6 | * You can obtain one at https://mozilla.org/MPL/2.0/. 7 | */ 8 | 9 | use wasm_bindgen::prelude::*; 10 | 11 | use cozo::*; 12 | 13 | mod utils; 14 | 15 | // When the `wee_alloc` feature is enabled, use `wee_alloc` as the global 16 | // allocator. 17 | #[cfg(feature = "wee_alloc")] 18 | #[global_allocator] 19 | static ALLOC: wee_alloc::WeeAlloc = wee_alloc::WeeAlloc::INIT; 20 | 21 | #[wasm_bindgen] 22 | extern "C" { 23 | fn alert(s: &str); 24 | } 25 | 26 | #[wasm_bindgen] 27 | pub struct CozoDb { 28 | db: DbInstance, 29 | } 30 | 31 | #[wasm_bindgen] 32 | impl CozoDb { 33 | #[allow(clippy::new_without_default)] 34 | pub fn new() -> Self { 35 | utils::set_panic_hook(); 36 | let db = DbInstance::new("mem", "", "").unwrap(); 37 | Self { db } 38 | } 39 | pub fn run(&self, script: &str, params: &str, immutable: bool) -> String { 40 | self.db.run_script_str(script, params, immutable) 41 | } 42 | pub fn export_relations(&self, data: &str) -> String { 43 | self.db.export_relations_str(data) 44 | } 45 | pub fn import_relations(&self, data: &str) -> String { 46 | self.db.import_relations_str(data) 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /cozo-lib-wasm/src/utils.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022, The Cozo Project Authors. 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 5 | * If a copy of the MPL was not distributed with this file, 6 | * You can obtain one at https://mozilla.org/MPL/2.0/. 7 | */ 8 | 9 | pub fn set_panic_hook() { 10 | // When the `console_error_panic_hook` feature is enabled, we can call the 11 | // `set_panic_hook` function at least once during initialization, and then 12 | // we will get better error messages if our code ever panics. 13 | // 14 | // For more details see 15 | // https://github.com/rustwasm/console_error_panic_hook#readme 16 | #[cfg(feature = "console_error_panic_hook")] 17 | console_error_panic_hook::set_once(); 18 | } 19 | -------------------------------------------------------------------------------- /cozo-lib-wasm/tests/web.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022, The Cozo Project Authors. 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 5 | * If a copy of the MPL was not distributed with this file, 6 | * You can obtain one at https://mozilla.org/MPL/2.0/. 7 | */ 8 | 9 | //! Test suite for the Web and headless browsers. 10 | 11 | #![cfg(target_arch = "wasm32")] 12 | 13 | extern crate wasm_bindgen_test; 14 | use wasm_bindgen_test::*; 15 | 16 | wasm_bindgen_test_configure!(run_in_browser); 17 | 18 | #[wasm_bindgen_test] 19 | fn pass() { 20 | assert_eq!(1 + 1, 2); 21 | } 22 | -------------------------------------------------------------------------------- /cozorocks/.gitignore: -------------------------------------------------------------------------------- 1 | ### CMake template 2 | CMakeLists.txt.user 3 | CMakeCache.txt 4 | CMakeFiles 5 | CMakeScripts 6 | Testing 7 | Makefile 8 | cmake_install.cmake 9 | install_manifest.txt 10 | compile_commands.json 11 | CTestTestfile.cmake 12 | _deps 13 | 14 | ### JetBrains template 15 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider 16 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 17 | 18 | # User-specific stuff 19 | .idea/**/workspace.xml 20 | .idea/**/tasks.xml 21 | .idea/**/usage.statistics.xml 22 | .idea/**/dictionaries 23 | .idea/**/shelf 24 | 25 | # Generated files 26 | .idea/**/contentModel.xml 27 | 28 | # Sensitive or high-churn files 29 | .idea/**/dataSources/ 30 | .idea/**/dataSources.ids 31 | .idea/**/dataSources.local.xml 32 | .idea/**/sqlDataSources.xml 33 | .idea/**/dynamic.xml 34 | .idea/**/uiDesigner.xml 35 | .idea/**/dbnavigator.xml 36 | 37 | # Gradle 38 | .idea/**/gradle.xml 39 | .idea/**/libraries 40 | 41 | # Gradle and Maven with auto-import 42 | # When using Gradle or Maven with auto-import, you should exclude module files, 43 | # since they will be recreated, and may cause churn. Uncomment if using 44 | # auto-import. 45 | # .idea/artifacts 46 | # .idea/compiler.xml 47 | # .idea/jarRepositories.xml 48 | # .idea/modules.xml 49 | # .idea/*.iml 50 | # .idea/modules 51 | # *.iml 52 | # *.ipr 53 | 54 | # CMake 55 | cmake-build-*/ 56 | 57 | # Mongo Explorer plugin 58 | .idea/**/mongoSettings.xml 59 | 60 | # File-based project format 61 | *.iws 62 | 63 | # IntelliJ 64 | out/ 65 | 66 | # mpeltonen/sbt-idea plugin 67 | .idea_modules/ 68 | 69 | # JIRA plugin 70 | atlassian-ide-plugin.xml 71 | 72 | # Cursive Clojure plugin 73 | .idea/replstate.xml 74 | 75 | # Crashlytics plugin (for Android Studio and IntelliJ) 76 | com_crashlytics_export_strings.xml 77 | crashlytics.properties 78 | crashlytics-build.properties 79 | fabric.properties 80 | 81 | # Editor-based Rest Client 82 | .idea/httpRequests 83 | 84 | # Android studio 3.1+ serialized cache file 85 | .idea/caches/build_file_checksums.ser 86 | 87 | -------------------------------------------------------------------------------- /cozorocks/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.22) 2 | project(cozorocks) 3 | 4 | set(CMAKE_CXX_STANDARD 17) 5 | 6 | include_directories("bridge") 7 | include_directories("./rocksdb/include") 8 | include_directories("../target/cxxbridge") 9 | 10 | add_library(cozorocks "bridge/bridge.h" "bridge/common.h" "bridge/db.cpp" "bridge/db.h" "bridge/iter.h" "bridge/opts.h" 11 | "bridge/slice.h" "bridge/status.cpp" "bridge/status.h" "bridge/tx.cpp" "bridge/tx.h") -------------------------------------------------------------------------------- /cozorocks/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "cozorocks" 3 | version = "0.1.7" 4 | edition = "2021" 5 | license = "MPL-2.0" 6 | authors = ["Ziyang Hu"] 7 | description = "Bindings to RocksDB used by the cozo crate" 8 | homepage = "https://www.cozodb.org" 9 | repository = "https://github.com/cozodb/cozo" 10 | documentation = "https://docs.cozodb.org" 11 | exclude = ["rocksdb/docs", "rocksdb/java"] 12 | 13 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 14 | 15 | 16 | [dependencies] 17 | cxx = "1.0.69" 18 | miette = "5.10.0" 19 | libc = "0.2" 20 | tikv-jemalloc-sys = { version = "0.5", features = ["unprefixed_malloc_on_supported_platforms"], optional = true } 21 | lz4-sys = { version = "1.9" } 22 | zstd-sys = { version = "2.0", features = ["zdict_builder"] } 23 | 24 | [features] 25 | jemalloc = ["tikv-jemalloc-sys"] 26 | io-uring = ["pkg-config"] 27 | 28 | [build-dependencies] 29 | cxx-build = "1.0.69" 30 | pkg-config = { version = "0.3.25", optional = true } 31 | cc = { version = "1.0", features = ["parallel"] } 32 | -------------------------------------------------------------------------------- /cozorocks/README-zh.md: -------------------------------------------------------------------------------- 1 | # Cozorocks 2 | 3 | 对 RocksDB C++ 接口的封装。 -------------------------------------------------------------------------------- /cozorocks/README.md: -------------------------------------------------------------------------------- 1 | # Cozorocks 2 | 3 | Bindings to RocksDB's C++ API. -------------------------------------------------------------------------------- /cozorocks/bridge/bridge.h: -------------------------------------------------------------------------------- 1 | // Copyright 2022, The Cozo Project Authors. 2 | // 3 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 4 | // If a copy of the MPL was not distributed with this file, 5 | // You can obtain one at https://mozilla.org/MPL/2.0/. 6 | 7 | #ifndef COZOROCKS_BRIDGE_H 8 | #define COZOROCKS_BRIDGE_H 9 | 10 | #include "db.h" 11 | #include "slice.h" 12 | #include "tx.h" 13 | #include "status.h" 14 | #include "opts.h" 15 | #include "iter.h" 16 | 17 | #endif //COZOROCKS_BRIDGE_H 18 | -------------------------------------------------------------------------------- /cozorocks/bridge/common.h: -------------------------------------------------------------------------------- 1 | // Copyright 2022, The Cozo Project Authors. 2 | // 3 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 4 | // If a copy of the MPL was not distributed with this file, 5 | // You can obtain one at https://mozilla.org/MPL/2.0/. 6 | 7 | #ifndef COZOROCKS_ROCKS_BRIDGE_H 8 | #define COZOROCKS_ROCKS_BRIDGE_H 9 | 10 | #include "rust/cxx.h" 11 | #include "rocksdb/db.h" 12 | #include "rocksdb/slice.h" 13 | #include "rocksdb/options.h" 14 | #include "rocksdb/utilities/transaction.h" 15 | #include "rocksdb/utilities/transaction_db.h" 16 | #include "rocksdb/utilities/optimistic_transaction_db.h" 17 | #include "rocksdb/table.h" 18 | #include "rocksdb/filter_policy.h" 19 | #include "rocksdb/slice_transform.h" 20 | 21 | using namespace rocksdb; 22 | using namespace std; 23 | 24 | struct RocksDbStatus; 25 | struct DbOpts; 26 | 27 | typedef Status::Code StatusCode; 28 | typedef Status::SubCode StatusSubCode; 29 | typedef Status::Severity StatusSeverity; 30 | typedef rust::Slice RustBytes; 31 | 32 | 33 | #endif //COZOROCKS_ROCKS_BRIDGE_H 34 | -------------------------------------------------------------------------------- /cozorocks/bridge/db.h: -------------------------------------------------------------------------------- 1 | // Copyright 2022, The Cozo Project Authors. 2 | // 3 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 4 | // If a copy of the MPL was not distributed with this file, 5 | // You can obtain one at https://mozilla.org/MPL/2.0/. 6 | 7 | #ifndef COZOROCKS_DB_H 8 | #define COZOROCKS_DB_H 9 | 10 | #include 11 | 12 | #include "iostream" 13 | #include "common.h" 14 | #include "tx.h" 15 | #include "slice.h" 16 | 17 | struct SnapshotBridge { 18 | const Snapshot *snapshot; 19 | DB *db; 20 | 21 | explicit SnapshotBridge(const Snapshot *snapshot_, DB *db_) : snapshot(snapshot_), db(db_) {} 22 | 23 | ~SnapshotBridge() { 24 | db->ReleaseSnapshot(snapshot); 25 | // printf("released snapshot\n"); 26 | } 27 | }; 28 | 29 | struct SstFileWriterBridge { 30 | SstFileWriter inner; 31 | 32 | SstFileWriterBridge(EnvOptions eopts, Options opts) : inner(eopts, opts) { 33 | } 34 | 35 | inline void finish(RocksDbStatus &status) { 36 | write_status(inner.Finish(), status); 37 | } 38 | 39 | inline void put(RustBytes key, RustBytes val, RocksDbStatus &status) { 40 | write_status(inner.Put(convert_slice(key), convert_slice(val)), status); 41 | } 42 | 43 | }; 44 | 45 | static WriteOptions DEFAULT_WRITE_OPTIONS = WriteOptions(); 46 | 47 | struct RocksDbBridge { 48 | unique_ptr db; 49 | 50 | bool destroy_on_exit; 51 | string db_path; 52 | 53 | inline unique_ptr get_sst_writer(rust::Str path, RocksDbStatus &status) const { 54 | DB *db_ = get_base_db(); 55 | auto cf = db->DefaultColumnFamily(); 56 | Options options_ = db_->GetOptions(cf); 57 | auto sst_file_writer = std::make_unique(EnvOptions(), options_); 58 | string path_(path); 59 | 60 | write_status(sst_file_writer->inner.Open(path_), status); 61 | return sst_file_writer; 62 | } 63 | 64 | inline void ingest_sst(rust::Str path, RocksDbStatus &status) const { 65 | IngestExternalFileOptions ifo; 66 | DB *db_ = get_base_db(); 67 | string path_(path); 68 | auto cf = db->DefaultColumnFamily(); 69 | write_status(db_->IngestExternalFile(cf, {std::move(path_)}, ifo), status); 70 | } 71 | 72 | [[nodiscard]] inline const string &get_db_path() const { 73 | return db_path; 74 | } 75 | 76 | 77 | [[nodiscard]] inline unique_ptr transact() const { 78 | auto ret = make_unique(&*this->db, db->DefaultColumnFamily()); 79 | return ret; 80 | } 81 | 82 | inline void del_range(RustBytes start, RustBytes end, RocksDbStatus &status) const { 83 | WriteBatch batch; 84 | auto cf = db->DefaultColumnFamily(); 85 | auto s = batch.DeleteRange(cf, convert_slice(start), convert_slice(end)); 86 | if (!s.ok()) { 87 | write_status(s, status); 88 | return; 89 | } 90 | WriteOptions w_opts; 91 | TransactionDBWriteOptimizations optimizations; 92 | optimizations.skip_concurrency_control = true; 93 | optimizations.skip_duplicate_key_check = true; 94 | auto s2 = db->Write(w_opts, optimizations, &batch); 95 | write_status(s2, status); 96 | } 97 | 98 | inline void put(RustBytes key, RustBytes val, RocksDbStatus &status) const { 99 | auto raw_db = this->get_base_db(); 100 | auto s = raw_db->Put(DEFAULT_WRITE_OPTIONS, convert_slice(key), convert_slice(val)); 101 | write_status(s, status); 102 | } 103 | 104 | void compact_range(RustBytes start, RustBytes end, RocksDbStatus &status) const { 105 | CompactRangeOptions options; 106 | auto cf = db->DefaultColumnFamily(); 107 | auto start_s = convert_slice(start); 108 | auto end_s = convert_slice(end); 109 | auto s = db->CompactRange(options, cf, &start_s, &end_s); 110 | write_status(s, status); 111 | } 112 | 113 | DB *get_base_db() const { 114 | return db->GetBaseDB(); 115 | } 116 | 117 | ~RocksDbBridge(); 118 | }; 119 | 120 | shared_ptr 121 | open_db(const DbOpts &opts, RocksDbStatus &status); 122 | 123 | #endif //COZOROCKS_DB_H 124 | -------------------------------------------------------------------------------- /cozorocks/bridge/iter.h: -------------------------------------------------------------------------------- 1 | // Copyright 2022, The Cozo Project Authors. 2 | // 3 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 4 | // If a copy of the MPL was not distributed with this file, 5 | // You can obtain one at https://mozilla.org/MPL/2.0/. 6 | 7 | #ifndef COZOROCKS_ITER_H 8 | #define COZOROCKS_ITER_H 9 | 10 | #include "common.h" 11 | #include "slice.h" 12 | #include "status.h" 13 | 14 | struct IterBridge { 15 | DB *db; 16 | Transaction *tx; 17 | unique_ptr iter; 18 | string lower_storage; 19 | string upper_storage; 20 | Slice lower_bound; 21 | Slice upper_bound; 22 | unique_ptr r_opts; 23 | 24 | explicit IterBridge(Transaction *tx_) : db(nullptr), tx(tx_), iter(nullptr), lower_bound(), 25 | upper_bound(), 26 | r_opts(new ReadOptions) { 27 | r_opts->ignore_range_deletions = true; 28 | r_opts->auto_prefix_mode = true; 29 | } 30 | 31 | inline void set_snapshot(const Snapshot *snapshot) { 32 | r_opts->snapshot = snapshot; 33 | } 34 | 35 | // inline ReadOptions &get_r_opts() { 36 | // return *r_opts; 37 | // } 38 | 39 | inline void verify_checksums(bool val) { 40 | r_opts->verify_checksums = val; 41 | } 42 | 43 | inline void fill_cache(bool val) { 44 | r_opts->fill_cache = val; 45 | } 46 | 47 | inline void tailing(bool val) { 48 | r_opts->tailing = val; 49 | } 50 | 51 | inline void total_order_seek(bool val) { 52 | r_opts->total_order_seek = val; 53 | } 54 | 55 | inline void auto_prefix_mode(bool val) { 56 | r_opts->auto_prefix_mode = val; 57 | } 58 | 59 | inline void prefix_same_as_start(bool val) { 60 | r_opts->prefix_same_as_start = val; 61 | } 62 | 63 | inline void pin_data(bool val) { 64 | r_opts->pin_data = val; 65 | } 66 | 67 | inline void clear_bounds() { 68 | r_opts->iterate_lower_bound = nullptr; 69 | r_opts->iterate_upper_bound = nullptr; 70 | lower_bound.clear(); 71 | upper_bound.clear(); 72 | } 73 | 74 | inline void set_lower_bound(RustBytes bound) { 75 | lower_storage = convert_slice_to_string(bound); 76 | lower_bound = lower_storage; 77 | r_opts->iterate_lower_bound = &lower_bound; 78 | } 79 | 80 | inline void set_upper_bound(RustBytes bound) { 81 | upper_storage = convert_slice_to_string(bound); 82 | upper_bound = upper_storage; 83 | r_opts->iterate_upper_bound = &upper_bound; 84 | } 85 | 86 | inline void start() { 87 | if (db == nullptr) { 88 | iter.reset(tx->GetIterator(*r_opts)); 89 | } else { 90 | iter.reset(db->NewIterator(*r_opts)); 91 | } 92 | } 93 | 94 | inline void reset() { 95 | iter.reset(); 96 | clear_bounds(); 97 | } 98 | 99 | inline void to_start() { 100 | iter->SeekToFirst(); 101 | } 102 | 103 | inline void to_end() { 104 | iter->SeekToLast(); 105 | } 106 | 107 | inline void seek(RustBytes key) { 108 | iter->Seek(convert_slice(key)); 109 | } 110 | 111 | inline void seek_backward(RustBytes key) { 112 | iter->SeekForPrev(convert_slice(key)); 113 | } 114 | 115 | inline bool is_valid() const { 116 | return iter->Valid(); 117 | } 118 | 119 | inline void next() { 120 | iter->Next(); 121 | } 122 | 123 | inline void prev() { 124 | iter->Prev(); 125 | } 126 | 127 | inline void status(RocksDbStatus &status) const { 128 | write_status(iter->status(), status); 129 | } 130 | 131 | [[nodiscard]] inline RustBytes key() const { 132 | return convert_slice_back(iter->key()); 133 | } 134 | 135 | [[nodiscard]] inline RustBytes val() const { 136 | return convert_slice_back(iter->value()); 137 | } 138 | }; 139 | 140 | #endif //COZOROCKS_ITER_H 141 | -------------------------------------------------------------------------------- /cozorocks/bridge/opts.h: -------------------------------------------------------------------------------- 1 | // Copyright 2022, The Cozo Project Authors. 2 | // 3 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 4 | // If a copy of the MPL was not distributed with this file, 5 | // You can obtain one at https://mozilla.org/MPL/2.0/. 6 | 7 | #ifndef COZOROCKS_OPTS_H 8 | #define COZOROCKS_OPTS_H 9 | 10 | #include "common.h" 11 | 12 | inline void set_w_opts_sync(WriteOptions& opts, bool val) { 13 | opts.sync = val; 14 | } 15 | 16 | inline void set_w_opts_disable_wal(WriteOptions& opts, bool val) { 17 | opts.disableWAL = val; 18 | } 19 | 20 | inline void set_w_opts_no_slowdown(WriteOptions& opts, bool val) { 21 | opts.no_slowdown = val; 22 | } 23 | 24 | #endif //COZOROCKS_OPTS_H 25 | -------------------------------------------------------------------------------- /cozorocks/bridge/slice.h: -------------------------------------------------------------------------------- 1 | // Copyright 2022, The Cozo Project Authors. 2 | // 3 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 4 | // If a copy of the MPL was not distributed with this file, 5 | // You can obtain one at https://mozilla.org/MPL/2.0/. 6 | 7 | #ifndef COZOROCKS_SLICE_H 8 | #define COZOROCKS_SLICE_H 9 | 10 | #include "common.h" 11 | 12 | inline Slice convert_slice(RustBytes d) { 13 | return {reinterpret_cast(d.data()), d.size()}; 14 | } 15 | 16 | inline string convert_vec_to_string(const rust::Vec &d) { 17 | return {reinterpret_cast(d.data()), d.size()}; 18 | } 19 | 20 | inline string convert_slice_to_string(RustBytes d) { 21 | return {reinterpret_cast(d.data()), d.size()}; 22 | } 23 | 24 | inline RustBytes convert_slice_back(const Slice &s) { 25 | return {reinterpret_cast(s.data()), s.size()}; 26 | } 27 | 28 | inline RustBytes convert_pinnable_slice_back(const PinnableSlice &s) { 29 | return {reinterpret_cast(s.data()), s.size()}; 30 | } 31 | 32 | #endif //COZOROCKS_SLICE_H 33 | -------------------------------------------------------------------------------- /cozorocks/bridge/status.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2022, The Cozo Project Authors. 2 | // 3 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 4 | // If a copy of the MPL was not distributed with this file, 5 | // You can obtain one at https://mozilla.org/MPL/2.0/. 6 | 7 | #include "status.h" 8 | #include "cozorocks/src/bridge/mod.rs.h" 9 | 10 | void write_status(const Status &rstatus, RocksDbStatus &status) { 11 | status.code = rstatus.code(); 12 | status.subcode = rstatus.subcode(); 13 | status.severity = rstatus.severity(); 14 | if (!rstatus.ok() && !rstatus.IsNotFound()) { 15 | status.message = rust::String::lossy(rstatus.ToString()); 16 | } 17 | } 18 | 19 | RocksDbStatus convert_status(const Status &status) { 20 | RocksDbStatus ret; 21 | write_status(status, ret); 22 | return ret; 23 | } -------------------------------------------------------------------------------- /cozorocks/bridge/status.h: -------------------------------------------------------------------------------- 1 | // Copyright 2022, The Cozo Project Authors. 2 | // 3 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 4 | // If a copy of the MPL was not distributed with this file, 5 | // You can obtain one at https://mozilla.org/MPL/2.0/. 6 | 7 | #ifndef COZOROCKS_STATUS_H 8 | #define COZOROCKS_STATUS_H 9 | 10 | #include "common.h" 11 | 12 | void write_status(const Status &rstatus, RocksDbStatus &status); 13 | 14 | RocksDbStatus convert_status(const Status &status); 15 | 16 | #endif //COZOROCKS_STATUS_H 17 | -------------------------------------------------------------------------------- /cozorocks/bridge/tx.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2022, The Cozo Project Authors. 2 | // 3 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 4 | // If a copy of the MPL was not distributed with this file, 5 | // You can obtain one at https://mozilla.org/MPL/2.0/. 6 | 7 | #include "tx.h" 8 | #include "cozorocks/src/bridge/mod.rs.h" 9 | 10 | void TxBridge::start() { 11 | if (odb != nullptr) { 12 | Transaction *txn = odb->BeginTransaction(*w_opts, *o_tx_opts); 13 | tx.reset(txn); 14 | } else if (tdb != nullptr) { 15 | Transaction *txn = tdb->BeginTransaction(*w_opts, *p_tx_opts); 16 | tx.reset(txn); 17 | } 18 | assert(tx); 19 | } -------------------------------------------------------------------------------- /cozorocks/bridge/tx.h: -------------------------------------------------------------------------------- 1 | // Copyright 2022, The Cozo Project Authors. 2 | // 3 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 4 | // If a copy of the MPL was not distributed with this file, 5 | // You can obtain one at https://mozilla.org/MPL/2.0/. 6 | 7 | #ifndef COZOROCKS_TX_H 8 | #define COZOROCKS_TX_H 9 | 10 | #include "common.h" 11 | #include "slice.h" 12 | #include "status.h" 13 | #include "iter.h" 14 | 15 | struct TxBridge { 16 | OptimisticTransactionDB *odb; 17 | TransactionDB *tdb; 18 | unique_ptr tx; 19 | unique_ptr w_opts; 20 | unique_ptr r_opts; 21 | unique_ptr o_tx_opts; 22 | unique_ptr p_tx_opts; 23 | ColumnFamilyHandle * cf_handle; 24 | 25 | explicit TxBridge(TransactionDB *tdb_, ColumnFamilyHandle * cf_handle_) : 26 | odb(nullptr), 27 | tdb(tdb_), 28 | tx(), 29 | w_opts(new WriteOptions), 30 | r_opts(new ReadOptions), 31 | o_tx_opts(nullptr), 32 | p_tx_opts(new TransactionOptions), 33 | cf_handle(cf_handle_) { 34 | r_opts->ignore_range_deletions = true; 35 | } 36 | 37 | inline WriteOptions &get_w_opts() { 38 | return *w_opts; 39 | } 40 | 41 | // inline ReadOptions &get_r_opts() { 42 | // return *r_opts; 43 | // } 44 | 45 | inline void verify_checksums(bool val) { 46 | r_opts->verify_checksums = val; 47 | } 48 | 49 | inline void fill_cache(bool val) { 50 | r_opts->fill_cache = val; 51 | } 52 | 53 | inline unique_ptr iterator() const { 54 | return make_unique(&*tx); 55 | }; 56 | 57 | inline void set_snapshot(bool val) { 58 | if (tx != nullptr) { 59 | if (val) { 60 | tx->SetSnapshot(); 61 | } 62 | } else if (o_tx_opts != nullptr) { 63 | o_tx_opts->set_snapshot = val; 64 | } else if (p_tx_opts != nullptr) { 65 | p_tx_opts->set_snapshot = val; 66 | } 67 | } 68 | 69 | inline void clear_snapshot() { 70 | tx->ClearSnapshot(); 71 | } 72 | 73 | [[nodiscard]] inline DB *get_db() const { 74 | if (tdb != nullptr) { 75 | return tdb; 76 | } else { 77 | return odb; 78 | } 79 | } 80 | 81 | void start(); 82 | 83 | inline unique_ptr get(RustBytes key, bool for_update, RocksDbStatus &status) const { 84 | Slice key_ = convert_slice(key); 85 | auto ret = make_unique(); 86 | if (for_update) { 87 | auto s = tx->GetForUpdate(*r_opts, cf_handle, key_, &*ret); 88 | write_status(s, status); 89 | } else { 90 | auto s = tx->Get(*r_opts, key_, &*ret); 91 | write_status(s, status); 92 | } 93 | return ret; 94 | } 95 | 96 | inline void exists(RustBytes key, bool for_update, RocksDbStatus &status) const { 97 | Slice key_ = convert_slice(key); 98 | auto ret = PinnableSlice(); 99 | if (for_update) { 100 | auto s = tx->GetForUpdate(*r_opts, cf_handle, key_, &ret); 101 | write_status(s, status); 102 | } else { 103 | auto s = tx->Get(*r_opts, key_, &ret); 104 | write_status(s, status); 105 | } 106 | } 107 | 108 | inline void put(RustBytes key, RustBytes val, RocksDbStatus &status) const { 109 | write_status(tx->Put(convert_slice(key), convert_slice(val)), status); 110 | } 111 | 112 | inline void del(RustBytes key, RocksDbStatus &status) const { 113 | write_status(tx->Delete(convert_slice(key)), status); 114 | } 115 | 116 | inline void commit(RocksDbStatus &status) { 117 | write_status(tx->Commit(), status); 118 | } 119 | 120 | inline void rollback(RocksDbStatus &status) { 121 | write_status(tx->Rollback(), status); 122 | } 123 | 124 | inline void rollback_to_savepoint(RocksDbStatus &status) { 125 | write_status(tx->RollbackToSavePoint(), status); 126 | } 127 | 128 | inline void pop_savepoint(RocksDbStatus &status) { 129 | write_status(tx->PopSavePoint(), status); 130 | } 131 | 132 | inline void set_savepoint() { 133 | tx->SetSavePoint(); 134 | } 135 | }; 136 | 137 | #endif //COZOROCKS_TX_H 138 | -------------------------------------------------------------------------------- /cozorocks/build_version.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. 2 | 3 | #include 4 | 5 | #include "rocksdb/version.h" 6 | #include "rocksdb/utilities/object_registry.h" 7 | #include "util/string_util.h" 8 | 9 | // The build script may replace these values with real values based 10 | // on whether or not GIT is available and the platform settings 11 | static const std::string rocksdb_build_git_sha = "eb9a80fe1f18017b4d7f4084e8f2554f12234822"; 12 | static const std::string rocksdb_build_git_tag = "rocksdb_build_git_tag:v7.7.3"; 13 | #define HAS_GIT_CHANGES 0 14 | #if HAS_GIT_CHANGES == 0 15 | // If HAS_GIT_CHANGES is 0, the GIT date is used. 16 | // Use the time the branch/tag was last modified 17 | static const std::string rocksdb_build_date = "rocksdb_build_date:2022-10-11 08:49:59"; 18 | #else 19 | // If HAS_GIT_CHANGES is > 0, the branch/tag has modifications. 20 | // Use the time the build was created. 21 | static const std::string rocksdb_build_date = "rocksdb_build_date:2022-07-19 08:49:59"; 22 | #endif 23 | 24 | #ifndef ROCKSDB_LITE 25 | extern "C" { 26 | 27 | } // extern "C" 28 | 29 | std::unordered_map ROCKSDB_NAMESPACE::ObjectRegistry::builtins_ = { 30 | 31 | }; 32 | #endif //ROCKSDB_LITE 33 | 34 | namespace ROCKSDB_NAMESPACE { 35 | static void AddProperty(std::unordered_map *props, const std::string& name) { 36 | size_t colon = name.find(":"); 37 | if (colon != std::string::npos && colon > 0 && colon < name.length() - 1) { 38 | // If we found a "@:", then this property was a build-time substitution that failed. Skip it 39 | size_t at = name.find("@", colon); 40 | if (at != colon + 1) { 41 | // Everything before the colon is the name, after is the value 42 | (*props)[name.substr(0, colon)] = name.substr(colon + 1); 43 | } 44 | } 45 | } 46 | 47 | static std::unordered_map* LoadPropertiesSet() { 48 | auto * properties = new std::unordered_map(); 49 | AddProperty(properties, rocksdb_build_git_sha); 50 | AddProperty(properties, rocksdb_build_git_tag); 51 | AddProperty(properties, rocksdb_build_date); 52 | return properties; 53 | } 54 | 55 | const std::unordered_map& GetRocksBuildProperties() { 56 | static std::unique_ptr> props(LoadPropertiesSet()); 57 | return *props; 58 | } 59 | 60 | std::string GetRocksVersionAsString(bool with_patch) { 61 | std::string version = std::to_string(ROCKSDB_MAJOR) + "." + std::to_string(ROCKSDB_MINOR); 62 | if (with_patch) { 63 | return version + "." + std::to_string(ROCKSDB_PATCH); 64 | } else { 65 | return version; 66 | } 67 | } 68 | 69 | std::string GetRocksBuildInfoAsString(const std::string& program, bool verbose) { 70 | std::string info = program + " (RocksDB) " + GetRocksVersionAsString(true); 71 | if (verbose) { 72 | for (const auto& it : GetRocksBuildProperties()) { 73 | info.append("\n "); 74 | info.append(it.first); 75 | info.append(": "); 76 | info.append(it.second); 77 | } 78 | } 79 | return info; 80 | } 81 | } // namespace ROCKSDB_NAMESPACE 82 | -------------------------------------------------------------------------------- /cozorocks/src/bridge/iter.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022, The Cozo Project Authors. 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 5 | * If a copy of the MPL was not distributed with this file, 6 | * You can obtain one at https://mozilla.org/MPL/2.0/. 7 | */ 8 | 9 | use cxx::UniquePtr; 10 | 11 | use crate::bridge::ffi::*; 12 | 13 | pub struct IterBuilder { 14 | pub(crate) inner: UniquePtr, 15 | } 16 | 17 | pub struct DbIter { 18 | pub(crate) inner: UniquePtr, 19 | } 20 | 21 | impl IterBuilder { 22 | pub fn start(mut self) -> DbIter { 23 | self.inner.pin_mut().start(); 24 | DbIter { inner: self.inner } 25 | } 26 | pub fn clear_bounds(&mut self) { 27 | self.inner.pin_mut().clear_bounds(); 28 | } 29 | pub fn lower_bound(mut self, bound: &[u8]) -> Self { 30 | self.inner.pin_mut().set_lower_bound(bound); 31 | self 32 | } 33 | pub fn upper_bound(mut self, bound: &[u8]) -> Self { 34 | self.inner.pin_mut().set_upper_bound(bound); 35 | self 36 | } 37 | 38 | #[inline] 39 | pub fn verify_checksums(mut self, val: bool) -> Self { 40 | self.inner.pin_mut().verify_checksums(val); 41 | self 42 | } 43 | 44 | #[inline] 45 | pub fn fill_cache(mut self, val: bool) -> Self { 46 | self.inner.pin_mut().fill_cache(val); 47 | self 48 | } 49 | 50 | #[inline] 51 | pub fn tailing(mut self, val: bool) -> Self { 52 | self.inner.pin_mut().tailing(val); 53 | self 54 | } 55 | 56 | #[inline] 57 | pub fn total_order_seek(mut self, val: bool) -> Self { 58 | self.inner.pin_mut().total_order_seek(val); 59 | self 60 | } 61 | #[inline] 62 | pub fn auto_prefix_mode(mut self, val: bool) -> Self { 63 | self.inner.pin_mut().auto_prefix_mode(val); 64 | self 65 | } 66 | #[inline] 67 | pub fn prefix_same_as_start(mut self, val: bool) -> Self { 68 | self.inner.pin_mut().prefix_same_as_start(val); 69 | self 70 | } 71 | #[inline] 72 | pub fn pin_data(mut self, val: bool) -> Self { 73 | self.inner.pin_mut().pin_data(val); 74 | self 75 | } 76 | } 77 | 78 | impl DbIter { 79 | #[inline] 80 | pub fn reset(mut self) -> IterBuilder { 81 | self.inner.pin_mut().reset(); 82 | IterBuilder { inner: self.inner } 83 | } 84 | #[inline] 85 | pub fn seek_to_start(&mut self) { 86 | self.inner.pin_mut().to_start(); 87 | } 88 | #[inline] 89 | pub fn seek_to_end(&mut self) { 90 | self.inner.pin_mut().to_end(); 91 | } 92 | #[inline] 93 | pub fn seek(&mut self, key: &[u8]) { 94 | self.inner.pin_mut().seek(key); 95 | } 96 | #[inline] 97 | pub fn seek_back(&mut self, key: &[u8]) { 98 | self.inner.pin_mut().seek_backward(key); 99 | } 100 | #[inline] 101 | pub fn is_valid(&self) -> bool { 102 | self.inner.is_valid() 103 | } 104 | #[inline] 105 | pub fn next(&mut self) { 106 | self.inner.pin_mut().next(); 107 | } 108 | #[inline] 109 | pub fn prev(&mut self) { 110 | self.inner.pin_mut().prev(); 111 | } 112 | #[inline] 113 | pub fn status(&self) -> RocksDbStatus { 114 | let mut status = RocksDbStatus::default(); 115 | self.inner.status(&mut status); 116 | status 117 | } 118 | #[inline] 119 | pub fn key(&self) -> Result, RocksDbStatus> { 120 | if self.is_valid() { 121 | Ok(Some(self.inner.key())) 122 | } else { 123 | let status = self.status(); 124 | if status.is_ok() { 125 | Ok(None) 126 | } else { 127 | Err(status) 128 | } 129 | } 130 | } 131 | #[inline] 132 | pub fn val(&self) -> Result, RocksDbStatus> { 133 | if self.is_valid() { 134 | Ok(Some(self.inner.val())) 135 | } else { 136 | let status = self.status(); 137 | if status.is_ok() { 138 | Ok(None) 139 | } else { 140 | Err(status) 141 | } 142 | } 143 | } 144 | #[inline] 145 | pub fn pair(&self) -> Result, RocksDbStatus> { 146 | if self.is_valid() { 147 | Ok(Some((self.inner.key(), self.inner.val()))) 148 | } else { 149 | let status = self.status(); 150 | if status.is_ok() { 151 | Ok(None) 152 | } else { 153 | Err(status) 154 | } 155 | } 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /cozorocks/src/lib.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022, The Cozo Project Authors. 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 5 | * If a copy of the MPL was not distributed with this file, 6 | * You can obtain one at https://mozilla.org/MPL/2.0/. 7 | */ 8 | 9 | #![warn(rust_2018_idioms, future_incompatible)] 10 | #![allow(clippy::type_complexity)] 11 | 12 | pub use bridge::db::DbBuilder; 13 | pub use bridge::db::RocksDb; 14 | pub use bridge::ffi::RocksDbStatus; 15 | pub use bridge::ffi::SnapshotBridge; 16 | pub use bridge::ffi::StatusCode; 17 | pub use bridge::ffi::StatusSeverity; 18 | pub use bridge::ffi::StatusSubCode; 19 | pub use bridge::iter::DbIter; 20 | pub use bridge::iter::IterBuilder; 21 | pub use bridge::tx::PinSlice; 22 | pub use bridge::tx::Tx; 23 | pub use bridge::tx::TxBuilder; 24 | 25 | pub(crate) mod bridge; 26 | -------------------------------------------------------------------------------- /scripts/build-docs.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | VERSION=$(cat ./VERSION) 6 | 7 | cargo doc -p cozo --no-default-features 8 | 9 | mv target/doc ~/cozodb_site/$VERSION/docs.rs -------------------------------------------------------------------------------- /scripts/build-release-linux.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | VERSION=$(cat ./VERSION) 6 | 7 | #rm -fr release 8 | mkdir -p release 9 | 10 | # python 11 | CARGO_NET_GIT_FETCH_WITH_CLI=true podman run --rm -v $(pwd):/io -w /io/cozo-lib-python ghcr.io/pyo3/maturin:latest build --release --strip -F compact -F storage-rocksdb 12 | # copy python 13 | cp target/wheels/*.whl release/ 14 | 15 | for TARGET in aarch64-unknown-linux-gnu x86_64-unknown-linux-gnu; do 16 | # standalone, c, java, nodejs 17 | CROSS_CONTAINER_ENGINE=podman CARGO_PROFILE_RELEASE_LTO=fat cross build --release -p cozo-bin -p cozo_c -p cozo_java -p cozo-node -F compact -F storage-rocksdb --target $TARGET 18 | cp target/$TARGET/release/cozo-bin release/cozo-$VERSION-$TARGET # standalone 19 | cp target/$TARGET/release/libcozo_c.a release/libcozo_c-$VERSION-$TARGET.a # c static 20 | cp target/$TARGET/release/libcozo_c.so release/libcozo_c-$VERSION-$TARGET.so # c dynamic 21 | cp target/$TARGET/release/libcozo_java.so release/libcozo_java-$VERSION-$TARGET.so # java 22 | cp target/$TARGET/release/libcozo_node.so release/libcozo_node-$VERSION-$TARGET.so # nodejs 23 | 24 | done 25 | 26 | #for TARGET in x86_64-unknown-linux-gnu; do 27 | # CROSS_CONTAINER_ENGINE=podman PROTOC=$PWD/tools/protoc CARGO_PROFILE_RELEASE_LTO=fat cross build --release -p cozo-bin \ 28 | # -F compact -F storage-rocksdb -F storage-tikv -F storage-sled --target $TARGET 29 | # cp target/$TARGET/release/cozo-bin release/cozo_all-$VERSION-$TARGET # standalone 30 | #done 31 | 32 | #for TARGET in aarch64-unknown-linux-musl x86_64-unknown-linux-musl; do 33 | # CROSS_CONTAINER_ENGINE=podman CARGO_PROFILE_RELEASE_LTO=fat cross build --release -p cozo-bin -p cozo_c -F compact -F storage-rocksdb --target $TARGET 34 | # cp target/$TARGET/release/cozo-bin release/cozo-$VERSION-$TARGET # standalone 35 | # cp target/$TARGET/release/libcozo_c.a release/libcozo_c-$VERSION-$TARGET.a # c static 36 | #done 37 | 38 | for TARGET in aarch64-linux-android armv7-linux-androideabi i686-linux-android x86_64-linux-android; do 39 | CROSS_CONTAINER_ENGINE=podman CARGO_PROFILE_RELEASE_LTO=fat cross build -p cozo_java --release --target=$TARGET 40 | cp target/$TARGET/release/libcozo_java.so release/libcozo_java-$VERSION-$TARGET.so # java 41 | done 42 | -------------------------------------------------------------------------------- /scripts/build-release-mac.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | VERSION=$(cat ./VERSION) 6 | export MACOSX_DEPLOYMENT_TARGET=10.14 7 | 8 | #rm -fr release 9 | mkdir -p release 10 | 11 | for TARGET in aarch64-apple-darwin x86_64-apple-darwin; do 12 | # standalone, c, java, nodejs 13 | CARGO_PROFILE_RELEASE_LTO=fat cargo build --release -p cozo-bin -p cozo_c -p cozo_java -p cozo-node -F compact -F storage-rocksdb --target $TARGET 14 | cp target/$TARGET/release/cozo-bin release/cozo-$VERSION-$TARGET # standalone 15 | cp target/$TARGET/release/libcozo_c.a release/libcozo_c-$VERSION-$TARGET.a # c static 16 | cp target/$TARGET/release/libcozo_c.dylib release/libcozo_c-$VERSION-$TARGET.dylib # c dynamic 17 | cp target/$TARGET/release/libcozo_java.dylib release/libcozo_java-$VERSION-$TARGET.dylib # java 18 | cp target/$TARGET/release/libcozo_node.dylib release/libcozo_node-$VERSION-$TARGET.dylib # nodejs 19 | 20 | # python 21 | cd cozo-lib-python 22 | CARGO_PROFILE_RELEASE_LTO=fat PYO3_NO_PYTHON=1 maturin build -F compact -F storage-rocksdb --release --strip --target $TARGET 23 | cd .. 24 | done 25 | 26 | # copy python 27 | cp target/wheels/*.whl release/ 28 | 29 | # swift 30 | cd cozo-lib-swift 31 | CARGO_PROFILE_RELEASE_LTO=fat ./build-rust.sh 32 | cd .. 33 | 34 | # with TiKV 35 | #for TARGET in aarch64-apple-darwin x86_64-apple-darwin; do 36 | # CARGO_PROFILE_RELEASE_LTO=fat cargo build --release -p cozo-bin \ 37 | # -F compact -F storage-rocksdb -F storage-tikv -F storage-sled --target $TARGET 38 | # cp target/$TARGET/release/cozo-bin release/cozo_all-$VERSION-$TARGET # standalone 39 | #done 40 | 41 | # WASM 42 | cd cozo-lib-wasm 43 | CARGO_PROFILE_RELEASE_LTO=fat ./build.sh 44 | cd .. 45 | 46 | cp cozo-lib-c/cozo_c.h release/ 47 | 48 | zip release/cozo_wasm-$VERSION-wasm32-unknown-unknown.zip $(find cozo-lib-wasm/pkg) -------------------------------------------------------------------------------- /scripts/build-release-windows.ps1: -------------------------------------------------------------------------------- 1 | $ErrorActionPreference="Stop" 2 | 3 | $VERSION = cat .\VERSION 4 | $env:CARGO_PROFILE_RELEASE_LTO = "fat" 5 | $TARGET = "x86_64-pc-windows-msvc" 6 | # $env:PYO3_NO_PYTHON = 1 7 | 8 | mkdir -force release > $null 9 | 10 | cd cozo-lib-python 11 | maturin build -F compact -F storage-rocksdb --release --strip --target $TARGET 12 | cd .. 13 | 14 | cargo build --release -p cozo-bin -p cozo_c -p cozo_java -p cozo-node -F compact -F storage-rocksdb --target $TARGET 15 | cp target/$TARGET/release/cozo-bin.exe release/cozo-$VERSION-$TARGET.exe # standalone 16 | cp target/$TARGET/release/cozo_c.lib release/libcozo_c-$VERSION-$TARGET.lib # c static 17 | cp target/$TARGET/release/cozo_c.dll release/libcozo_c-$VERSION-$TARGET.dll # c dynamic 18 | cp target/$TARGET/release/cozo_java.dll release/libcozo_java-$VERSION-$TARGET.dll # java 19 | cp target/$TARGET/release/cozo_node.dll release/libcozo_node-$VERSION-$TARGET.dll # nodejs 20 | 21 | cp target/wheels/*.whl release/ 22 | 23 | # $TARGET = "x86_64-pc-windows-gnu" 24 | # cargo build --release -p cozo-bin -p cozo_c -p cozo_java -p cozo-node -F compact -F storage-rocksdb --target $TARGET 25 | # cp target/$TARGET/release/cozo-bin.exe release/cozo-$VERSION-$TARGET.exe # standalone 26 | # cp target/$TARGET/release/libcozo_c.a release/libcozo_c-$VERSION-$TARGET.a # c static 27 | # cp target/$TARGET/release/cozo_c.dll release/libcozo_c-$VERSION-$TARGET.dll # c dynamic 28 | # cp target/$TARGET/release/cozo_java.dll release/libcozo_java-$VERSION-$TARGET.dll # java 29 | # cp target/$TARGET/release/cozo_node.dll release/libcozo_node-$VERSION-$TARGET.dll # nodejs 30 | -------------------------------------------------------------------------------- /scripts/compress.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | VERSION=$(cat ./VERSION) 4 | 5 | gzip release/*java*.dll 6 | gzip release/libcozo_node* 7 | 8 | cd release 9 | 10 | for f in *.exe *.dll *.lib; do 11 | zip $f.zip $f 12 | rm $f 13 | done 14 | 15 | cd .. 16 | 17 | gzip release/*.a release/*.so release/*.dylib release/*-darwin release/*-gnu 18 | 19 | NODE_DIR=cozo-lib-nodejs/build/stage/$VERSION/ 20 | NODE_DIR_INNER=cozo-lib-nodejs/build/stage/$VERSION/6 21 | 22 | rm -fr $NODE_DIR_INNER 23 | mkdir -p $NODE_DIR_INNER 24 | cp release/libcozo_node-$VERSION-aarch64-apple-darwin.dylib.gz $NODE_DIR_INNER/cozo_node_prebuilt.node.gz 25 | pushd $NODE_DIR 26 | gunzip 6/cozo_node_prebuilt.node.gz 27 | tar cvzf 6-darwin-arm64.tar.gz 6/ 28 | popd 29 | 30 | rm -fr $NODE_DIR_INNER 31 | mkdir -p $NODE_DIR_INNER 32 | cp release/libcozo_node-$VERSION-x86_64-apple-darwin.dylib.gz $NODE_DIR_INNER/cozo_node_prebuilt.node.gz 33 | pushd $NODE_DIR 34 | gunzip 6/cozo_node_prebuilt.node.gz 35 | tar cvzf 6-darwin-x64.tar.gz 6/ 36 | popd 37 | 38 | rm -fr $NODE_DIR_INNER 39 | mkdir -p $NODE_DIR_INNER 40 | cp release/libcozo_node-$VERSION-x86_64-unknown-linux-gnu.so.gz $NODE_DIR_INNER/cozo_node_prebuilt.node.gz 41 | pushd $NODE_DIR 42 | gunzip 6/cozo_node_prebuilt.node.gz 43 | tar cvzf 6-linux-x64.tar.gz 6/ 44 | popd 45 | 46 | rm -fr $NODE_DIR_INNER 47 | mkdir -p $NODE_DIR_INNER 48 | cp release/libcozo_node-$VERSION-aarch64-unknown-linux-gnu.so.gz $NODE_DIR_INNER/cozo_node_prebuilt.node.gz 49 | pushd $NODE_DIR 50 | gunzip 6/cozo_node_prebuilt.node.gz 51 | tar cvzf 6-linux-arm64.tar.gz 6/ 52 | popd 53 | 54 | rm -fr $NODE_DIR_INNER 55 | mkdir -p $NODE_DIR_INNER 56 | cp release/libcozo_node-$VERSION-x86_64-pc-windows-msvc.dll.gz $NODE_DIR_INNER/cozo_node_prebuilt.node.gz 57 | pushd $NODE_DIR 58 | gunzip 6/cozo_node_prebuilt.node.gz 59 | tar cvzf 6-win32-x64.tar.gz 6/ 60 | popd 61 | 62 | for f in release/*; do 63 | gpg --armor --detach-sign $f 64 | done -------------------------------------------------------------------------------- /scripts/update-images.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright 2022, The Cozo Project Authors. 4 | # 5 | # This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 6 | # If a copy of the MPL was not distributed with this file, 7 | # You can obtain one at https://mozilla.org/MPL/2.0/. 8 | # 9 | 10 | podman pull ghcr.io/pyo3/maturin:latest 11 | podman pull ghcr.io/cross-rs/x86_64-unknown-linux-musl:main 12 | podman pull ghcr.io/cross-rs/aarch64-unknown-linux-musl:main 13 | podman pull ghcr.io/cross-rs/x86_64-linux-android:main 14 | podman pull ghcr.io/cross-rs/i686-linux-android:main 15 | podman pull ghcr.io/cross-rs/aarch64-linux-android:main 16 | podman pull ghcr.io/cross-rs/armv7-linux-androideabi:main 17 | podman pull ghcr.io/cross-rs/x86_64-unknown-linux-gnu:main 18 | podman pull ghcr.io/cross-rs/aarch64-unknown-linux-gnu:main -------------------------------------------------------------------------------- /signatures/version1/cla.json: -------------------------------------------------------------------------------- 1 | { 2 | "signedContributors": [ 3 | { 4 | "name": "redbar0n", 5 | "id": 459282, 6 | "comment_id": 1409499803, 7 | "created_at": "2023-01-30T23:11:47Z", 8 | "repoId": 551374215, 9 | "pullRequestNo": 38 10 | }, 11 | { 12 | "name": "GoldsteinE", 13 | "id": 12019211, 14 | "comment_id": 1412614986, 15 | "created_at": "2023-02-01T19:34:43Z", 16 | "repoId": 551374215, 17 | "pullRequestNo": 39 18 | }, 19 | { 20 | "name": "chuanqisun", 21 | "id": 1895289, 22 | "comment_id": 1519422437, 23 | "created_at": "2023-04-24T05:55:45Z", 24 | "repoId": 551374215, 25 | "pullRequestNo": 63 26 | }, 27 | { 28 | "name": "Avi-D-coder", 29 | "id": 29133776, 30 | "comment_id": 1544410132, 31 | "created_at": "2023-05-11T17:33:21Z", 32 | "repoId": 551374215, 33 | "pullRequestNo": 107 34 | }, 35 | { 36 | "name": "turnerdev", 37 | "id": 48462986, 38 | "comment_id": 1633419301, 39 | "created_at": "2023-07-13T01:52:22Z", 40 | "repoId": 551374215, 41 | "pullRequestNo": 145 42 | }, 43 | { 44 | "name": "pegesund", 45 | "id": 198916, 46 | "comment_id": 1642413475, 47 | "created_at": "2023-07-19T16:39:15Z", 48 | "repoId": 551374215, 49 | "pullRequestNo": 151 50 | }, 51 | { 52 | "name": "CrowdHailer", 53 | "id": 5862302, 54 | "comment_id": 1656326030, 55 | "created_at": "2023-07-28T20:55:41Z", 56 | "repoId": 551374215, 57 | "pullRequestNo": 164 58 | }, 59 | { 60 | "name": "michaelsbradleyjr", 61 | "id": 194260, 62 | "comment_id": 1742139428, 63 | "created_at": "2023-10-01T17:01:55Z", 64 | "repoId": 551374215, 65 | "pullRequestNo": 190 66 | } 67 | ] 68 | } -------------------------------------------------------------------------------- /static/logo_c.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cozodb/cozo/481af058abac9444ea8c9c52c78f096ed4b5bfc4/static/logo_c.png --------------------------------------------------------------------------------