├── .dockerignore ├── meilisearch-http ├── tests │ ├── settings │ │ ├── mod.rs │ │ └── distinct.rs │ ├── documents │ │ ├── mod.rs │ │ └── delete_documents.rs │ ├── index │ │ ├── mod.rs │ │ ├── stats.rs │ │ ├── delete_index.rs │ │ ├── update_index.rs │ │ └── create_index.rs │ ├── assets │ │ ├── v1_v0.20.0_movies.dump │ │ ├── v2_v0.21.1_movies.dump │ │ ├── v3_v0.24.0_movies.dump │ │ ├── v4_v0.25.2_movies.dump │ │ ├── v5_v0.28.0_test_dump.dump │ │ ├── v1_v0.20.0_movies_with_settings.dump │ │ ├── v2_v0.21.1_movies_with_settings.dump │ │ ├── v3_v0.24.0_movies_with_settings.dump │ │ ├── v4_v0.25.2_movies_with_settings.dump │ │ ├── v1_v0.20.0_rubygems_with_settings.dump │ │ ├── v2_v0.21.1_rubygems_with_settings.dump │ │ ├── v3_v0.24.0_rubygems_with_settings.dump │ │ ├── v4_v0.25.2_rubygems_with_settings.dump │ │ └── dumps │ │ │ └── v1 │ │ │ ├── metadata.json │ │ │ └── test │ │ │ ├── updates.jsonl │ │ │ └── settings.json │ ├── integration.rs │ ├── dashboard │ │ └── mod.rs │ ├── common │ │ ├── mod.rs │ │ └── server.rs │ ├── auth │ │ └── mod.rs │ ├── stats │ │ └── mod.rs │ ├── dumps │ │ └── data.rs │ └── snapshot │ │ └── mod.rs ├── src │ ├── helpers │ │ ├── mod.rs │ │ └── env.rs │ ├── extractors │ │ ├── mod.rs │ │ ├── authentication │ │ │ └── error.rs │ │ ├── payload.rs │ │ └── sequential_extractor.rs │ ├── routes │ │ ├── dump.rs │ │ ├── api_key.rs │ │ └── indexes │ │ │ └── mod.rs │ ├── analytics │ │ ├── mock_analytics.rs │ │ └── mod.rs │ └── error.rs ├── build.rs └── Cargo.toml ├── meilisearch-types ├── src │ ├── lib.rs │ ├── index_uid.rs │ └── star_or.rs └── Cargo.toml ├── assets ├── trumen-fast.gif ├── crates-io-demo.gif ├── logo.svg └── do-btn-blue.svg ├── meilisearch-lib ├── src │ ├── dump │ │ ├── loaders │ │ │ ├── mod.rs │ │ │ ├── v1.rs │ │ │ ├── v5.rs │ │ │ ├── v4.rs │ │ │ └── v3.rs │ │ ├── compat │ │ │ ├── mod.rs │ │ │ ├── v4.rs │ │ │ └── v2.rs │ │ └── error.rs │ ├── analytics.rs │ ├── tasks │ │ ├── handlers │ │ │ ├── empty_handler.rs │ │ │ ├── snapshot_handler.rs │ │ │ ├── mod.rs │ │ │ └── dump_handler.rs │ │ ├── error.rs │ │ ├── mod.rs │ │ ├── batch.rs │ │ └── update_loop.rs │ ├── index_controller │ │ ├── versioning │ │ │ ├── error.rs │ │ │ └── mod.rs │ │ ├── updates │ │ │ └── error.rs │ │ └── error.rs │ ├── compression.rs │ ├── lib.rs │ ├── index │ │ ├── error.rs │ │ └── dump.rs │ ├── error.rs │ ├── index_resolver │ │ ├── error.rs │ │ └── index_store.rs │ └── document_formats.rs ├── proptest-regressions │ ├── tasks │ │ └── task_store │ │ │ └── store.txt │ └── index_resolver │ │ └── mod.txt └── Cargo.toml ├── .gitignore ├── Cross.toml ├── .github ├── dependabot.yml ├── workflows │ ├── flaky.yml │ ├── coverage.yml │ ├── publish-deb-brew-pkg.yml │ ├── publish-docker-images.yml │ ├── rust.yml │ └── publish-binaries.yml ├── ISSUE_TEMPLATE │ ├── config.yml │ └── bug_report.md └── is-latest-release.sh ├── bors.toml ├── permissive-json-pointer ├── Cargo.toml └── README.md ├── Cargo.toml ├── meilisearch-auth ├── Cargo.toml └── src │ ├── dump.rs │ ├── error.rs │ └── action.rs ├── LICENSE ├── Dockerfile ├── SECURITY.md ├── CODE_OF_CONDUCT.md └── CONTRIBUTING.md /.dockerignore: -------------------------------------------------------------------------------- 1 | target 2 | Dockerfile 3 | .dockerignore 4 | .gitignore 5 | -------------------------------------------------------------------------------- /meilisearch-http/tests/settings/mod.rs: -------------------------------------------------------------------------------- 1 | mod distinct; 2 | mod get_settings; 3 | -------------------------------------------------------------------------------- /meilisearch-http/src/helpers/mod.rs: -------------------------------------------------------------------------------- 1 | mod env; 2 | 3 | pub use env::EnvSizer; 4 | -------------------------------------------------------------------------------- /meilisearch-types/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod error; 2 | pub mod index_uid; 3 | pub mod star_or; 4 | -------------------------------------------------------------------------------- /assets/trumen-fast.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thinkcmf/meilisearch/main/assets/trumen-fast.gif -------------------------------------------------------------------------------- /meilisearch-lib/src/dump/loaders/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod v2; 2 | pub mod v3; 3 | pub mod v4; 4 | pub mod v5; 5 | -------------------------------------------------------------------------------- /assets/crates-io-demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thinkcmf/meilisearch/main/assets/crates-io-demo.gif -------------------------------------------------------------------------------- /meilisearch-http/tests/documents/mod.rs: -------------------------------------------------------------------------------- 1 | mod add_documents; 2 | mod delete_documents; 3 | mod get_documents; 4 | -------------------------------------------------------------------------------- /meilisearch-http/src/extractors/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod payload; 2 | #[macro_use] 3 | pub mod authentication; 4 | pub mod sequential_extractor; 5 | -------------------------------------------------------------------------------- /meilisearch-http/tests/index/mod.rs: -------------------------------------------------------------------------------- 1 | mod create_index; 2 | mod delete_index; 3 | mod get_index; 4 | mod stats; 5 | mod update_index; 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | **/*.csv 3 | **/*.json_lines 4 | **/*.rs.bk 5 | /*.mdb 6 | /query-history.txt 7 | /data.ms 8 | /snapshots 9 | /dumps 10 | -------------------------------------------------------------------------------- /Cross.toml: -------------------------------------------------------------------------------- 1 | [build.env] 2 | passthrough = [ 3 | "RUST_BACKTRACE", 4 | "CARGO_TERM_COLOR", 5 | "RUSTFLAGS", 6 | "JEMALLOC_SYS_WITH_LG_PAGE" 7 | ] 8 | -------------------------------------------------------------------------------- /meilisearch-http/tests/assets/v1_v0.20.0_movies.dump: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thinkcmf/meilisearch/main/meilisearch-http/tests/assets/v1_v0.20.0_movies.dump -------------------------------------------------------------------------------- /meilisearch-http/tests/assets/v2_v0.21.1_movies.dump: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thinkcmf/meilisearch/main/meilisearch-http/tests/assets/v2_v0.21.1_movies.dump -------------------------------------------------------------------------------- /meilisearch-http/tests/assets/v3_v0.24.0_movies.dump: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thinkcmf/meilisearch/main/meilisearch-http/tests/assets/v3_v0.24.0_movies.dump -------------------------------------------------------------------------------- /meilisearch-http/tests/assets/v4_v0.25.2_movies.dump: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thinkcmf/meilisearch/main/meilisearch-http/tests/assets/v4_v0.25.2_movies.dump -------------------------------------------------------------------------------- /meilisearch-http/tests/assets/v5_v0.28.0_test_dump.dump: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thinkcmf/meilisearch/main/meilisearch-http/tests/assets/v5_v0.28.0_test_dump.dump -------------------------------------------------------------------------------- /meilisearch-http/tests/assets/v1_v0.20.0_movies_with_settings.dump: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thinkcmf/meilisearch/main/meilisearch-http/tests/assets/v1_v0.20.0_movies_with_settings.dump -------------------------------------------------------------------------------- /meilisearch-http/tests/assets/v2_v0.21.1_movies_with_settings.dump: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thinkcmf/meilisearch/main/meilisearch-http/tests/assets/v2_v0.21.1_movies_with_settings.dump -------------------------------------------------------------------------------- /meilisearch-http/tests/assets/v3_v0.24.0_movies_with_settings.dump: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thinkcmf/meilisearch/main/meilisearch-http/tests/assets/v3_v0.24.0_movies_with_settings.dump -------------------------------------------------------------------------------- /meilisearch-http/tests/assets/v4_v0.25.2_movies_with_settings.dump: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thinkcmf/meilisearch/main/meilisearch-http/tests/assets/v4_v0.25.2_movies_with_settings.dump -------------------------------------------------------------------------------- /meilisearch-http/tests/assets/v1_v0.20.0_rubygems_with_settings.dump: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thinkcmf/meilisearch/main/meilisearch-http/tests/assets/v1_v0.20.0_rubygems_with_settings.dump -------------------------------------------------------------------------------- /meilisearch-http/tests/assets/v2_v0.21.1_rubygems_with_settings.dump: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thinkcmf/meilisearch/main/meilisearch-http/tests/assets/v2_v0.21.1_rubygems_with_settings.dump -------------------------------------------------------------------------------- /meilisearch-http/tests/assets/v3_v0.24.0_rubygems_with_settings.dump: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thinkcmf/meilisearch/main/meilisearch-http/tests/assets/v3_v0.24.0_rubygems_with_settings.dump -------------------------------------------------------------------------------- /meilisearch-http/tests/assets/v4_v0.25.2_rubygems_with_settings.dump: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thinkcmf/meilisearch/main/meilisearch-http/tests/assets/v4_v0.25.2_rubygems_with_settings.dump -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # Set update schedule for GitHub Actions only 2 | 3 | version: 2 4 | updates: 5 | 6 | - package-ecosystem: "github-actions" 7 | directory: "/" 8 | schedule: 9 | interval: "monthly" 10 | -------------------------------------------------------------------------------- /bors.toml: -------------------------------------------------------------------------------- 1 | status = [ 2 | 'Tests on ubuntu-18.04', 3 | 'Tests on macos-latest', 4 | 'Tests on windows-latest', 5 | 'Run Clippy', 6 | 'Run Rustfmt', 7 | 'Run tests in debug', 8 | ] 9 | pr_status = ['Milestone Check'] 10 | # 3 hours timeout 11 | timeout-sec = 10800 12 | -------------------------------------------------------------------------------- /permissive-json-pointer/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "permissive-json-pointer" 3 | version = "0.2.0" 4 | edition = "2021" 5 | description = "A permissive json pointer" 6 | readme = "README.md" 7 | 8 | [dependencies] 9 | serde_json = "1.0" 10 | 11 | [dev-dependencies] 12 | big_s = "1.0" 13 | -------------------------------------------------------------------------------- /meilisearch-http/tests/assets/dumps/v1/metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "indices": [{ 3 | "uid": "test", 4 | "primaryKey": "id" 5 | }, { 6 | "uid": "test2", 7 | "primaryKey": "test2_id" 8 | } 9 | ], 10 | "dbVersion": "0.13.0", 11 | "dumpVersion": "1" 12 | } 13 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | resolver = "2" 3 | members = [ 4 | "meilisearch-http", 5 | "meilisearch-types", 6 | "meilisearch-lib", 7 | "meilisearch-auth", 8 | "permissive-json-pointer", 9 | ] 10 | 11 | [profile.dev.package.flate2] 12 | opt-level = 3 13 | 14 | [profile.dev.package.milli] 15 | opt-level = 3 16 | -------------------------------------------------------------------------------- /meilisearch-lib/src/analytics.rs: -------------------------------------------------------------------------------- 1 | use std::{fs, path::Path}; 2 | 3 | /// Copy the `instance-uid` contained in one db to another. Ignore all errors. 4 | pub fn copy_user_id(src: &Path, dst: &Path) { 5 | if let Ok(user_id) = fs::read_to_string(src.join("instance-uid")) { 6 | let _ = fs::write(dst.join("instance-uid"), &user_id); 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /.github/workflows/flaky.yml: -------------------------------------------------------------------------------- 1 | name: Look for flaky tests 2 | on: 3 | schedule: 4 | - cron: "0 12 * * FRI" # every friday at 12:00PM 5 | 6 | jobs: 7 | flaky: 8 | runs-on: ubuntu-18.04 9 | 10 | steps: 11 | - uses: actions/checkout@v3 12 | - name: Install cargo-flaky 13 | run: cargo install cargo-flaky 14 | - name: Run cargo flaky 100 times 15 | run: cargo flaky -i 100 --release 16 | -------------------------------------------------------------------------------- /meilisearch-lib/proptest-regressions/tasks/task_store/store.txt: -------------------------------------------------------------------------------- 1 | # Seeds for failure cases proptest has generated in the past. It is 2 | # automatically read and these particular cases re-run before any 3 | # novel cases are generated. 4 | # 5 | # It is recommended to check this file in to source control so that 6 | # everyone who runs the test benefits from these saved cases. 7 | cc 8cbd6c45ce8c5611ec3f2f94fd485f6a8eeccc470fa426e59bdfd4d9e7fce0e1 # shrinks to bytes = [] 8 | -------------------------------------------------------------------------------- /meilisearch-http/src/helpers/env.rs: -------------------------------------------------------------------------------- 1 | use meilisearch_lib::heed::Env; 2 | use walkdir::WalkDir; 3 | 4 | pub trait EnvSizer { 5 | fn size(&self) -> u64; 6 | } 7 | 8 | impl EnvSizer for Env { 9 | fn size(&self) -> u64 { 10 | WalkDir::new(self.path()) 11 | .into_iter() 12 | .filter_map(|entry| entry.ok()) 13 | .filter_map(|entry| entry.metadata().ok()) 14 | .filter(|metadata| metadata.is_file()) 15 | .fold(0, |acc, m| acc + m.len()) 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /meilisearch-http/tests/integration.rs: -------------------------------------------------------------------------------- 1 | mod auth; 2 | mod common; 3 | mod dashboard; 4 | mod documents; 5 | mod dumps; 6 | mod index; 7 | mod search; 8 | mod settings; 9 | mod snapshot; 10 | mod stats; 11 | mod tasks; 12 | 13 | // Tests are isolated by features in different modules to allow better readability, test 14 | // targetability, and improved incremental compilation times. 15 | // 16 | // All the integration tests live in the same root module so only one test executable is generated, 17 | // thus improving linking time. 18 | -------------------------------------------------------------------------------- /meilisearch-types/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "meilisearch-types" 3 | version = "0.28.0" 4 | authors = ["marin "] 5 | edition = "2021" 6 | 7 | [dependencies] 8 | actix-web = { version = "4.0.1", default-features = false } 9 | proptest = { version = "1.0.0", optional = true } 10 | proptest-derive = { version = "0.3.0", optional = true } 11 | serde = { version = "1.0.136", features = ["derive"] } 12 | serde_json = "1.0.79" 13 | 14 | [features] 15 | test-traits = ["proptest", "proptest-derive"] 16 | -------------------------------------------------------------------------------- /meilisearch-lib/src/tasks/handlers/empty_handler.rs: -------------------------------------------------------------------------------- 1 | use crate::tasks::batch::{Batch, BatchContent}; 2 | use crate::tasks::BatchHandler; 3 | 4 | /// A sink handler for empty tasks. 5 | pub struct EmptyBatchHandler; 6 | 7 | #[async_trait::async_trait] 8 | impl BatchHandler for EmptyBatchHandler { 9 | fn accept(&self, batch: &Batch) -> bool { 10 | matches!(batch.content, BatchContent::Empty) 11 | } 12 | 13 | async fn process_batch(&self, batch: Batch) -> Batch { 14 | batch 15 | } 16 | 17 | async fn finish(&self, _: &Batch) {} 18 | } 19 | -------------------------------------------------------------------------------- /meilisearch-lib/src/dump/compat/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod v2; 2 | pub mod v3; 3 | pub mod v4; 4 | 5 | /// Parses the v1 version of the Asc ranking rules `asc(price)`and returns the field name. 6 | pub fn asc_ranking_rule(text: &str) -> Option<&str> { 7 | text.split_once("asc(") 8 | .and_then(|(_, tail)| tail.rsplit_once(')')) 9 | .map(|(field, _)| field) 10 | } 11 | 12 | /// Parses the v1 version of the Desc ranking rules `desc(price)`and returns the field name. 13 | pub fn desc_ranking_rule(text: &str) -> Option<&str> { 14 | text.split_once("desc(") 15 | .and_then(|(_, tail)| tail.rsplit_once(')')) 16 | .map(|(field, _)| field) 17 | } 18 | -------------------------------------------------------------------------------- /meilisearch-auth/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "meilisearch-auth" 3 | version = "0.28.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | base64 = "0.13.0" 8 | enum-iterator = "0.7.0" 9 | hmac = "0.12.1" 10 | meilisearch-types = { path = "../meilisearch-types" } 11 | milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.29.3" } 12 | rand = "0.8.4" 13 | serde = { version = "1.0.136", features = ["derive"] } 14 | serde_json = { version = "1.0.79", features = ["preserve_order"] } 15 | sha2 = "0.10.2" 16 | thiserror = "1.0.30" 17 | time = { version = "0.3.7", features = ["serde-well-known", "formatting", "parsing", "macros"] } 18 | uuid = { version = "0.8.2", features = ["serde", "v4"] } 19 | -------------------------------------------------------------------------------- /meilisearch-http/tests/dashboard/mod.rs: -------------------------------------------------------------------------------- 1 | use crate::common::Server; 2 | 3 | #[actix_rt::test] 4 | async fn dashboard_assets_load() { 5 | let server = Server::new().await; 6 | 7 | mod generated { 8 | include!(concat!(env!("OUT_DIR"), "/generated.rs")); 9 | } 10 | 11 | let generated = generated::generate(); 12 | 13 | for (path, _) in generated.into_iter() { 14 | let path = if path == "index.html" { 15 | // "index.html" redirects to "/" 16 | "/".to_owned() 17 | } else { 18 | "/".to_owned() + path 19 | }; 20 | 21 | let (_, status_code) = server.service.get(&path).await; 22 | assert_eq!(status_code, 200); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | contact_links: 2 | - name: Feature request & feedback 3 | url: https://github.com/meilisearch/product/discussions/categories/feedback-feature-proposal 4 | about: The feature requests and feedback regarding the already existing features are not managed in this repository. Please open a discussion in our dedicated product repository 5 | - name: Documentation issue 6 | url: https://github.com/meilisearch/documentation/issues/new 7 | about: For documentation issues, open an issue or a PR in the documentation repository 8 | - name: Support questions & other 9 | url: https://github.com/meilisearch/meilisearch/discussions/new 10 | about: For any other question, open a discussion in this repository 11 | -------------------------------------------------------------------------------- /meilisearch-http/tests/assets/dumps/v1/test/updates.jsonl: -------------------------------------------------------------------------------- 1 | {"status": "processed","updateId": 0,"type": {"name":"Settings","settings":{"ranking_rules":{"Update":["Typo","Words","Proximity","Attribute","WordsPosition","Exactness"]},"distinct_attribute":"Nothing","primary_key":"Nothing","searchable_attributes":{"Update":["balance","picture","age","color","name","gender","email","phone","address","about","registered","latitude","longitude","tags"]},"displayed_attributes":{"Update":["about","address","age","balance","color","email","gender","id","isActive","latitude","longitude","name","phone","picture","registered","tags"]},"stop_words":"Nothing","synonyms":"Nothing","filterable_attributes":"Nothing"}}} 2 | {"status": "processed", "updateId": 1, "type": { "name": "DocumentsAddition"}} 3 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Meilisearch version:** [e.g. v0.20.0] 27 | 28 | **Additional context** 29 | Additional information that may be relevant to the issue. 30 | [e.g. architecture, device, OS, browser] 31 | -------------------------------------------------------------------------------- /meilisearch-lib/src/tasks/handlers/snapshot_handler.rs: -------------------------------------------------------------------------------- 1 | use crate::tasks::batch::{Batch, BatchContent}; 2 | use crate::tasks::BatchHandler; 3 | 4 | pub struct SnapshotHandler; 5 | 6 | #[async_trait::async_trait] 7 | impl BatchHandler for SnapshotHandler { 8 | fn accept(&self, batch: &Batch) -> bool { 9 | matches!(batch.content, BatchContent::Snapshot(_)) 10 | } 11 | 12 | async fn process_batch(&self, batch: Batch) -> Batch { 13 | match batch.content { 14 | BatchContent::Snapshot(job) => { 15 | if let Err(e) = job.run().await { 16 | log::error!("snapshot error: {e}"); 17 | } 18 | } 19 | _ => unreachable!(), 20 | } 21 | 22 | Batch::empty() 23 | } 24 | 25 | async fn finish(&self, _: &Batch) {} 26 | } 27 | -------------------------------------------------------------------------------- /meilisearch-lib/src/index_controller/versioning/error.rs: -------------------------------------------------------------------------------- 1 | #[derive(thiserror::Error, Debug)] 2 | pub enum VersionFileError { 3 | #[error( 4 | "Meilisearch (v{}) failed to infer the version of the database. Please consider using a dump to load your data.", 5 | env!("CARGO_PKG_VERSION").to_string() 6 | )] 7 | MissingVersionFile, 8 | #[error("Version file is corrupted and thus Meilisearch is unable to determine the version of the database.")] 9 | MalformedVersionFile, 10 | #[error( 11 | "Expected Meilisearch engine version: {major}.{minor}.{patch}, current engine version: {}. To update Meilisearch use a dump.", 12 | env!("CARGO_PKG_VERSION").to_string() 13 | )] 14 | VersionMismatch { 15 | major: String, 16 | minor: String, 17 | patch: String, 18 | }, 19 | } 20 | -------------------------------------------------------------------------------- /meilisearch-lib/src/dump/loaders/v1.rs: -------------------------------------------------------------------------------- 1 | use std::path::Path; 2 | 3 | use serde::{Deserialize, Serialize}; 4 | 5 | use crate::index_controller::IndexMetadata; 6 | 7 | #[derive(Serialize, Deserialize, Debug)] 8 | #[serde(rename_all = "camelCase")] 9 | pub struct MetadataV1 { 10 | pub db_version: String, 11 | indexes: Vec, 12 | } 13 | 14 | impl MetadataV1 { 15 | #[allow(dead_code, unreachable_code, unused_variables)] 16 | pub fn load_dump( 17 | self, 18 | src: impl AsRef, 19 | dst: impl AsRef, 20 | size: usize, 21 | indexer_options: &IndexerOpts, 22 | ) -> anyhow::Result<()> { 23 | anyhow::bail!("The version 1 of the dumps is not supported anymore. You can re-export your dump from a version between 0.21 and 0.24, or start fresh from a version 0.25 onwards.") 24 | } 25 | -------------------------------------------------------------------------------- /meilisearch-http/src/extractors/authentication/error.rs: -------------------------------------------------------------------------------- 1 | use meilisearch_types::error::{Code, ErrorCode}; 2 | 3 | #[derive(Debug, thiserror::Error)] 4 | pub enum AuthenticationError { 5 | #[error("The Authorization header is missing. It must use the bearer authorization method.")] 6 | MissingAuthorizationHeader, 7 | #[error("The provided API key is invalid.")] 8 | InvalidToken, 9 | // Triggered on configuration error. 10 | #[error("An internal error has occurred. `Irretrievable state`.")] 11 | IrretrievableState, 12 | } 13 | 14 | impl ErrorCode for AuthenticationError { 15 | fn error_code(&self) -> Code { 16 | match self { 17 | AuthenticationError::MissingAuthorizationHeader => Code::MissingAuthorizationHeader, 18 | AuthenticationError::InvalidToken => Code::InvalidToken, 19 | AuthenticationError::IrretrievableState => Code::Internal, 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /meilisearch-lib/src/compression.rs: -------------------------------------------------------------------------------- 1 | use std::fs::{create_dir_all, File}; 2 | use std::io::Write; 3 | use std::path::Path; 4 | 5 | use flate2::{read::GzDecoder, write::GzEncoder, Compression}; 6 | use tar::{Archive, Builder}; 7 | 8 | pub fn to_tar_gz(src: impl AsRef, dest: impl AsRef) -> anyhow::Result<()> { 9 | let mut f = File::create(dest)?; 10 | let gz_encoder = GzEncoder::new(&mut f, Compression::default()); 11 | let mut tar_encoder = Builder::new(gz_encoder); 12 | tar_encoder.append_dir_all(".", src)?; 13 | let gz_encoder = tar_encoder.into_inner()?; 14 | gz_encoder.finish()?; 15 | f.flush()?; 16 | Ok(()) 17 | } 18 | 19 | pub fn from_tar_gz(src: impl AsRef, dest: impl AsRef) -> anyhow::Result<()> { 20 | let f = File::open(&src)?; 21 | let gz = GzDecoder::new(f); 22 | let mut ar = Archive::new(gz); 23 | create_dir_all(&dest)?; 24 | ar.unpack(&dest)?; 25 | Ok(()) 26 | } 27 | -------------------------------------------------------------------------------- /meilisearch-lib/src/tasks/error.rs: -------------------------------------------------------------------------------- 1 | use meilisearch_types::error::{Code, ErrorCode}; 2 | use meilisearch_types::internal_error; 3 | use tokio::task::JoinError; 4 | 5 | use crate::update_file_store::UpdateFileStoreError; 6 | 7 | use super::task::TaskId; 8 | 9 | pub type Result = std::result::Result; 10 | 11 | #[derive(Debug, thiserror::Error)] 12 | pub enum TaskError { 13 | #[error("Task `{0}` not found.")] 14 | UnexistingTask(TaskId), 15 | #[error("Internal error: {0}")] 16 | Internal(Box), 17 | } 18 | 19 | internal_error!( 20 | TaskError: milli::heed::Error, 21 | JoinError, 22 | std::io::Error, 23 | serde_json::Error, 24 | UpdateFileStoreError 25 | ); 26 | 27 | impl ErrorCode for TaskError { 28 | fn error_code(&self) -> Code { 29 | match self { 30 | TaskError::UnexistingTask(_) => Code::TaskNotFound, 31 | TaskError::Internal(_) => Code::Internal, 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /meilisearch-http/src/routes/dump.rs: -------------------------------------------------------------------------------- 1 | use actix_web::{web, HttpRequest, HttpResponse}; 2 | use log::debug; 3 | use meilisearch_lib::MeiliSearch; 4 | use meilisearch_types::error::ResponseError; 5 | use serde_json::json; 6 | 7 | use crate::analytics::Analytics; 8 | use crate::extractors::authentication::{policies::*, GuardedData}; 9 | use crate::extractors::sequential_extractor::SeqHandler; 10 | use crate::task::SummarizedTaskView; 11 | 12 | pub fn configure(cfg: &mut web::ServiceConfig) { 13 | cfg.service(web::resource("").route(web::post().to(SeqHandler(create_dump)))); 14 | } 15 | 16 | pub async fn create_dump( 17 | meilisearch: GuardedData, MeiliSearch>, 18 | req: HttpRequest, 19 | analytics: web::Data, 20 | ) -> Result { 21 | analytics.publish("Dump Created".to_string(), json!({}), Some(&req)); 22 | 23 | let res: SummarizedTaskView = meilisearch.register_dump_task().await?.into(); 24 | 25 | debug!("returns: {:?}", res); 26 | Ok(HttpResponse::Accepted().json(res)) 27 | } 28 | -------------------------------------------------------------------------------- /.github/workflows/coverage.yml: -------------------------------------------------------------------------------- 1 | --- 2 | on: 3 | workflow_dispatch: 4 | 5 | name: Execute code coverage 6 | 7 | jobs: 8 | nightly-coverage: 9 | runs-on: ubuntu-18.04 10 | steps: 11 | - uses: actions/checkout@v3 12 | - uses: actions-rs/toolchain@v1 13 | with: 14 | toolchain: nightly 15 | override: true 16 | - uses: actions-rs/cargo@v1 17 | with: 18 | command: clean 19 | - uses: actions-rs/cargo@v1 20 | with: 21 | command: test 22 | args: --all-features --no-fail-fast 23 | env: 24 | CARGO_INCREMENTAL: "0" 25 | RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Cpanic=unwind -Zpanic_abort_tests" 26 | - uses: actions-rs/grcov@v0.1 27 | - name: Upload coverage to Codecov 28 | uses: codecov/codecov-action@v3 29 | with: 30 | token: ${{ secrets.CODECOV_TOKEN }} 31 | file: ${{ steps.coverage.outputs.report }} 32 | yml: ./codecov.yml 33 | fail_ci_if_error: true 34 | -------------------------------------------------------------------------------- /meilisearch-lib/src/tasks/handlers/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod dump_handler; 2 | pub mod empty_handler; 3 | mod index_resolver_handler; 4 | pub mod snapshot_handler; 5 | 6 | #[cfg(test)] 7 | mod test { 8 | use time::OffsetDateTime; 9 | 10 | use crate::tasks::{ 11 | batch::{Batch, BatchContent}, 12 | task::{Task, TaskContent}, 13 | }; 14 | 15 | pub fn task_to_batch(task: Task) -> Batch { 16 | let content = match task.content { 17 | TaskContent::DocumentAddition { .. } => { 18 | BatchContent::DocumentsAdditionBatch(vec![task]) 19 | } 20 | TaskContent::DocumentDeletion { .. } 21 | | TaskContent::SettingsUpdate { .. } 22 | | TaskContent::IndexDeletion { .. } 23 | | TaskContent::IndexCreation { .. } 24 | | TaskContent::IndexUpdate { .. } => BatchContent::IndexUpdate(task), 25 | TaskContent::Dump { .. } => BatchContent::Dump(task), 26 | }; 27 | 28 | Batch { 29 | id: Some(1), 30 | created_at: OffsetDateTime::now_utc(), 31 | content, 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019-2022 Meili SAS 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /meilisearch-lib/src/dump/error.rs: -------------------------------------------------------------------------------- 1 | use meilisearch_auth::error::AuthControllerError; 2 | use meilisearch_types::error::{Code, ErrorCode}; 3 | use meilisearch_types::internal_error; 4 | 5 | use crate::{index_resolver::error::IndexResolverError, tasks::error::TaskError}; 6 | 7 | pub type Result = std::result::Result; 8 | 9 | #[derive(thiserror::Error, Debug)] 10 | pub enum DumpError { 11 | #[error("An internal error has occurred. `{0}`.")] 12 | Internal(Box), 13 | #[error("{0}")] 14 | IndexResolver(#[from] IndexResolverError), 15 | } 16 | 17 | internal_error!( 18 | DumpError: milli::heed::Error, 19 | std::io::Error, 20 | tokio::task::JoinError, 21 | tokio::sync::oneshot::error::RecvError, 22 | serde_json::error::Error, 23 | tempfile::PersistError, 24 | fs_extra::error::Error, 25 | AuthControllerError, 26 | TaskError 27 | ); 28 | 29 | impl ErrorCode for DumpError { 30 | fn error_code(&self) -> Code { 31 | match self { 32 | DumpError::Internal(_) => Code::Internal, 33 | DumpError::IndexResolver(e) => e.error_code(), 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /meilisearch-http/tests/assets/dumps/v1/test/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "rankingRules": [ 3 | "typo", 4 | "words", 5 | "proximity", 6 | "attribute", 7 | "wordsPosition", 8 | "exactness" 9 | ], 10 | "distinctAttribute": "email", 11 | "searchableAttributes": [ 12 | "balance", 13 | "picture", 14 | "age", 15 | "color", 16 | "name", 17 | "gender", 18 | "email", 19 | "phone", 20 | "address", 21 | "about", 22 | "registered", 23 | "latitude", 24 | "longitude", 25 | "tags" 26 | ], 27 | "displayedAttributes": [ 28 | "id", 29 | "isActive", 30 | "balance", 31 | "picture", 32 | "age", 33 | "color", 34 | "name", 35 | "gender", 36 | "email", 37 | "phone", 38 | "address", 39 | "about", 40 | "registered", 41 | "latitude", 42 | "longitude", 43 | "tags" 44 | ], 45 | "stopWords": [ 46 | "in", 47 | "ad" 48 | ], 49 | "synonyms": { 50 | "wolverine": ["xmen", "logan"], 51 | "logan": ["wolverine", "xmen"] 52 | }, 53 | "filterableAttributes": [ 54 | "gender", 55 | "color", 56 | "tags", 57 | "isActive" 58 | ] 59 | } 60 | -------------------------------------------------------------------------------- /assets/logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /meilisearch-http/tests/common/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod index; 2 | pub mod server; 3 | pub mod service; 4 | 5 | pub use index::{GetAllDocumentsOptions, GetDocumentOptions}; 6 | pub use server::{default_settings, Server}; 7 | 8 | /// Performs a search test on both post and get routes 9 | #[macro_export] 10 | macro_rules! test_post_get_search { 11 | ($server:expr, $query:expr, |$response:ident, $status_code:ident | $block:expr) => { 12 | let post_query: meilisearch_http::routes::search::SearchQueryPost = 13 | serde_json::from_str(&$query.clone().to_string()).unwrap(); 14 | let get_query: meilisearch_http::routes::search::SearchQuery = post_query.into(); 15 | let get_query = ::serde_url_params::to_string(&get_query).unwrap(); 16 | let ($response, $status_code) = $server.search_get(&get_query).await; 17 | let _ = ::std::panic::catch_unwind(|| $block).map_err(|e| { 18 | panic!( 19 | "panic in get route: {:?}", 20 | e.downcast_ref::<&str>().unwrap() 21 | ) 22 | }); 23 | let ($response, $status_code) = $server.search_post($query).await; 24 | let _ = ::std::panic::catch_unwind(|| $block).map_err(|e| { 25 | panic!( 26 | "panic in post route: {:?}", 27 | e.downcast_ref::<&str>().unwrap() 28 | ) 29 | }); 30 | }; 31 | } 32 | -------------------------------------------------------------------------------- /.github/workflows/publish-deb-brew-pkg.yml: -------------------------------------------------------------------------------- 1 | name: Publish deb pkg to GitHub release & APT repository & Homebrew 2 | 3 | on: 4 | release: 5 | types: [released] 6 | 7 | jobs: 8 | debian: 9 | name: Publish debian packagge 10 | runs-on: ubuntu-18.04 11 | steps: 12 | - uses: hecrj/setup-rust-action@master 13 | with: 14 | rust-version: stable 15 | - name: Install cargo-deb 16 | run: cargo install cargo-deb 17 | - uses: actions/checkout@v3 18 | - name: Build deb package 19 | run: cargo deb -p meilisearch-http -o target/debian/meilisearch.deb 20 | - name: Upload debian pkg to release 21 | uses: svenstaro/upload-release-action@v1-release 22 | with: 23 | repo_token: ${{ secrets.GITHUB_TOKEN }} 24 | file: target/debian/meilisearch.deb 25 | asset_name: meilisearch.deb 26 | tag: ${{ github.ref }} 27 | - name: Upload debian pkg to apt repository 28 | run: curl -F package=@target/debian/meilisearch.deb https://${{ secrets.GEMFURY_PUSH_TOKEN }}@push.fury.io/meilisearch/ 29 | 30 | homebrew: 31 | name: Bump Homebrew formula 32 | runs-on: ubuntu-18.04 33 | steps: 34 | - name: Create PR to Homebrew 35 | uses: mislav/bump-homebrew-formula-action@v1 36 | with: 37 | formula-name: meilisearch 38 | env: 39 | COMMITTER_TOKEN: ${{ secrets.HOMEBREW_COMMITTER_TOKEN }} 40 | -------------------------------------------------------------------------------- /meilisearch-lib/src/dump/loaders/v5.rs: -------------------------------------------------------------------------------- 1 | use std::{path::Path, sync::Arc}; 2 | 3 | use log::info; 4 | use meilisearch_auth::AuthController; 5 | use milli::heed::EnvOpenOptions; 6 | 7 | use crate::analytics; 8 | use crate::dump::Metadata; 9 | use crate::index_resolver::IndexResolver; 10 | use crate::options::IndexerOpts; 11 | use crate::tasks::TaskStore; 12 | use crate::update_file_store::UpdateFileStore; 13 | 14 | pub fn load_dump( 15 | meta: Metadata, 16 | src: impl AsRef, 17 | dst: impl AsRef, 18 | index_db_size: usize, 19 | meta_env_size: usize, 20 | indexing_options: &IndexerOpts, 21 | ) -> anyhow::Result<()> { 22 | info!( 23 | "Loading dump from {}, dump database version: {}, dump version: V5", 24 | meta.dump_date, meta.db_version 25 | ); 26 | 27 | let mut options = EnvOpenOptions::new(); 28 | options.map_size(meta_env_size); 29 | options.max_dbs(100); 30 | let env = Arc::new(options.open(&dst)?); 31 | 32 | IndexResolver::load_dump( 33 | src.as_ref(), 34 | &dst, 35 | index_db_size, 36 | env.clone(), 37 | indexing_options, 38 | )?; 39 | UpdateFileStore::load_dump(src.as_ref(), &dst)?; 40 | TaskStore::load_dump(&src, env)?; 41 | AuthController::load_dump(&src, &dst)?; 42 | analytics::copy_user_id(src.as_ref(), dst.as_ref()); 43 | 44 | info!("Loading indexes."); 45 | 46 | Ok(()) 47 | } 48 | -------------------------------------------------------------------------------- /meilisearch-auth/src/dump.rs: -------------------------------------------------------------------------------- 1 | use serde_json::Deserializer; 2 | 3 | use std::fs::File; 4 | use std::io::BufReader; 5 | use std::io::Write; 6 | use std::path::Path; 7 | 8 | use crate::{AuthController, HeedAuthStore, Result}; 9 | 10 | const KEYS_PATH: &str = "keys"; 11 | 12 | impl AuthController { 13 | pub fn dump(src: impl AsRef, dst: impl AsRef) -> Result<()> { 14 | let mut store = HeedAuthStore::new(&src)?; 15 | 16 | // do not attempt to close the database on drop! 17 | store.set_drop_on_close(false); 18 | 19 | let keys_file_path = dst.as_ref().join(KEYS_PATH); 20 | 21 | let keys = store.list_api_keys()?; 22 | let mut keys_file = File::create(&keys_file_path)?; 23 | for key in keys { 24 | serde_json::to_writer(&mut keys_file, &key)?; 25 | keys_file.write_all(b"\n")?; 26 | } 27 | 28 | Ok(()) 29 | } 30 | 31 | pub fn load_dump(src: impl AsRef, dst: impl AsRef) -> Result<()> { 32 | let store = HeedAuthStore::new(&dst)?; 33 | 34 | let keys_file_path = src.as_ref().join(KEYS_PATH); 35 | 36 | if !keys_file_path.exists() { 37 | return Ok(()); 38 | } 39 | 40 | let reader = BufReader::new(File::open(&keys_file_path)?); 41 | for key in Deserializer::from_reader(reader).into_iter() { 42 | store.put_api_key(key?)?; 43 | } 44 | 45 | Ok(()) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /meilisearch-http/tests/settings/distinct.rs: -------------------------------------------------------------------------------- 1 | use crate::common::Server; 2 | use serde_json::json; 3 | 4 | #[actix_rt::test] 5 | async fn set_and_reset_distinct_attribute() { 6 | let server = Server::new().await; 7 | let index = server.index("test"); 8 | 9 | let (_response, _code) = index 10 | .update_settings(json!({ "distinctAttribute": "test"})) 11 | .await; 12 | index.wait_task(0).await; 13 | 14 | let (response, _) = index.settings().await; 15 | 16 | assert_eq!(response["distinctAttribute"], "test"); 17 | 18 | index 19 | .update_settings(json!({ "distinctAttribute": null })) 20 | .await; 21 | 22 | index.wait_task(1).await; 23 | 24 | let (response, _) = index.settings().await; 25 | 26 | assert_eq!(response["distinctAttribute"], json!(null)); 27 | } 28 | 29 | #[actix_rt::test] 30 | async fn set_and_reset_distinct_attribute_with_dedicated_route() { 31 | let server = Server::new().await; 32 | let index = server.index("test"); 33 | 34 | let (_response, _code) = index.update_distinct_attribute(json!("test")).await; 35 | index.wait_task(0).await; 36 | 37 | let (response, _) = index.get_distinct_attribute().await; 38 | 39 | assert_eq!(response, "test"); 40 | 41 | index.update_distinct_attribute(json!(null)).await; 42 | 43 | index.wait_task(1).await; 44 | 45 | let (response, _) = index.get_distinct_attribute().await; 46 | 47 | assert_eq!(response, json!(null)); 48 | } 49 | -------------------------------------------------------------------------------- /meilisearch-lib/src/lib.rs: -------------------------------------------------------------------------------- 1 | #[macro_use] 2 | pub mod error; 3 | pub mod options; 4 | 5 | mod analytics; 6 | mod dump; 7 | pub mod index; 8 | pub mod index_controller; 9 | mod index_resolver; 10 | mod snapshot; 11 | pub mod tasks; 12 | mod update_file_store; 13 | 14 | use std::path::Path; 15 | 16 | pub use index_controller::MeiliSearch; 17 | pub use milli; 18 | pub use milli::heed; 19 | 20 | mod compression; 21 | pub mod document_formats; 22 | 23 | use walkdir::WalkDir; 24 | 25 | pub trait EnvSizer { 26 | fn size(&self) -> u64; 27 | } 28 | 29 | impl EnvSizer for milli::heed::Env { 30 | fn size(&self) -> u64 { 31 | WalkDir::new(self.path()) 32 | .into_iter() 33 | .filter_map(|entry| entry.ok()) 34 | .filter_map(|entry| entry.metadata().ok()) 35 | .filter(|metadata| metadata.is_file()) 36 | .fold(0, |acc, m| acc + m.len()) 37 | } 38 | } 39 | 40 | /// Check if a db is empty. It does not provide any information on the 41 | /// validity of the data in it. 42 | /// We consider a database as non empty when it's a non empty directory. 43 | pub fn is_empty_db(db_path: impl AsRef) -> bool { 44 | let db_path = db_path.as_ref(); 45 | 46 | if !db_path.exists() { 47 | true 48 | // if we encounter an error or if the db is a file we consider the db non empty 49 | } else if let Ok(dir) = db_path.read_dir() { 50 | dir.count() == 0 51 | } else { 52 | true 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Compile 2 | FROM rust:alpine3.14 AS compiler 3 | 4 | RUN apk add -q --update-cache --no-cache build-base openssl-dev 5 | 6 | WORKDIR /meilisearch 7 | 8 | ARG COMMIT_SHA 9 | ARG COMMIT_DATE 10 | ENV COMMIT_SHA=${COMMIT_SHA} COMMIT_DATE=${COMMIT_DATE} 11 | ENV RUSTFLAGS="-C target-feature=-crt-static" 12 | 13 | COPY . . 14 | RUN set -eux; \ 15 | apkArch="$(apk --print-arch)"; \ 16 | if [ "$apkArch" = "aarch64" ]; then \ 17 | export JEMALLOC_SYS_WITH_LG_PAGE=16; \ 18 | fi && \ 19 | cargo build --release 20 | 21 | # Run 22 | FROM alpine:3.14 23 | 24 | ENV MEILI_HTTP_ADDR 0.0.0.0:7700 25 | ENV MEILI_SERVER_PROVIDER docker 26 | 27 | RUN apk update --quiet \ 28 | && apk add -q --no-cache libgcc tini curl 29 | 30 | # add meilisearch to the `/bin` so you can run it from anywhere and it's easy 31 | # to find. 32 | COPY --from=compiler /meilisearch/target/release/meilisearch /bin/meilisearch 33 | # To stay compatible with the older version of the container (pre v0.27.0) we're 34 | # going to symlink the meilisearch binary in the path to `/meilisearch` 35 | RUN ln -s /bin/meilisearch /meilisearch 36 | 37 | # This directory should hold all the data related to meilisearch so we're going 38 | # to move our PWD in there. 39 | # We don't want to put the meilisearch binary 40 | WORKDIR /meili_data 41 | 42 | 43 | EXPOSE 7700/tcp 44 | 45 | ENTRYPOINT ["tini", "--"] 46 | CMD /bin/meilisearch 47 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security 2 | 3 | Meilisearch takes the security of our software products and services seriously. 4 | 5 | If you believe you have found a security vulnerability in any Meilisearch-owned repository, please report it to us as described below. 6 | 7 | ## Supported versions 8 | 9 | As long as we are pre-v1.0, only the latest version of Meilisearch will be supported with security updates. 10 | 11 | ## Reporting security issues 12 | 13 | ⚠️ Please do not report security vulnerabilities through public GitHub issues. ⚠️ 14 | 15 | Instead, please kindly email us at security@meilisearch.com 16 | 17 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 18 | 19 | - Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 20 | - Full paths of source file(s) related to the manifestation of the issue 21 | - The location of the affected source code (tag/branch/commit or direct URL) 22 | - Any special configuration required to reproduce the issue 23 | - Step-by-step instructions to reproduce the issue 24 | - Proof-of-concept or exploit code (if possible) 25 | - Impact of the issue, including how an attacker might exploit the issue 26 | 27 | This information will help us triage your report more quickly. 28 | 29 | You will receive a response from us within 72 hours. If the issue is confirmed, we will release a patch as soon as possible depending on complexity. 30 | 31 | ## Preferred languages 32 | 33 | We prefer all communications to be in English. 34 | -------------------------------------------------------------------------------- /meilisearch-http/src/analytics/mock_analytics.rs: -------------------------------------------------------------------------------- 1 | use std::{any::Any, sync::Arc}; 2 | 3 | use actix_web::HttpRequest; 4 | use serde_json::Value; 5 | 6 | use crate::{routes::indexes::documents::UpdateDocumentsQuery, Opt}; 7 | 8 | use super::{find_user_id, Analytics}; 9 | 10 | pub struct MockAnalytics; 11 | 12 | #[derive(Default)] 13 | pub struct SearchAggregator {} 14 | 15 | #[allow(dead_code)] 16 | impl SearchAggregator { 17 | pub fn from_query(_: &dyn Any, _: &dyn Any) -> Self { 18 | Self::default() 19 | } 20 | 21 | pub fn succeed(&mut self, _: &dyn Any) {} 22 | } 23 | 24 | impl MockAnalytics { 25 | #[allow(clippy::new_ret_no_self)] 26 | pub fn new(opt: &Opt) -> (Arc, String) { 27 | let user = find_user_id(&opt.db_path).unwrap_or_default(); 28 | (Arc::new(Self), user) 29 | } 30 | } 31 | 32 | impl Analytics for MockAnalytics { 33 | // These methods are noop and should be optimized out 34 | fn publish(&self, _event_name: String, _send: Value, _request: Option<&HttpRequest>) {} 35 | fn get_search(&self, _aggregate: super::SearchAggregator) {} 36 | fn post_search(&self, _aggregate: super::SearchAggregator) {} 37 | fn add_documents( 38 | &self, 39 | _documents_query: &UpdateDocumentsQuery, 40 | _index_creation: bool, 41 | _request: &HttpRequest, 42 | ) { 43 | } 44 | fn update_documents( 45 | &self, 46 | _documents_query: &UpdateDocumentsQuery, 47 | _index_creation: bool, 48 | _request: &HttpRequest, 49 | ) { 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /meilisearch-lib/src/index/error.rs: -------------------------------------------------------------------------------- 1 | use std::error::Error; 2 | 3 | use meilisearch_types::error::{Code, ErrorCode}; 4 | use meilisearch_types::internal_error; 5 | use serde_json::Value; 6 | 7 | use crate::{error::MilliError, update_file_store}; 8 | 9 | pub type Result = std::result::Result; 10 | 11 | #[derive(Debug, thiserror::Error)] 12 | pub enum IndexError { 13 | #[error("An internal error has occurred. `{0}`.")] 14 | Internal(Box), 15 | #[error("Document `{0}` not found.")] 16 | DocumentNotFound(String), 17 | #[error("{0}")] 18 | Facet(#[from] FacetError), 19 | #[error("{0}")] 20 | Milli(#[from] milli::Error), 21 | } 22 | 23 | internal_error!( 24 | IndexError: std::io::Error, 25 | milli::heed::Error, 26 | fst::Error, 27 | serde_json::Error, 28 | update_file_store::UpdateFileStoreError, 29 | milli::documents::Error 30 | ); 31 | 32 | impl ErrorCode for IndexError { 33 | fn error_code(&self) -> Code { 34 | match self { 35 | IndexError::Internal(_) => Code::Internal, 36 | IndexError::DocumentNotFound(_) => Code::DocumentNotFound, 37 | IndexError::Facet(e) => e.error_code(), 38 | IndexError::Milli(e) => MilliError(e).error_code(), 39 | } 40 | } 41 | } 42 | 43 | #[derive(Debug, thiserror::Error)] 44 | pub enum FacetError { 45 | #[error("Invalid syntax for the filter parameter: `expected {}, found: {1}`.", .0.join(", "))] 46 | InvalidExpression(&'static [&'static str], Value), 47 | } 48 | 49 | impl ErrorCode for FacetError { 50 | fn error_code(&self) -> Code { 51 | match self { 52 | FacetError::InvalidExpression(_, _) => Code::Filter, 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /meilisearch-lib/src/tasks/mod.rs: -------------------------------------------------------------------------------- 1 | use async_trait::async_trait; 2 | 3 | pub use handlers::empty_handler::EmptyBatchHandler; 4 | pub use handlers::snapshot_handler::SnapshotHandler; 5 | pub use scheduler::Scheduler; 6 | pub use task_store::TaskFilter; 7 | 8 | #[cfg(test)] 9 | pub use task_store::test::MockTaskStore as TaskStore; 10 | #[cfg(not(test))] 11 | pub use task_store::TaskStore; 12 | 13 | use batch::Batch; 14 | use error::Result; 15 | 16 | pub mod batch; 17 | pub mod error; 18 | mod handlers; 19 | mod scheduler; 20 | pub mod task; 21 | mod task_store; 22 | pub mod update_loop; 23 | 24 | #[cfg_attr(test, mockall::automock(type Error=test::DebugError;))] 25 | #[async_trait] 26 | pub trait BatchHandler: Sync + Send + 'static { 27 | /// return whether this handler can accept this batch 28 | fn accept(&self, batch: &Batch) -> bool; 29 | 30 | /// Processes the `Task` batch returning the batch with the `Task` updated. 31 | /// 32 | /// It is ok for this function to panic if a batch is handed that hasn't been verified by 33 | /// `accept` beforehand. 34 | async fn process_batch(&self, batch: Batch) -> Batch; 35 | 36 | /// `finish` is called when the result of `process` has been committed to the task store. This 37 | /// method can be used to perform cleanup after the update has been completed for example. 38 | async fn finish(&self, batch: &Batch); 39 | } 40 | 41 | #[cfg(test)] 42 | mod test { 43 | use serde::{Deserialize, Serialize}; 44 | use std::fmt::Display; 45 | 46 | #[derive(Debug, Serialize, Deserialize)] 47 | pub struct DebugError; 48 | 49 | impl Display for DebugError { 50 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 51 | f.write_str("an error") 52 | } 53 | } 54 | 55 | impl std::error::Error for DebugError {} 56 | } 57 | -------------------------------------------------------------------------------- /meilisearch-http/tests/index/stats.rs: -------------------------------------------------------------------------------- 1 | use serde_json::json; 2 | 3 | use crate::common::Server; 4 | 5 | #[actix_rt::test] 6 | async fn stats() { 7 | let server = Server::new().await; 8 | let index = server.index("test"); 9 | let (_, code) = index.create(Some("id")).await; 10 | 11 | assert_eq!(code, 202); 12 | 13 | index.wait_task(0).await; 14 | 15 | let (response, code) = index.stats().await; 16 | 17 | assert_eq!(code, 200); 18 | assert_eq!(response["numberOfDocuments"], 0); 19 | assert!(response["isIndexing"] == false); 20 | assert!(response["fieldDistribution"] 21 | .as_object() 22 | .unwrap() 23 | .is_empty()); 24 | 25 | let documents = json!([ 26 | { 27 | "id": 1, 28 | "name": "Alexey", 29 | }, 30 | { 31 | "id": 2, 32 | "age": 45, 33 | } 34 | ]); 35 | 36 | let (response, code) = index.add_documents(documents, None).await; 37 | assert_eq!(code, 202); 38 | assert_eq!(response["taskUid"], 1); 39 | 40 | index.wait_task(1).await; 41 | 42 | let (response, code) = index.stats().await; 43 | 44 | assert_eq!(code, 200); 45 | assert_eq!(response["numberOfDocuments"], 2); 46 | assert!(response["isIndexing"] == false); 47 | assert_eq!(response["fieldDistribution"]["id"], 2); 48 | assert_eq!(response["fieldDistribution"]["name"], 1); 49 | assert_eq!(response["fieldDistribution"]["age"], 1); 50 | } 51 | 52 | #[actix_rt::test] 53 | async fn error_get_stats_unexisting_index() { 54 | let server = Server::new().await; 55 | let (response, code) = server.index("test").stats().await; 56 | 57 | let expected_response = json!({ 58 | "message": "Index `test` not found.", 59 | "code": "index_not_found", 60 | "type": "invalid_request", 61 | "link": "https://docs.meilisearch.com/errors#index_not_found" 62 | }); 63 | 64 | assert_eq!(response, expected_response); 65 | assert_eq!(code, 404); 66 | } 67 | -------------------------------------------------------------------------------- /meilisearch-http/src/extractors/payload.rs: -------------------------------------------------------------------------------- 1 | use std::pin::Pin; 2 | use std::task::{Context, Poll}; 3 | 4 | use actix_web::error::PayloadError; 5 | use actix_web::{dev, web, FromRequest, HttpRequest}; 6 | use futures::future::{ready, Ready}; 7 | use futures::Stream; 8 | 9 | pub struct Payload { 10 | payload: dev::Payload, 11 | limit: usize, 12 | } 13 | 14 | pub struct PayloadConfig { 15 | limit: usize, 16 | } 17 | 18 | impl PayloadConfig { 19 | pub fn new(limit: usize) -> Self { 20 | Self { limit } 21 | } 22 | } 23 | 24 | impl Default for PayloadConfig { 25 | fn default() -> Self { 26 | Self { limit: 256 * 1024 } 27 | } 28 | } 29 | 30 | impl FromRequest for Payload { 31 | type Error = PayloadError; 32 | 33 | type Future = Ready>; 34 | 35 | #[inline] 36 | fn from_request(req: &HttpRequest, payload: &mut dev::Payload) -> Self::Future { 37 | let limit = req 38 | .app_data::() 39 | .map(|c| c.limit) 40 | .unwrap_or(PayloadConfig::default().limit); 41 | ready(Ok(Payload { 42 | payload: payload.take(), 43 | limit, 44 | })) 45 | } 46 | } 47 | 48 | impl Stream for Payload { 49 | type Item = Result; 50 | 51 | #[inline] 52 | fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { 53 | match Pin::new(&mut self.payload).poll_next(cx) { 54 | Poll::Ready(Some(result)) => match result { 55 | Ok(bytes) => match self.limit.checked_sub(bytes.len()) { 56 | Some(new_limit) => { 57 | self.limit = new_limit; 58 | Poll::Ready(Some(Ok(bytes))) 59 | } 60 | None => Poll::Ready(Some(Err(PayloadError::Overflow))), 61 | }, 62 | x => Poll::Ready(Some(x)), 63 | }, 64 | otherwise => otherwise, 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /meilisearch-lib/src/index_controller/versioning/mod.rs: -------------------------------------------------------------------------------- 1 | use std::fs; 2 | use std::io::ErrorKind; 3 | use std::path::Path; 4 | 5 | use self::error::VersionFileError; 6 | 7 | mod error; 8 | 9 | pub const VERSION_FILE_NAME: &str = "VERSION"; 10 | 11 | static VERSION_MAJOR: &str = env!("CARGO_PKG_VERSION_MAJOR"); 12 | static VERSION_MINOR: &str = env!("CARGO_PKG_VERSION_MINOR"); 13 | static VERSION_PATCH: &str = env!("CARGO_PKG_VERSION_PATCH"); 14 | 15 | // Persists the version of the current Meilisearch binary to a VERSION file 16 | pub fn create_version_file(db_path: &Path) -> anyhow::Result<()> { 17 | let version_path = db_path.join(VERSION_FILE_NAME); 18 | fs::write( 19 | version_path, 20 | format!("{}.{}.{}", VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH), 21 | )?; 22 | 23 | Ok(()) 24 | } 25 | 26 | // Ensures Meilisearch version is compatible with the database, returns an error versions mismatch. 27 | pub fn check_version_file(db_path: &Path) -> anyhow::Result<()> { 28 | let version_path = db_path.join(VERSION_FILE_NAME); 29 | 30 | match fs::read_to_string(&version_path) { 31 | Ok(version) => { 32 | let version_components = version.split('.').collect::>(); 33 | let (major, minor, patch) = match &version_components[..] { 34 | [major, minor, patch] => (major.to_string(), minor.to_string(), patch.to_string()), 35 | _ => return Err(VersionFileError::MalformedVersionFile.into()), 36 | }; 37 | 38 | if major != VERSION_MAJOR || minor != VERSION_MINOR { 39 | return Err(VersionFileError::VersionMismatch { 40 | major, 41 | minor, 42 | patch, 43 | } 44 | .into()); 45 | } 46 | } 47 | Err(error) => { 48 | return match error.kind() { 49 | ErrorKind::NotFound => Err(VersionFileError::MissingVersionFile.into()), 50 | _ => Err(error.into()), 51 | } 52 | } 53 | } 54 | 55 | Ok(()) 56 | } 57 | -------------------------------------------------------------------------------- /meilisearch-http/tests/index/delete_index.rs: -------------------------------------------------------------------------------- 1 | use serde_json::json; 2 | 3 | use crate::common::Server; 4 | 5 | #[actix_rt::test] 6 | async fn create_and_delete_index() { 7 | let server = Server::new().await; 8 | let index = server.index("test"); 9 | let (_response, code) = index.create(None).await; 10 | 11 | assert_eq!(code, 202); 12 | 13 | index.wait_task(0).await; 14 | 15 | assert_eq!(index.get().await.1, 200); 16 | 17 | let (_response, code) = index.delete().await; 18 | 19 | assert_eq!(code, 202); 20 | 21 | index.wait_task(1).await; 22 | 23 | assert_eq!(index.get().await.1, 404); 24 | } 25 | 26 | #[actix_rt::test] 27 | async fn error_delete_unexisting_index() { 28 | let server = Server::new().await; 29 | let index = server.index("test"); 30 | let (_, code) = index.delete().await; 31 | 32 | assert_eq!(code, 202); 33 | 34 | let expected_response = json!({ 35 | "message": "Index `test` not found.", 36 | "code": "index_not_found", 37 | "type": "invalid_request", 38 | "link": "https://docs.meilisearch.com/errors#index_not_found" 39 | }); 40 | 41 | let response = index.wait_task(0).await; 42 | assert_eq!(response["status"], "failed"); 43 | assert_eq!(response["error"], expected_response); 44 | } 45 | 46 | #[actix_rt::test] 47 | #[cfg_attr(target_os = "windows", ignore)] 48 | async fn loop_delete_add_documents() { 49 | let server = Server::new().await; 50 | let index = server.index("test"); 51 | let documents = json!([{"id": 1, "field1": "hello"}]); 52 | let mut tasks = Vec::new(); 53 | for _ in 0..50 { 54 | let (response, code) = index.add_documents(documents.clone(), None).await; 55 | tasks.push(response["taskUid"].as_u64().unwrap()); 56 | assert_eq!(code, 202, "{}", response); 57 | let (response, code) = index.delete().await; 58 | tasks.push(response["taskUid"].as_u64().unwrap()); 59 | assert_eq!(code, 202, "{}", response); 60 | } 61 | 62 | for task in tasks { 63 | let response = index.wait_task(task).await; 64 | assert_eq!(response["status"], "succeeded", "{}", response); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /.github/workflows/publish-docker-images.yml: -------------------------------------------------------------------------------- 1 | --- 2 | on: 3 | schedule: 4 | - cron: '0 4 * * *' # Every day at 4:00am 5 | push: 6 | tags: 7 | - '*' 8 | release: 9 | types: [released] 10 | 11 | name: Publish tagged images to Docker Hub 12 | 13 | jobs: 14 | docker: 15 | runs-on: docker 16 | steps: 17 | - name: Set up QEMU 18 | uses: docker/setup-qemu-action@v2 19 | 20 | - name: Set up Docker Buildx 21 | uses: docker/setup-buildx-action@v2 22 | 23 | - name: Login to DockerHub 24 | if: github.event_name != 'schedule' 25 | uses: docker/login-action@v1 26 | with: 27 | username: ${{ secrets.DOCKER_USERNAME }} 28 | password: ${{ secrets.DOCKER_PASSWORD }} 29 | 30 | - name: Check tag format 31 | id: check-tag-format 32 | run: | 33 | # Escape submitted tag name 34 | escaped_tag=$(printf "%q" ${{ github.ref_name }}) 35 | 36 | # Check if tag has format v.. and set output.match 37 | # to create a vX.Y (without patch version) Docker tag 38 | if [[ $escaped_tag =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then 39 | echo ::set-output name=match::true 40 | else 41 | echo ::set-output name=match::false 42 | fi 43 | 44 | - name: Docker meta 45 | id: meta 46 | uses: docker/metadata-action@v4 47 | with: 48 | images: getmeili/meilisearch 49 | # The lastest tag is only pushed for the official Meilisearch release 50 | # See https://github.com/docker/metadata-action#latest-tag 51 | flavor: latest=false 52 | tags: | 53 | type=ref,event=tag 54 | type=semver,pattern=v{{major}}.{{minor}},enable=${{ steps.check-tag-format.outputs.match }} 55 | type=raw,value=latest,enable=${{ github.event_name == 'release' }} 56 | 57 | - name: Build and push 58 | id: docker_build 59 | uses: docker/build-push-action@v2 60 | with: 61 | # We do not push tags for the cron jobs, this is only for test purposes 62 | push: ${{ github.event_name != 'schedule' }} 63 | platforms: linux/amd64,linux/arm64 64 | tags: ${{ steps.meta.outputs.tags }} 65 | -------------------------------------------------------------------------------- /meilisearch-http/tests/auth/mod.rs: -------------------------------------------------------------------------------- 1 | mod api_keys; 2 | mod authorization; 3 | mod payload; 4 | mod tenant_token; 5 | 6 | use crate::common::Server; 7 | use actix_web::http::StatusCode; 8 | 9 | use serde_json::{json, Value}; 10 | 11 | impl Server { 12 | pub fn use_api_key(&mut self, api_key: impl AsRef) { 13 | self.service.api_key = Some(api_key.as_ref().to_string()); 14 | } 15 | 16 | /// Fetch and use the default admin key for nexts http requests. 17 | pub async fn use_admin_key(&mut self, master_key: impl AsRef) { 18 | self.use_api_key(master_key); 19 | let (response, code) = self.list_api_keys().await; 20 | assert_eq!(200, code, "{:?}", response); 21 | let admin_key = &response["results"][1]["key"]; 22 | self.use_api_key(admin_key.as_str().unwrap()); 23 | } 24 | 25 | pub async fn add_api_key(&self, content: Value) -> (Value, StatusCode) { 26 | let url = "/keys"; 27 | self.service.post(url, content).await 28 | } 29 | 30 | pub async fn get_api_key(&self, key: impl AsRef) -> (Value, StatusCode) { 31 | let url = format!("/keys/{}", key.as_ref()); 32 | self.service.get(url).await 33 | } 34 | 35 | pub async fn patch_api_key(&self, key: impl AsRef, content: Value) -> (Value, StatusCode) { 36 | let url = format!("/keys/{}", key.as_ref()); 37 | self.service.patch(url, content).await 38 | } 39 | 40 | pub async fn list_api_keys(&self) -> (Value, StatusCode) { 41 | let url = "/keys"; 42 | self.service.get(url).await 43 | } 44 | 45 | pub async fn delete_api_key(&self, key: impl AsRef) -> (Value, StatusCode) { 46 | let url = format!("/keys/{}", key.as_ref()); 47 | self.service.delete(url).await 48 | } 49 | 50 | pub async fn dummy_request( 51 | &self, 52 | method: impl AsRef, 53 | url: impl AsRef, 54 | ) -> (Value, StatusCode) { 55 | match method.as_ref() { 56 | "POST" => self.service.post(url, json!({})).await, 57 | "PUT" => self.service.put(url, json!({})).await, 58 | "PATCH" => self.service.patch(url, json!({})).await, 59 | "GET" => self.service.get(url).await, 60 | "DELETE" => self.service.delete(url).await, 61 | _ => unreachable!(), 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /assets/do-btn-blue.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | do-btn-blue 5 | Created with Sketch. 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | Create a Droplet 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /meilisearch-lib/src/tasks/batch.rs: -------------------------------------------------------------------------------- 1 | use time::OffsetDateTime; 2 | 3 | use crate::snapshot::SnapshotJob; 4 | 5 | use super::task::{Task, TaskEvent}; 6 | 7 | pub type BatchId = u32; 8 | 9 | #[derive(Debug)] 10 | pub enum BatchContent { 11 | DocumentsAdditionBatch(Vec), 12 | IndexUpdate(Task), 13 | Dump(Task), 14 | Snapshot(SnapshotJob), 15 | // Symbolizes a empty batch. This can occur when we were woken, but there wasn't any work to do. 16 | Empty, 17 | } 18 | 19 | impl BatchContent { 20 | pub fn first(&self) -> Option<&Task> { 21 | match self { 22 | BatchContent::DocumentsAdditionBatch(ts) => ts.first(), 23 | BatchContent::Dump(t) | BatchContent::IndexUpdate(t) => Some(t), 24 | BatchContent::Snapshot(_) | BatchContent::Empty => None, 25 | } 26 | } 27 | 28 | pub fn push_event(&mut self, event: TaskEvent) { 29 | match self { 30 | BatchContent::DocumentsAdditionBatch(ts) => { 31 | ts.iter_mut().for_each(|t| t.events.push(event.clone())) 32 | } 33 | BatchContent::IndexUpdate(t) | BatchContent::Dump(t) => t.events.push(event), 34 | BatchContent::Snapshot(_) | BatchContent::Empty => (), 35 | } 36 | } 37 | } 38 | 39 | #[derive(Debug)] 40 | pub struct Batch { 41 | // Only batches that contains a persistent tasks are given an id. Snapshot batches don't have 42 | // an id. 43 | pub id: Option, 44 | pub created_at: OffsetDateTime, 45 | pub content: BatchContent, 46 | } 47 | 48 | impl Batch { 49 | pub fn new(id: Option, content: BatchContent) -> Self { 50 | Self { 51 | id, 52 | created_at: OffsetDateTime::now_utc(), 53 | content, 54 | } 55 | } 56 | pub fn len(&self) -> usize { 57 | match self.content { 58 | BatchContent::DocumentsAdditionBatch(ref ts) => ts.len(), 59 | BatchContent::IndexUpdate(_) | BatchContent::Dump(_) | BatchContent::Snapshot(_) => 1, 60 | BatchContent::Empty => 0, 61 | } 62 | } 63 | 64 | pub fn is_empty(&self) -> bool { 65 | self.len() == 0 66 | } 67 | 68 | pub fn empty() -> Self { 69 | Self { 70 | id: None, 71 | created_at: OffsetDateTime::now_utc(), 72 | content: BatchContent::Empty, 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /meilisearch-types/src/index_uid.rs: -------------------------------------------------------------------------------- 1 | use serde::{Deserialize, Serialize}; 2 | use std::error::Error; 3 | use std::fmt; 4 | use std::str::FromStr; 5 | 6 | /// An index uid is composed of only ascii alphanumeric characters, - and _, between 1 and 400 7 | /// bytes long 8 | #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] 9 | #[cfg_attr(feature = "test-traits", derive(proptest_derive::Arbitrary))] 10 | pub struct IndexUid( 11 | #[cfg_attr(feature = "test-traits", proptest(regex("[a-zA-Z0-9_-]{1,400}")))] String, 12 | ); 13 | 14 | impl IndexUid { 15 | pub fn new_unchecked(s: impl AsRef) -> Self { 16 | Self(s.as_ref().to_string()) 17 | } 18 | 19 | pub fn into_inner(self) -> String { 20 | self.0 21 | } 22 | 23 | /// Return a reference over the inner str. 24 | pub fn as_str(&self) -> &str { 25 | &self.0 26 | } 27 | } 28 | 29 | impl std::ops::Deref for IndexUid { 30 | type Target = str; 31 | 32 | fn deref(&self) -> &Self::Target { 33 | &self.0 34 | } 35 | } 36 | 37 | impl TryFrom for IndexUid { 38 | type Error = IndexUidFormatError; 39 | 40 | fn try_from(uid: String) -> Result { 41 | if !uid 42 | .chars() 43 | .all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_') 44 | || uid.is_empty() 45 | || uid.len() > 400 46 | { 47 | Err(IndexUidFormatError { invalid_uid: uid }) 48 | } else { 49 | Ok(IndexUid(uid)) 50 | } 51 | } 52 | } 53 | 54 | impl FromStr for IndexUid { 55 | type Err = IndexUidFormatError; 56 | 57 | fn from_str(uid: &str) -> Result { 58 | uid.to_string().try_into() 59 | } 60 | } 61 | 62 | impl From for String { 63 | fn from(uid: IndexUid) -> Self { 64 | uid.into_inner() 65 | } 66 | } 67 | 68 | #[derive(Debug)] 69 | pub struct IndexUidFormatError { 70 | pub invalid_uid: String, 71 | } 72 | 73 | impl fmt::Display for IndexUidFormatError { 74 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 75 | write!( 76 | f, 77 | "invalid index uid `{}`, the uid must be an integer \ 78 | or a string containing only alphanumeric characters \ 79 | a-z A-Z 0-9, hyphens - and underscores _.", 80 | self.invalid_uid, 81 | ) 82 | } 83 | } 84 | 85 | impl Error for IndexUidFormatError {} 86 | -------------------------------------------------------------------------------- /meilisearch-lib/src/error.rs: -------------------------------------------------------------------------------- 1 | use std::error::Error; 2 | use std::fmt; 3 | 4 | use meilisearch_types::error::{Code, ErrorCode}; 5 | use milli::UserError; 6 | 7 | #[derive(Debug)] 8 | pub struct MilliError<'a>(pub &'a milli::Error); 9 | 10 | impl Error for MilliError<'_> {} 11 | 12 | impl fmt::Display for MilliError<'_> { 13 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 14 | self.0.fmt(f) 15 | } 16 | } 17 | 18 | impl ErrorCode for MilliError<'_> { 19 | fn error_code(&self) -> Code { 20 | match self.0 { 21 | milli::Error::InternalError(_) => Code::Internal, 22 | milli::Error::IoError(_) => Code::Internal, 23 | milli::Error::UserError(ref error) => { 24 | match error { 25 | // TODO: wait for spec for new error codes. 26 | UserError::SerdeJson(_) 27 | | UserError::DocumentLimitReached 28 | | UserError::UnknownInternalDocumentId { .. } => Code::Internal, 29 | UserError::InvalidStoreFile => Code::InvalidStore, 30 | UserError::NoSpaceLeftOnDevice => Code::NoSpaceLeftOnDevice, 31 | UserError::MaxDatabaseSizeReached => Code::DatabaseSizeLimitReached, 32 | UserError::AttributeLimitReached => Code::MaxFieldsLimitExceeded, 33 | UserError::InvalidFilter(_) => Code::Filter, 34 | UserError::MissingDocumentId { .. } => Code::MissingDocumentId, 35 | UserError::InvalidDocumentId { .. } => Code::InvalidDocumentId, 36 | UserError::MissingPrimaryKey => Code::MissingPrimaryKey, 37 | UserError::PrimaryKeyCannotBeChanged(_) => Code::PrimaryKeyAlreadyPresent, 38 | UserError::SortRankingRuleMissing => Code::Sort, 39 | UserError::InvalidFacetsDistribution { .. } => Code::BadRequest, 40 | UserError::InvalidSortableAttribute { .. } => Code::Sort, 41 | UserError::CriterionError(_) => Code::InvalidRankingRule, 42 | UserError::InvalidGeoField { .. } => Code::InvalidGeoField, 43 | UserError::SortError(_) => Code::Sort, 44 | UserError::InvalidMinTypoWordLenSetting(_, _) => { 45 | Code::InvalidMinWordLengthForTypo 46 | } 47 | } 48 | } 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /meilisearch-lib/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "meilisearch-lib" 3 | version = "0.28.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | actix-web = { version = "4.0.1", default-features = false } 10 | anyhow = { version = "1.0.56", features = ["backtrace"] } 11 | async-stream = "0.3.3" 12 | async-trait = "0.1.52" 13 | atomic_refcell = "0.1.8" 14 | byte-unit = { version = "4.0.14", default-features = false, features = ["std"] } 15 | bytes = "1.1.0" 16 | clap = { version = "3.1.6", features = ["derive", "env"] } 17 | crossbeam-channel = "0.5.2" 18 | csv = "1.1.6" 19 | derivative = "2.2.0" 20 | either = "1.6.1" 21 | flate2 = "1.0.22" 22 | fs_extra = "1.2.0" 23 | fst = "0.4.7" 24 | futures = "0.3.21" 25 | futures-util = "0.3.21" 26 | http = "0.2.6" 27 | indexmap = { version = "1.8.0", features = ["serde-1"] } 28 | itertools = "0.10.3" 29 | lazy_static = "1.4.0" 30 | log = "0.4.14" 31 | meilisearch-auth = { path = "../meilisearch-auth" } 32 | meilisearch-types = { path = "../meilisearch-types" } 33 | milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.29.3" } 34 | mime = "0.3.16" 35 | num_cpus = "1.13.1" 36 | obkv = "0.2.0" 37 | once_cell = "1.10.0" 38 | parking_lot = "0.12.0" 39 | permissive-json-pointer = { path = "../permissive-json-pointer" } 40 | rand = "0.8.5" 41 | rayon = "1.5.1" 42 | regex = "1.5.5" 43 | reqwest = { version = "0.11.9", features = ["json", "rustls-tls"], default-features = false, optional = true } 44 | roaring = "0.9.0" 45 | rustls = "0.20.4" 46 | serde = { version = "1.0.136", features = ["derive"] } 47 | serde_json = { version = "1.0.79", features = ["preserve_order"] } 48 | siphasher = "0.3.10" 49 | slice-group-by = "0.3.0" 50 | sysinfo = "0.23.5" 51 | tar = "0.4.38" 52 | tempfile = "3.3.0" 53 | thiserror = "1.0.30" 54 | time = { version = "0.3.7", features = ["serde-well-known", "formatting", "parsing", "macros"] } 55 | tokio = { version = "1.17.0", features = ["full"] } 56 | uuid = { version = "0.8.2", features = ["serde", "v4"] } 57 | walkdir = "2.3.2" 58 | whoami = { version = "1.2.1", optional = true } 59 | 60 | [dev-dependencies] 61 | actix-rt = "2.7.0" 62 | meilisearch-types = { path = "../meilisearch-types", features = ["test-traits"] } 63 | mockall = "0.11.0" 64 | nelson = { git = "https://github.com/meilisearch/nelson.git", rev = "675f13885548fb415ead8fbb447e9e6d9314000a"} 65 | paste = "1.0.6" 66 | proptest = "1.0.0" 67 | proptest-derive = "0.3.0" 68 | -------------------------------------------------------------------------------- /.github/workflows/rust.yml: -------------------------------------------------------------------------------- 1 | name: Rust 2 | 3 | on: 4 | workflow_dispatch: 5 | pull_request: 6 | push: 7 | # trying and staging branches are for Bors config 8 | branches: 9 | - trying 10 | - staging 11 | 12 | env: 13 | CARGO_TERM_COLOR: always 14 | RUST_BACKTRACE: 1 15 | RUSTFLAGS: "-D warnings" 16 | 17 | jobs: 18 | tests: 19 | name: Tests on ${{ matrix.os }} 20 | runs-on: ${{ matrix.os }} 21 | strategy: 22 | fail-fast: false 23 | matrix: 24 | os: [ubuntu-18.04, macos-latest, windows-latest] 25 | steps: 26 | - uses: actions/checkout@v3 27 | - name: Cache dependencies 28 | uses: Swatinem/rust-cache@v1.3.0 29 | - name: Run cargo check without any default features 30 | uses: actions-rs/cargo@v1 31 | with: 32 | command: build 33 | args: --locked --release --no-default-features 34 | - name: Run cargo test 35 | uses: actions-rs/cargo@v1 36 | with: 37 | command: test 38 | args: --locked --release 39 | 40 | # We run tests in debug also, to make sure that the debug_assertions are hit 41 | test-debug: 42 | name: Run tests in debug 43 | runs-on: ubuntu-18.04 44 | steps: 45 | - uses: actions/checkout@v3 46 | - uses: actions-rs/toolchain@v1 47 | with: 48 | profile: minimal 49 | toolchain: stable 50 | override: true 51 | - name: Cache dependencies 52 | uses: Swatinem/rust-cache@v1.3.0 53 | - name: Run tests in debug 54 | uses: actions-rs/cargo@v1 55 | with: 56 | command: test 57 | args: --locked 58 | 59 | clippy: 60 | name: Run Clippy 61 | runs-on: ubuntu-18.04 62 | steps: 63 | - uses: actions/checkout@v3 64 | - uses: actions-rs/toolchain@v1 65 | with: 66 | profile: minimal 67 | toolchain: stable 68 | override: true 69 | components: clippy 70 | - name: Cache dependencies 71 | uses: Swatinem/rust-cache@v1.3.0 72 | - name: Run cargo clippy 73 | uses: actions-rs/cargo@v1 74 | with: 75 | command: clippy 76 | args: --all-targets -- --deny warnings 77 | 78 | fmt: 79 | name: Run Rustfmt 80 | runs-on: ubuntu-18.04 81 | steps: 82 | - uses: actions/checkout@v3 83 | - uses: actions-rs/toolchain@v1 84 | with: 85 | profile: minimal 86 | toolchain: stable 87 | override: true 88 | components: rustfmt 89 | - name: Cache dependencies 90 | uses: Swatinem/rust-cache@v1.3.0 91 | - name: Run cargo fmt 92 | run: cargo fmt --all -- --check 93 | -------------------------------------------------------------------------------- /meilisearch-http/tests/stats/mod.rs: -------------------------------------------------------------------------------- 1 | use serde_json::json; 2 | use time::{format_description::well_known::Rfc3339, OffsetDateTime}; 3 | 4 | use crate::common::Server; 5 | 6 | #[actix_rt::test] 7 | async fn get_settings_unexisting_index() { 8 | let server = Server::new().await; 9 | let (response, code) = server.version().await; 10 | assert_eq!(code, 200); 11 | let version = response.as_object().unwrap(); 12 | assert!(version.get("commitSha").is_some()); 13 | assert!(version.get("commitDate").is_some()); 14 | assert!(version.get("pkgVersion").is_some()); 15 | } 16 | 17 | #[actix_rt::test] 18 | async fn test_healthyness() { 19 | let server = Server::new().await; 20 | 21 | let (response, status_code) = server.service.get("/health").await; 22 | assert_eq!(status_code, 200); 23 | assert_eq!(response["status"], "available"); 24 | } 25 | 26 | #[actix_rt::test] 27 | async fn stats() { 28 | let server = Server::new().await; 29 | let index = server.index("test"); 30 | let (_, code) = index.create(Some("id")).await; 31 | 32 | assert_eq!(code, 202); 33 | index.wait_task(0).await; 34 | 35 | let (response, code) = server.stats().await; 36 | 37 | assert_eq!(code, 200); 38 | assert!(response.get("databaseSize").is_some()); 39 | assert!(response.get("lastUpdate").is_some()); 40 | assert!(response["indexes"].get("test").is_some()); 41 | assert_eq!(response["indexes"]["test"]["numberOfDocuments"], 0); 42 | assert!(response["indexes"]["test"]["isIndexing"] == false); 43 | 44 | let documents = json!([ 45 | { 46 | "id": 1, 47 | "name": "Alexey", 48 | }, 49 | { 50 | "id": 2, 51 | "age": 45, 52 | } 53 | ]); 54 | 55 | let (response, code) = index.add_documents(documents, None).await; 56 | assert_eq!(code, 202, "{}", response); 57 | assert_eq!(response["taskUid"], 1); 58 | 59 | index.wait_task(1).await; 60 | 61 | let timestamp = OffsetDateTime::now_utc(); 62 | let (response, code) = server.stats().await; 63 | 64 | assert_eq!(code, 200); 65 | assert!(response["databaseSize"].as_u64().unwrap() > 0); 66 | let last_update = 67 | OffsetDateTime::parse(response["lastUpdate"].as_str().unwrap(), &Rfc3339).unwrap(); 68 | assert!(last_update - timestamp < time::Duration::SECOND); 69 | 70 | assert_eq!(response["indexes"]["test"]["numberOfDocuments"], 2); 71 | assert!(response["indexes"]["test"]["isIndexing"] == false); 72 | assert_eq!(response["indexes"]["test"]["fieldDistribution"]["id"], 2); 73 | assert_eq!(response["indexes"]["test"]["fieldDistribution"]["name"], 1); 74 | assert_eq!(response["indexes"]["test"]["fieldDistribution"]["age"], 1); 75 | } 76 | -------------------------------------------------------------------------------- /meilisearch-http/tests/dumps/data.rs: -------------------------------------------------------------------------------- 1 | use std::path::PathBuf; 2 | 3 | use manifest_dir_macros::exist_relative_path; 4 | 5 | pub enum GetDump { 6 | MoviesRawV1, 7 | MoviesWithSettingsV1, 8 | RubyGemsWithSettingsV1, 9 | 10 | MoviesRawV2, 11 | MoviesWithSettingsV2, 12 | RubyGemsWithSettingsV2, 13 | 14 | MoviesRawV3, 15 | MoviesWithSettingsV3, 16 | RubyGemsWithSettingsV3, 17 | 18 | MoviesRawV4, 19 | MoviesWithSettingsV4, 20 | RubyGemsWithSettingsV4, 21 | 22 | TestV5, 23 | } 24 | 25 | impl GetDump { 26 | pub fn path(&self) -> PathBuf { 27 | match self { 28 | GetDump::MoviesRawV1 => { 29 | exist_relative_path!("tests/assets/v1_v0.20.0_movies.dump").into() 30 | } 31 | GetDump::MoviesWithSettingsV1 => { 32 | exist_relative_path!("tests/assets/v1_v0.20.0_movies_with_settings.dump").into() 33 | } 34 | GetDump::RubyGemsWithSettingsV1 => { 35 | exist_relative_path!("tests/assets/v1_v0.20.0_rubygems_with_settings.dump").into() 36 | } 37 | 38 | GetDump::MoviesRawV2 => { 39 | exist_relative_path!("tests/assets/v2_v0.21.1_movies.dump").into() 40 | } 41 | GetDump::MoviesWithSettingsV2 => { 42 | exist_relative_path!("tests/assets/v2_v0.21.1_movies_with_settings.dump").into() 43 | } 44 | 45 | GetDump::RubyGemsWithSettingsV2 => { 46 | exist_relative_path!("tests/assets/v2_v0.21.1_rubygems_with_settings.dump").into() 47 | } 48 | 49 | GetDump::MoviesRawV3 => { 50 | exist_relative_path!("tests/assets/v3_v0.24.0_movies.dump").into() 51 | } 52 | GetDump::MoviesWithSettingsV3 => { 53 | exist_relative_path!("tests/assets/v3_v0.24.0_movies_with_settings.dump").into() 54 | } 55 | GetDump::RubyGemsWithSettingsV3 => { 56 | exist_relative_path!("tests/assets/v3_v0.24.0_rubygems_with_settings.dump").into() 57 | } 58 | 59 | GetDump::MoviesRawV4 => { 60 | exist_relative_path!("tests/assets/v4_v0.25.2_movies.dump").into() 61 | } 62 | GetDump::MoviesWithSettingsV4 => { 63 | exist_relative_path!("tests/assets/v4_v0.25.2_movies_with_settings.dump").into() 64 | } 65 | GetDump::RubyGemsWithSettingsV4 => { 66 | exist_relative_path!("tests/assets/v4_v0.25.2_rubygems_with_settings.dump").into() 67 | } 68 | GetDump::TestV5 => { 69 | exist_relative_path!("tests/assets/v5_v0.28.0_test_dump.dump").into() 70 | } 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /meilisearch-http/tests/snapshot/mod.rs: -------------------------------------------------------------------------------- 1 | use std::time::Duration; 2 | 3 | use crate::common::server::default_settings; 4 | use crate::common::GetAllDocumentsOptions; 5 | use crate::common::Server; 6 | use tokio::time::sleep; 7 | 8 | use meilisearch_http::Opt; 9 | 10 | macro_rules! verify_snapshot { 11 | ( 12 | $orig:expr, 13 | $snapshot: expr, 14 | |$server:ident| => 15 | $($e:expr,)+) => { 16 | use std::sync::Arc; 17 | let snapshot = Arc::new($snapshot); 18 | let orig = Arc::new($orig); 19 | $( 20 | { 21 | let test= |$server: Arc| async move { 22 | $e.await 23 | }; 24 | let (snapshot, _) = test(snapshot.clone()).await; 25 | let (orig, _) = test(orig.clone()).await; 26 | assert_eq!(snapshot, orig); 27 | } 28 | )* 29 | }; 30 | } 31 | 32 | #[actix_rt::test] 33 | async fn perform_snapshot() { 34 | let temp = tempfile::tempdir().unwrap(); 35 | let snapshot_dir = tempfile::tempdir().unwrap(); 36 | 37 | let options = Opt { 38 | snapshot_dir: snapshot_dir.path().to_owned(), 39 | snapshot_interval_sec: 1, 40 | schedule_snapshot: true, 41 | ..default_settings(temp.path()) 42 | }; 43 | 44 | let server = Server::new_with_options(options).await.unwrap(); 45 | 46 | let index = server.index("test"); 47 | index 48 | .update_settings(serde_json::json! ({ 49 | "searchableAttributes": [], 50 | })) 51 | .await; 52 | 53 | index.load_test_set().await; 54 | 55 | server.index("test1").create(Some("prim")).await; 56 | 57 | index.wait_task(2).await; 58 | 59 | sleep(Duration::from_secs(2)).await; 60 | 61 | let temp = tempfile::tempdir().unwrap(); 62 | 63 | let snapshot_path = snapshot_dir.path().to_owned().join("db.snapshot"); 64 | 65 | let options = Opt { 66 | import_snapshot: Some(snapshot_path), 67 | ..default_settings(temp.path()) 68 | }; 69 | 70 | let snapshot_server = Server::new_with_options(options).await.unwrap(); 71 | 72 | verify_snapshot!(server, snapshot_server, |server| => 73 | server.list_indexes(None, None), 74 | // for some reason the db sizes differ. this may be due to the compaction options we have 75 | // set when performing the snapshot 76 | //server.stats(), 77 | server.tasks(), 78 | server.index("test").get_all_documents(GetAllDocumentsOptions::default()), 79 | server.index("test").settings(), 80 | server.index("test1").get_all_documents(GetAllDocumentsOptions::default()), 81 | server.index("test1").settings(), 82 | ); 83 | } 84 | -------------------------------------------------------------------------------- /meilisearch-lib/src/index_resolver/error.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | 3 | use meilisearch_types::error::{Code, ErrorCode}; 4 | use meilisearch_types::index_uid::IndexUidFormatError; 5 | use meilisearch_types::internal_error; 6 | use tokio::sync::mpsc::error::SendError as MpscSendError; 7 | use tokio::sync::oneshot::error::RecvError as OneshotRecvError; 8 | use uuid::Uuid; 9 | 10 | use crate::{error::MilliError, index::error::IndexError, update_file_store::UpdateFileStoreError}; 11 | 12 | pub type Result = std::result::Result; 13 | 14 | #[derive(thiserror::Error, Debug)] 15 | pub enum IndexResolverError { 16 | #[error("{0}")] 17 | IndexError(#[from] IndexError), 18 | #[error("Index `{0}` already exists.")] 19 | IndexAlreadyExists(String), 20 | #[error("Index `{0}` not found.")] 21 | UnexistingIndex(String), 22 | #[error("A primary key is already present. It's impossible to update it")] 23 | ExistingPrimaryKey, 24 | #[error("An internal error has occurred. `{0}`.")] 25 | Internal(Box), 26 | #[error("The creation of the `{0}` index has failed due to `Index uuid is already assigned`.")] 27 | UuidAlreadyExists(Uuid), 28 | #[error("{0}")] 29 | Milli(#[from] milli::Error), 30 | #[error("{0}")] 31 | BadlyFormatted(#[from] IndexUidFormatError), 32 | } 33 | 34 | impl From> for IndexResolverError 35 | where 36 | T: Send + Sync + 'static + fmt::Debug, 37 | { 38 | fn from(other: tokio::sync::mpsc::error::SendError) -> Self { 39 | Self::Internal(Box::new(other)) 40 | } 41 | } 42 | 43 | impl From for IndexResolverError { 44 | fn from(other: tokio::sync::oneshot::error::RecvError) -> Self { 45 | Self::Internal(Box::new(other)) 46 | } 47 | } 48 | 49 | internal_error!( 50 | IndexResolverError: milli::heed::Error, 51 | uuid::Error, 52 | std::io::Error, 53 | tokio::task::JoinError, 54 | serde_json::Error, 55 | UpdateFileStoreError 56 | ); 57 | 58 | impl ErrorCode for IndexResolverError { 59 | fn error_code(&self) -> Code { 60 | match self { 61 | IndexResolverError::IndexError(e) => e.error_code(), 62 | IndexResolverError::IndexAlreadyExists(_) => Code::IndexAlreadyExists, 63 | IndexResolverError::UnexistingIndex(_) => Code::IndexNotFound, 64 | IndexResolverError::ExistingPrimaryKey => Code::PrimaryKeyAlreadyPresent, 65 | IndexResolverError::Internal(_) => Code::Internal, 66 | IndexResolverError::UuidAlreadyExists(_) => Code::CreateIndex, 67 | IndexResolverError::Milli(e) => MilliError(e).error_code(), 68 | IndexResolverError::BadlyFormatted(_) => Code::InvalidIndexUid, 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /meilisearch-lib/src/index_controller/updates/error.rs: -------------------------------------------------------------------------------- 1 | use std::error::Error; 2 | use std::fmt; 3 | 4 | use meilisearch_types::{internal_error, Code, ErrorCode}; 5 | 6 | use crate::{ 7 | document_formats::DocumentFormatError, 8 | index::error::IndexError, 9 | index_controller::{update_file_store::UpdateFileStoreError, DocumentAdditionFormat}, 10 | }; 11 | 12 | pub type Result = std::result::Result; 13 | 14 | #[derive(Debug, thiserror::Error)] 15 | #[allow(clippy::large_enum_variant)] 16 | pub enum UpdateLoopError { 17 | #[error("Task `{0}` not found.")] 18 | UnexistingUpdate(u64), 19 | #[error("An internal error has occurred. `{0}`.")] 20 | Internal(Box), 21 | #[error( 22 | "update store was shut down due to a fatal error, please check your logs for more info." 23 | )] 24 | FatalUpdateStoreError, 25 | #[error("{0}")] 26 | DocumentFormatError(#[from] DocumentFormatError), 27 | #[error("The provided payload reached the size limit.")] 28 | PayloadTooLarge, 29 | #[error("A {0} payload is missing.")] 30 | MissingPayload(DocumentAdditionFormat), 31 | #[error("{0}")] 32 | IndexError(#[from] IndexError), 33 | } 34 | 35 | impl From> for UpdateLoopError 36 | where 37 | T: Sync + Send + 'static + fmt::Debug, 38 | { 39 | fn from(other: tokio::sync::mpsc::error::SendError) -> Self { 40 | Self::Internal(Box::new(other)) 41 | } 42 | } 43 | 44 | impl From for UpdateLoopError { 45 | fn from(other: tokio::sync::oneshot::error::RecvError) -> Self { 46 | Self::Internal(Box::new(other)) 47 | } 48 | } 49 | 50 | impl From for UpdateLoopError { 51 | fn from(other: actix_web::error::PayloadError) -> Self { 52 | match other { 53 | actix_web::error::PayloadError::Overflow => Self::PayloadTooLarge, 54 | _ => Self::Internal(Box::new(other)), 55 | } 56 | } 57 | } 58 | 59 | internal_error!( 60 | UpdateLoopError: heed::Error, 61 | std::io::Error, 62 | serde_json::Error, 63 | tokio::task::JoinError, 64 | UpdateFileStoreError 65 | ); 66 | 67 | impl ErrorCode for UpdateLoopError { 68 | fn error_code(&self) -> Code { 69 | match self { 70 | Self::UnexistingUpdate(_) => Code::TaskNotFound, 71 | Self::Internal(_) => Code::Internal, 72 | Self::FatalUpdateStoreError => Code::Internal, 73 | Self::DocumentFormatError(error) => error.error_code(), 74 | Self::PayloadTooLarge => Code::PayloadTooLarge, 75 | Self::MissingPayload(_) => Code::MissingPayload, 76 | Self::IndexError(e) => e.error_code(), 77 | } 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /meilisearch-lib/src/index_controller/error.rs: -------------------------------------------------------------------------------- 1 | use std::error::Error; 2 | 3 | use meilisearch_types::error::{Code, ErrorCode}; 4 | use meilisearch_types::index_uid::IndexUidFormatError; 5 | use meilisearch_types::internal_error; 6 | use tokio::task::JoinError; 7 | 8 | use super::DocumentAdditionFormat; 9 | use crate::document_formats::DocumentFormatError; 10 | use crate::dump::error::DumpError; 11 | use crate::index::error::IndexError; 12 | use crate::tasks::error::TaskError; 13 | use crate::update_file_store::UpdateFileStoreError; 14 | 15 | use crate::index_resolver::error::IndexResolverError; 16 | 17 | pub type Result = std::result::Result; 18 | 19 | #[derive(Debug, thiserror::Error)] 20 | pub enum IndexControllerError { 21 | #[error("Index creation must have an uid")] 22 | MissingUid, 23 | #[error("{0}")] 24 | IndexResolver(#[from] IndexResolverError), 25 | #[error("{0}")] 26 | IndexError(#[from] IndexError), 27 | #[error("An internal error has occurred. `{0}`.")] 28 | Internal(Box), 29 | #[error("{0}")] 30 | TaskError(#[from] TaskError), 31 | #[error("{0}")] 32 | DumpError(#[from] DumpError), 33 | #[error("{0}")] 34 | DocumentFormatError(#[from] DocumentFormatError), 35 | #[error("A {0} payload is missing.")] 36 | MissingPayload(DocumentAdditionFormat), 37 | #[error("The provided payload reached the size limit.")] 38 | PayloadTooLarge, 39 | } 40 | 41 | internal_error!(IndexControllerError: JoinError, UpdateFileStoreError); 42 | 43 | impl From for IndexControllerError { 44 | fn from(other: actix_web::error::PayloadError) -> Self { 45 | match other { 46 | actix_web::error::PayloadError::Overflow => Self::PayloadTooLarge, 47 | _ => Self::Internal(Box::new(other)), 48 | } 49 | } 50 | } 51 | 52 | impl ErrorCode for IndexControllerError { 53 | fn error_code(&self) -> Code { 54 | match self { 55 | IndexControllerError::MissingUid => Code::BadRequest, 56 | IndexControllerError::IndexResolver(e) => e.error_code(), 57 | IndexControllerError::IndexError(e) => e.error_code(), 58 | IndexControllerError::Internal(_) => Code::Internal, 59 | IndexControllerError::TaskError(e) => e.error_code(), 60 | IndexControllerError::DocumentFormatError(e) => e.error_code(), 61 | IndexControllerError::MissingPayload(_) => Code::MissingPayload, 62 | IndexControllerError::PayloadTooLarge => Code::PayloadTooLarge, 63 | IndexControllerError::DumpError(e) => e.error_code(), 64 | } 65 | } 66 | } 67 | 68 | impl From for IndexControllerError { 69 | fn from(err: IndexUidFormatError) -> Self { 70 | IndexResolverError::from(err).into() 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /meilisearch-auth/src/error.rs: -------------------------------------------------------------------------------- 1 | use std::error::Error; 2 | 3 | use meilisearch_types::error::{Code, ErrorCode}; 4 | use meilisearch_types::internal_error; 5 | use serde_json::Value; 6 | 7 | pub type Result = std::result::Result; 8 | 9 | #[derive(Debug, thiserror::Error)] 10 | pub enum AuthControllerError { 11 | #[error("`{0}` field is mandatory.")] 12 | MissingParameter(&'static str), 13 | #[error("`actions` field value `{0}` is invalid. It should be an array of string representing action names.")] 14 | InvalidApiKeyActions(Value), 15 | #[error("`indexes` field value `{0}` is invalid. It should be an array of string representing index names.")] 16 | InvalidApiKeyIndexes(Value), 17 | #[error("`expiresAt` field value `{0}` is invalid. It should follow the RFC 3339 format to represents a date or datetime in the future or specified as a null value. e.g. 'YYYY-MM-DD' or 'YYYY-MM-DD HH:MM:SS'.")] 18 | InvalidApiKeyExpiresAt(Value), 19 | #[error("`description` field value `{0}` is invalid. It should be a string or specified as a null value.")] 20 | InvalidApiKeyDescription(Value), 21 | #[error( 22 | "`name` field value `{0}` is invalid. It should be a string or specified as a null value." 23 | )] 24 | InvalidApiKeyName(Value), 25 | #[error("`uid` field value `{0}` is invalid. It should be a valid UUID v4 string or omitted.")] 26 | InvalidApiKeyUid(Value), 27 | #[error("API key `{0}` not found.")] 28 | ApiKeyNotFound(String), 29 | #[error("`uid` field value `{0}` is already an existing API key.")] 30 | ApiKeyAlreadyExists(String), 31 | #[error("The `{0}` field cannot be modified for the given resource.")] 32 | ImmutableField(String), 33 | #[error("Internal error: {0}")] 34 | Internal(Box), 35 | } 36 | 37 | internal_error!( 38 | AuthControllerError: milli::heed::Error, 39 | std::io::Error, 40 | serde_json::Error, 41 | std::str::Utf8Error 42 | ); 43 | 44 | impl ErrorCode for AuthControllerError { 45 | fn error_code(&self) -> Code { 46 | match self { 47 | Self::MissingParameter(_) => Code::MissingParameter, 48 | Self::InvalidApiKeyActions(_) => Code::InvalidApiKeyActions, 49 | Self::InvalidApiKeyIndexes(_) => Code::InvalidApiKeyIndexes, 50 | Self::InvalidApiKeyExpiresAt(_) => Code::InvalidApiKeyExpiresAt, 51 | Self::InvalidApiKeyDescription(_) => Code::InvalidApiKeyDescription, 52 | Self::InvalidApiKeyName(_) => Code::InvalidApiKeyName, 53 | Self::ApiKeyNotFound(_) => Code::ApiKeyNotFound, 54 | Self::InvalidApiKeyUid(_) => Code::InvalidApiKeyUid, 55 | Self::ApiKeyAlreadyExists(_) => Code::ApiKeyAlreadyExists, 56 | Self::ImmutableField(_) => Code::ImmutableField, 57 | Self::Internal(_) => Code::Internal, 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /permissive-json-pointer/README.md: -------------------------------------------------------------------------------- 1 | # Permissive json pointer 2 | 3 | This crate provide an interface a little bit similar to what you know as “json pointer”. 4 | But it’s actually doing something quite different. 5 | 6 | ## The API 7 | 8 | The crate provide only one function called [`select_values`]. 9 | It takes one object in parameter and a list of selectors. 10 | It then returns a new object containing only the fields you selected. 11 | 12 | ## The selectors 13 | 14 | The syntax for the selector is easier than with other API. 15 | There is only ONE special symbol, it’s the `.`. 16 | 17 | If you write `dog` and provide the following object; 18 | ```json 19 | { 20 | "dog": "bob", 21 | "cat": "michel" 22 | } 23 | ``` 24 | You’ll get back; 25 | ```json 26 | { 27 | "dog": "bob", 28 | } 29 | ``` 30 | 31 | Easy right? 32 | 33 | Now the dot can either be used as a field name, or as a nested object. 34 | 35 | For example, if you have the following json; 36 | ```json 37 | { 38 | "dog.name": "jean", 39 | "dog": { 40 | "name": "bob", 41 | "age": 6 42 | } 43 | } 44 | ``` 45 | 46 | What a crappy json! But never underestimate your users, they [_WILL_](https://xkcd.com/1172/) 47 | somehow base their entire workflow on this kind of json. 48 | Here with the `dog.name` selector both fields will be 49 | selected and the following json will be returned; 50 | ```json 51 | { 52 | "dog.name": "jean", 53 | "dog": { 54 | "name": "bob", 55 | } 56 | } 57 | ``` 58 | 59 | And as you can guess, this crate is as permissive as possible. 60 | It’ll match everything it can! 61 | Consider this even more crappy json; 62 | ```json 63 | { 64 | "pet.dog.name": "jean", 65 | "pet.dog": { 66 | "name": "bob" 67 | }, 68 | "pet": { 69 | "dog.name": "michel" 70 | }, 71 | "pet": { 72 | "dog": { 73 | "name": "milan" 74 | } 75 | } 76 | } 77 | ``` 78 | If you write `pet.dog.name` everything will be selected. 79 | 80 | ## Matching arrays 81 | 82 | With this kind of selectors you can’t match a specific element in an array. 83 | Your selector will be applied to all the element _in_ the array. 84 | 85 | Consider the following json; 86 | ```json 87 | { 88 | "pets": [ 89 | { 90 | "animal": "dog", 91 | "race": "bernese mountain", 92 | }, 93 | { 94 | "animal": "dog", 95 | "race": "golden retriever", 96 | }, 97 | { 98 | "animal": "cat", 99 | "age": 8, 100 | } 101 | ] 102 | } 103 | ``` 104 | 105 | With the filter `pets.animal` you’ll get; 106 | ```json 107 | { 108 | "pets": [ 109 | { 110 | "animal": "dog", 111 | }, 112 | { 113 | "animal": "dog", 114 | }, 115 | { 116 | "animal": "cat", 117 | } 118 | ] 119 | } 120 | ``` 121 | 122 | The empty element in an array gets removed. So if you were to look 123 | for `pets.age` you would only get; 124 | ```json 125 | { 126 | "pets": [ 127 | { 128 | "age": 8, 129 | } 130 | ] 131 | } 132 | ``` 133 | 134 | And I think that’s all you need to know 🎉 -------------------------------------------------------------------------------- /meilisearch-http/build.rs: -------------------------------------------------------------------------------- 1 | use vergen::{vergen, Config}; 2 | 3 | fn main() { 4 | if let Err(e) = vergen(Config::default()) { 5 | println!("cargo:warning=vergen: {}", e); 6 | } 7 | 8 | #[cfg(feature = "mini-dashboard")] 9 | mini_dashboard::setup_mini_dashboard().expect("Could not load the mini-dashboard assets"); 10 | } 11 | 12 | #[cfg(feature = "mini-dashboard")] 13 | mod mini_dashboard { 14 | use std::env; 15 | use std::fs::{create_dir_all, File, OpenOptions}; 16 | use std::io::{Cursor, Read, Write}; 17 | use std::path::PathBuf; 18 | 19 | use anyhow::Context; 20 | use cargo_toml::Manifest; 21 | use reqwest::blocking::get; 22 | use sha1::{Digest, Sha1}; 23 | use static_files::resource_dir; 24 | 25 | pub fn setup_mini_dashboard() -> anyhow::Result<()> { 26 | let cargo_manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); 27 | let cargo_toml = cargo_manifest_dir.join("Cargo.toml"); 28 | let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); 29 | 30 | let sha1_path = out_dir.join(".mini-dashboard.sha1"); 31 | let dashboard_dir = out_dir.join("mini-dashboard"); 32 | 33 | let manifest = Manifest::from_path(cargo_toml).unwrap(); 34 | 35 | let meta = &manifest 36 | .package 37 | .as_ref() 38 | .context("package not specified in Cargo.toml")? 39 | .metadata 40 | .as_ref() 41 | .context("no metadata specified in Cargo.toml")?["mini-dashboard"]; 42 | 43 | // Check if there already is a dashboard built, and if it is up to date. 44 | if sha1_path.exists() && dashboard_dir.exists() { 45 | let mut sha1_file = File::open(&sha1_path)?; 46 | let mut sha1 = String::new(); 47 | sha1_file.read_to_string(&mut sha1)?; 48 | if sha1 == meta["sha1"].as_str().unwrap() { 49 | // Nothing to do. 50 | return Ok(()); 51 | } 52 | } 53 | 54 | let url = meta["assets-url"].as_str().unwrap(); 55 | 56 | let dashboard_assets_bytes = get(url)?.bytes()?; 57 | 58 | let mut hasher = Sha1::new(); 59 | hasher.update(&dashboard_assets_bytes); 60 | let sha1 = hex::encode(hasher.finalize()); 61 | 62 | assert_eq!( 63 | meta["sha1"].as_str().unwrap(), 64 | sha1, 65 | "Downloaded mini-dashboard shasum differs from the one specified in the Cargo.toml" 66 | ); 67 | 68 | create_dir_all(&dashboard_dir)?; 69 | let cursor = Cursor::new(&dashboard_assets_bytes); 70 | let mut zip = zip::read::ZipArchive::new(cursor)?; 71 | zip.extract(&dashboard_dir)?; 72 | resource_dir(&dashboard_dir).build()?; 73 | 74 | // Write the sha1 for the dashboard back to file. 75 | let mut file = OpenOptions::new() 76 | .write(true) 77 | .create(true) 78 | .truncate(true) 79 | .open(sha1_path)?; 80 | 81 | file.write_all(sha1.as_bytes())?; 82 | file.flush()?; 83 | 84 | Ok(()) 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /meilisearch-lib/src/dump/loaders/v4.rs: -------------------------------------------------------------------------------- 1 | use std::fs::{self, create_dir_all, File}; 2 | use std::io::{BufReader, Write}; 3 | use std::path::Path; 4 | 5 | use fs_extra::dir::{self, CopyOptions}; 6 | use log::info; 7 | use serde_json::{Deserializer, Map, Value}; 8 | use tempfile::tempdir; 9 | use uuid::Uuid; 10 | 11 | use crate::dump::{compat, Metadata}; 12 | use crate::options::IndexerOpts; 13 | use crate::tasks::task::Task; 14 | 15 | pub fn load_dump( 16 | meta: Metadata, 17 | src: impl AsRef, 18 | dst: impl AsRef, 19 | index_db_size: usize, 20 | meta_env_size: usize, 21 | indexing_options: &IndexerOpts, 22 | ) -> anyhow::Result<()> { 23 | info!("Patching dump V4 to dump V5..."); 24 | 25 | let patched_dir = tempdir()?; 26 | let options = CopyOptions::default(); 27 | 28 | // Indexes 29 | dir::copy(src.as_ref().join("indexes"), &patched_dir, &options)?; 30 | 31 | // Index uuids 32 | dir::copy(src.as_ref().join("index_uuids"), &patched_dir, &options)?; 33 | 34 | // Metadata 35 | fs::copy( 36 | src.as_ref().join("metadata.json"), 37 | patched_dir.path().join("metadata.json"), 38 | )?; 39 | 40 | // Updates 41 | patch_updates(&src, &patched_dir)?; 42 | 43 | // Keys 44 | patch_keys(&src, &patched_dir)?; 45 | 46 | super::v5::load_dump( 47 | meta, 48 | &patched_dir, 49 | dst, 50 | index_db_size, 51 | meta_env_size, 52 | indexing_options, 53 | ) 54 | } 55 | 56 | fn patch_updates(src: impl AsRef, dst: impl AsRef) -> anyhow::Result<()> { 57 | let updates_path = src.as_ref().join("updates/data.jsonl"); 58 | let output_updates_path = dst.as_ref().join("updates/data.jsonl"); 59 | create_dir_all(output_updates_path.parent().unwrap())?; 60 | let udpates_file = File::open(updates_path)?; 61 | let mut output_update_file = File::create(output_updates_path)?; 62 | 63 | serde_json::Deserializer::from_reader(udpates_file) 64 | .into_iter::() 65 | .try_for_each(|task| -> anyhow::Result<()> { 66 | let task: Task = task?.into(); 67 | 68 | serde_json::to_writer(&mut output_update_file, &task)?; 69 | output_update_file.write_all(b"\n")?; 70 | 71 | Ok(()) 72 | })?; 73 | 74 | output_update_file.flush()?; 75 | 76 | Ok(()) 77 | } 78 | 79 | fn patch_keys(src: impl AsRef, dst: impl AsRef) -> anyhow::Result<()> { 80 | let keys_file_src = src.as_ref().join("keys"); 81 | 82 | if !keys_file_src.exists() { 83 | return Ok(()); 84 | } 85 | 86 | fs::create_dir_all(&dst)?; 87 | let keys_file_dst = dst.as_ref().join("keys"); 88 | let mut writer = File::create(&keys_file_dst)?; 89 | 90 | let reader = BufReader::new(File::open(&keys_file_src)?); 91 | for key in Deserializer::from_reader(reader).into_iter() { 92 | let mut key: Map = key?; 93 | 94 | // generate a new uuid v4 and insert it in the key. 95 | let uid = serde_json::to_value(Uuid::new_v4()).unwrap(); 96 | key.insert("uid".to_string(), uid); 97 | 98 | serde_json::to_writer(&mut writer, &key)?; 99 | writer.write_all(b"\n")?; 100 | } 101 | 102 | Ok(()) 103 | } 104 | -------------------------------------------------------------------------------- /meilisearch-http/tests/index/update_index.rs: -------------------------------------------------------------------------------- 1 | use crate::common::Server; 2 | use serde_json::json; 3 | use time::{format_description::well_known::Rfc3339, OffsetDateTime}; 4 | 5 | #[actix_rt::test] 6 | async fn update_primary_key() { 7 | let server = Server::new().await; 8 | let index = server.index("test"); 9 | let (_, code) = index.create(None).await; 10 | 11 | assert_eq!(code, 202); 12 | 13 | index.update(Some("primary")).await; 14 | 15 | let response = index.wait_task(1).await; 16 | 17 | assert_eq!(response["status"], "succeeded"); 18 | 19 | let (response, code) = index.get().await; 20 | 21 | assert_eq!(code, 200); 22 | 23 | assert_eq!(response["uid"], "test"); 24 | assert!(response.get("createdAt").is_some()); 25 | assert!(response.get("updatedAt").is_some()); 26 | 27 | let created_at = 28 | OffsetDateTime::parse(response["createdAt"].as_str().unwrap(), &Rfc3339).unwrap(); 29 | let updated_at = 30 | OffsetDateTime::parse(response["updatedAt"].as_str().unwrap(), &Rfc3339).unwrap(); 31 | assert!(created_at < updated_at); 32 | 33 | assert_eq!(response["primaryKey"], "primary"); 34 | assert_eq!(response.as_object().unwrap().len(), 4); 35 | } 36 | 37 | #[actix_rt::test] 38 | async fn update_nothing() { 39 | let server = Server::new().await; 40 | let index = server.index("test"); 41 | let (_, code) = index.create(None).await; 42 | 43 | assert_eq!(code, 202); 44 | 45 | index.wait_task(0).await; 46 | 47 | let (_, code) = index.update(None).await; 48 | 49 | assert_eq!(code, 202); 50 | 51 | let response = index.wait_task(1).await; 52 | 53 | assert_eq!(response["status"], "succeeded"); 54 | } 55 | 56 | #[actix_rt::test] 57 | async fn error_update_existing_primary_key() { 58 | let server = Server::new().await; 59 | let index = server.index("test"); 60 | let (_response, code) = index.create(Some("id")).await; 61 | 62 | assert_eq!(code, 202); 63 | 64 | let documents = json!([ 65 | { 66 | "id": "11", 67 | "content": "foobar" 68 | } 69 | ]); 70 | index.add_documents(documents, None).await; 71 | 72 | let (_, code) = index.update(Some("primary")).await; 73 | 74 | assert_eq!(code, 202); 75 | 76 | let response = index.wait_task(2).await; 77 | 78 | let expected_response = json!({ 79 | "message": "Index already has a primary key: `id`.", 80 | "code": "index_primary_key_already_exists", 81 | "type": "invalid_request", 82 | "link": "https://docs.meilisearch.com/errors#index_primary_key_already_exists" 83 | }); 84 | 85 | assert_eq!(response["error"], expected_response); 86 | } 87 | 88 | #[actix_rt::test] 89 | async fn error_update_unexisting_index() { 90 | let server = Server::new().await; 91 | let (_, code) = server.index("test").update(None).await; 92 | 93 | assert_eq!(code, 202); 94 | 95 | let response = server.index("test").wait_task(0).await; 96 | 97 | let expected_response = json!({ 98 | "message": "Index `test` not found.", 99 | "code": "index_not_found", 100 | "type": "invalid_request", 101 | "link": "https://docs.meilisearch.com/errors#index_not_found" 102 | }); 103 | 104 | assert_eq!(response["error"], expected_response); 105 | } 106 | -------------------------------------------------------------------------------- /meilisearch-http/src/analytics/mod.rs: -------------------------------------------------------------------------------- 1 | mod mock_analytics; 2 | // if we are in release mode and the feature analytics was enabled 3 | #[cfg(all(not(debug_assertions), feature = "analytics"))] 4 | mod segment_analytics; 5 | 6 | use std::fs; 7 | use std::path::{Path, PathBuf}; 8 | 9 | use actix_web::HttpRequest; 10 | use once_cell::sync::Lazy; 11 | use platform_dirs::AppDirs; 12 | use serde_json::Value; 13 | 14 | use crate::routes::indexes::documents::UpdateDocumentsQuery; 15 | 16 | pub use mock_analytics::MockAnalytics; 17 | 18 | // if we are in debug mode OR the analytics feature is disabled 19 | // the `SegmentAnalytics` point to the mock instead of the real analytics 20 | #[cfg(any(debug_assertions, not(feature = "analytics")))] 21 | pub type SegmentAnalytics = mock_analytics::MockAnalytics; 22 | #[cfg(any(debug_assertions, not(feature = "analytics")))] 23 | pub type SearchAggregator = mock_analytics::SearchAggregator; 24 | 25 | // if we are in release mode and the feature analytics was enabled 26 | // we use the real analytics 27 | #[cfg(all(not(debug_assertions), feature = "analytics"))] 28 | pub type SegmentAnalytics = segment_analytics::SegmentAnalytics; 29 | #[cfg(all(not(debug_assertions), feature = "analytics"))] 30 | pub type SearchAggregator = segment_analytics::SearchAggregator; 31 | 32 | /// The Meilisearch config dir: 33 | /// `~/.config/Meilisearch` on *NIX or *BSD. 34 | /// `~/Library/ApplicationSupport` on macOS. 35 | /// `%APPDATA` (= `C:\Users%USERNAME%\AppData\Roaming`) on windows. 36 | static MEILISEARCH_CONFIG_PATH: Lazy> = 37 | Lazy::new(|| AppDirs::new(Some("Meilisearch"), false).map(|appdir| appdir.config_dir)); 38 | 39 | fn config_user_id_path(db_path: &Path) -> Option { 40 | db_path 41 | .canonicalize() 42 | .ok() 43 | .map(|path| { 44 | path.join("instance-uid") 45 | .display() 46 | .to_string() 47 | .replace('/', "-") 48 | }) 49 | .zip(MEILISEARCH_CONFIG_PATH.as_ref()) 50 | .map(|(filename, config_path)| config_path.join(filename.trim_start_matches('-'))) 51 | } 52 | 53 | /// Look for the instance-uid in the `data.ms` or in `~/.config/Meilisearch/path-to-db-instance-uid` 54 | fn find_user_id(db_path: &Path) -> Option { 55 | fs::read_to_string(db_path.join("instance-uid")) 56 | .ok() 57 | .or_else(|| fs::read_to_string(&config_user_id_path(db_path)?).ok()) 58 | } 59 | 60 | pub trait Analytics: Sync + Send { 61 | /// The method used to publish most analytics that do not need to be batched every hours 62 | fn publish(&self, event_name: String, send: Value, request: Option<&HttpRequest>); 63 | 64 | /// This method should be called to aggregate a get search 65 | fn get_search(&self, aggregate: SearchAggregator); 66 | 67 | /// This method should be called to aggregate a post search 68 | fn post_search(&self, aggregate: SearchAggregator); 69 | 70 | // this method should be called to aggregate a add documents request 71 | fn add_documents( 72 | &self, 73 | documents_query: &UpdateDocumentsQuery, 74 | index_creation: bool, 75 | request: &HttpRequest, 76 | ); 77 | // this method should be called to batch a update documents request 78 | fn update_documents( 79 | &self, 80 | documents_query: &UpdateDocumentsQuery, 81 | index_creation: bool, 82 | request: &HttpRequest, 83 | ); 84 | } 85 | -------------------------------------------------------------------------------- /meilisearch-lib/src/tasks/update_loop.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | use std::time::Duration; 3 | 4 | use time::OffsetDateTime; 5 | use tokio::sync::{watch, RwLock}; 6 | use tokio::time::interval_at; 7 | 8 | use super::batch::Batch; 9 | use super::error::Result; 10 | use super::{BatchHandler, Scheduler}; 11 | use crate::tasks::task::TaskEvent; 12 | 13 | /// The update loop sequentially performs batches of updates by asking the scheduler for a batch, 14 | /// and handing it to the `TaskPerformer`. 15 | pub struct UpdateLoop { 16 | scheduler: Arc>, 17 | performers: Vec>, 18 | 19 | notifier: Option>, 20 | debounce_duration: Option, 21 | } 22 | 23 | impl UpdateLoop { 24 | pub fn new( 25 | scheduler: Arc>, 26 | performers: Vec>, 27 | debuf_duration: Option, 28 | notifier: watch::Receiver<()>, 29 | ) -> Self { 30 | Self { 31 | scheduler, 32 | performers, 33 | debounce_duration: debuf_duration, 34 | notifier: Some(notifier), 35 | } 36 | } 37 | 38 | pub async fn run(mut self) { 39 | let mut notifier = self.notifier.take().unwrap(); 40 | 41 | loop { 42 | if notifier.changed().await.is_err() { 43 | break; 44 | } 45 | 46 | if let Some(t) = self.debounce_duration { 47 | let mut interval = interval_at(tokio::time::Instant::now() + t, t); 48 | interval.tick().await; 49 | }; 50 | 51 | if let Err(e) = self.process_next_batch().await { 52 | log::error!("an error occurred while processing an update batch: {}", e); 53 | } 54 | } 55 | } 56 | 57 | async fn process_next_batch(&self) -> Result<()> { 58 | let mut batch = { self.scheduler.write().await.prepare().await? }; 59 | let performer = self 60 | .performers 61 | .iter() 62 | .find(|p| p.accept(&batch)) 63 | .expect("No performer found for batch") 64 | .clone(); 65 | 66 | batch 67 | .content 68 | .push_event(TaskEvent::Processing(OffsetDateTime::now_utc())); 69 | 70 | batch.content = { 71 | self.scheduler 72 | .read() 73 | .await 74 | .update_tasks(batch.content) 75 | .await? 76 | }; 77 | 78 | let batch = performer.process_batch(batch).await; 79 | 80 | self.handle_batch_result(batch, performer).await?; 81 | 82 | Ok(()) 83 | } 84 | 85 | /// Handles the result from a processed batch. 86 | /// 87 | /// When a task is processed, the result of the process is pushed to its event list. The 88 | /// `handle_batch_result` make sure that the new state is saved to the store. 89 | /// The tasks are then removed from the processing queue. 90 | async fn handle_batch_result( 91 | &self, 92 | mut batch: Batch, 93 | performer: Arc, 94 | ) -> Result<()> { 95 | let mut scheduler = self.scheduler.write().await; 96 | let content = scheduler.update_tasks(batch.content).await?; 97 | scheduler.finish(); 98 | drop(scheduler); 99 | batch.content = content; 100 | performer.finish(&batch).await; 101 | Ok(()) 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at bonjour@meilisearch.com. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | 75 | For answers to common questions about this code of conduct, see 76 | https://www.contributor-covenant.org/faq 77 | -------------------------------------------------------------------------------- /.github/workflows/publish-binaries.yml: -------------------------------------------------------------------------------- 1 | on: 2 | release: 3 | types: [published] 4 | 5 | name: Publish binaries to release 6 | 7 | jobs: 8 | publish: 9 | name: Publish binary for ${{ matrix.os }} 10 | runs-on: ${{ matrix.os }} 11 | strategy: 12 | fail-fast: false 13 | matrix: 14 | os: [ubuntu-18.04, macos-latest, windows-latest] 15 | include: 16 | - os: ubuntu-18.04 17 | artifact_name: meilisearch 18 | asset_name: meilisearch-linux-amd64 19 | - os: macos-latest 20 | artifact_name: meilisearch 21 | asset_name: meilisearch-macos-amd64 22 | - os: windows-latest 23 | artifact_name: meilisearch.exe 24 | asset_name: meilisearch-windows-amd64.exe 25 | 26 | steps: 27 | - uses: hecrj/setup-rust-action@master 28 | with: 29 | rust-version: stable 30 | - uses: actions/checkout@v3 31 | - name: Build 32 | run: cargo build --release --locked 33 | - name: Upload binaries to release 34 | uses: svenstaro/upload-release-action@v1-release 35 | with: 36 | repo_token: ${{ secrets.PUBLISH_TOKEN }} 37 | file: target/release/${{ matrix.artifact_name }} 38 | asset_name: ${{ matrix.asset_name }} 39 | tag: ${{ github.ref }} 40 | 41 | publish-aarch64: 42 | name: Publish binary for aarch64 43 | runs-on: ${{ matrix.os }} 44 | continue-on-error: false 45 | strategy: 46 | fail-fast: false 47 | matrix: 48 | include: 49 | - build: aarch64 50 | os: ubuntu-18.04 51 | target: aarch64-unknown-linux-gnu 52 | linker: gcc-aarch64-linux-gnu 53 | use-cross: true 54 | asset_name: meilisearch-linux-aarch64 55 | 56 | steps: 57 | - name: Checkout repository 58 | uses: actions/checkout@v3 59 | 60 | - name: Installing Rust toolchain 61 | uses: actions-rs/toolchain@v1 62 | with: 63 | toolchain: stable 64 | profile: minimal 65 | target: ${{ matrix.target }} 66 | override: true 67 | 68 | - name: APT update 69 | run: | 70 | sudo apt update 71 | 72 | - name: Install target specific tools 73 | if: matrix.use-cross 74 | run: | 75 | sudo apt-get install -y ${{ matrix.linker }} 76 | 77 | - name: Configure target aarch64 GNU 78 | if: matrix.target == 'aarch64-unknown-linux-gnu' 79 | ## Environment variable is not passed using env: 80 | ## LD gold won't work with MUSL 81 | # env: 82 | # JEMALLOC_SYS_WITH_LG_PAGE: 16 83 | # RUSTFLAGS: '-Clink-arg=-fuse-ld=gold' 84 | run: | 85 | echo '[target.aarch64-unknown-linux-gnu]' >> ~/.cargo/config 86 | echo 'linker = "aarch64-linux-gnu-gcc"' >> ~/.cargo/config 87 | echo 'JEMALLOC_SYS_WITH_LG_PAGE=16' >> $GITHUB_ENV 88 | echo RUSTFLAGS="-Clink-arg=-fuse-ld=gold" >> $GITHUB_ENV 89 | 90 | - name: Cargo build 91 | uses: actions-rs/cargo@v1 92 | with: 93 | command: build 94 | use-cross: ${{ matrix.use-cross }} 95 | args: --release --target ${{ matrix.target }} 96 | 97 | - name: List target output files 98 | run: ls -lR ./target 99 | 100 | - name: Upload the binary to release 101 | uses: svenstaro/upload-release-action@v1-release 102 | with: 103 | repo_token: ${{ secrets.PUBLISH_TOKEN }} 104 | file: target/${{ matrix.target }}/release/meilisearch 105 | asset_name: ${{ matrix.asset_name }} 106 | tag: ${{ github.ref }} 107 | -------------------------------------------------------------------------------- /meilisearch-http/src/error.rs: -------------------------------------------------------------------------------- 1 | use actix_web as aweb; 2 | use aweb::error::{JsonPayloadError, QueryPayloadError}; 3 | use meilisearch_types::error::{Code, ErrorCode, ResponseError}; 4 | 5 | #[derive(Debug, thiserror::Error)] 6 | pub enum MeilisearchHttpError { 7 | #[error("A Content-Type header is missing. Accepted values for the Content-Type header are: {}", 8 | .0.iter().map(|s| format!("`{}`", s)).collect::>().join(", "))] 9 | MissingContentType(Vec), 10 | #[error( 11 | "The Content-Type `{0}` is invalid. Accepted values for the Content-Type header are: {}", 12 | .1.iter().map(|s| format!("`{}`", s)).collect::>().join(", ") 13 | )] 14 | InvalidContentType(String, Vec), 15 | } 16 | 17 | impl ErrorCode for MeilisearchHttpError { 18 | fn error_code(&self) -> Code { 19 | match self { 20 | MeilisearchHttpError::MissingContentType(_) => Code::MissingContentType, 21 | MeilisearchHttpError::InvalidContentType(_, _) => Code::InvalidContentType, 22 | } 23 | } 24 | } 25 | 26 | impl From for aweb::Error { 27 | fn from(other: MeilisearchHttpError) -> Self { 28 | aweb::Error::from(ResponseError::from(other)) 29 | } 30 | } 31 | 32 | #[derive(Debug, thiserror::Error)] 33 | pub enum PayloadError { 34 | #[error("{0}")] 35 | Json(JsonPayloadError), 36 | #[error("{0}")] 37 | Query(QueryPayloadError), 38 | #[error("The json payload provided is malformed. `{0}`.")] 39 | MalformedPayload(serde_json::error::Error), 40 | #[error("A json payload is missing.")] 41 | MissingPayload, 42 | } 43 | 44 | impl ErrorCode for PayloadError { 45 | fn error_code(&self) -> Code { 46 | match self { 47 | PayloadError::Json(err) => match err { 48 | JsonPayloadError::Overflow { .. } => Code::PayloadTooLarge, 49 | JsonPayloadError::ContentType => Code::UnsupportedMediaType, 50 | JsonPayloadError::Payload(aweb::error::PayloadError::Overflow) => { 51 | Code::PayloadTooLarge 52 | } 53 | JsonPayloadError::Payload(_) => Code::BadRequest, 54 | JsonPayloadError::Deserialize(_) => Code::BadRequest, 55 | JsonPayloadError::Serialize(_) => Code::Internal, 56 | _ => Code::Internal, 57 | }, 58 | PayloadError::Query(err) => match err { 59 | QueryPayloadError::Deserialize(_) => Code::BadRequest, 60 | _ => Code::Internal, 61 | }, 62 | PayloadError::MissingPayload => Code::MissingPayload, 63 | PayloadError::MalformedPayload(_) => Code::MalformedPayload, 64 | } 65 | } 66 | } 67 | 68 | impl From for PayloadError { 69 | fn from(other: JsonPayloadError) -> Self { 70 | match other { 71 | JsonPayloadError::Deserialize(e) 72 | if e.classify() == serde_json::error::Category::Eof 73 | && e.line() == 1 74 | && e.column() == 0 => 75 | { 76 | Self::MissingPayload 77 | } 78 | JsonPayloadError::Deserialize(e) 79 | if e.classify() != serde_json::error::Category::Data => 80 | { 81 | Self::MalformedPayload(e) 82 | } 83 | _ => Self::Json(other), 84 | } 85 | } 86 | } 87 | 88 | impl From for PayloadError { 89 | fn from(other: QueryPayloadError) -> Self { 90 | Self::Query(other) 91 | } 92 | } 93 | 94 | impl From for aweb::Error { 95 | fn from(other: PayloadError) -> Self { 96 | aweb::Error::from(ResponseError::from(other)) 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /meilisearch-http/tests/index/create_index.rs: -------------------------------------------------------------------------------- 1 | use crate::common::Server; 2 | use serde_json::{json, Value}; 3 | 4 | #[actix_rt::test] 5 | async fn create_index_no_primary_key() { 6 | let server = Server::new().await; 7 | let index = server.index("test"); 8 | let (response, code) = index.create(None).await; 9 | 10 | assert_eq!(code, 202); 11 | 12 | assert_eq!(response["status"], "enqueued"); 13 | 14 | let response = index.wait_task(0).await; 15 | 16 | assert_eq!(response["status"], "succeeded"); 17 | assert_eq!(response["type"], "indexCreation"); 18 | assert_eq!(response["details"]["primaryKey"], Value::Null); 19 | } 20 | 21 | #[actix_rt::test] 22 | async fn create_index_with_primary_key() { 23 | let server = Server::new().await; 24 | let index = server.index("test"); 25 | let (response, code) = index.create(Some("primary")).await; 26 | 27 | assert_eq!(code, 202); 28 | 29 | assert_eq!(response["status"], "enqueued"); 30 | 31 | let response = index.wait_task(0).await; 32 | 33 | assert_eq!(response["status"], "succeeded"); 34 | assert_eq!(response["type"], "indexCreation"); 35 | assert_eq!(response["details"]["primaryKey"], "primary"); 36 | } 37 | 38 | #[actix_rt::test] 39 | async fn create_index_with_invalid_primary_key() { 40 | let document = json!([ { "id": 2, "title": "Pride and Prejudice" } ]); 41 | 42 | let server = Server::new().await; 43 | let index = server.index("movies"); 44 | let (_response, code) = index.add_documents(document, Some("title")).await; 45 | assert_eq!(code, 202); 46 | 47 | index.wait_task(0).await; 48 | 49 | let (response, code) = index.get().await; 50 | assert_eq!(code, 200); 51 | assert_eq!(response["primaryKey"], Value::Null); 52 | } 53 | 54 | #[actix_rt::test] 55 | async fn test_create_multiple_indexes() { 56 | let server = Server::new().await; 57 | let index1 = server.index("test1"); 58 | let index2 = server.index("test2"); 59 | let index3 = server.index("test3"); 60 | let index4 = server.index("test4"); 61 | 62 | index1.create(None).await; 63 | index2.create(None).await; 64 | index3.create(None).await; 65 | 66 | index1.wait_task(0).await; 67 | index1.wait_task(1).await; 68 | index1.wait_task(2).await; 69 | 70 | assert_eq!(index1.get().await.1, 200); 71 | assert_eq!(index2.get().await.1, 200); 72 | assert_eq!(index3.get().await.1, 200); 73 | assert_eq!(index4.get().await.1, 404); 74 | } 75 | 76 | #[actix_rt::test] 77 | async fn error_create_existing_index() { 78 | let server = Server::new().await; 79 | let index = server.index("test"); 80 | let (_, code) = index.create(Some("primary")).await; 81 | 82 | assert_eq!(code, 202); 83 | 84 | index.create(Some("primary")).await; 85 | 86 | let response = index.wait_task(1).await; 87 | 88 | let expected_response = json!({ 89 | "message": "Index `test` already exists.", 90 | "code": "index_already_exists", 91 | "type": "invalid_request", 92 | "link":"https://docs.meilisearch.com/errors#index_already_exists" 93 | }); 94 | 95 | assert_eq!(response["error"], expected_response); 96 | } 97 | 98 | #[actix_rt::test] 99 | async fn error_create_with_invalid_index_uid() { 100 | let server = Server::new().await; 101 | let index = server.index("test test#!"); 102 | let (response, code) = index.create(None).await; 103 | 104 | let expected_response = json!({ 105 | "message": "invalid index uid `test test#!`, the uid must be an integer or a string containing only alphanumeric characters a-z A-Z 0-9, hyphens - and underscores _.", 106 | "code": "invalid_index_uid", 107 | "type": "invalid_request", 108 | "link": "https://docs.meilisearch.com/errors#invalid_index_uid" 109 | }); 110 | 111 | assert_eq!(response, expected_response); 112 | assert_eq!(code, 400); 113 | } 114 | -------------------------------------------------------------------------------- /meilisearch-lib/proptest-regressions/index_resolver/mod.txt: -------------------------------------------------------------------------------- 1 | # Seeds for failure cases proptest has generated in the past. It is 2 | # automatically read and these particular cases re-run before any 3 | # novel cases are generated. 4 | # 5 | # It is recommended to check this file in to source control so that 6 | # everyone who runs the test benefits from these saved cases. 7 | cc 6f3ae3cba934ba3e328e2306218c32f27a46ce2d54a1258b05fef65663208662 # shrinks to task = Task { id: 0, index_uid: IndexUid("a"), content: DocumentAddition { content_uuid: 37bc137d-2038-47f0-819f-b133233daadc, merge_strategy: ReplaceDocuments, primary_key: None, documents_count: 0 }, events: [] } 8 | cc b726f7d9f44a9216aad302ddba0f04e7108817e741d656a4759aea8562de4d63 # shrinks to task = Task { id: 0, index_uid: IndexUid("_"), content: IndexDeletion, events: [] }, index_exists = false, index_op_fails = false, any_int = 0 9 | cc 427ec2dde3260b1ab334207bdc22adef28a5b8532b9902c84b55fd2c017ea7e1 # shrinks to task = Task { id: 0, index_uid: IndexUid("A"), content: IndexDeletion, events: [] }, index_exists = true, index_op_fails = false, any_int = 0 10 | cc c24f3d42f0f36fbdbf4e9d4327e75529b163ac580d63a5934ca05e9b5bd23a65 # shrinks to task = Task { id: 0, index_uid: IndexUid("a"), content: IndexDeletion, events: [] }, index_exists = true, index_op_fails = true, any_int = 0 11 | cc 8084e2410801b997533b0bcbad75cd212873cfc2677f26847f68c568ead1604c # shrinks to task = Task { id: 0, index_uid: IndexUid("A"), content: SettingsUpdate { settings: Settings { displayed_attributes: NotSet, searchable_attributes: NotSet, filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, synonyms: NotSet, distinct_attribute: NotSet, _kind: PhantomData }, is_deletion: false }, events: [] }, index_exists = false, index_op_fails = false, any_int = 0 12 | cc 330085e0200a9a2ddfdd764a03d768aa95c431bcaafbd530c8c949425beed18b # shrinks to task = Task { id: 0, index_uid: IndexUid("a"), content: CreateIndex { primary_key: None }, events: [] }, index_exists = false, index_op_fails = true, any_int = 0 13 | cc c70e901576ef2fb9622e814bdecd11e4747cd70d71a9a6ce771b5b7256a187c0 # shrinks to task = Task { id: 0, index_uid: IndexUid("a"), content: SettingsUpdate { settings: Settings { displayed_attributes: NotSet, searchable_attributes: NotSet, filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, synonyms: NotSet, distinct_attribute: NotSet, _kind: PhantomData }, is_deletion: true }, events: [] }, index_exists = false, index_op_fails = false, any_int = 0 14 | cc 3fe2c38cbc2cca34ecde321472141d386056f0cd332cbf700773657715a382b5 # shrinks to task = Task { id: 0, index_uid: IndexUid("a"), content: UpdateIndex { primary_key: None }, events: [] }, index_exists = false, index_op_fails = false, any_int = 0 15 | cc c31cf86692968483f1ab08a6a9d4667ccb9635c306998551bf1eb1f135ef0d4b # shrinks to task = Task { id: 0, index_uid: IndexUid("a"), content: UpdateIndex { primary_key: Some("") }, events: [] }, index_exists = true, index_op_fails = false, any_int = 0 16 | cc 3a01c78db082434b8a4f8914abf0d1059d39f4426d16df20d72e1bd7ebb94a6a # shrinks to task = Task { id: 0, index_uid: IndexUid("0"), content: UpdateIndex { primary_key: None }, events: [] }, index_exists = true, index_op_fails = true, any_int = 0 17 | cc c450806df3921d1e6fe9b6af93d999e8196d0175b69b64f1810802582421e94a # shrinks to task = Task { id: 0, index_uid: IndexUid("a"), content: CreateIndex { primary_key: Some("") }, events: [] }, index_exists = false, index_op_fails = false, any_int = 0 18 | cc fb6b98947cbdbdee05ed3c0bf2923aad2c311edc276253642eb43a0c0ec4888a # shrinks to task = Task { id: 0, index_uid: IndexUid("A"), content: CreateIndex { primary_key: Some("") }, events: [] }, index_exists = false, index_op_fails = true, any_int = 0 19 | cc 1aa59d8e22484e9915efbb5818e1e1ab684aa61b166dc82130d6221663ba00bf # shrinks to task = Task { id: 0, index_uid: IndexUid("a"), content: DocumentDeletion(Clear), events: [] }, index_exists = true, index_op_fails = false, any_int = 0 20 | -------------------------------------------------------------------------------- /meilisearch-lib/src/index_resolver/index_store.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | use std::convert::TryFrom; 3 | use std::path::{Path, PathBuf}; 4 | use std::sync::Arc; 5 | 6 | use milli::update::IndexerConfig; 7 | use tokio::fs; 8 | use tokio::sync::RwLock; 9 | use tokio::task::spawn_blocking; 10 | use uuid::Uuid; 11 | 12 | use super::error::{IndexResolverError, Result}; 13 | use crate::index::Index; 14 | use crate::options::IndexerOpts; 15 | 16 | type AsyncMap = Arc>>; 17 | 18 | #[async_trait::async_trait] 19 | #[cfg_attr(test, mockall::automock)] 20 | pub trait IndexStore { 21 | async fn create(&self, uuid: Uuid) -> Result; 22 | async fn get(&self, uuid: Uuid) -> Result>; 23 | async fn delete(&self, uuid: Uuid) -> Result>; 24 | } 25 | 26 | pub struct MapIndexStore { 27 | index_store: AsyncMap, 28 | path: PathBuf, 29 | index_size: usize, 30 | indexer_config: Arc, 31 | } 32 | 33 | impl MapIndexStore { 34 | pub fn new( 35 | path: impl AsRef, 36 | index_size: usize, 37 | indexer_opts: &IndexerOpts, 38 | ) -> anyhow::Result { 39 | let indexer_config = Arc::new(IndexerConfig::try_from(indexer_opts)?); 40 | let path = path.as_ref().join("indexes/"); 41 | let index_store = Arc::new(RwLock::new(HashMap::new())); 42 | Ok(Self { 43 | index_store, 44 | path, 45 | index_size, 46 | indexer_config, 47 | }) 48 | } 49 | } 50 | 51 | #[async_trait::async_trait] 52 | impl IndexStore for MapIndexStore { 53 | async fn create(&self, uuid: Uuid) -> Result { 54 | // We need to keep the lock until we are sure the db file has been opened correclty, to 55 | // ensure that another db is not created at the same time. 56 | let mut lock = self.index_store.write().await; 57 | 58 | if let Some(index) = lock.get(&uuid) { 59 | return Ok(index.clone()); 60 | } 61 | let path = self.path.join(format!("{}", uuid)); 62 | if path.exists() { 63 | return Err(IndexResolverError::UuidAlreadyExists(uuid)); 64 | } 65 | 66 | let index_size = self.index_size; 67 | let update_handler = self.indexer_config.clone(); 68 | let index = spawn_blocking(move || -> Result { 69 | let index = Index::open(path, index_size, uuid, update_handler)?; 70 | Ok(index) 71 | }) 72 | .await??; 73 | 74 | lock.insert(uuid, index.clone()); 75 | 76 | Ok(index) 77 | } 78 | 79 | async fn get(&self, uuid: Uuid) -> Result> { 80 | let guard = self.index_store.read().await; 81 | match guard.get(&uuid) { 82 | Some(index) => Ok(Some(index.clone())), 83 | None => { 84 | // drop the guard here so we can perform the write after without deadlocking; 85 | drop(guard); 86 | let path = self.path.join(format!("{}", uuid)); 87 | if !path.exists() { 88 | return Ok(None); 89 | } 90 | 91 | let index_size = self.index_size; 92 | let update_handler = self.indexer_config.clone(); 93 | let index = 94 | spawn_blocking(move || Index::open(path, index_size, uuid, update_handler)) 95 | .await??; 96 | self.index_store.write().await.insert(uuid, index.clone()); 97 | Ok(Some(index)) 98 | } 99 | } 100 | } 101 | 102 | async fn delete(&self, uuid: Uuid) -> Result> { 103 | let db_path = self.path.join(format!("{}", uuid)); 104 | fs::remove_dir_all(db_path).await?; 105 | let index = self.index_store.write().await.remove(&uuid); 106 | Ok(index) 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /meilisearch-http/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | authors = ["Quentin de Quelen ", "Clément Renault "] 3 | description = "Meilisearch HTTP server" 4 | edition = "2021" 5 | license = "MIT" 6 | name = "meilisearch-http" 7 | version = "0.28.0" 8 | 9 | [[bin]] 10 | name = "meilisearch" 11 | path = "src/main.rs" 12 | 13 | [build-dependencies] 14 | anyhow = { version = "1.0.56", optional = true } 15 | cargo_toml = { version = "0.11.4", optional = true } 16 | hex = { version = "0.4.3", optional = true } 17 | reqwest = { version = "0.11.9", features = ["blocking", "rustls-tls"], default-features = false, optional = true } 18 | sha-1 = { version = "0.10.0", optional = true } 19 | static-files = { version = "0.2.3", optional = true } 20 | tempfile = { version = "3.3.0", optional = true } 21 | vergen = { version = "7.0.0", default-features = false, features = ["git"] } 22 | zip = { version = "0.5.13", optional = true } 23 | 24 | [dependencies] 25 | actix-cors = "0.6.1" 26 | actix-web = { version = "4.0.1", default-features = false, features = ["macros", "compress-brotli", "compress-gzip", "cookies", "rustls"] } 27 | actix-web-static-files = { git = "https://github.com/kilork/actix-web-static-files.git", rev = "2d3b6160", optional = true } 28 | anyhow = { version = "1.0.56", features = ["backtrace"] } 29 | async-stream = "0.3.3" 30 | async-trait = "0.1.52" 31 | bstr = "0.2.17" 32 | byte-unit = { version = "4.0.14", default-features = false, features = ["std", "serde"] } 33 | bytes = "1.1.0" 34 | clap = { version = "3.1.6", features = ["derive", "env"] } 35 | crossbeam-channel = "0.5.2" 36 | either = "1.6.1" 37 | env_logger = "0.9.0" 38 | flate2 = "1.0.22" 39 | fst = "0.4.7" 40 | futures = "0.3.21" 41 | futures-util = "0.3.21" 42 | http = "0.2.6" 43 | indexmap = { version = "1.8.0", features = ["serde-1"] } 44 | itertools = "0.10.3" 45 | jsonwebtoken = "8.0.1" 46 | log = "0.4.14" 47 | meilisearch-auth = { path = "../meilisearch-auth" } 48 | meilisearch-types = { path = "../meilisearch-types" } 49 | meilisearch-lib = { path = "../meilisearch-lib" } 50 | mime = "0.3.16" 51 | num_cpus = "1.13.1" 52 | obkv = "0.2.0" 53 | once_cell = "1.10.0" 54 | parking_lot = "0.12.0" 55 | pin-project-lite = "0.2.8" 56 | platform-dirs = "0.3.0" 57 | rand = "0.8.5" 58 | rayon = "1.5.1" 59 | regex = "1.5.5" 60 | reqwest = { version = "0.11.4", features = ["rustls-tls", "json"], default-features = false } 61 | rustls = "0.20.4" 62 | rustls-pemfile = "0.3.0" 63 | segment = { version = "0.2.0", optional = true } 64 | serde = { version = "1.0.136", features = ["derive"] } 65 | serde-cs = "0.2.3" 66 | serde_json = { version = "1.0.79", features = ["preserve_order"] } 67 | sha2 = "0.10.2" 68 | siphasher = "0.3.10" 69 | slice-group-by = "0.3.0" 70 | static-files = { version = "0.2.3", optional = true } 71 | sysinfo = "0.23.5" 72 | tar = "0.4.38" 73 | tempfile = "3.3.0" 74 | thiserror = "1.0.30" 75 | time = { version = "0.3.7", features = ["serde-well-known", "formatting", "parsing", "macros"] } 76 | tokio = { version = "1.17.0", features = ["full"] } 77 | tokio-stream = "0.1.8" 78 | uuid = { version = "0.8.2", features = ["serde", "v4"] } 79 | walkdir = "2.3.2" 80 | 81 | [dev-dependencies] 82 | actix-rt = "2.7.0" 83 | assert-json-diff = "2.0.1" 84 | manifest-dir-macros = "0.1.14" 85 | maplit = "1.0.2" 86 | serde_url_params = "0.2.1" 87 | urlencoding = "2.1.0" 88 | 89 | [features] 90 | default = ["analytics", "mini-dashboard"] 91 | analytics = ["segment"] 92 | mini-dashboard = [ 93 | "actix-web-static-files", 94 | "static-files", 95 | "anyhow", 96 | "cargo_toml", 97 | "hex", 98 | "reqwest", 99 | "sha-1", 100 | "tempfile", 101 | "zip", 102 | ] 103 | 104 | [target.'cfg(target_os = "linux")'.dependencies] 105 | tikv-jemallocator = "0.4.3" 106 | 107 | [package.metadata.mini-dashboard] 108 | assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.1.10/build.zip" 109 | sha1 = "1adf96592c267425c110bfefc36b7fc6bfb0f93d" 110 | -------------------------------------------------------------------------------- /meilisearch-lib/src/dump/loaders/v3.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | use std::fs::{self, File}; 3 | use std::io::{BufReader, BufWriter, Write}; 4 | use std::path::Path; 5 | 6 | use anyhow::Context; 7 | use fs_extra::dir::{self, CopyOptions}; 8 | use log::info; 9 | use tempfile::tempdir; 10 | use uuid::Uuid; 11 | 12 | use crate::dump::compat::{self, v3}; 13 | use crate::dump::Metadata; 14 | use crate::index_resolver::meta_store::{DumpEntry, IndexMeta}; 15 | use crate::options::IndexerOpts; 16 | use crate::tasks::task::TaskId; 17 | 18 | /// dump structure for V3: 19 | /// . 20 | /// ├── indexes 21 | /// │   └── 25f10bb8-6ea8-42f0-bd48-ad5857f77648 22 | /// │   ├── documents.jsonl 23 | /// │   └── meta.json 24 | /// ├── index_uuids 25 | /// │   └── data.jsonl 26 | /// ├── metadata.json 27 | /// └── updates 28 | /// └── data.jsonl 29 | 30 | pub fn load_dump( 31 | meta: Metadata, 32 | src: impl AsRef, 33 | dst: impl AsRef, 34 | index_db_size: usize, 35 | meta_env_size: usize, 36 | indexing_options: &IndexerOpts, 37 | ) -> anyhow::Result<()> { 38 | info!("Patching dump V3 to dump V4..."); 39 | 40 | let patched_dir = tempdir()?; 41 | 42 | let options = CopyOptions::default(); 43 | dir::copy(src.as_ref().join("indexes"), patched_dir.path(), &options)?; 44 | dir::copy( 45 | src.as_ref().join("index_uuids"), 46 | patched_dir.path(), 47 | &options, 48 | )?; 49 | 50 | let uuid_map = patch_index_meta( 51 | src.as_ref().join("index_uuids/data.jsonl"), 52 | patched_dir.path(), 53 | )?; 54 | 55 | fs::copy( 56 | src.as_ref().join("metadata.json"), 57 | patched_dir.path().join("metadata.json"), 58 | )?; 59 | 60 | patch_updates(&src, patched_dir.path(), uuid_map)?; 61 | 62 | super::v4::load_dump( 63 | meta, 64 | patched_dir.path(), 65 | dst, 66 | index_db_size, 67 | meta_env_size, 68 | indexing_options, 69 | ) 70 | } 71 | 72 | fn patch_index_meta( 73 | path: impl AsRef, 74 | dst: impl AsRef, 75 | ) -> anyhow::Result> { 76 | let file = BufReader::new(File::open(path)?); 77 | let dst = dst.as_ref().join("index_uuids"); 78 | fs::create_dir_all(&dst)?; 79 | let mut dst_file = File::create(dst.join("data.jsonl"))?; 80 | 81 | let map = serde_json::Deserializer::from_reader(file) 82 | .into_iter::() 83 | .try_fold(HashMap::new(), |mut map, entry| -> anyhow::Result<_> { 84 | let entry = entry?; 85 | map.insert(entry.uuid, entry.uid.clone()); 86 | let meta = IndexMeta { 87 | uuid: entry.uuid, 88 | // This is lost information, we patch it to 0; 89 | creation_task_id: 0, 90 | }; 91 | let entry = DumpEntry { 92 | uid: entry.uid, 93 | index_meta: meta, 94 | }; 95 | serde_json::to_writer(&mut dst_file, &entry)?; 96 | dst_file.write_all(b"\n")?; 97 | Ok(map) 98 | })?; 99 | 100 | dst_file.flush()?; 101 | 102 | Ok(map) 103 | } 104 | 105 | fn patch_updates( 106 | src: impl AsRef, 107 | dst: impl AsRef, 108 | uuid_map: HashMap, 109 | ) -> anyhow::Result<()> { 110 | let dst = dst.as_ref().join("updates"); 111 | fs::create_dir_all(&dst)?; 112 | 113 | let mut dst_file = BufWriter::new(File::create(dst.join("data.jsonl"))?); 114 | let src_file = BufReader::new(File::open(src.as_ref().join("updates/data.jsonl"))?); 115 | 116 | serde_json::Deserializer::from_reader(src_file) 117 | .into_iter::() 118 | .enumerate() 119 | .try_for_each(|(task_id, entry)| -> anyhow::Result<()> { 120 | let entry = entry?; 121 | let name = uuid_map 122 | .get(&entry.uuid) 123 | .with_context(|| format!("Unknown index uuid: {}", entry.uuid))? 124 | .clone(); 125 | serde_json::to_writer( 126 | &mut dst_file, 127 | &compat::v4::Task::from((entry.update, name, task_id as TaskId)), 128 | )?; 129 | dst_file.write_all(b"\n")?; 130 | Ok(()) 131 | })?; 132 | 133 | dst_file.flush()?; 134 | 135 | Ok(()) 136 | } 137 | -------------------------------------------------------------------------------- /meilisearch-types/src/star_or.rs: -------------------------------------------------------------------------------- 1 | use serde::de::Visitor; 2 | use serde::{Deserialize, Deserializer, Serialize, Serializer}; 3 | use std::fmt::{Display, Formatter}; 4 | use std::marker::PhantomData; 5 | use std::ops::Deref; 6 | use std::str::FromStr; 7 | 8 | /// A type that tries to match either a star (*) or 9 | /// any other thing that implements `FromStr`. 10 | #[derive(Debug)] 11 | pub enum StarOr { 12 | Star, 13 | Other(T), 14 | } 15 | 16 | impl FromStr for StarOr { 17 | type Err = T::Err; 18 | 19 | fn from_str(s: &str) -> Result { 20 | if s.trim() == "*" { 21 | Ok(StarOr::Star) 22 | } else { 23 | T::from_str(s).map(StarOr::Other) 24 | } 25 | } 26 | } 27 | 28 | impl> Deref for StarOr { 29 | type Target = str; 30 | 31 | fn deref(&self) -> &Self::Target { 32 | match self { 33 | Self::Star => "*", 34 | Self::Other(t) => t.deref(), 35 | } 36 | } 37 | } 38 | 39 | impl> From> for String { 40 | fn from(s: StarOr) -> Self { 41 | match s { 42 | StarOr::Star => "*".to_string(), 43 | StarOr::Other(t) => t.into(), 44 | } 45 | } 46 | } 47 | 48 | impl PartialEq for StarOr { 49 | fn eq(&self, other: &Self) -> bool { 50 | match (self, other) { 51 | (Self::Star, Self::Star) => true, 52 | (Self::Other(left), Self::Other(right)) if left.eq(right) => true, 53 | _ => false, 54 | } 55 | } 56 | } 57 | 58 | impl Eq for StarOr {} 59 | 60 | impl<'de, T, E> Deserialize<'de> for StarOr 61 | where 62 | T: FromStr, 63 | E: Display, 64 | { 65 | fn deserialize(deserializer: D) -> Result 66 | where 67 | D: Deserializer<'de>, 68 | { 69 | /// Serde can't differentiate between `StarOr::Star` and `StarOr::Other` without a tag. 70 | /// Simply using `#[serde(untagged)]` + `#[serde(rename="*")]` will lead to attempting to 71 | /// deserialize everything as a `StarOr::Other`, including "*". 72 | /// [`#[serde(other)]`](https://serde.rs/variant-attrs.html#other) might have helped but is 73 | /// not supported on untagged enums. 74 | struct StarOrVisitor(PhantomData); 75 | 76 | impl<'de, T, FE> Visitor<'de> for StarOrVisitor 77 | where 78 | T: FromStr, 79 | FE: Display, 80 | { 81 | type Value = StarOr; 82 | 83 | fn expecting(&self, formatter: &mut Formatter) -> std::fmt::Result { 84 | formatter.write_str("a string") 85 | } 86 | 87 | fn visit_str(self, v: &str) -> Result 88 | where 89 | SE: serde::de::Error, 90 | { 91 | match v { 92 | "*" => Ok(StarOr::Star), 93 | v => { 94 | let other = FromStr::from_str(v).map_err(|e: T::Err| { 95 | SE::custom(format!("Invalid `other` value: {}", e)) 96 | })?; 97 | Ok(StarOr::Other(other)) 98 | } 99 | } 100 | } 101 | } 102 | 103 | deserializer.deserialize_str(StarOrVisitor(PhantomData)) 104 | } 105 | } 106 | 107 | impl Serialize for StarOr 108 | where 109 | T: Deref, 110 | { 111 | fn serialize(&self, serializer: S) -> Result 112 | where 113 | S: Serializer, 114 | { 115 | match self { 116 | StarOr::Star => serializer.serialize_str("*"), 117 | StarOr::Other(other) => serializer.serialize_str(other.deref()), 118 | } 119 | } 120 | } 121 | 122 | #[cfg(test)] 123 | mod tests { 124 | use super::*; 125 | use serde_json::{json, Value}; 126 | 127 | #[test] 128 | fn star_or_serde_roundtrip() { 129 | fn roundtrip(content: Value, expected: StarOr) { 130 | let deserialized: StarOr = serde_json::from_value(content.clone()).unwrap(); 131 | assert_eq!(deserialized, expected); 132 | assert_eq!(content, serde_json::to_value(deserialized).unwrap()); 133 | } 134 | 135 | roundtrip(json!("products"), StarOr::Other("products".to_string())); 136 | roundtrip(json!("*"), StarOr::Star); 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /.github/is-latest-release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Checks if the current tag should be the latest (in terms of semver and not of release date). 4 | # Ex: previous tag -> v0.10.1 5 | # new tag -> v0.8.12 6 | # The new tag should not be the latest 7 | # So it returns "false", the CI should not run for the release v0.8.2 8 | 9 | # Used in GHA in publish-docker-latest.yml 10 | # Returns "true" or "false" (as a string) to be used in the `if` in GHA 11 | 12 | # GLOBAL 13 | GREP_SEMVER_REGEXP='v\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)$' # i.e. v[number].[number].[number] 14 | 15 | # FUNCTIONS 16 | 17 | # semverParseInto and semverLT from https://github.com/cloudflare/semver_bash/blob/master/semver.sh 18 | 19 | # usage: semverParseInto version major minor patch special 20 | # version: the string version 21 | # major, minor, patch, special: will be assigned by the function 22 | semverParseInto() { 23 | local RE='[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\([0-9A-Za-z-]*\)' 24 | #MAJOR 25 | eval $2=`echo $1 | sed -e "s#$RE#\1#"` 26 | #MINOR 27 | eval $3=`echo $1 | sed -e "s#$RE#\2#"` 28 | #MINOR 29 | eval $4=`echo $1 | sed -e "s#$RE#\3#"` 30 | #SPECIAL 31 | eval $5=`echo $1 | sed -e "s#$RE#\4#"` 32 | } 33 | 34 | # usage: semverLT version1 version2 35 | semverLT() { 36 | local MAJOR_A=0 37 | local MINOR_A=0 38 | local PATCH_A=0 39 | local SPECIAL_A=0 40 | 41 | local MAJOR_B=0 42 | local MINOR_B=0 43 | local PATCH_B=0 44 | local SPECIAL_B=0 45 | 46 | semverParseInto $1 MAJOR_A MINOR_A PATCH_A SPECIAL_A 47 | semverParseInto $2 MAJOR_B MINOR_B PATCH_B SPECIAL_B 48 | 49 | if [ $MAJOR_A -lt $MAJOR_B ]; then 50 | return 0 51 | fi 52 | if [ $MAJOR_A -le $MAJOR_B ] && [ $MINOR_A -lt $MINOR_B ]; then 53 | return 0 54 | fi 55 | if [ $MAJOR_A -le $MAJOR_B ] && [ $MINOR_A -le $MINOR_B ] && [ $PATCH_A -lt $PATCH_B ]; then 56 | return 0 57 | fi 58 | if [ "_$SPECIAL_A" == "_" ] && [ "_$SPECIAL_B" == "_" ] ; then 59 | return 1 60 | fi 61 | if [ "_$SPECIAL_A" == "_" ] && [ "_$SPECIAL_B" != "_" ] ; then 62 | return 1 63 | fi 64 | if [ "_$SPECIAL_A" != "_" ] && [ "_$SPECIAL_B" == "_" ] ; then 65 | return 0 66 | fi 67 | if [ "_$SPECIAL_A" < "_$SPECIAL_B" ]; then 68 | return 0 69 | fi 70 | 71 | return 1 72 | } 73 | 74 | # Returns the tag of the latest stable release (in terms of semver and not of release date) 75 | get_latest() { 76 | temp_file='temp_file' # temp_file needed because the grep would start before the download is over 77 | curl -s 'https://api.github.com/repos/meilisearch/meilisearch/releases' > "$temp_file" 78 | releases=$(cat "$temp_file" | \ 79 | grep -E "tag_name|draft|prerelease" \ 80 | | tr -d ',"' | cut -d ':' -f2 | tr -d ' ') 81 | # Returns a list of [tag_name draft_boolean prerelease_boolean ...] 82 | # Ex: v0.10.1 false false v0.9.1-rc.1 false true v0.9.0 false false... 83 | 84 | i=0 85 | latest="" 86 | current_tag="" 87 | for release_info in $releases; do 88 | if [ $i -eq 0 ]; then # Cheking tag_name 89 | if echo "$release_info" | grep -q "$GREP_SEMVER_REGEXP"; then # If it's not an alpha or beta release 90 | current_tag=$release_info 91 | else 92 | current_tag="" 93 | fi 94 | i=1 95 | elif [ $i -eq 1 ]; then # Checking draft boolean 96 | if [ "$release_info" = "true" ]; then 97 | current_tag="" 98 | fi 99 | i=2 100 | elif [ $i -eq 2 ]; then # Checking prerelease boolean 101 | if [ "$release_info" = "true" ]; then 102 | current_tag="" 103 | fi 104 | i=0 105 | if [ "$current_tag" != "" ]; then # If the current_tag is valid 106 | if [ "$latest" = "" ]; then # If there is no latest yet 107 | latest="$current_tag" 108 | else 109 | semverLT $current_tag $latest # Comparing latest and the current tag 110 | if [ $? -eq 1 ]; then 111 | latest="$current_tag" 112 | fi 113 | fi 114 | fi 115 | fi 116 | done 117 | 118 | rm -f "$temp_file" 119 | echo $latest 120 | } 121 | 122 | # MAIN 123 | current_tag="$(echo $GITHUB_REF | tr -d 'refs/tags/')" 124 | latest="$(get_latest)" 125 | 126 | if [ "$current_tag" != "$latest" ]; then 127 | # The current release tag is not the latest 128 | echo "false" 129 | else 130 | # The current release tag is the latest 131 | echo "true" 132 | fi 133 | -------------------------------------------------------------------------------- /meilisearch-lib/src/document_formats.rs: -------------------------------------------------------------------------------- 1 | use std::borrow::Borrow; 2 | use std::fmt::{self, Debug, Display}; 3 | use std::io::{self, BufRead, BufReader, BufWriter, Cursor, Read, Seek, Write}; 4 | 5 | use meilisearch_types::error::{Code, ErrorCode}; 6 | use meilisearch_types::internal_error; 7 | use milli::documents::DocumentBatchBuilder; 8 | 9 | type Result = std::result::Result; 10 | 11 | #[derive(Debug)] 12 | pub enum PayloadType { 13 | Ndjson, 14 | Json, 15 | Csv, 16 | } 17 | 18 | impl fmt::Display for PayloadType { 19 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 20 | match self { 21 | PayloadType::Ndjson => write!(f, "ndjson"), 22 | PayloadType::Json => write!(f, "json"), 23 | PayloadType::Csv => write!(f, "csv"), 24 | } 25 | } 26 | } 27 | 28 | #[derive(Debug)] 29 | pub enum DocumentFormatError { 30 | Internal(Box), 31 | MalformedPayload(Box, PayloadType), 32 | } 33 | 34 | impl Display for DocumentFormatError { 35 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 36 | match self { 37 | Self::Internal(e) => write!(f, "An internal error has occurred: `{}`.", e), 38 | Self::MalformedPayload(me, b) => match me.borrow() { 39 | milli::documents::Error::JsonError(se) => { 40 | // https://github.com/meilisearch/meilisearch/issues/2107 41 | // The user input maybe insanely long. We need to truncate it. 42 | let mut serde_msg = se.to_string(); 43 | let ellipsis = "..."; 44 | if serde_msg.len() > 100 + ellipsis.len() { 45 | serde_msg.replace_range(50..serde_msg.len() - 85, ellipsis); 46 | } 47 | 48 | write!( 49 | f, 50 | "The `{}` payload provided is malformed. `Couldn't serialize document value: {}`.", 51 | b, serde_msg 52 | ) 53 | } 54 | _ => write!(f, "The `{}` payload provided is malformed: `{}`.", b, me), 55 | }, 56 | } 57 | } 58 | } 59 | 60 | impl std::error::Error for DocumentFormatError {} 61 | 62 | impl From<(PayloadType, milli::documents::Error)> for DocumentFormatError { 63 | fn from((ty, error): (PayloadType, milli::documents::Error)) -> Self { 64 | match error { 65 | milli::documents::Error::Io(e) => Self::Internal(Box::new(e)), 66 | e => Self::MalformedPayload(Box::new(e), ty), 67 | } 68 | } 69 | } 70 | 71 | impl ErrorCode for DocumentFormatError { 72 | fn error_code(&self) -> Code { 73 | match self { 74 | DocumentFormatError::Internal(_) => Code::Internal, 75 | DocumentFormatError::MalformedPayload(_, _) => Code::MalformedPayload, 76 | } 77 | } 78 | } 79 | 80 | internal_error!(DocumentFormatError: io::Error); 81 | 82 | /// reads csv from input and write an obkv batch to writer. 83 | pub fn read_csv(input: impl Read, writer: impl Write + Seek) -> Result { 84 | let writer = BufWriter::new(writer); 85 | let builder = 86 | DocumentBatchBuilder::from_csv(input, writer).map_err(|e| (PayloadType::Csv, e))?; 87 | 88 | let count = builder.finish().map_err(|e| (PayloadType::Csv, e))?; 89 | 90 | Ok(count) 91 | } 92 | 93 | /// reads jsonl from input and write an obkv batch to writer. 94 | pub fn read_ndjson(input: impl Read, writer: impl Write + Seek) -> Result { 95 | let mut reader = BufReader::new(input); 96 | let writer = BufWriter::new(writer); 97 | 98 | let mut builder = DocumentBatchBuilder::new(writer).map_err(|e| (PayloadType::Ndjson, e))?; 99 | let mut buf = String::new(); 100 | 101 | while reader.read_line(&mut buf)? > 0 { 102 | // skip empty lines 103 | if buf == "\n" { 104 | buf.clear(); 105 | continue; 106 | } 107 | builder 108 | .extend_from_json(Cursor::new(&buf.as_bytes())) 109 | .map_err(|e| (PayloadType::Ndjson, e))?; 110 | buf.clear(); 111 | } 112 | 113 | let count = builder.finish().map_err(|e| (PayloadType::Ndjson, e))?; 114 | 115 | Ok(count) 116 | } 117 | 118 | /// reads json from input and write an obkv batch to writer. 119 | pub fn read_json(input: impl Read, writer: impl Write + Seek) -> Result { 120 | let writer = BufWriter::new(writer); 121 | let mut builder = DocumentBatchBuilder::new(writer).map_err(|e| (PayloadType::Json, e))?; 122 | builder 123 | .extend_from_json(input) 124 | .map_err(|e| (PayloadType::Json, e))?; 125 | 126 | let count = builder.finish().map_err(|e| (PayloadType::Json, e))?; 127 | 128 | Ok(count) 129 | } 130 | -------------------------------------------------------------------------------- /meilisearch-lib/src/tasks/handlers/dump_handler.rs: -------------------------------------------------------------------------------- 1 | use crate::dump::DumpHandler; 2 | use crate::index_resolver::index_store::IndexStore; 3 | use crate::index_resolver::meta_store::IndexMetaStore; 4 | use crate::tasks::batch::{Batch, BatchContent}; 5 | use crate::tasks::task::{Task, TaskContent, TaskEvent, TaskResult}; 6 | use crate::tasks::BatchHandler; 7 | 8 | #[async_trait::async_trait] 9 | impl BatchHandler for DumpHandler 10 | where 11 | U: IndexMetaStore + Sync + Send + 'static, 12 | I: IndexStore + Sync + Send + 'static, 13 | { 14 | fn accept(&self, batch: &Batch) -> bool { 15 | matches!(batch.content, BatchContent::Dump { .. }) 16 | } 17 | 18 | async fn process_batch(&self, mut batch: Batch) -> Batch { 19 | match &batch.content { 20 | BatchContent::Dump(Task { 21 | content: TaskContent::Dump { uid }, 22 | .. 23 | }) => { 24 | match self.run(uid.clone()).await { 25 | Ok(_) => { 26 | batch 27 | .content 28 | .push_event(TaskEvent::succeeded(TaskResult::Other)); 29 | } 30 | Err(e) => batch.content.push_event(TaskEvent::failed(e)), 31 | } 32 | batch 33 | } 34 | _ => unreachable!("invalid batch content for dump"), 35 | } 36 | } 37 | 38 | async fn finish(&self, _: &Batch) {} 39 | } 40 | 41 | #[cfg(test)] 42 | mod test { 43 | use crate::dump::error::{DumpError, Result as DumpResult}; 44 | use crate::index_resolver::{index_store::MockIndexStore, meta_store::MockIndexMetaStore}; 45 | use crate::tasks::handlers::test::task_to_batch; 46 | 47 | use super::*; 48 | 49 | use nelson::Mocker; 50 | use proptest::prelude::*; 51 | 52 | proptest! { 53 | #[test] 54 | fn finish_does_nothing( 55 | task in any::(), 56 | ) { 57 | let rt = tokio::runtime::Runtime::new().unwrap(); 58 | let handle = rt.spawn(async { 59 | let batch = task_to_batch(task); 60 | 61 | let mocker = Mocker::default(); 62 | let dump_handler = DumpHandler::::mock(mocker); 63 | 64 | dump_handler.finish(&batch).await; 65 | }); 66 | 67 | rt.block_on(handle).unwrap(); 68 | } 69 | 70 | #[test] 71 | fn test_handle_dump_success( 72 | task in any::(), 73 | ) { 74 | let rt = tokio::runtime::Runtime::new().unwrap(); 75 | let handle = rt.spawn(async { 76 | let batch = task_to_batch(task); 77 | let should_accept = matches!(batch.content, BatchContent::Dump { .. }); 78 | 79 | let mocker = Mocker::default(); 80 | if should_accept { 81 | mocker.when::>("run") 82 | .once() 83 | .then(|_| Ok(())); 84 | } 85 | 86 | let dump_handler = DumpHandler::::mock(mocker); 87 | 88 | let accept = dump_handler.accept(&batch); 89 | assert_eq!(accept, should_accept); 90 | 91 | if accept { 92 | let batch = dump_handler.process_batch(batch).await; 93 | let last_event = batch.content.first().unwrap().events.last().unwrap(); 94 | assert!(matches!(last_event, TaskEvent::Succeeded { .. })); 95 | } 96 | }); 97 | 98 | rt.block_on(handle).unwrap(); 99 | } 100 | 101 | #[test] 102 | fn test_handle_dump_error( 103 | task in any::(), 104 | ) { 105 | let rt = tokio::runtime::Runtime::new().unwrap(); 106 | let handle = rt.spawn(async { 107 | let batch = task_to_batch(task); 108 | let should_accept = matches!(batch.content, BatchContent::Dump { .. }); 109 | 110 | let mocker = Mocker::default(); 111 | if should_accept { 112 | mocker.when::>("run") 113 | .once() 114 | .then(|_| Err(DumpError::Internal("error".into()))); 115 | } 116 | 117 | let dump_handler = DumpHandler::::mock(mocker); 118 | 119 | let accept = dump_handler.accept(&batch); 120 | assert_eq!(accept, should_accept); 121 | 122 | if accept { 123 | let batch = dump_handler.process_batch(batch).await; 124 | let last_event = batch.content.first().unwrap().events.last().unwrap(); 125 | assert!(matches!(last_event, TaskEvent::Failed { .. })); 126 | } 127 | }); 128 | 129 | rt.block_on(handle).unwrap(); 130 | } 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /meilisearch-lib/src/dump/compat/v4.rs: -------------------------------------------------------------------------------- 1 | use meilisearch_types::error::ResponseError; 2 | use meilisearch_types::index_uid::IndexUid; 3 | use milli::update::IndexDocumentsMethod; 4 | use serde::{Deserialize, Serialize}; 5 | use time::OffsetDateTime; 6 | use uuid::Uuid; 7 | 8 | use crate::index::{Settings, Unchecked}; 9 | use crate::tasks::batch::BatchId; 10 | use crate::tasks::task::{ 11 | DocumentDeletion, TaskContent as NewTaskContent, TaskEvent as NewTaskEvent, TaskId, TaskResult, 12 | }; 13 | 14 | #[derive(Debug, Serialize, Deserialize)] 15 | pub struct Task { 16 | pub id: TaskId, 17 | pub index_uid: IndexUid, 18 | pub content: TaskContent, 19 | pub events: Vec, 20 | } 21 | 22 | impl From for crate::tasks::task::Task { 23 | fn from(other: Task) -> Self { 24 | Self { 25 | id: other.id, 26 | content: NewTaskContent::from((other.index_uid, other.content)), 27 | events: other.events.into_iter().map(Into::into).collect(), 28 | } 29 | } 30 | } 31 | 32 | #[derive(Debug, Serialize, Deserialize)] 33 | pub enum TaskEvent { 34 | Created(#[serde(with = "time::serde::rfc3339")] OffsetDateTime), 35 | Batched { 36 | #[serde(with = "time::serde::rfc3339")] 37 | timestamp: OffsetDateTime, 38 | batch_id: BatchId, 39 | }, 40 | Processing(#[serde(with = "time::serde::rfc3339")] OffsetDateTime), 41 | Succeded { 42 | result: TaskResult, 43 | #[serde(with = "time::serde::rfc3339")] 44 | timestamp: OffsetDateTime, 45 | }, 46 | Failed { 47 | error: ResponseError, 48 | #[serde(with = "time::serde::rfc3339")] 49 | timestamp: OffsetDateTime, 50 | }, 51 | } 52 | 53 | impl From for NewTaskEvent { 54 | fn from(other: TaskEvent) -> Self { 55 | match other { 56 | TaskEvent::Created(x) => NewTaskEvent::Created(x), 57 | TaskEvent::Batched { 58 | timestamp, 59 | batch_id, 60 | } => NewTaskEvent::Batched { 61 | timestamp, 62 | batch_id, 63 | }, 64 | TaskEvent::Processing(x) => NewTaskEvent::Processing(x), 65 | TaskEvent::Succeded { result, timestamp } => { 66 | NewTaskEvent::Succeeded { result, timestamp } 67 | } 68 | TaskEvent::Failed { error, timestamp } => NewTaskEvent::Failed { error, timestamp }, 69 | } 70 | } 71 | } 72 | 73 | #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] 74 | #[allow(clippy::large_enum_variant)] 75 | pub enum TaskContent { 76 | DocumentAddition { 77 | content_uuid: Uuid, 78 | merge_strategy: IndexDocumentsMethod, 79 | primary_key: Option, 80 | documents_count: usize, 81 | allow_index_creation: bool, 82 | }, 83 | DocumentDeletion(DocumentDeletion), 84 | SettingsUpdate { 85 | settings: Settings, 86 | /// Indicates whether the task was a deletion 87 | is_deletion: bool, 88 | allow_index_creation: bool, 89 | }, 90 | IndexDeletion, 91 | IndexCreation { 92 | primary_key: Option, 93 | }, 94 | IndexUpdate { 95 | primary_key: Option, 96 | }, 97 | Dump { 98 | uid: String, 99 | }, 100 | } 101 | 102 | impl From<(IndexUid, TaskContent)> for NewTaskContent { 103 | fn from((index_uid, content): (IndexUid, TaskContent)) -> Self { 104 | match content { 105 | TaskContent::DocumentAddition { 106 | content_uuid, 107 | merge_strategy, 108 | primary_key, 109 | documents_count, 110 | allow_index_creation, 111 | } => NewTaskContent::DocumentAddition { 112 | index_uid, 113 | content_uuid, 114 | merge_strategy, 115 | primary_key, 116 | documents_count, 117 | allow_index_creation, 118 | }, 119 | TaskContent::DocumentDeletion(deletion) => NewTaskContent::DocumentDeletion { 120 | index_uid, 121 | deletion, 122 | }, 123 | TaskContent::SettingsUpdate { 124 | settings, 125 | is_deletion, 126 | allow_index_creation, 127 | } => NewTaskContent::SettingsUpdate { 128 | index_uid, 129 | settings, 130 | is_deletion, 131 | allow_index_creation, 132 | }, 133 | TaskContent::IndexDeletion => NewTaskContent::IndexDeletion { index_uid }, 134 | TaskContent::IndexCreation { primary_key } => NewTaskContent::IndexCreation { 135 | index_uid, 136 | primary_key, 137 | }, 138 | TaskContent::IndexUpdate { primary_key } => NewTaskContent::IndexUpdate { 139 | index_uid, 140 | primary_key, 141 | }, 142 | TaskContent::Dump { uid } => NewTaskContent::Dump { uid }, 143 | } 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /meilisearch-lib/src/dump/compat/v2.rs: -------------------------------------------------------------------------------- 1 | use anyhow::bail; 2 | use meilisearch_types::error::Code; 3 | use milli::update::IndexDocumentsMethod; 4 | use serde::{Deserialize, Serialize}; 5 | use time::OffsetDateTime; 6 | use uuid::Uuid; 7 | 8 | use crate::index::{Settings, Unchecked}; 9 | 10 | #[derive(Serialize, Deserialize)] 11 | pub struct UpdateEntry { 12 | pub uuid: Uuid, 13 | pub update: UpdateStatus, 14 | } 15 | 16 | #[derive(Debug, Clone, Serialize, Deserialize)] 17 | pub enum UpdateFormat { 18 | Json, 19 | } 20 | 21 | #[derive(Debug, Serialize, Deserialize, Clone)] 22 | pub struct DocumentAdditionResult { 23 | pub nb_documents: usize, 24 | } 25 | 26 | #[derive(Debug, Clone, Serialize, Deserialize)] 27 | pub enum UpdateResult { 28 | DocumentsAddition(DocumentAdditionResult), 29 | DocumentDeletion { deleted: u64 }, 30 | Other, 31 | } 32 | 33 | #[allow(clippy::large_enum_variant)] 34 | #[derive(Debug, Clone, Serialize, Deserialize)] 35 | #[serde(tag = "type")] 36 | pub enum UpdateMeta { 37 | DocumentsAddition { 38 | method: IndexDocumentsMethod, 39 | format: UpdateFormat, 40 | primary_key: Option, 41 | }, 42 | ClearDocuments, 43 | DeleteDocuments { 44 | ids: Vec, 45 | }, 46 | Settings(Settings), 47 | } 48 | 49 | #[derive(Debug, Serialize, Deserialize, Clone)] 50 | #[serde(rename_all = "camelCase")] 51 | pub struct Enqueued { 52 | pub update_id: u64, 53 | pub meta: UpdateMeta, 54 | #[serde(with = "time::serde::rfc3339")] 55 | pub enqueued_at: OffsetDateTime, 56 | pub content: Option, 57 | } 58 | 59 | #[derive(Debug, Serialize, Deserialize, Clone)] 60 | #[serde(rename_all = "camelCase")] 61 | pub struct Processed { 62 | pub success: UpdateResult, 63 | #[serde(with = "time::serde::rfc3339")] 64 | pub processed_at: OffsetDateTime, 65 | #[serde(flatten)] 66 | pub from: Processing, 67 | } 68 | 69 | #[derive(Debug, Serialize, Deserialize, Clone)] 70 | #[serde(rename_all = "camelCase")] 71 | pub struct Processing { 72 | #[serde(flatten)] 73 | pub from: Enqueued, 74 | #[serde(with = "time::serde::rfc3339")] 75 | pub started_processing_at: OffsetDateTime, 76 | } 77 | 78 | #[derive(Debug, Serialize, Deserialize, Clone)] 79 | #[serde(rename_all = "camelCase")] 80 | pub struct Aborted { 81 | #[serde(flatten)] 82 | pub from: Enqueued, 83 | #[serde(with = "time::serde::rfc3339")] 84 | pub aborted_at: OffsetDateTime, 85 | } 86 | 87 | #[derive(Debug, Serialize, Deserialize)] 88 | #[serde(rename_all = "camelCase")] 89 | pub struct Failed { 90 | #[serde(flatten)] 91 | pub from: Processing, 92 | pub error: ResponseError, 93 | #[serde(with = "time::serde::rfc3339")] 94 | pub failed_at: OffsetDateTime, 95 | } 96 | 97 | #[derive(Debug, Serialize, Deserialize)] 98 | #[serde(tag = "status", rename_all = "camelCase")] 99 | pub enum UpdateStatus { 100 | Processing(Processing), 101 | Enqueued(Enqueued), 102 | Processed(Processed), 103 | Aborted(Aborted), 104 | Failed(Failed), 105 | } 106 | 107 | type StatusCode = (); 108 | 109 | #[derive(Debug, Serialize, Deserialize, Clone)] 110 | #[serde(rename_all = "camelCase")] 111 | pub struct ResponseError { 112 | #[serde(skip)] 113 | pub code: StatusCode, 114 | pub message: String, 115 | pub error_code: String, 116 | pub error_type: String, 117 | pub error_link: String, 118 | } 119 | 120 | pub fn error_code_from_str(s: &str) -> anyhow::Result { 121 | let code = match s { 122 | "index_creation_failed" => Code::CreateIndex, 123 | "index_already_exists" => Code::IndexAlreadyExists, 124 | "index_not_found" => Code::IndexNotFound, 125 | "invalid_index_uid" => Code::InvalidIndexUid, 126 | "invalid_state" => Code::InvalidState, 127 | "missing_primary_key" => Code::MissingPrimaryKey, 128 | "primary_key_already_present" => Code::PrimaryKeyAlreadyPresent, 129 | "invalid_request" => Code::InvalidRankingRule, 130 | "max_fields_limit_exceeded" => Code::MaxFieldsLimitExceeded, 131 | "missing_document_id" => Code::MissingDocumentId, 132 | "invalid_facet" => Code::Filter, 133 | "invalid_filter" => Code::Filter, 134 | "invalid_sort" => Code::Sort, 135 | "bad_parameter" => Code::BadParameter, 136 | "bad_request" => Code::BadRequest, 137 | "document_not_found" => Code::DocumentNotFound, 138 | "internal" => Code::Internal, 139 | "invalid_geo_field" => Code::InvalidGeoField, 140 | "invalid_token" => Code::InvalidToken, 141 | "missing_authorization_header" => Code::MissingAuthorizationHeader, 142 | "payload_too_large" => Code::PayloadTooLarge, 143 | "unretrievable_document" => Code::RetrieveDocument, 144 | "search_error" => Code::SearchDocuments, 145 | "unsupported_media_type" => Code::UnsupportedMediaType, 146 | "dump_already_in_progress" => Code::DumpAlreadyInProgress, 147 | "dump_process_failed" => Code::DumpProcessFailed, 148 | _ => bail!("unknow error code."), 149 | }; 150 | 151 | Ok(code) 152 | } 153 | -------------------------------------------------------------------------------- /meilisearch-auth/src/action.rs: -------------------------------------------------------------------------------- 1 | use enum_iterator::IntoEnumIterator; 2 | use serde::{Deserialize, Serialize}; 3 | 4 | #[derive(IntoEnumIterator, Copy, Clone, Serialize, Deserialize, Debug, Eq, PartialEq)] 5 | #[repr(u8)] 6 | pub enum Action { 7 | #[serde(rename = "*")] 8 | All = actions::ALL, 9 | #[serde(rename = "search")] 10 | Search = actions::SEARCH, 11 | #[serde(rename = "documents.*")] 12 | DocumentsAll = actions::DOCUMENTS_ALL, 13 | #[serde(rename = "documents.add")] 14 | DocumentsAdd = actions::DOCUMENTS_ADD, 15 | #[serde(rename = "documents.get")] 16 | DocumentsGet = actions::DOCUMENTS_GET, 17 | #[serde(rename = "documents.delete")] 18 | DocumentsDelete = actions::DOCUMENTS_DELETE, 19 | #[serde(rename = "indexes.create")] 20 | IndexesAdd = actions::INDEXES_CREATE, 21 | #[serde(rename = "indexes.get")] 22 | IndexesGet = actions::INDEXES_GET, 23 | #[serde(rename = "indexes.update")] 24 | IndexesUpdate = actions::INDEXES_UPDATE, 25 | #[serde(rename = "indexes.delete")] 26 | IndexesDelete = actions::INDEXES_DELETE, 27 | #[serde(rename = "tasks.get")] 28 | TasksGet = actions::TASKS_GET, 29 | #[serde(rename = "settings.get")] 30 | SettingsGet = actions::SETTINGS_GET, 31 | #[serde(rename = "settings.update")] 32 | SettingsUpdate = actions::SETTINGS_UPDATE, 33 | #[serde(rename = "stats.get")] 34 | StatsGet = actions::STATS_GET, 35 | #[serde(rename = "dumps.create")] 36 | DumpsCreate = actions::DUMPS_CREATE, 37 | #[serde(rename = "version")] 38 | Version = actions::VERSION, 39 | #[serde(rename = "keys.create")] 40 | KeysAdd = actions::KEYS_CREATE, 41 | #[serde(rename = "keys.get")] 42 | KeysGet = actions::KEYS_GET, 43 | #[serde(rename = "keys.update")] 44 | KeysUpdate = actions::KEYS_UPDATE, 45 | #[serde(rename = "keys.delete")] 46 | KeysDelete = actions::KEYS_DELETE, 47 | } 48 | 49 | impl Action { 50 | pub fn from_repr(repr: u8) -> Option { 51 | use actions::*; 52 | match repr { 53 | ALL => Some(Self::All), 54 | SEARCH => Some(Self::Search), 55 | DOCUMENTS_ALL => Some(Self::DocumentsAll), 56 | DOCUMENTS_ADD => Some(Self::DocumentsAdd), 57 | DOCUMENTS_GET => Some(Self::DocumentsGet), 58 | DOCUMENTS_DELETE => Some(Self::DocumentsDelete), 59 | INDEXES_CREATE => Some(Self::IndexesAdd), 60 | INDEXES_GET => Some(Self::IndexesGet), 61 | INDEXES_UPDATE => Some(Self::IndexesUpdate), 62 | INDEXES_DELETE => Some(Self::IndexesDelete), 63 | TASKS_GET => Some(Self::TasksGet), 64 | SETTINGS_GET => Some(Self::SettingsGet), 65 | SETTINGS_UPDATE => Some(Self::SettingsUpdate), 66 | STATS_GET => Some(Self::StatsGet), 67 | DUMPS_CREATE => Some(Self::DumpsCreate), 68 | VERSION => Some(Self::Version), 69 | KEYS_CREATE => Some(Self::KeysAdd), 70 | KEYS_GET => Some(Self::KeysGet), 71 | KEYS_UPDATE => Some(Self::KeysUpdate), 72 | KEYS_DELETE => Some(Self::KeysDelete), 73 | _otherwise => None, 74 | } 75 | } 76 | 77 | pub fn repr(&self) -> u8 { 78 | use actions::*; 79 | match self { 80 | Self::All => ALL, 81 | Self::Search => SEARCH, 82 | Self::DocumentsAll => DOCUMENTS_ALL, 83 | Self::DocumentsAdd => DOCUMENTS_ADD, 84 | Self::DocumentsGet => DOCUMENTS_GET, 85 | Self::DocumentsDelete => DOCUMENTS_DELETE, 86 | Self::IndexesAdd => INDEXES_CREATE, 87 | Self::IndexesGet => INDEXES_GET, 88 | Self::IndexesUpdate => INDEXES_UPDATE, 89 | Self::IndexesDelete => INDEXES_DELETE, 90 | Self::TasksGet => TASKS_GET, 91 | Self::SettingsGet => SETTINGS_GET, 92 | Self::SettingsUpdate => SETTINGS_UPDATE, 93 | Self::StatsGet => STATS_GET, 94 | Self::DumpsCreate => DUMPS_CREATE, 95 | Self::Version => VERSION, 96 | Self::KeysAdd => KEYS_CREATE, 97 | Self::KeysGet => KEYS_GET, 98 | Self::KeysUpdate => KEYS_UPDATE, 99 | Self::KeysDelete => KEYS_DELETE, 100 | } 101 | } 102 | } 103 | 104 | pub mod actions { 105 | pub(crate) const ALL: u8 = 0; 106 | pub const SEARCH: u8 = 1; 107 | pub const DOCUMENTS_ALL: u8 = 2; 108 | pub const DOCUMENTS_ADD: u8 = 3; 109 | pub const DOCUMENTS_GET: u8 = 4; 110 | pub const DOCUMENTS_DELETE: u8 = 5; 111 | pub const INDEXES_CREATE: u8 = 6; 112 | pub const INDEXES_GET: u8 = 7; 113 | pub const INDEXES_UPDATE: u8 = 8; 114 | pub const INDEXES_DELETE: u8 = 9; 115 | pub const TASKS_GET: u8 = 10; 116 | pub const SETTINGS_GET: u8 = 11; 117 | pub const SETTINGS_UPDATE: u8 = 12; 118 | pub const STATS_GET: u8 = 13; 119 | pub const DUMPS_CREATE: u8 = 14; 120 | pub const VERSION: u8 = 15; 121 | pub const KEYS_CREATE: u8 = 16; 122 | pub const KEYS_GET: u8 = 17; 123 | pub const KEYS_UPDATE: u8 = 18; 124 | pub const KEYS_DELETE: u8 = 19; 125 | } 126 | -------------------------------------------------------------------------------- /meilisearch-http/tests/documents/delete_documents.rs: -------------------------------------------------------------------------------- 1 | use serde_json::json; 2 | 3 | use crate::common::{GetAllDocumentsOptions, Server}; 4 | 5 | #[actix_rt::test] 6 | async fn delete_one_document_unexisting_index() { 7 | let server = Server::new().await; 8 | let index = server.index("test"); 9 | let (_response, code) = index.delete_document(0).await; 10 | assert_eq!(code, 202); 11 | 12 | let response = index.wait_task(0).await; 13 | 14 | assert_eq!(response["status"], "failed"); 15 | } 16 | 17 | #[actix_rt::test] 18 | async fn delete_one_unexisting_document() { 19 | let server = Server::new().await; 20 | let index = server.index("test"); 21 | index.create(None).await; 22 | let (response, code) = index.delete_document(0).await; 23 | assert_eq!(code, 202, "{}", response); 24 | let update = index.wait_task(0).await; 25 | assert_eq!(update["status"], "succeeded"); 26 | } 27 | 28 | #[actix_rt::test] 29 | async fn delete_one_document() { 30 | let server = Server::new().await; 31 | let index = server.index("test"); 32 | index 33 | .add_documents(json!([{ "id": 0, "content": "foobar" }]), None) 34 | .await; 35 | index.wait_task(0).await; 36 | let (_response, code) = server.index("test").delete_document(0).await; 37 | assert_eq!(code, 202); 38 | index.wait_task(1).await; 39 | 40 | let (_response, code) = index.get_document(0, None).await; 41 | assert_eq!(code, 404); 42 | } 43 | 44 | #[actix_rt::test] 45 | async fn clear_all_documents_unexisting_index() { 46 | let server = Server::new().await; 47 | let index = server.index("test"); 48 | let (_response, code) = index.clear_all_documents().await; 49 | assert_eq!(code, 202); 50 | 51 | let response = index.wait_task(0).await; 52 | 53 | assert_eq!(response["status"], "failed"); 54 | } 55 | 56 | #[actix_rt::test] 57 | async fn clear_all_documents() { 58 | let server = Server::new().await; 59 | let index = server.index("test"); 60 | index 61 | .add_documents( 62 | json!([{ "id": 1, "content": "foobar" }, { "id": 0, "content": "foobar" }]), 63 | None, 64 | ) 65 | .await; 66 | index.wait_task(0).await; 67 | let (_response, code) = index.clear_all_documents().await; 68 | assert_eq!(code, 202); 69 | 70 | let _update = index.wait_task(1).await; 71 | let (response, code) = index 72 | .get_all_documents(GetAllDocumentsOptions::default()) 73 | .await; 74 | assert_eq!(code, 200); 75 | assert!(response["results"].as_array().unwrap().is_empty()); 76 | } 77 | 78 | #[actix_rt::test] 79 | async fn clear_all_documents_empty_index() { 80 | let server = Server::new().await; 81 | let index = server.index("test"); 82 | index.create(None).await; 83 | 84 | let (_response, code) = index.clear_all_documents().await; 85 | assert_eq!(code, 202); 86 | 87 | let _update = index.wait_task(0).await; 88 | let (response, code) = index 89 | .get_all_documents(GetAllDocumentsOptions::default()) 90 | .await; 91 | assert_eq!(code, 200); 92 | assert!(response["results"].as_array().unwrap().is_empty()); 93 | } 94 | 95 | #[actix_rt::test] 96 | async fn error_delete_batch_unexisting_index() { 97 | let server = Server::new().await; 98 | let index = server.index("test"); 99 | let (_, code) = index.delete_batch(vec![]).await; 100 | let expected_response = json!({ 101 | "message": "Index `test` not found.", 102 | "code": "index_not_found", 103 | "type": "invalid_request", 104 | "link": "https://docs.meilisearch.com/errors#index_not_found" 105 | }); 106 | assert_eq!(code, 202); 107 | 108 | let response = index.wait_task(0).await; 109 | 110 | assert_eq!(response["status"], "failed"); 111 | assert_eq!(response["error"], expected_response); 112 | } 113 | 114 | #[actix_rt::test] 115 | async fn delete_batch() { 116 | let server = Server::new().await; 117 | let index = server.index("test"); 118 | index.add_documents(json!([{ "id": 1, "content": "foobar" }, { "id": 0, "content": "foobar" }, { "id": 3, "content": "foobar" }]), Some("id")).await; 119 | index.wait_task(0).await; 120 | let (_response, code) = index.delete_batch(vec![1, 0]).await; 121 | assert_eq!(code, 202); 122 | 123 | let _update = index.wait_task(1).await; 124 | let (response, code) = index 125 | .get_all_documents(GetAllDocumentsOptions::default()) 126 | .await; 127 | assert_eq!(code, 200); 128 | assert_eq!(response["results"].as_array().unwrap().len(), 1); 129 | assert_eq!(response["results"][0]["id"], json!(3)); 130 | } 131 | 132 | #[actix_rt::test] 133 | async fn delete_no_document_batch() { 134 | let server = Server::new().await; 135 | let index = server.index("test"); 136 | index.add_documents(json!([{ "id": 1, "content": "foobar" }, { "id": 0, "content": "foobar" }, { "id": 3, "content": "foobar" }]), Some("id")).await; 137 | index.wait_task(0).await; 138 | let (_response, code) = index.delete_batch(vec![]).await; 139 | assert_eq!(code, 202, "{}", _response); 140 | 141 | let _update = index.wait_task(1).await; 142 | let (response, code) = index 143 | .get_all_documents(GetAllDocumentsOptions::default()) 144 | .await; 145 | assert_eq!(code, 200); 146 | assert_eq!(response["results"].as_array().unwrap().len(), 3); 147 | } 148 | -------------------------------------------------------------------------------- /meilisearch-lib/src/index/dump.rs: -------------------------------------------------------------------------------- 1 | use std::fs::{create_dir_all, File}; 2 | use std::io::{BufReader, Seek, SeekFrom, Write}; 3 | use std::path::Path; 4 | 5 | use anyhow::Context; 6 | use indexmap::IndexMap; 7 | use milli::documents::DocumentBatchReader; 8 | use milli::heed::{EnvOpenOptions, RoTxn}; 9 | use milli::update::{IndexDocumentsConfig, IndexerConfig}; 10 | use serde::{Deserialize, Serialize}; 11 | 12 | use crate::document_formats::read_ndjson; 13 | use crate::index::updates::apply_settings_to_builder; 14 | 15 | use super::error::Result; 16 | use super::{index::Index, Settings, Unchecked}; 17 | 18 | #[derive(Serialize, Deserialize)] 19 | struct DumpMeta { 20 | settings: Settings, 21 | primary_key: Option, 22 | } 23 | 24 | const META_FILE_NAME: &str = "meta.json"; 25 | const DATA_FILE_NAME: &str = "documents.jsonl"; 26 | 27 | impl Index { 28 | pub fn dump(&self, path: impl AsRef) -> Result<()> { 29 | // acquire write txn make sure any ongoing write is finished before we start. 30 | let txn = self.env.write_txn()?; 31 | let path = path.as_ref().join(format!("indexes/{}", self.uuid)); 32 | 33 | create_dir_all(&path)?; 34 | 35 | self.dump_documents(&txn, &path)?; 36 | self.dump_meta(&txn, &path)?; 37 | 38 | Ok(()) 39 | } 40 | 41 | fn dump_documents(&self, txn: &RoTxn, path: impl AsRef) -> Result<()> { 42 | let document_file_path = path.as_ref().join(DATA_FILE_NAME); 43 | let mut document_file = File::create(&document_file_path)?; 44 | 45 | let documents = self.all_documents(txn)?; 46 | let fields_ids_map = self.fields_ids_map(txn)?; 47 | 48 | // dump documents 49 | let mut json_map = IndexMap::new(); 50 | for document in documents { 51 | let (_, reader) = document?; 52 | 53 | for (fid, bytes) in reader.iter() { 54 | if let Some(name) = fields_ids_map.name(fid) { 55 | json_map.insert(name, serde_json::from_slice::(bytes)?); 56 | } 57 | } 58 | 59 | serde_json::to_writer(&mut document_file, &json_map)?; 60 | document_file.write_all(b"\n")?; 61 | 62 | json_map.clear(); 63 | } 64 | 65 | Ok(()) 66 | } 67 | 68 | fn dump_meta(&self, txn: &RoTxn, path: impl AsRef) -> Result<()> { 69 | let meta_file_path = path.as_ref().join(META_FILE_NAME); 70 | let mut meta_file = File::create(&meta_file_path)?; 71 | 72 | let settings = self.settings_txn(txn)?.into_unchecked(); 73 | let primary_key = self.primary_key(txn)?.map(String::from); 74 | let meta = DumpMeta { 75 | settings, 76 | primary_key, 77 | }; 78 | 79 | serde_json::to_writer(&mut meta_file, &meta)?; 80 | 81 | Ok(()) 82 | } 83 | 84 | pub fn load_dump( 85 | src: impl AsRef, 86 | dst: impl AsRef, 87 | size: usize, 88 | indexer_config: &IndexerConfig, 89 | ) -> anyhow::Result<()> { 90 | let dir_name = src 91 | .as_ref() 92 | .file_name() 93 | .with_context(|| format!("invalid dump index: {}", src.as_ref().display()))?; 94 | 95 | let dst_dir_path = dst.as_ref().join("indexes").join(dir_name); 96 | create_dir_all(&dst_dir_path)?; 97 | 98 | let meta_path = src.as_ref().join(META_FILE_NAME); 99 | let meta_file = File::open(meta_path)?; 100 | let DumpMeta { 101 | settings, 102 | primary_key, 103 | } = serde_json::from_reader(meta_file)?; 104 | let settings = settings.check(); 105 | 106 | let mut options = EnvOpenOptions::new(); 107 | options.map_size(size); 108 | let index = milli::Index::new(options, &dst_dir_path)?; 109 | 110 | let mut txn = index.write_txn()?; 111 | 112 | // Apply settings first 113 | let mut builder = milli::update::Settings::new(&mut txn, &index, indexer_config); 114 | 115 | if let Some(primary_key) = primary_key { 116 | builder.set_primary_key(primary_key); 117 | } 118 | 119 | apply_settings_to_builder(&settings, &mut builder); 120 | 121 | builder.execute(|_| ())?; 122 | 123 | let document_file_path = src.as_ref().join(DATA_FILE_NAME); 124 | let reader = BufReader::new(File::open(&document_file_path)?); 125 | 126 | let mut tmp_doc_file = tempfile::tempfile()?; 127 | 128 | let empty = match read_ndjson(reader, &mut tmp_doc_file) { 129 | // if there was no document in the file it's because the index was empty 130 | Ok(0) => true, 131 | Ok(_) => false, 132 | Err(e) => return Err(e.into()), 133 | }; 134 | 135 | if !empty { 136 | tmp_doc_file.seek(SeekFrom::Start(0))?; 137 | 138 | let documents_reader = DocumentBatchReader::from_reader(tmp_doc_file)?; 139 | 140 | //If the document file is empty, we don't perform the document addition, to prevent 141 | //a primary key error to be thrown. 142 | let config = IndexDocumentsConfig::default(); 143 | let mut builder = milli::update::IndexDocuments::new( 144 | &mut txn, 145 | &index, 146 | indexer_config, 147 | config, 148 | |_| (), 149 | )?; 150 | builder.add_documents(documents_reader)?; 151 | builder.execute()?; 152 | } 153 | 154 | txn.commit()?; 155 | index.prepare_for_closing().wait(); 156 | 157 | Ok(()) 158 | } 159 | } 160 | -------------------------------------------------------------------------------- /meilisearch-http/tests/common/server.rs: -------------------------------------------------------------------------------- 1 | #![allow(dead_code)] 2 | 3 | use clap::Parser; 4 | use std::path::Path; 5 | 6 | use actix_web::http::StatusCode; 7 | use byte_unit::{Byte, ByteUnit}; 8 | use meilisearch_auth::AuthController; 9 | use meilisearch_http::setup_meilisearch; 10 | use meilisearch_lib::options::{IndexerOpts, MaxMemory}; 11 | use once_cell::sync::Lazy; 12 | use serde_json::Value; 13 | use tempfile::TempDir; 14 | 15 | use meilisearch_http::option::Opt; 16 | 17 | use super::index::Index; 18 | use super::service::Service; 19 | 20 | pub struct Server { 21 | pub service: Service, 22 | // hold ownership to the tempdir while we use the server instance. 23 | _dir: Option, 24 | } 25 | 26 | pub static TEST_TEMP_DIR: Lazy = Lazy::new(|| TempDir::new().unwrap()); 27 | 28 | impl Server { 29 | pub async fn new() -> Self { 30 | let dir = TempDir::new().unwrap(); 31 | 32 | if cfg!(windows) { 33 | std::env::set_var("TMP", TEST_TEMP_DIR.path()); 34 | } else { 35 | std::env::set_var("TMPDIR", TEST_TEMP_DIR.path()); 36 | } 37 | 38 | let options = default_settings(dir.path()); 39 | 40 | let meilisearch = setup_meilisearch(&options).unwrap(); 41 | let auth = AuthController::new(&options.db_path, &options.master_key).unwrap(); 42 | let service = Service { 43 | meilisearch, 44 | auth, 45 | options, 46 | api_key: None, 47 | }; 48 | 49 | Server { 50 | service, 51 | _dir: Some(dir), 52 | } 53 | } 54 | 55 | pub async fn new_auth_with_options(mut options: Opt, dir: TempDir) -> Self { 56 | if cfg!(windows) { 57 | std::env::set_var("TMP", TEST_TEMP_DIR.path()); 58 | } else { 59 | std::env::set_var("TMPDIR", TEST_TEMP_DIR.path()); 60 | } 61 | 62 | options.master_key = Some("MASTER_KEY".to_string()); 63 | 64 | let meilisearch = setup_meilisearch(&options).unwrap(); 65 | let auth = AuthController::new(&options.db_path, &options.master_key).unwrap(); 66 | let service = Service { 67 | meilisearch, 68 | auth, 69 | options, 70 | api_key: None, 71 | }; 72 | 73 | Server { 74 | service, 75 | _dir: Some(dir), 76 | } 77 | } 78 | 79 | pub async fn new_auth() -> Self { 80 | let dir = TempDir::new().unwrap(); 81 | let options = default_settings(dir.path()); 82 | Self::new_auth_with_options(options, dir).await 83 | } 84 | 85 | pub async fn new_with_options(options: Opt) -> Result { 86 | let meilisearch = setup_meilisearch(&options)?; 87 | let auth = AuthController::new(&options.db_path, &options.master_key)?; 88 | let service = Service { 89 | meilisearch, 90 | auth, 91 | options, 92 | api_key: None, 93 | }; 94 | 95 | Ok(Server { 96 | service, 97 | _dir: None, 98 | }) 99 | } 100 | 101 | /// Returns a view to an index. There is no guarantee that the index exists. 102 | pub fn index(&self, uid: impl AsRef) -> Index<'_> { 103 | Index { 104 | uid: uid.as_ref().to_string(), 105 | service: &self.service, 106 | } 107 | } 108 | 109 | pub async fn list_indexes( 110 | &self, 111 | offset: Option, 112 | limit: Option, 113 | ) -> (Value, StatusCode) { 114 | let (offset, limit) = ( 115 | offset.map(|offset| format!("offset={offset}")), 116 | limit.map(|limit| format!("limit={limit}")), 117 | ); 118 | let query_parameter = offset 119 | .as_ref() 120 | .zip(limit.as_ref()) 121 | .map(|(offset, limit)| format!("{offset}&{limit}")) 122 | .or_else(|| offset.xor(limit)); 123 | if let Some(query_parameter) = query_parameter { 124 | self.service 125 | .get(format!("/indexes?{query_parameter}")) 126 | .await 127 | } else { 128 | self.service.get("/indexes").await 129 | } 130 | } 131 | 132 | pub async fn version(&self) -> (Value, StatusCode) { 133 | self.service.get("/version").await 134 | } 135 | 136 | pub async fn stats(&self) -> (Value, StatusCode) { 137 | self.service.get("/stats").await 138 | } 139 | 140 | pub async fn tasks(&self) -> (Value, StatusCode) { 141 | self.service.get("/tasks").await 142 | } 143 | 144 | pub async fn get_dump_status(&self, uid: &str) -> (Value, StatusCode) { 145 | self.service.get(format!("/dumps/{}/status", uid)).await 146 | } 147 | } 148 | 149 | pub fn default_settings(dir: impl AsRef) -> Opt { 150 | Opt { 151 | db_path: dir.as_ref().join("db"), 152 | dumps_dir: dir.as_ref().join("dump"), 153 | env: "development".to_owned(), 154 | #[cfg(all(not(debug_assertions), feature = "analytics"))] 155 | no_analytics: true, 156 | max_index_size: Byte::from_unit(100.0, ByteUnit::MiB).unwrap(), 157 | max_task_db_size: Byte::from_unit(1.0, ByteUnit::GiB).unwrap(), 158 | http_payload_size_limit: Byte::from_unit(10.0, ByteUnit::MiB).unwrap(), 159 | snapshot_dir: ".".into(), 160 | indexer_options: IndexerOpts { 161 | // memory has to be unlimited because several meilisearch are running in test context. 162 | max_indexing_memory: MaxMemory::unlimited(), 163 | ..Parser::parse_from(None as Option<&str>) 164 | }, 165 | ..Parser::parse_from(None as Option<&str>) 166 | } 167 | } 168 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | First, thank you for contributing to Meilisearch! The goal of this document is to provide everything you need to start contributing to Meilisearch. 4 | 5 | Remember that there are many ways to contribute other than writing code: writing [tutorials or blog posts](https://github.com/meilisearch/awesome-meilisearch), improving [the documentation](https://github.com/meilisearch/documentation), submitting [bug reports](https://github.com/meilisearch/meilisearch/issues/new?assignees=&labels=&template=bug_report.md&title=) and [feature requests](https://github.com/meilisearch/product/discussions/categories/feedback-feature-proposal)... 6 | 7 | ## Table of Contents 8 | 9 | - [Assumptions](#assumptions) 10 | - [How to Contribute](#how-to-contribute) 11 | - [Development Workflow](#development-workflow) 12 | - [Git Guidelines](#git-guidelines) 13 | - [Release Process (for internal team only)](#release-process-for-internal-team-only) 14 | 15 | ## Assumptions 16 | 17 | 1. **You're familiar with [GitHub](https://github.com) and the [Pull Requests](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/about-pull-requests)(PR) workflow.** 18 | 2. **You've read the Meilisearch [documentation](https://docs.meilisearch.com).** 19 | 3. **You know about the [Meilisearch community](https://docs.meilisearch.com/learn/what_is_meilisearch/contact.html). 20 | Please use this for help.** 21 | 22 | ## How to Contribute 23 | 24 | 1. Ensure your change has an issue! Find an 25 | [existing issue](https://github.com/meilisearch/meilisearch/issues/) or [open a new issue](https://github.com/meilisearch/meilisearch/issues/new). 26 | * This is where you can get a feel if the change will be accepted or not. 27 | 2. Once approved, [fork the Meilisearch repository](https://help.github.com/en/github/getting-started-with-github/fork-a-repo) in your own GitHub account. 28 | 3. [Create a new Git branch](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-and-deleting-branches-within-your-repository) 29 | 4. Review the [Development Workflow](#development-workflow) section that describes the steps to maintain the repository. 30 | 5. Make your changes on your branch. 31 | 6. [Submit the branch as a Pull Request](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request-from-a-fork) pointing to the `main` branch of the Meilisearch repository. A maintainer should comment and/or review your Pull Request within a few days. Although depending on the circumstances, it may take longer. 32 | 33 | ## Development Workflow 34 | 35 | ### Setup and run Meilisearch 36 | 37 | ```bash 38 | cargo run --release 39 | ``` 40 | 41 | We recommend using the `--release` flag to test the full performance of Meilisearch. 42 | 43 | ### Test 44 | 45 | ```bash 46 | cargo test 47 | ``` 48 | 49 | This command will be triggered to each PR as a requirement for merging it. 50 | 51 | If you get a "Too many open files" error you might want to increase the open file limit using this command: 52 | 53 | ```bash 54 | ulimit -Sn 3000 55 | ``` 56 | 57 | ## Git Guidelines 58 | 59 | ### Git Branches 60 | 61 | All changes must be made in a branch and submitted as PR. 62 | 63 | We do not enforce any branch naming style, but please use something descriptive of your changes. 64 | 65 | ### Git Commits 66 | 67 | As minimal requirements, your commit message should: 68 | - be capitalized 69 | - not finish by a dot or any other punctuation character (!,?) 70 | - start with a verb so that we can read your commit message this way: "This commit will ...", where "..." is the commit message. 71 | e.g.: "Fix the home page button" or "Add more tests for create_index method" 72 | 73 | We don't follow any other convention, but if you want to use one, we recommend [the Chris Beams one](https://chris.beams.io/posts/git-commit/). 74 | 75 | ### GitHub Pull Requests 76 | 77 | Some notes on GitHub PRs: 78 | 79 | - All PRs must be reviewed and approved by at least one maintainer. 80 | - The PR title should be accurate and descriptive of the changes. 81 | - [Convert your PR as a draft](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/changing-the-stage-of-a-pull-request) if your changes are a work in progress: no one will review it until you pass your PR as ready for review.
82 | The draft PRs are recommended when you want to show that you are working on something and make your work visible. 83 | - The branch related to the PR must be **up-to-date with `main`** before merging. Fortunately, this project uses [Bors](https://github.com/bors-ng/bors-ng) to automatically enforce this requirement without the PR author having to rebase manually. 84 | 85 | ## Release Process (for internal team only) 86 | 87 | Meilisearch tools follow the [Semantic Versioning Convention](https://semver.org/). 88 | 89 | ### Automation to rebase and Merge the PRs 90 | 91 | This project integrates a bot that helps us manage pull requests merging.
92 | _[Read more about this](https://github.com/meilisearch/integration-guides/blob/main/resources/bors.md)._ 93 | 94 | ### How to Publish a new Release 95 | 96 | The full Meilisearch release process is described in [this guide](https://github.com/meilisearch/core-team/blob/main/resources/meilisearch-release.md). Please follow it carefully before doing any release. 97 | 98 | ### Release assets 99 | 100 | For each release, the following assets are created: 101 | - Binaries for differents platforms (Linux, MacOS, Windows and ARM architectures) are attached to the GitHub release 102 | - Binaries are pushed to HomeBrew and APT (not published for RC) 103 | - Docker tags are created/updated: 104 | - `vX.Y.Z` 105 | - `vX.Y` (not published for RC) 106 | - `latest` (not published for RC) 107 | 108 |
109 | 110 | Thank you again for reading this through, we can not wait to begin to work with you if you made your way through this contributing guide ❤️ 111 | -------------------------------------------------------------------------------- /meilisearch-http/src/routes/api_key.rs: -------------------------------------------------------------------------------- 1 | use std::str; 2 | 3 | use actix_web::{web, HttpRequest, HttpResponse}; 4 | use serde::{Deserialize, Serialize}; 5 | use serde_json::Value; 6 | use time::OffsetDateTime; 7 | use uuid::Uuid; 8 | 9 | use meilisearch_auth::{error::AuthControllerError, Action, AuthController, Key}; 10 | use meilisearch_types::error::{Code, ResponseError}; 11 | 12 | use crate::extractors::{ 13 | authentication::{policies::*, GuardedData}, 14 | sequential_extractor::SeqHandler, 15 | }; 16 | use crate::routes::Pagination; 17 | 18 | pub fn configure(cfg: &mut web::ServiceConfig) { 19 | cfg.service( 20 | web::resource("") 21 | .route(web::post().to(SeqHandler(create_api_key))) 22 | .route(web::get().to(SeqHandler(list_api_keys))), 23 | ) 24 | .service( 25 | web::resource("/{key}") 26 | .route(web::get().to(SeqHandler(get_api_key))) 27 | .route(web::patch().to(SeqHandler(patch_api_key))) 28 | .route(web::delete().to(SeqHandler(delete_api_key))), 29 | ); 30 | } 31 | 32 | pub async fn create_api_key( 33 | auth_controller: GuardedData, AuthController>, 34 | body: web::Json, 35 | _req: HttpRequest, 36 | ) -> Result { 37 | let v = body.into_inner(); 38 | let res = tokio::task::spawn_blocking(move || -> Result<_, AuthControllerError> { 39 | let key = auth_controller.create_key(v)?; 40 | Ok(KeyView::from_key(key, &auth_controller)) 41 | }) 42 | .await 43 | .map_err(|e| ResponseError::from_msg(e.to_string(), Code::Internal))??; 44 | 45 | Ok(HttpResponse::Created().json(res)) 46 | } 47 | 48 | pub async fn list_api_keys( 49 | auth_controller: GuardedData, AuthController>, 50 | paginate: web::Query, 51 | ) -> Result { 52 | let page_view = tokio::task::spawn_blocking(move || -> Result<_, AuthControllerError> { 53 | let keys = auth_controller.list_keys()?; 54 | let page_view = paginate.auto_paginate_sized( 55 | keys.into_iter() 56 | .map(|k| KeyView::from_key(k, &auth_controller)), 57 | ); 58 | 59 | Ok(page_view) 60 | }) 61 | .await 62 | .map_err(|e| ResponseError::from_msg(e.to_string(), Code::Internal))??; 63 | 64 | Ok(HttpResponse::Ok().json(page_view)) 65 | } 66 | 67 | pub async fn get_api_key( 68 | auth_controller: GuardedData, AuthController>, 69 | path: web::Path, 70 | ) -> Result { 71 | let key = path.into_inner().key; 72 | 73 | let res = tokio::task::spawn_blocking(move || -> Result<_, AuthControllerError> { 74 | let uid = 75 | Uuid::parse_str(&key).or_else(|_| auth_controller.get_uid_from_encoded_key(&key))?; 76 | let key = auth_controller.get_key(uid)?; 77 | 78 | Ok(KeyView::from_key(key, &auth_controller)) 79 | }) 80 | .await 81 | .map_err(|e| ResponseError::from_msg(e.to_string(), Code::Internal))??; 82 | 83 | Ok(HttpResponse::Ok().json(res)) 84 | } 85 | 86 | pub async fn patch_api_key( 87 | auth_controller: GuardedData, AuthController>, 88 | body: web::Json, 89 | path: web::Path, 90 | ) -> Result { 91 | let key = path.into_inner().key; 92 | let body = body.into_inner(); 93 | let res = tokio::task::spawn_blocking(move || -> Result<_, AuthControllerError> { 94 | let uid = 95 | Uuid::parse_str(&key).or_else(|_| auth_controller.get_uid_from_encoded_key(&key))?; 96 | let key = auth_controller.update_key(uid, body)?; 97 | 98 | Ok(KeyView::from_key(key, &auth_controller)) 99 | }) 100 | .await 101 | .map_err(|e| ResponseError::from_msg(e.to_string(), Code::Internal))??; 102 | 103 | Ok(HttpResponse::Ok().json(res)) 104 | } 105 | 106 | pub async fn delete_api_key( 107 | auth_controller: GuardedData, AuthController>, 108 | path: web::Path, 109 | ) -> Result { 110 | let key = path.into_inner().key; 111 | tokio::task::spawn_blocking(move || { 112 | let uid = 113 | Uuid::parse_str(&key).or_else(|_| auth_controller.get_uid_from_encoded_key(&key))?; 114 | auth_controller.delete_key(uid) 115 | }) 116 | .await 117 | .map_err(|e| ResponseError::from_msg(e.to_string(), Code::Internal))??; 118 | 119 | Ok(HttpResponse::NoContent().finish()) 120 | } 121 | 122 | #[derive(Deserialize)] 123 | pub struct AuthParam { 124 | key: String, 125 | } 126 | 127 | #[derive(Debug, Serialize)] 128 | #[serde(rename_all = "camelCase")] 129 | struct KeyView { 130 | name: Option, 131 | description: Option, 132 | key: String, 133 | uid: Uuid, 134 | actions: Vec, 135 | indexes: Vec, 136 | #[serde(serialize_with = "time::serde::rfc3339::option::serialize")] 137 | expires_at: Option, 138 | #[serde(serialize_with = "time::serde::rfc3339::serialize")] 139 | created_at: OffsetDateTime, 140 | #[serde(serialize_with = "time::serde::rfc3339::serialize")] 141 | updated_at: OffsetDateTime, 142 | } 143 | 144 | impl KeyView { 145 | fn from_key(key: Key, auth: &AuthController) -> Self { 146 | let generated_key = auth.generate_key(key.uid).unwrap_or_default(); 147 | 148 | KeyView { 149 | name: key.name, 150 | description: key.description, 151 | key: generated_key, 152 | uid: key.uid, 153 | actions: key.actions, 154 | indexes: key.indexes.into_iter().map(String::from).collect(), 155 | expires_at: key.expires_at, 156 | created_at: key.created_at, 157 | updated_at: key.updated_at, 158 | } 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /meilisearch-http/src/extractors/sequential_extractor.rs: -------------------------------------------------------------------------------- 1 | #![allow(non_snake_case)] 2 | use std::{future::Future, pin::Pin, task::Poll}; 3 | 4 | use actix_web::{dev::Payload, FromRequest, Handler, HttpRequest}; 5 | use pin_project_lite::pin_project; 6 | 7 | /// `SeqHandler` is an actix `Handler` that enforces that extractors errors are returned in the 8 | /// same order as they are defined in the wrapped handler. This is needed because, by default, actix 9 | /// resolves the extractors concurrently, whereas we always need the authentication extractor to 10 | /// throw first. 11 | #[derive(Clone)] 12 | pub struct SeqHandler(pub H); 13 | 14 | pub struct SeqFromRequest(T); 15 | 16 | /// This macro implements `FromRequest` for arbitrary arity handler, except for one, which is 17 | /// useless anyway. 18 | macro_rules! gen_seq { 19 | ($ty:ident; $($T:ident)+) => { 20 | pin_project! { 21 | pub struct $ty<$($T: FromRequest), +> { 22 | $( 23 | #[pin] 24 | $T: ExtractFuture<$T::Future, $T, $T::Error>, 25 | )+ 26 | } 27 | } 28 | 29 | impl<$($T: FromRequest), +> Future for $ty<$($T),+> { 30 | type Output = Result, actix_web::Error>; 31 | 32 | fn poll(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll { 33 | let mut this = self.project(); 34 | 35 | let mut count_fut = 0; 36 | let mut count_finished = 0; 37 | 38 | $( 39 | count_fut += 1; 40 | match this.$T.as_mut().project() { 41 | ExtractProj::Future { fut } => match fut.poll(cx) { 42 | Poll::Ready(Ok(output)) => { 43 | count_finished += 1; 44 | let _ = this 45 | .$T 46 | .as_mut() 47 | .project_replace(ExtractFuture::Done { output }); 48 | } 49 | Poll::Ready(Err(error)) => { 50 | count_finished += 1; 51 | let _ = this 52 | .$T 53 | .as_mut() 54 | .project_replace(ExtractFuture::Error { error }); 55 | } 56 | Poll::Pending => (), 57 | }, 58 | ExtractProj::Done { .. } => count_finished += 1, 59 | ExtractProj::Error { .. } => { 60 | // short circuit if all previous are finished and we had an error. 61 | if count_finished == count_fut { 62 | match this.$T.project_replace(ExtractFuture::Empty) { 63 | ExtractReplaceProj::Error { error } => { 64 | return Poll::Ready(Err(error.into())) 65 | } 66 | _ => unreachable!("Invalid future state"), 67 | } 68 | } else { 69 | count_finished += 1; 70 | } 71 | } 72 | ExtractProj::Empty => unreachable!("From request polled after being finished. {}", stringify!($T)), 73 | } 74 | )+ 75 | 76 | if count_fut == count_finished { 77 | let result = ( 78 | $( 79 | match this.$T.project_replace(ExtractFuture::Empty) { 80 | ExtractReplaceProj::Done { output } => output, 81 | ExtractReplaceProj::Error { error } => return Poll::Ready(Err(error.into())), 82 | _ => unreachable!("Invalid future state"), 83 | }, 84 | )+ 85 | ); 86 | 87 | Poll::Ready(Ok(SeqFromRequest(result))) 88 | } else { 89 | Poll::Pending 90 | } 91 | } 92 | } 93 | 94 | impl<$($T: FromRequest,)+> FromRequest for SeqFromRequest<($($T,)+)> { 95 | type Error = actix_web::Error; 96 | 97 | type Future = $ty<$($T),+>; 98 | 99 | fn from_request(req: &HttpRequest, payload: &mut Payload) -> Self::Future { 100 | $ty { 101 | $( 102 | $T: ExtractFuture::Future { 103 | fut: $T::from_request(req, payload), 104 | }, 105 | )+ 106 | } 107 | } 108 | } 109 | 110 | impl Handler> for SeqHandler 111 | where 112 | Han: Handler<($($T),+)>, 113 | { 114 | type Output = Han::Output; 115 | type Future = Han::Future; 116 | 117 | fn call(&self, args: SeqFromRequest<($($T),+)>) -> Self::Future { 118 | self.0.call(args.0) 119 | } 120 | } 121 | }; 122 | } 123 | 124 | // Not working for a single argument, but then, it is not really necessary. 125 | // gen_seq! { SeqFromRequestFut1; A } 126 | gen_seq! { SeqFromRequestFut2; A B } 127 | gen_seq! { SeqFromRequestFut3; A B C } 128 | gen_seq! { SeqFromRequestFut4; A B C D } 129 | gen_seq! { SeqFromRequestFut5; A B C D E } 130 | gen_seq! { SeqFromRequestFut6; A B C D E F } 131 | 132 | pin_project! { 133 | #[project = ExtractProj] 134 | #[project_replace = ExtractReplaceProj] 135 | enum ExtractFuture { 136 | Future { 137 | #[pin] 138 | fut: Fut, 139 | }, 140 | Done { 141 | output: Res, 142 | }, 143 | Error { 144 | error: Err, 145 | }, 146 | Empty, 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /meilisearch-http/src/routes/indexes/mod.rs: -------------------------------------------------------------------------------- 1 | use actix_web::{web, HttpRequest, HttpResponse}; 2 | use log::debug; 3 | use meilisearch_lib::index_controller::Update; 4 | use meilisearch_lib::MeiliSearch; 5 | use meilisearch_types::error::ResponseError; 6 | use serde::{Deserialize, Serialize}; 7 | use serde_json::json; 8 | use time::OffsetDateTime; 9 | 10 | use crate::analytics::Analytics; 11 | use crate::extractors::authentication::{policies::*, GuardedData}; 12 | use crate::extractors::sequential_extractor::SeqHandler; 13 | use crate::task::SummarizedTaskView; 14 | 15 | use super::Pagination; 16 | 17 | pub mod documents; 18 | pub mod search; 19 | pub mod settings; 20 | 21 | pub fn configure(cfg: &mut web::ServiceConfig) { 22 | cfg.service( 23 | web::resource("") 24 | .route(web::get().to(list_indexes)) 25 | .route(web::post().to(SeqHandler(create_index))), 26 | ) 27 | .service( 28 | web::scope("/{index_uid}") 29 | .service( 30 | web::resource("") 31 | .route(web::get().to(SeqHandler(get_index))) 32 | .route(web::patch().to(SeqHandler(update_index))) 33 | .route(web::delete().to(SeqHandler(delete_index))), 34 | ) 35 | .service(web::resource("/stats").route(web::get().to(SeqHandler(get_index_stats)))) 36 | .service(web::scope("/documents").configure(documents::configure)) 37 | .service(web::scope("/search").configure(search::configure)) 38 | .service(web::scope("/settings").configure(settings::configure)), 39 | ); 40 | } 41 | 42 | pub async fn list_indexes( 43 | data: GuardedData, MeiliSearch>, 44 | paginate: web::Query, 45 | ) -> Result { 46 | let search_rules = &data.filters().search_rules; 47 | let indexes: Vec<_> = data.list_indexes().await?; 48 | let nb_indexes = indexes.len(); 49 | let iter = indexes 50 | .into_iter() 51 | .filter(|i| search_rules.is_index_authorized(&i.uid)); 52 | let ret = paginate 53 | .into_inner() 54 | .auto_paginate_unsized(nb_indexes, iter); 55 | 56 | debug!("returns: {:?}", ret); 57 | Ok(HttpResponse::Ok().json(ret)) 58 | } 59 | 60 | #[derive(Debug, Deserialize)] 61 | #[serde(rename_all = "camelCase", deny_unknown_fields)] 62 | pub struct IndexCreateRequest { 63 | uid: String, 64 | primary_key: Option, 65 | } 66 | 67 | pub async fn create_index( 68 | meilisearch: GuardedData, MeiliSearch>, 69 | body: web::Json, 70 | req: HttpRequest, 71 | analytics: web::Data, 72 | ) -> Result { 73 | let IndexCreateRequest { 74 | primary_key, uid, .. 75 | } = body.into_inner(); 76 | 77 | analytics.publish( 78 | "Index Created".to_string(), 79 | json!({ "primary_key": primary_key }), 80 | Some(&req), 81 | ); 82 | 83 | let update = Update::CreateIndex { primary_key }; 84 | let task: SummarizedTaskView = meilisearch.register_update(uid, update).await?.into(); 85 | 86 | Ok(HttpResponse::Accepted().json(task)) 87 | } 88 | 89 | #[derive(Debug, Deserialize)] 90 | #[serde(rename_all = "camelCase", deny_unknown_fields)] 91 | #[allow(dead_code)] 92 | pub struct UpdateIndexRequest { 93 | uid: Option, 94 | primary_key: Option, 95 | } 96 | 97 | #[derive(Debug, Serialize)] 98 | #[serde(rename_all = "camelCase")] 99 | pub struct UpdateIndexResponse { 100 | name: String, 101 | uid: String, 102 | #[serde(serialize_with = "time::serde::rfc3339::serialize")] 103 | created_at: OffsetDateTime, 104 | #[serde(serialize_with = "time::serde::rfc3339::serialize")] 105 | updated_at: OffsetDateTime, 106 | #[serde(serialize_with = "time::serde::rfc3339::serialize")] 107 | primary_key: OffsetDateTime, 108 | } 109 | 110 | pub async fn get_index( 111 | meilisearch: GuardedData, MeiliSearch>, 112 | path: web::Path, 113 | ) -> Result { 114 | let meta = meilisearch.get_index(path.into_inner()).await?; 115 | debug!("returns: {:?}", meta); 116 | Ok(HttpResponse::Ok().json(meta)) 117 | } 118 | 119 | pub async fn update_index( 120 | meilisearch: GuardedData, MeiliSearch>, 121 | path: web::Path, 122 | body: web::Json, 123 | req: HttpRequest, 124 | analytics: web::Data, 125 | ) -> Result { 126 | debug!("called with params: {:?}", body); 127 | let body = body.into_inner(); 128 | analytics.publish( 129 | "Index Updated".to_string(), 130 | json!({ "primary_key": body.primary_key}), 131 | Some(&req), 132 | ); 133 | 134 | let update = Update::UpdateIndex { 135 | primary_key: body.primary_key, 136 | }; 137 | 138 | let task: SummarizedTaskView = meilisearch 139 | .register_update(path.into_inner(), update) 140 | .await? 141 | .into(); 142 | 143 | debug!("returns: {:?}", task); 144 | Ok(HttpResponse::Accepted().json(task)) 145 | } 146 | 147 | pub async fn delete_index( 148 | meilisearch: GuardedData, MeiliSearch>, 149 | path: web::Path, 150 | ) -> Result { 151 | let uid = path.into_inner(); 152 | let update = Update::DeleteIndex; 153 | let task: SummarizedTaskView = meilisearch.register_update(uid, update).await?.into(); 154 | 155 | Ok(HttpResponse::Accepted().json(task)) 156 | } 157 | 158 | pub async fn get_index_stats( 159 | meilisearch: GuardedData, MeiliSearch>, 160 | path: web::Path, 161 | ) -> Result { 162 | let response = meilisearch.get_index_stats(path.into_inner()).await?; 163 | 164 | debug!("returns: {:?}", response); 165 | Ok(HttpResponse::Ok().json(response)) 166 | } 167 | --------------------------------------------------------------------------------