├── .gitignore
├── examples
    └── searchable
    │   ├── .gitignore
    │   ├── dioxus_search
    │       └── index_searchable.bin
    │   ├── Cargo.toml
    │   ├── static
    │       ├── index.html
    │       ├── _dioxus
    │       │   └── ws
    │       │   │   └── index.html
    │       ├── favicon.ico
    │       │   └── index.html
    │       ├── static
    │       │   └── favicon.png
    │       │   │   └── index.html
    │       ├── blog
    │       │   └── index.html
    │       └── other
    │       │   └── index.html
    │   └── src
    │       └── main.rs
├── search-macro
    ├── .gitignore
    ├── Cargo.toml
    └── src
    │   └── lib.rs
├── search-shared
    ├── .gitignore
    ├── Cargo.toml
    └── src
    │   └── lib.rs
├── src
    └── lib.rs
├── Cargo.toml
└── README.md


/.gitignore:
--------------------------------------------------------------------------------
1 | /target
2 | /Cargo.lock
3 | 


--------------------------------------------------------------------------------
/examples/searchable/.gitignore:
--------------------------------------------------------------------------------
1 | /dist


--------------------------------------------------------------------------------
/search-macro/.gitignore:
--------------------------------------------------------------------------------
1 | /target
2 | /Cargo.lock
3 | 


--------------------------------------------------------------------------------
/search-shared/.gitignore:
--------------------------------------------------------------------------------
1 | /target
2 | /Cargo.lock
3 | 


--------------------------------------------------------------------------------
/examples/searchable/dioxus_search/index_searchable.bin:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DioxusLabs/dioxus-search/master/examples/searchable/dioxus_search/index_searchable.bin


--------------------------------------------------------------------------------
/src/lib.rs:
--------------------------------------------------------------------------------
1 | pub use once_cell;
2 | pub use yazi;
3 | 
4 | pub use dioxus_search_macro::load_search_index;
5 | pub use dioxus_search_shared::*;
6 | 
7 | pub type LazySearchIndex<R> = once_cell::sync::Lazy<SearchIndex<R>>;
8 | 


--------------------------------------------------------------------------------
/search-macro/Cargo.toml:
--------------------------------------------------------------------------------
 1 | [package]
 2 | name = "dioxus-search-macro"
 3 | version = "0.1.0"
 4 | edition = "2021"
 5 | 
 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 7 | 
 8 | [lib]
 9 | proc-macro = true
10 | 
11 | [dependencies]
12 | quote = "1.0"
13 | syn = "2.0"
14 | 


--------------------------------------------------------------------------------
/examples/searchable/Cargo.toml:
--------------------------------------------------------------------------------
 1 | [package]
 2 | name = "searchable"
 3 | version = "0.1.0"
 4 | edition = "2021"
 5 | 
 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 7 | 
 8 | [dependencies]
 9 | dioxus = { version = "0.6.0", features = ["fullstack", "router"] }
10 | dioxus-search = { path = "../../" }
11 | axum = { version = "0.6.12", optional = true }
12 | serde = { version = "1.0.174", features = ["derive"] }
13 | getrandom = { version = "0.2" }
14 | tokio = { version = "1.29.1", features = ["full"], optional = true }
15 | simple_logger = "4.2.0"
16 | log = "0.4.19"
17 | 
18 | [features]
19 | web = ["getrandom/js", "dioxus/web"]
20 | ssr = ["dioxus/server", "tokio", "axum"]
21 | 


--------------------------------------------------------------------------------
/search-shared/Cargo.toml:
--------------------------------------------------------------------------------
 1 | [package]
 2 | name = "dioxus-search-shared"
 3 | version = "0.1.0"
 4 | edition = "2021"
 5 | 
 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 7 | 
 8 | [dependencies]
 9 | serde = { version = "1.0.163", features = ["derive"] }
10 | serde_json = "1.0.96"
11 | toml = "0.7.4"
12 | stork-lib = { version = "1.6.0", features = ["build-v3"], default-features = false }
13 | bytes = { version = "1.3.0", features = ["serde"] }
14 | slab = "0.4.8"
15 | dioxus-router = { version = "0.6.0" }
16 | yazi = "0.1.5"
17 | scraper = "0.17.1"
18 | log = "0.4.19"
19 | 
20 | [target.'cfg(target_family = "wasm")'.dependencies]
21 | getrandom = { version = "0.2", features = ["js"] }
22 | 


--------------------------------------------------------------------------------
/examples/searchable/static/index.html:
--------------------------------------------------------------------------------
 1 | <!DOCTYPE html>
 2 | <html>
 3 | <head>
 4 |   <title>dioxus | ⛺</title>
 5 |   <meta content="text/html;charset=utf-8" http-equiv="Content-Type" />
 6 |   <meta name="viewport" content="width=device-width, initial-scale=1">
 7 |   <meta charset="UTF-8" />
 8 |   
 9 | </head>
10 | <body>
11 |   <div id="main"><input value=""/><ul><pre></pre></ul><meta hidden="true" id="dioxus-storage-props" data-serialized="AQ==" /><meta hidden="true" id="dioxus-storage-data" data-serialized="AA==" /></div>
12 |   <script type="module">
13 |     import init from "/./assets/dioxus/name.js";
14 |     init("/./assets/dioxus/name_bg.wasm").then(wasm => {
15 |       if (wasm.__wbindgen_start == undefined) {
16 |         wasm.main();
17 |       }
18 |     });
19 |   </script>
20 |   
21 | </body>
22 | </html>


--------------------------------------------------------------------------------
/examples/searchable/static/_dioxus/ws/index.html:
--------------------------------------------------------------------------------
 1 | <!DOCTYPE html>
 2 | <html>
 3 | <head>
 4 |   <title>dioxus | ⛺</title>
 5 |   <meta content="text/html;charset=utf-8" http-equiv="Content-Type" />
 6 |   <meta name="viewport" content="width=device-width, initial-scale=1">
 7 |   <meta charset="UTF-8" />
 8 |   
 9 | </head>
10 | <body>
11 |   <div id="main"><input value=""/><ul><pre></pre></ul><meta hidden="true" id="dioxus-storage-props" data-serialized="AQ==" /><meta hidden="true" id="dioxus-storage-data" data-serialized="AA==" /></div>
12 |   <script type="module">
13 |     import init from "/./assets/dioxus/name.js";
14 |     init("/./assets/dioxus/name_bg.wasm").then(wasm => {
15 |       if (wasm.__wbindgen_start == undefined) {
16 |         wasm.main();
17 |       }
18 |     });
19 |   </script>
20 |   
21 | </body>
22 | </html>


--------------------------------------------------------------------------------
/examples/searchable/static/favicon.ico/index.html:
--------------------------------------------------------------------------------
 1 | <!DOCTYPE html>
 2 | <html>
 3 | <head>
 4 |   <title>dioxus | ⛺</title>
 5 |   <meta content="text/html;charset=utf-8" http-equiv="Content-Type" />
 6 |   <meta name="viewport" content="width=device-width, initial-scale=1">
 7 |   <meta charset="UTF-8" />
 8 |   
 9 | </head>
10 | <body>
11 |   <div id="main"><input value=""/><ul><pre></pre></ul><meta hidden="true" id="dioxus-storage-props" data-serialized="AQ==" /><meta hidden="true" id="dioxus-storage-data" data-serialized="AA==" /></div>
12 |   <script type="module">
13 |     import init from "/./assets/dioxus/dioxus.js";
14 |     init("/./assets/dioxus/dioxus_bg.wasm").then(wasm => {
15 |       if (wasm.__wbindgen_start == undefined) {
16 |         wasm.main();
17 |       }
18 |     });
19 |   </script>
20 |   
21 | </body>
22 | </html>


--------------------------------------------------------------------------------
/examples/searchable/static/static/favicon.png/index.html:
--------------------------------------------------------------------------------
 1 | <!DOCTYPE html>
 2 | <html>
 3 | <head>
 4 |   <title>dioxus | ⛺</title>
 5 |   <meta content="text/html;charset=utf-8" http-equiv="Content-Type" />
 6 |   <meta name="viewport" content="width=device-width, initial-scale=1">
 7 |   <meta charset="UTF-8" />
 8 |   
 9 | </head>
10 | <body>
11 |   <div id="main"><input value=""/><ul><pre></pre></ul><meta hidden="true" id="dioxus-storage-props" data-serialized="AQ==" /><meta hidden="true" id="dioxus-storage-data" data-serialized="AA==" /></div>
12 |   <script type="module">
13 |     import init from "/./assets/dioxus/name.js";
14 |     init("/./assets/dioxus/name_bg.wasm").then(wasm => {
15 |       if (wasm.__wbindgen_start == undefined) {
16 |         wasm.main();
17 |       }
18 |     });
19 |   </script>
20 |   
21 | </body>
22 | </html>


--------------------------------------------------------------------------------
/examples/searchable/static/blog/index.html:
--------------------------------------------------------------------------------
 1 | <!DOCTYPE html>
 2 | <html>
 3 | <head>
 4 |   <title>dioxus | ⛺</title>
 5 |   <meta content="text/html;charset=utf-8" http-equiv="Content-Type" />
 6 |   <meta name="viewport" content="width=device-width, initial-scale=1">
 7 |   <meta charset="UTF-8" />
 8 |   
 9 | </head>
10 | <body>
11 |   <div id="main"><div><h1>Hello World</h1><p>This is a blog post</p></div><meta hidden="true" id="dioxus-storage-props" data-serialized="AQ==" /><meta hidden="true" id="dioxus-storage-data" data-serialized="AA==" /></div>
12 |   <script type="module">
13 |     import init from "/./assets/dioxus/name.js";
14 |     init("/./assets/dioxus/name_bg.wasm").then(wasm => {
15 |       if (wasm.__wbindgen_start == undefined) {
16 |         wasm.main();
17 |       }
18 |     });
19 |   </script>
20 |   
21 | </body>
22 | </html>


--------------------------------------------------------------------------------
/examples/searchable/static/other/index.html:
--------------------------------------------------------------------------------
 1 | <!DOCTYPE html>
 2 | <html>
 3 | <head>
 4 |   <title>dioxus | ⛺</title>
 5 |   <meta content="text/html;charset=utf-8" http-equiv="Content-Type" />
 6 |   <meta name="viewport" content="width=device-width, initial-scale=1">
 7 |   <meta charset="UTF-8" />
 8 |   
 9 | </head>
10 | <body>
11 |   <div id="main"><div><h1>Goodbye</h1><p>This is another blog post</p></div><meta hidden="true" id="dioxus-storage-props" data-serialized="AQ==" /><meta hidden="true" id="dioxus-storage-data" data-serialized="AA==" /></div>
12 |   <script type="module">
13 |     import init from "/./assets/dioxus/name.js";
14 |     init("/./assets/dioxus/name_bg.wasm").then(wasm => {
15 |       if (wasm.__wbindgen_start == undefined) {
16 |         wasm.main();
17 |       }
18 |     });
19 |   </script>
20 |   
21 | </body>
22 | </html>


--------------------------------------------------------------------------------
/Cargo.toml:
--------------------------------------------------------------------------------
 1 | [package]
 2 | name = "dioxus-search"
 3 | version = "0.1.0"
 4 | edition = "2021"
 5 | 
 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 7 | 
 8 | [dependencies]
 9 | serde = { version = "1.0.163", features = ["derive"] }
10 | serde_json = "1.0.96"
11 | toml = "0.7.4"
12 | stork-lib = { version = "1.6.0", features = ["build-v3"], default-features = false }
13 | bytes = { version = "1.3.0", features = ["serde"] }
14 | slab = "0.4.8"
15 | dioxus-router = { version = "0.6.0" }
16 | yazi = "0.1.5"
17 | once_cell = "1.18.0"
18 | dioxus-search-macro = { path = "search-macro" }
19 | dioxus-search-shared = { path = "search-shared" }
20 | 
21 | [target.'cfg(target_family = "wasm")'.dependencies]
22 | getrandom = { version = "0.2", features = ["js"] }
23 | 
24 | [workspace]
25 | members = ["search-macro", "search-shared", "examples/searchable"]
26 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # This crate has moved!
 2 | 
 3 | https://github.com/DioxusLabs/docsite/tree/main/packages/search
 4 | 
 5 | # Dioxus Search
 6 | 
 7 | Dioxus search creates a prebaked search index for all your static Dioxus routes.
 8 | 
 9 | It integrates with the Dioxus router to find all the static routes in your application and search for any rendered HTML files for those files.
10 | 
11 | Example:
12 | 
13 | ```rust
14 | #[cfg(feature = "ssr")]
15 | {
16 |     // Generate all static routes in the ./static folder using Dioxus fullstack
17 |     // ...
18 | 
19 |     // Create search index
20 |     dioxus_search::SearchIndex::<Route>::create(
21 |         "searchable",
22 |         dioxus_search::BaseDirectoryMapping::new("./static")
23 |     );
24 | }
25 | 
26 | // After the first build the search index is cached at compile time inline in your program
27 | static SEARCH_INDEX: dioxus_search::LazySearchIndex<Route> = dioxus_search::load_search_index! {
28 |     "searchable"
29 | };
30 | 
31 | 
32 | #[component]
33 | fn Homepage() -> Element {
34 |     let search_text = use_signal(String::new);
35 |     let results = SEARCH_INDEX.search(&search_text.get());
36 | 
37 |     render!{
38 |         input {
39 |             oninput: move |e| {
40 |                 search_text.set(e.value.clone());
41 |             },
42 |             value: "{search_text}",
43 |         }
44 |         ul {
45 |             for result in results.map(|i| i.into_iter()).ok().into_iter().flatten() {
46 |                 li {
47 |                     Link {
48 |                         target: result.route.clone(),
49 |                         "{result:#?}"
50 |                     }
51 |                 }
52 |             }
53 |         }
54 |     }
55 | }
56 | ```
57 | 
58 | For a full working demo, see the [searchable example](./examples/searchable/).
59 | 


--------------------------------------------------------------------------------
/search-macro/src/lib.rs:
--------------------------------------------------------------------------------
 1 | use std::path::PathBuf;
 2 | 
 3 | use proc_macro::TokenStream;
 4 | use quote::quote;
 5 | use syn::LitStr;
 6 | 
 7 | #[proc_macro]
 8 | pub fn load_search_index(input: TokenStream) -> TokenStream {
 9 |     match syn::parse::<LitStr>(input) {
10 |         Ok(input) => generate_search_index(input).into(),
11 |         Err(err) => err.to_compile_error().into(),
12 |     }
13 | }
14 | 
15 | /// Transforms the book to use enum routes instead of paths
16 | fn generate_search_index(id: LitStr) -> TokenStream {
17 |     let target_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap();
18 | 
19 |     let name = id.value();
20 |     let index_path = PathBuf::from(format!("{}/dioxus_search/index_{}.bin", target_dir, name));
21 |     let index_str = index_path.to_str().unwrap();
22 |     if index_path.exists() && std::fs::read(&index_path).unwrap().len() > 0 {
23 |         quote!{
24 |             {
25 |                 const INDEX_BYTES: &[u8] = include_bytes!(#index_str);
26 | 
27 |                 dioxus_search::once_cell::sync::Lazy::new(|| {
28 |                     let (bytes, _) = dioxus_search::yazi::decompress(INDEX_BYTES, dioxus_search::yazi::Format::Zlib).unwrap();
29 |                     dioxus_search::SearchIndex::from_bytes(#name, bytes)
30 |                 })
31 |             }
32 |         }.into()
33 |     } else {
34 |         // create a blank index file
35 |         std::fs::create_dir_all(index_path.parent().unwrap()).unwrap();
36 |         std::fs::write(&index_path, vec![]).unwrap();
37 |         println!("It looks like you haven't built the index yet, so we created an empty index file at {:?}", index_path);
38 |         quote! {
39 |             {
40 |                 // let rust know that we care about this file
41 |                 const INDEX_BYTES: &[u8] = include_bytes!(#index_str);
42 | 
43 |                 dioxus_search::once_cell::sync::Lazy::new(|| {
44 |                     eprintln!("Index not found: {:?}, have you built the index yet?", #index_str);
45 |                     dioxus_search::SearchIndex::default()
46 |                 })
47 |             }
48 |         }
49 |         .into()
50 |     }
51 | }
52 | 


--------------------------------------------------------------------------------
/examples/searchable/src/main.rs:
--------------------------------------------------------------------------------
  1 | // Run with:
  2 | // dx build --release --features web
  3 | // cargo run --release --features ssr
  4 | //
  5 | // Note: The first time you run the build, the search index will be empty. You need to rebuild the build again to fill the search index.
  6 | 
  7 | use dioxus::prelude::*;
  8 | 
  9 | fn main() {
 10 |     #[cfg(feature = "ssr")]
 11 |     {
 12 |         use log::LevelFilter;
 13 |         simple_logger::SimpleLogger::new()
 14 |             .with_level(LevelFilter::Info)
 15 |             .with_module_level("dioxus_search_macro", LevelFilter::Trace)
 16 |             .with_module_level("dioxus_search_shared", LevelFilter::Trace)
 17 |             .init()
 18 |             .unwrap();
 19 |         tokio::runtime::Runtime::new()
 20 |             .unwrap()
 21 |             .block_on(async move {
 22 |                 pre_cache_static_routes_with_props(
 23 |                     &ServeConfigBuilder::new_with_router(
 24 |                         dioxus_fullstack::router::FullstackRouterConfig::<Route>::default(),
 25 |                     )
 26 |                     .incremental(IncrementalRendererConfig::default())
 27 |                     .build(),
 28 |                 )
 29 |                 .await
 30 |                 .unwrap();
 31 |             });
 32 |         println!("finished prebuilding static routes");
 33 | 
 34 |         dioxus_search::SearchIndex::<Route>::create(
 35 |             "searchable",
 36 |             dioxus_search::BaseDirectoryMapping::new("./static"),
 37 |         );
 38 |         println!("finished creating search index");
 39 |     }
 40 | 
 41 |     launch(|| rsx! { Router::<Route> {} });
 42 | }
 43 | 
 44 | #[derive(Clone, Routable, PartialEq, Eq, Debug, serde::Serialize, serde::Deserialize)]
 45 | #[rustfmt::skip]
 46 | pub enum Route {
 47 |     #[route("/")]
 48 |     Homepage {},
 49 | 
 50 |     #[route("/blog")]
 51 |     BlogPost {},
 52 | 
 53 |     #[route("/other")]
 54 |     OtherPost {},
 55 | }
 56 | 
 57 | fn Homepage() -> Element {
 58 |     let mut search_text = use_signal(String::new);
 59 |     let results = SEARCH_INDEX.search(&search_text());
 60 | 
 61 |     rsx! {
 62 |         input {
 63 |             oninput: move |e| {
 64 |                 search_text.set(e.value());
 65 |             },
 66 |             value: "{search_text}",
 67 |         }
 68 |         ul {
 69 |             for result in results.map(|i| i.into_iter()).ok().into_iter().flatten() {
 70 |                 li {
 71 |                     Link {
 72 |                         to: result.route.clone(),
 73 |                         "{result:#?}"
 74 |                     }
 75 |                 }
 76 |             }
 77 |         }
 78 |     }
 79 | }
 80 | 
 81 | fn BlogPost() -> Element {
 82 |     rsx! {
 83 |         div {
 84 |             h1 { "Hello World" }
 85 |             p { "This is a blog post" }
 86 |         }
 87 |     }
 88 | }
 89 | 
 90 | fn OtherPost() -> Element {
 91 |     rsx! {
 92 |         div {
 93 |             h1 { "Goodbye" }
 94 |             p { "This is another blog post" }
 95 |         }
 96 |     }
 97 | }
 98 | 
 99 | static SEARCH_INDEX: dioxus_search::LazySearchIndex<Route> = dioxus_search::load_search_index! {
100 |     "searchable"
101 | };
102 | 


--------------------------------------------------------------------------------
/search-shared/src/lib.rs:
--------------------------------------------------------------------------------
  1 | use std::{
  2 |     collections::HashMap,
  3 |     fmt::{Debug, Display},
  4 |     num::ParseIntError,
  5 |     path::PathBuf,
  6 |     str::FromStr,
  7 | };
  8 | 
  9 | use bytes::Bytes;
 10 | use dioxus_router::routable::Routable;
 11 | use scraper::{Html, Selector};
 12 | use serde::{Deserialize, Serialize};
 13 | use stork_lib::{build_index, SearchError};
 14 | 
 15 | #[derive(Deserialize, Serialize)]
 16 | pub struct SearchIndex<R> {
 17 |     index: Bytes,
 18 |     name: String,
 19 |     mock: bool,
 20 |     _marker: std::marker::PhantomData<R>,
 21 | }
 22 | 
 23 | impl<R> Default for SearchIndex<R> {
 24 |     fn default() -> Self {
 25 |         Self {
 26 |             index: Bytes::new(),
 27 |             name: String::new(),
 28 |             mock: true,
 29 |             _marker: std::marker::PhantomData,
 30 |         }
 31 |     }
 32 | }
 33 | 
 34 | impl<R> Debug for SearchIndex<R> {
 35 |     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
 36 |         f.debug_struct("SearchIndex").finish()
 37 |     }
 38 | }
 39 | 
 40 | impl<R: Routable> SearchIndex<R>
 41 | where
 42 |     <R as FromStr>::Err: Display,
 43 | {
 44 |     pub fn create(name: impl AsRef<str>, mapping: impl SearchIndexMapping<R>) -> Self {
 45 |         let name = name.as_ref().to_string();
 46 |         let asset_format = Config::from_route(mapping);
 47 | 
 48 |         let toml = toml::to_string(&asset_format).unwrap();
 49 |         let bytes = build_index(&stork_lib::Config::try_from(&*toml).unwrap())
 50 |             .unwrap()
 51 |             .bytes;
 52 | 
 53 |         stork_lib::register_index(&format!("index_{name}"), bytes.clone()).unwrap();
 54 | 
 55 |         let myself = Self {
 56 |             index: bytes,
 57 |             name,
 58 |             mock: false,
 59 |             _marker: std::marker::PhantomData,
 60 |         };
 61 | 
 62 |         let target_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap_or_else(|_| ".".into());
 63 | 
 64 |         let path = format!("{}/dioxus_search/index_{}.bin", target_dir, myself.name);
 65 |         std::fs::create_dir_all(std::path::Path::new(&path).parent().unwrap()).unwrap();
 66 |         let compressed = yazi::compress(
 67 |             &myself.index,
 68 |             yazi::Format::Zlib,
 69 |             yazi::CompressionLevel::Default,
 70 |         )
 71 |         .unwrap();
 72 |         std::fs::write(path, compressed).unwrap();
 73 | 
 74 |         myself
 75 |     }
 76 | 
 77 |     pub fn from_bytes<T: Into<Bytes>>(name: impl AsRef<str>, bytes: T) -> Self {
 78 |         let name = name.as_ref().to_string();
 79 |         let bytes = bytes.into();
 80 |         stork_lib::register_index(&format!("index_{name}"), bytes.clone()).unwrap();
 81 |         Self {
 82 |             index: bytes,
 83 |             name,
 84 |             mock: false,
 85 |             _marker: std::marker::PhantomData,
 86 |         }
 87 |     }
 88 | 
 89 |     pub fn search(&self, text: &str) -> Result<Vec<SearchResult<R>>, SearchError> {
 90 |         if self.mock {
 91 |             return Ok(Vec::new());
 92 |         }
 93 | 
 94 |         let id = &self.name;
 95 |         let output = stork_lib::search_from_cache(&format!("index_{id}"), text)?;
 96 |         let mut results = Vec::new();
 97 |         for result in output.results {
 98 |             match result.entry.url.parse() {
 99 |                 Ok(route) => {
100 |                     let excerpts = result
101 |                         .excerpts
102 |                         .into_iter()
103 |                         .map(|excerpt| {
104 |                             let mut segments = Vec::new();
105 |                             let mut char_index = 0;
106 |                             let mut chars = excerpt.text.chars();
107 |                             let mut current_segment = String::new();
108 |                             for highlight_range in excerpt.highlight_ranges {
109 |                                 let start = highlight_range.beginning;
110 |                                 while char_index < start.saturating_sub(1) {
111 |                                     if let Some(c) = chars.next() {
112 |                                         current_segment.push(c);
113 |                                         char_index += 1;
114 |                                     } else {
115 |                                         break;
116 |                                     }
117 |                                 }
118 |                                 // add the current segment as a plain text segment
119 |                                 if !current_segment.is_empty() {
120 |                                     segments.push(Segment {
121 |                                         text: std::mem::take(&mut current_segment),
122 |                                         highlighted: false,
123 |                                     });
124 |                                 }
125 |                                 let end = highlight_range.end;
126 |                                 while char_index < end {
127 |                                     if let Some(c) = chars.next() {
128 |                                         current_segment.push(c);
129 |                                         char_index += 1;
130 |                                     } else {
131 |                                         break;
132 |                                     }
133 |                                 }
134 |                                 // add the current segment as a highlighted segment
135 |                                 if !current_segment.is_empty() {
136 |                                     segments.push(Segment {
137 |                                         text: std::mem::take(&mut current_segment),
138 |                                         highlighted: true,
139 |                                     });
140 |                                 }
141 |                             }
142 |                             Excerpt {
143 |                                 text: segments,
144 |                                 score: excerpt.score,
145 |                             }
146 |                         })
147 |                         .collect();
148 |                     results.push(SearchResult {
149 |                         route,
150 |                         excerpts,
151 |                         title: result.entry.title,
152 |                         score: result.score,
153 |                     })
154 |                 }
155 |                 Err(err) => {
156 |                     log::error!("Failed to parse url ({}): {err}", result.entry.url);
157 |                 }
158 |             }
159 |         }
160 | 
161 |         results.sort_by_key(|result| !result.score);
162 | 
163 |         Ok(results)
164 |     }
165 | }
166 | 
167 | #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
168 | pub struct SearchResult<R: Routable> {
169 |     pub route: R,
170 |     pub title: String,
171 |     pub excerpts: Vec<Excerpt>,
172 |     pub score: usize,
173 | }
174 | 
175 | #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
176 | pub struct Excerpt {
177 |     pub text: Vec<Segment>,
178 |     pub score: usize,
179 | }
180 | 
181 | #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
182 | pub struct Segment {
183 |     pub text: String,
184 |     pub highlighted: bool,
185 | }
186 | 
187 | #[derive(Serialize, Deserialize)]
188 | #[serde(deny_unknown_fields)]
189 | pub struct Config {
190 |     pub input: InputConfig,
191 | }
192 | 
193 | impl Config {
194 |     fn from_route<R: Routable>(mapping: impl SearchIndexMapping<R>) -> Self
195 |     where
196 |         <R as FromStr>::Err: Display,
197 |     {
198 |         let mut files = Vec::new();
199 |         let base_directory = mapping.base_directory();
200 | 
201 |         // Collect all the static routes
202 |         let static_routes = R::static_routes();
203 |         // Add the routes to the index
204 |         for route in static_routes {
205 |             let url = route.to_string();
206 |             if let Some(path) = mapping.map_route(route) {
207 |                 let path = &path.strip_prefix("/").unwrap_or(&path);
208 |                 let absolute_path = base_directory.join(path);
209 |                 log::trace!("Adding {:?} to search index", absolute_path);
210 |                 match std::fs::read_to_string(&absolute_path) {
211 |                     Ok(contents) => {
212 |                         let document = Html::parse_document(&contents);
213 |                         let title = document
214 |                             .select(&Selector::parse("h1").unwrap())
215 |                             .next()
216 |                             .map(|title| title.text().collect::<String>())
217 |                             .unwrap_or_else(|| {
218 |                                 document
219 |                                     .select(&Selector::parse("title").unwrap())
220 |                                     .next()
221 |                                     .map(|title| title.text().collect::<String>())
222 |                                     .unwrap_or_else(|| {
223 |                                         let mut title = String::new();
224 |                                         for segment in path.iter() {
225 |                                             title.push_str(&segment.to_string_lossy());
226 |                                             title.push(' ');
227 |                                         }
228 |                                         title
229 |                                     })
230 |                             });
231 |                         files.push(File {
232 |                             path: path.to_string_lossy().into(),
233 |                             url,
234 |                             title,
235 |                             fields: HashMap::new(),
236 |                             explicit_source: None,
237 |                         })
238 |                     }
239 |                     Err(err) => {
240 |                         log::error!("Error reading file: {:?}: {}", absolute_path, err);
241 |                     }
242 |                 }
243 |             }
244 |         }
245 | 
246 |         Self {
247 |             input: InputConfig {
248 |                 base_directory: base_directory.to_string_lossy().into(),
249 |                 url_prefix: "".into(),
250 |                 html_selector: Some("#main".into()),
251 |                 files,
252 |                 break_on_file_error: false,
253 |                 minimum_indexed_substring_length: 3,
254 |                 minimum_index_ideographic_substring_length: 1,
255 |             },
256 |         }
257 |     }
258 | }
259 | 
260 | #[derive(Serialize, Deserialize)]
261 | #[serde(deny_unknown_fields)]
262 | pub struct InputConfig {
263 |     base_directory: String,
264 |     url_prefix: String,
265 |     html_selector: Option<String>,
266 |     files: Vec<File>,
267 |     break_on_file_error: bool,
268 |     minimum_indexed_substring_length: u8,
269 |     minimum_index_ideographic_substring_length: u8,
270 | }
271 | 
272 | #[derive(Deserialize, Serialize)]
273 | #[serde(deny_unknown_fields)]
274 | struct File {
275 |     path: String,
276 |     url: String,
277 |     title: String,
278 |     #[serde(flatten, default)]
279 |     pub fields: HashMap<String, String>,
280 |     #[serde(flatten)]
281 |     pub explicit_source: Option<DataSource>,
282 | }
283 | 
284 | #[derive(Serialize, Deserialize)]
285 | pub enum DataSource {
286 |     #[serde(rename = "contents")]
287 |     Contents(String),
288 | 
289 |     #[serde(rename = "src_url")]
290 |     #[allow(clippy::upper_case_acronyms)]
291 |     URL(String),
292 | 
293 |     #[serde(rename = "path")]
294 |     FilePath(String),
295 | }
296 | 
297 | pub trait SearchIndexMapping<R: Routable> {
298 |     fn base_directory(&self) -> PathBuf;
299 |     fn map_route(&self, route: R) -> Option<PathBuf>;
300 | }
301 | 
302 | pub struct Mapped<F: Fn(R) -> Option<PathBuf>, R: Routable> {
303 |     base_directory: PathBuf,
304 |     map: F,
305 |     _marker: std::marker::PhantomData<R>,
306 | }
307 | 
308 | impl<F: Fn(R) -> Option<PathBuf>, R: Routable> SearchIndexMapping<R> for Mapped<F, R> {
309 |     fn base_directory(&self) -> PathBuf {
310 |         self.base_directory.clone()
311 |     }
312 | 
313 |     fn map_route(&self, route: R) -> Option<PathBuf> {
314 |         (self.map)(route)
315 |     }
316 | }
317 | 
318 | pub struct BaseDirectoryMapping {
319 |     base_directory: PathBuf,
320 | }
321 | 
322 | impl<R: Routable> SearchIndexMapping<R> for BaseDirectoryMapping {
323 |     fn base_directory(&self) -> PathBuf {
324 |         self.base_directory.clone()
325 |     }
326 | 
327 |     fn map_route(&self, route: R) -> Option<PathBuf> {
328 |         let route = route.to_string();
329 |         let (route, _) = route.split_once('#').unwrap_or((&route, ""));
330 |         let (route, _) = route.split_once('?').unwrap_or((&route, ""));
331 |         let route = PathBuf::from(route).join("index.html");
332 |         Some(route)
333 |     }
334 | }
335 | 
336 | impl BaseDirectoryMapping {
337 |     pub fn new(base_directory: impl Into<PathBuf>) -> Self {
338 |         Self {
339 |             base_directory: base_directory.into(),
340 |         }
341 |     }
342 | 
343 |     pub fn map<F: Fn(R) -> Option<PathBuf>, R: Routable>(self, map: F) -> Mapped<F, R> {
344 |         Mapped {
345 |             base_directory: self.base_directory,
346 |             map,
347 |             _marker: std::marker::PhantomData,
348 |         }
349 |     }
350 | }
351 | 
352 | impl From<PathBuf> for BaseDirectoryMapping {
353 |     fn from(base_directory: PathBuf) -> Self {
354 |         Self::new(base_directory)
355 |     }
356 | }
357 | 


--------------------------------------------------------------------------------