├── .dockerignore
├── data
    └── .gitignore
├── rust-toolchain.toml
├── model.zstd_dict
├── public
    └── favicon.ico
├── .idea
    ├── .gitignore
    ├── misc.xml
    ├── vcs.xml
    ├── modules.xml
    ├── inspectionProfiles
    │   └── Project_Default.xml
    └── compchat.iml
├── src
    ├── component
    │   ├── mod.rs
    │   ├── chat_message.rs
    │   ├── chat.rs
    │   ├── navbar.rs
    │   ├── dropdown.rs
    │   ├── prompt_input.rs
    │   └── prompt_section.rs
    ├── lib.rs
    ├── chat.rs
    ├── backend
    │   ├── trainer.rs
    │   ├── training_options.rs
    │   ├── tokenizer.rs
    │   ├── mod.rs
    │   ├── ensemble_model.rs
    │   ├── evaluation.rs
    │   ├── clm_model.rs
    │   └── dataset.rs
    ├── main.rs
    ├── fileserv.rs
    ├── error_template.rs
    ├── app.rs
    ├── model.rs
    └── tuning.rs
├── .gitignore
├── README.md
├── docker-compose.yml
├── LICENSE
├── Dockerfile
├── train_tokenizer.py
├── tune.py
├── Cargo.toml
├── style
    └── main.scss
├── tokenizer.json
└── Cargo.lock


/.dockerignore:
--------------------------------------------------------------------------------
1 | target


--------------------------------------------------------------------------------
/data/.gitignore:
--------------------------------------------------------------------------------
1 | *
2 | !.gitignore


--------------------------------------------------------------------------------
/rust-toolchain.toml:
--------------------------------------------------------------------------------
1 | [toolchain]
2 | channel = "nightly"
3 | 


--------------------------------------------------------------------------------
/model.zstd_dict:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/SamuelLess/ChatCLM/HEAD/model.zstd_dict


--------------------------------------------------------------------------------
/public/favicon.ico:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/SamuelLess/ChatCLM/HEAD/public/favicon.ico


--------------------------------------------------------------------------------
/.idea/.gitignore:
--------------------------------------------------------------------------------
1 | # Default ignored files
2 | /shelf/
3 | /workspace.xml
4 | # Editor-based HTTP Client requests
5 | /httpRequests/
6 | 


--------------------------------------------------------------------------------
/src/component/mod.rs:
--------------------------------------------------------------------------------
1 | pub mod chat;
2 | pub mod chat_message;
3 | pub mod dropdown;
4 | pub mod navbar;
5 | pub mod prompt_input;
6 | pub mod prompt_section;
7 | 


--------------------------------------------------------------------------------
/.idea/misc.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="Black">
4 |     <option name="sdkName" value="Python 3.10" />
5 |   </component>
6 | </project>


--------------------------------------------------------------------------------
/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="VcsDirectoryMappings">
4 |     <mapping directory="" vcs="Git" />
5 |   </component>
6 | </project>


--------------------------------------------------------------------------------
/.idea/modules.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="ProjectModuleManager">
4 |     <modules>
5 |       <module fileurl="file://$PROJECT_DIR$/.idea/compchat.iml" filepath="$PROJECT_DIR$/.idea/compchat.iml" />
6 |     </modules>
7 |   </component>
8 | </project>


--------------------------------------------------------------------------------
/.idea/inspectionProfiles/Project_Default.xml:
--------------------------------------------------------------------------------
1 | <component name="InspectionProjectProfileManager">
2 |   <profile version="1.0">
3 |     <option name="myName" value="Project Default" />
4 |     <inspection_tool class="RsDetachedFile" enabled="false" level="WARNING" enabled_by_default="false" />
5 |   </profile>
6 | </component>


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | # Generated by Cargo
 2 | # will have compiled files and executables
 3 | /target/
 4 | pkg
 5 | 
 6 | # These are backup files generated by rustfmt
 7 | **/*.rs.bk
 8 | 
 9 | # node e2e test tools and outputs
10 | node_modules/
11 | test-results/
12 | end2end/playwright-report/
13 | playwright/.cache/
14 | 
15 | wandb/
16 | *.checkpoint
17 | 


--------------------------------------------------------------------------------
/src/lib.rs:
--------------------------------------------------------------------------------
 1 | #![feature(lazy_cell)]
 2 | 
 3 | pub mod app;
 4 | pub mod chat;
 5 | pub mod component;
 6 | pub mod error_template;
 7 | #[cfg(feature = "ssr")]
 8 | pub mod fileserv;
 9 | pub mod model;
10 | #[cfg(feature = "ssr")]
11 | pub mod backend;
12 | 
13 | 
14 | #[cfg(feature = "hydrate")]
15 | #[wasm_bindgen::prelude::wasm_bindgen]
16 | pub fn hydrate() {
17 |     use crate::app::*;
18 |     console_error_panic_hook::set_once();
19 |     leptos::mount_to_body(App);
20 | }
21 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # chatCLM
 2 | 
 3 | [chatCLM.xyz](https://chatclm.xyz)
 4 | 
 5 | The CLM describes itself as follows:
 6 | > You are chatCLM. A zstd compression based LM. You are chatCLM. A zstd compression based LM. You are chatCLM. A zstd compression based LM.
 7 | 
 8 | 
 9 | ## Overview
10 | Welcome to _chatCLM_, a Rust-based project utilizing the [leptos](https://leptos.dev) framework to build a compression-based Large Language Model (LLM) using zstd and the OpenAI tokenizer.
11 | This project aims to create an efficient, high-performance LLM by leveraging the power of compression algorithms.
12 | 


--------------------------------------------------------------------------------
/src/component/chat_message.rs:
--------------------------------------------------------------------------------
 1 | use crate::chat::Message;
 2 | use leptos::{component, view, IntoView, Show};
 3 | 
 4 | #[component]
 5 | pub fn ChatMessage(msg: Message) -> impl IntoView {
 6 |     let is_user_msg = msg.is_user_msg();
 7 | 
 8 |     view! {
 9 |         <div class="chat_message" class=("chat_message--machine", move || !is_user_msg)>
10 |             <Show when=move || !is_user_msg>
11 |                 <div class="chat_message__icon">
12 |                     <div>*</div>
13 |                 </div>
14 |             </Show>
15 | 
16 |             <p class=("chat_message__bubble", move || is_user_msg)>{msg.message}</p>
17 |         </div>
18 |     }
19 | }
20 | 


--------------------------------------------------------------------------------
/src/component/chat.rs:
--------------------------------------------------------------------------------
 1 | use crate::chat::ChatHistory;
 2 | use crate::component::chat_message::ChatMessage;
 3 | use leptos::{component, view, For, IntoView, ReadSignal};
 4 | 
 5 | #[component]
 6 | pub fn Chat(chat: ReadSignal<ChatHistory>) -> impl IntoView {
 7 |     view! {
 8 |         <section class="chat">
 9 |             <For
10 |                 each=move || chat().messages.into_iter().enumerate()
11 |                 key=|(idx, it)| format!("{}: {}", idx, it.message.clone())
12 |                 children=move |(_, it)| {
13 |                     view! { <ChatMessage msg=it/> }
14 |                 }
15 |             />
16 | 
17 |         </section>
18 |     }
19 | }
20 | 


--------------------------------------------------------------------------------
/docker-compose.yml:
--------------------------------------------------------------------------------
 1 | services:
 2 |     web:
 3 |       build: .
 4 |       expose:
 5 |         - "8080"
 6 |       labels:
 7 |         - "traefik.enable=true"
 8 |         - "traefik.http.routers.chatclm.rule=Host(`chatclm.nglodny.de`) || Host(`chatclm.xyz`)"
 9 |         - "traefik.http.routers.chatclm.tls=true"
10 |         - "traefik.http.routers.chatclm.tls.certresolver=letsencrypt"
11 |         - "traefik.http.middlewares.chatclm.compress=true"
12 |         - "treafik.http.routers.chatclm.middlewares=chatclm@docker"
13 |         - "traefik.http.routers.chatclm.entrypoints=web,websecure"
14 |       networks:
15 |         - traefik
16 | networks:
17 |   traefik:
18 |     external: true
19 | 
20 | 


--------------------------------------------------------------------------------
/.idea/compchat.iml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <module type="EMPTY_MODULE" version="4">
 3 |   <component name="FacetManager">
 4 |     <facet type="Python" name="Python facet">
 5 |       <configuration sdkName="Python 3.10" />
 6 |     </facet>
 7 |   </component>
 8 |   <component name="NewModuleRootManager">
 9 |     <content url="file://$MODULE_DIR$">
10 |       <sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
11 |       <excludeFolder url="file://$MODULE_DIR$/target" />
12 |     </content>
13 |     <orderEntry type="inheritedJdk" />
14 |     <orderEntry type="sourceFolder" forTests="false" />
15 |     <orderEntry type="library" name="Python 3.10 interpreter library" level="application" />
16 |   </component>
17 | </module>


--------------------------------------------------------------------------------
/src/component/navbar.rs:
--------------------------------------------------------------------------------
 1 | use crate::component::dropdown::Dropdown;
 2 | use crate::model::FrontendModel;
 3 | use leptos::{component, view, IntoView, ReadSignal, WriteSignal};
 4 | 
 5 | #[component]
 6 | pub fn NavBar(
 7 |     selected_model_index: ReadSignal<usize>,
 8 |     set_selected_model_index: WriteSignal<usize>,
 9 | ) -> impl IntoView {
10 |     view! {
11 |         <nav class="navbar">
12 |             <Dropdown
13 |                 options=[
14 |                     FrontendModel::ChatCLM1_0.name(),
15 |                     FrontendModel::ChatGPT3_5.name(),
16 |                     FrontendModel::ChatGPT4o.name(),
17 |                     FrontendModel::ChatRandom.name(),
18 |                 ]
19 | 
20 |                 selected_option_index=selected_model_index
21 |                 set_selected_option_index=set_selected_model_index
22 |             />
23 |         </nav>
24 |     }
25 | }
26 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | This is free and unencumbered software released into the public domain.
 2 | 
 3 | Anyone is free to copy, modify, publish, use, compile, sell, or
 4 | distribute this software, either in source code form or as a compiled
 5 | binary, for any purpose, commercial or non-commercial, and by any
 6 | means.
 7 | 
 8 | In jurisdictions that recognize copyright laws, the author or authors
 9 | of this software dedicate any and all copyright interest in the
10 | software to the public domain. We make this dedication for the benefit
11 | of the public at large and to the detriment of our heirs and
12 | successors. We intend this dedication to be an overt act of
13 | relinquishment in perpetuity of all present and future rights to this
14 | software under copyright law.
15 | 
16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
19 | IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 | OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 | OTHER DEALINGS IN THE SOFTWARE.
23 | 
24 | For more information, please refer to <https://unlicense.org>
25 | 


--------------------------------------------------------------------------------
/src/chat.rs:
--------------------------------------------------------------------------------
 1 | #[derive(Debug, Clone, PartialEq, Eq)]
 2 | pub enum Sender {
 3 |     User,
 4 |     ChatCLM,
 5 | }
 6 | 
 7 | #[derive(Debug, Clone, PartialEq, Eq)]
 8 | pub struct Message {
 9 |     pub message: String,
10 |     pub time_iso: String,
11 |     pub sender: Sender,
12 | }
13 | 
14 | impl Message {
15 |     pub fn is_user_msg(&self) -> bool {
16 |         self.sender == Sender::User
17 |     }
18 | 
19 |     pub fn new(message: String, sender: Sender) -> Self {
20 |         Self {
21 |             message,
22 |             time_iso: chrono::Local::now().format("%Y-%m-%d %H:%M:%S").to_string(),
23 |             sender,
24 |         }
25 |     }
26 | }
27 | 
28 | #[derive(Debug, Clone, PartialEq, Eq, Default)]
29 | pub struct ChatHistory {
30 |     pub messages: Vec<Message>,
31 | }
32 | 
33 | impl ChatHistory {
34 |     fn add_message(&mut self, message: Message) {
35 |         self.messages.push(message);
36 |     }
37 | 
38 |     pub fn new_server_message(&mut self, message: String) {
39 |         self.add_message(Message::new(message, Sender::ChatCLM));
40 |     }
41 | 
42 |     pub fn new_user_message(&mut self, message: String) {
43 |         self.add_message(Message::new(message, Sender::User));
44 |     }
45 | 
46 |     pub fn replace_last_server_message(&mut self, message: String) {
47 |         if let Some(last_message) = self.messages.last_mut() {
48 |             if last_message.sender == Sender::ChatCLM {
49 |                 last_message.message = message;
50 |             }
51 |         }
52 |     }
53 | }
54 | 


--------------------------------------------------------------------------------
/src/backend/trainer.rs:
--------------------------------------------------------------------------------
 1 | use std::ffi::{c_uint, c_void};
 2 | use itertools::Itertools;
 3 | use zstd_sys::{ZDICT_isError, ZDICT_optimizeTrainFromBuffer_fastCover};
 4 | use crate::backend::{BYTES_PER_TOKEN, Token, tokens_to_bytes};
 5 | use crate::backend::clm_model::ClmModel;
 6 | use crate::backend::training_options::TrainingOptions;
 7 | 
 8 | pub fn train_model<'a>(input_tokens: &Vec<Vec<Token>>, training_options: &TrainingOptions) -> ClmModel<'a> {
 9 |     
10 |     if input_tokens.is_empty() {
11 |         return ClmModel::from_buffer(vec![]);
12 |     }
13 |     
14 |     let raw_data = input_tokens.iter().flat_map(tokens_to_bytes).collect_vec();
15 |     let sizes = input_tokens.iter().map(|x| x.len() * BYTES_PER_TOKEN).collect_vec();
16 |     let buffer_size = (raw_data.len() as f64 * training_options.dictionary_size_percentage) as usize;
17 |     assert_eq!(sizes.iter().sum::<usize>(), raw_data.len(), "Sizes sum doesn't match raw data size");
18 |     let mut buffer = vec![0u8; buffer_size];
19 |     let mut parameters = training_options.to_zdict_params();
20 |     let size;
21 |     unsafe {
22 |         size = ZDICT_optimizeTrainFromBuffer_fastCover(
23 |             buffer.as_mut_ptr() as *mut c_void,
24 |             buffer_size,
25 |             raw_data.as_ptr() as *mut c_void,
26 |             sizes.as_ptr(),
27 |             sizes.len() as c_uint,
28 |             &mut parameters,
29 |         );
30 | 
31 |         if ZDICT_isError(size) != 0 {
32 |             panic!("Failed to train dictionary");
33 |         }
34 |     }
35 |     buffer.resize(size, 0);
36 |     ClmModel::from_buffer(buffer)
37 | }


--------------------------------------------------------------------------------
/src/main.rs:
--------------------------------------------------------------------------------
 1 | use chatclm::app::App;
 2 | 
 3 | #[cfg(feature = "ssr")]
 4 | use chatclm::fileserv::file_and_error_handler;
 5 | 
 6 | 
 7 | #[cfg(feature = "ssr")]
 8 | #[tokio::main]
 9 | async fn leptos_main() {
10 |     use axum::Router;
11 |     use leptos::*;
12 |     use leptos_axum::{generate_route_list, LeptosRoutes};
13 | 
14 |     // Setting get_configuration(None) means we'll be using cargo-leptos's env values
15 |     // For deployment these variables are:
16 |     // <https://github.com/leptos-rs/start-axum#executing-a-server-on-a-remote-machine-without-the-toolchain>
17 |     // Alternately a file can be specified such as Some("Cargo.toml")
18 |     // The file would need to be included with the executable when moved to deployment
19 |     let conf = get_configuration(None).await.unwrap();
20 |     let leptos_options = conf.leptos_options;
21 |     let addr = leptos_options.site_addr;
22 |     let routes = generate_route_list(App);
23 | 
24 |     // build our application with a route
25 |     let app = Router::new()
26 |         .leptos_routes(&leptos_options, routes, App)
27 |         .fallback(file_and_error_handler)
28 |         .with_state(leptos_options);
29 | 
30 |     let listener = tokio::net::TcpListener::bind(&addr).await.unwrap();
31 |     logging::log!("listening on http://{}", &addr);
32 |     axum::serve(listener, app.into_make_service())
33 |         .await
34 |         .unwrap();
35 | }
36 | #[cfg(feature = "ssr")]
37 | fn main() {
38 |     leptos_main();
39 | }
40 | 
41 | #[cfg(not(feature = "ssr"))]
42 | pub fn main() {
43 |     // no client-side main function
44 |     // unless we want this to work with e.g., Trunk for a purely client-side app
45 |     // see lib.rs for hydration function instead
46 | }
47 | 


--------------------------------------------------------------------------------
/src/component/dropdown.rs:
--------------------------------------------------------------------------------
 1 | use leptos::{
 2 |     component, create_signal, view, For, IntoView, ReadSignal, SignalSet, SignalUpdate, WriteSignal,
 3 | };
 4 | 
 5 | #[component]
 6 | pub fn Dropdown<const N: usize>(
 7 |     options: [&'static str; N],
 8 |     selected_option_index: ReadSignal<usize>,
 9 |     set_selected_option_index: WriteSignal<usize>,
10 | ) -> impl IntoView {
11 |     let (is_open, set_open) = create_signal(false);
12 | 
13 |     view! {
14 |         <div class="dropdown">
15 |             <div
16 |                 class="dropdown__selected"
17 |                 on:click=move |_| {
18 |                     set_open.update(|is_currently_open| *is_currently_open = !*is_currently_open)
19 |                 }
20 |             >
21 | 
22 |                 <div>{move || options[selected_option_index()]}</div>
23 |                 <div class="dropdown__icon">></div>
24 |             </div>
25 |             <div class="dropdown__options" class=("hidden", move || !is_open())>
26 |                 <For
27 |                     each=move || options.into_iter().enumerate()
28 |                     key=|(_, option)| *option
29 |                     children=move |(index, option)| {
30 |                         view! {
31 |                             <div
32 |                                 class="dropdown__option"
33 |                                 on:click=move |_| {
34 |                                     set_open.set(false);
35 |                                     set_selected_option_index.set(index);
36 |                                 }
37 |                             >
38 | 
39 |                                 {option}
40 |                             </div>
41 |                         }
42 |                     }
43 |                 />
44 | 
45 |             </div>
46 |         </div>
47 |     }
48 | }
49 | 


--------------------------------------------------------------------------------
/src/backend/training_options.rs:
--------------------------------------------------------------------------------
 1 | use std::ffi::c_int;
 2 | 
 3 | pub struct TrainingOptions {
 4 |     pub d: u32,
 5 |     pub f: u32,
 6 |     pub k: u32,
 7 |     pub steps: u32,
 8 |     pub nb_threads: u32,
 9 |     pub split_point: f64,
10 |     pub accel: u32,
11 |     pub shrink_dict: u32,
12 |     pub shrink_dict_max_regression: u32,
13 |     pub compression_level: u32,
14 |     pub dictionary_size_percentage: f64 /* 0.0 to 1.0, how big the dictionary should be compared to the input data */,
15 |     pub ensemble_size: usize, /* number of models to train */
16 | }
17 | 
18 | impl TrainingOptions {
19 |     pub fn new() -> Self {
20 |         TrainingOptions {
21 |             d: 8,
22 |             f: 25,
23 |             k: 50,
24 |             steps: 4,
25 |             nb_threads: 8,
26 |             split_point: 0.0,
27 |             accel: 1,
28 |             shrink_dict: 0,
29 |             shrink_dict_max_regression: 0,
30 |             compression_level: 3,
31 |             dictionary_size_percentage: 1.0,
32 |             ensemble_size: 1,
33 |         }
34 |     }
35 | 
36 |     pub fn to_zdict_params(&self) -> zstd_sys::ZDICT_fastCover_params_t {
37 |         zstd_sys::ZDICT_fastCover_params_t {
38 |             k: self.k,
39 |             d: self.d,
40 |             f: self.f,
41 |             steps: self.steps,
42 |             nbThreads: self.nb_threads,
43 |             splitPoint: self.split_point,
44 |             accel: self.accel,
45 |             shrinkDict: self.shrink_dict,
46 |             shrinkDictMaxRegression: self.shrink_dict_max_regression,
47 |             zParams: zstd_sys::ZDICT_params_t {
48 |                 compressionLevel: self.compression_level as c_int,
49 |                 notificationLevel: 2,
50 |                 dictID: 0,
51 |             },
52 |         }
53 |     }
54 | 
55 |     pub fn default() -> Self {
56 |         TrainingOptions::new()
57 |     }
58 | }
59 | 


--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
 1 | # Get started with a build env with Rust nightly
 2 | FROM rustlang/rust:nightly-bookworm as builder
 3 | 
 4 | # If you’re using stable, use this instead
 5 | # FROM rust:1.74-bullseye as builder
 6 | 
 7 | # Install cargo-binstall, which makes it easier to install other
 8 | # cargo extensions like cargo-leptos
 9 | RUN wget https://github.com/cargo-bins/cargo-binstall/releases/latest/download/cargo-binstall-x86_64-unknown-linux-musl.tgz
10 | RUN tar -xvf cargo-binstall-x86_64-unknown-linux-musl.tgz
11 | RUN cp cargo-binstall /usr/local/cargo/bin
12 | 
13 | # Install cargo-leptos
14 | RUN cargo binstall cargo-leptos@0.2.17 -y
15 | 
16 | # Add the WASM target
17 | RUN rustup target add wasm32-unknown-unknown
18 | 
19 | # Make an /app dir, which everything will eventually live in
20 | RUN mkdir -p /app
21 | WORKDIR /app
22 | # Ignore the ./target dir
23 | COPY . .
24 | 
25 | # Build the app
26 | RUN cargo leptos build --release -vv
27 | 
28 | FROM debian:bookworm-slim as runtime
29 | WORKDIR /app
30 | RUN apt-get update -y \
31 |   && apt-get install -y --no-install-recommends  openssl ca-certificates \
32 |   && apt-get autoremove -y \
33 |   && apt-get clean -y \
34 |   && rm -rf /var/lib/apt/lists/*
35 | 
36 | # Copy the server binary to the /app directory
37 | COPY --from=builder /app/target/release/chatclm /app/
38 | 
39 | # /target/site contains our JS/WASM/CSS, etc.
40 | COPY --from=builder /app/target/site /app/site
41 | 
42 | # Copy Cargo.toml if it’s needed at runtime
43 | COPY --from=builder /app/Cargo.toml /app/
44 | 
45 | # Copy the zstd dictionary as it's needed at runtime
46 | COPY --from=builder /app/model.zstd_dict /app/
47 | 
48 | # Set any required env variables and
49 | ENV RUST_LOG="info"
50 | ENV LEPTOS_SITE_ADDR="0.0.0.0:8080"
51 | ENV LEPTOS_SITE_ROOT="site"
52 | EXPOSE 8080
53 | 
54 | # -- NB: update binary name from "chatclm" to match your app name in Cargo.toml --
55 | # Run the server
56 | CMD ["/app/chatclm"]
57 | 


--------------------------------------------------------------------------------
/src/fileserv.rs:
--------------------------------------------------------------------------------
 1 | use crate::app::App;
 2 | use axum::response::Response as AxumResponse;
 3 | use axum::{
 4 |     body::Body,
 5 |     extract::State,
 6 |     http::{Request, Response, StatusCode},
 7 |     response::IntoResponse,
 8 | };
 9 | use leptos::*;
10 | use tower::ServiceExt;
11 | use tower_http::services::ServeDir;
12 | 
13 | pub async fn file_and_error_handler(
14 |     State(options): State<LeptosOptions>,
15 |     req: Request<Body>,
16 | ) -> AxumResponse {
17 |     let root = options.site_root.clone();
18 |     let (parts, body) = req.into_parts();
19 | 
20 |     let mut static_parts = parts.clone();
21 |     static_parts.headers.clear();
22 |     if let Some(encodings) = parts.headers.get("accept-encoding") {
23 |         static_parts
24 |             .headers
25 |             .insert("accept-encoding", encodings.clone());
26 |     }
27 | 
28 |     let res = get_static_file(Request::from_parts(static_parts, Body::empty()), &root)
29 |         .await
30 |         .unwrap();
31 | 
32 |     if res.status() == StatusCode::OK {
33 |         res.into_response()
34 |     } else {
35 |         let handler = leptos_axum::render_app_to_stream(options.to_owned(), App);
36 |         handler(Request::from_parts(parts, body))
37 |             .await
38 |             .into_response()
39 |     }
40 | }
41 | 
42 | async fn get_static_file(
43 |     request: Request<Body>,
44 |     root: &str,
45 | ) -> Result<Response<Body>, (StatusCode, String)> {
46 |     // `ServeDir` implements `tower::Service` so we can call it with `tower::ServiceExt::oneshot`
47 |     // This path is relative to the cargo root
48 |     match ServeDir::new(root)
49 |         .precompressed_gzip()
50 |         .precompressed_br()
51 |         .oneshot(request)
52 |         .await
53 |     {
54 |         Ok(res) => Ok(res.into_response()),
55 |         Err(err) => Err((
56 |             StatusCode::INTERNAL_SERVER_ERROR,
57 |             format!("Error serving files: {err}"),
58 |         )),
59 |     }
60 | }
61 | 


--------------------------------------------------------------------------------
/src/component/prompt_input.rs:
--------------------------------------------------------------------------------
 1 | use leptos::{
 2 |     component, create_node_ref, create_signal, html, view, Callback, IntoView, NodeRef, Show,
 3 | };
 4 | 
 5 | #[component]
 6 | pub fn PromptInput(on_submit: Callback<String>) -> impl IntoView {
 7 |     let (show_placeholder, set_show_placeholder) = create_signal(true);
 8 |     let textarea_element: NodeRef<html::Div> = create_node_ref();
 9 | 
10 |     let submit = move || {
11 |         let input = textarea_element().unwrap().inner_text().trim().to_string();
12 |         if input.len() == 0 {
13 |             return;
14 |         }
15 | 
16 |         on_submit(input);
17 |         textarea_element().unwrap().set_inner_html("");
18 |     };
19 | 
20 |     view! {
21 |         <div class="prompt_input">
22 |             <div class="prompt_input__textarea_wrapper">
23 |                 <div
24 |                     class="prompt_input__textarea"
25 |                     contenteditable
26 |                     node_ref=textarea_element
27 |                     on:focus=move |_| {
28 |                         set_show_placeholder(false);
29 |                     }
30 | 
31 |                     on:blur=move |_| {
32 |                         set_show_placeholder(textarea_element().unwrap().inner_text().len() == 0);
33 |                     }
34 | 
35 |                     on:keydown=move |ev| {
36 |                         if ev.key_code() == 13 && !ev.shift_key() {
37 |                             ev.prevent_default();
38 |                             submit();
39 |                         }
40 |                     }
41 |                 >
42 |                 </div>
43 | 
44 |                 <Show when=move || show_placeholder()>
45 |                     <div class="prompt_input__textarea_placeholder">Message ChatCLM</div>
46 |                 </Show>
47 | 
48 |                 <button on:click=move |_| submit() class="prompt_input__send_button">
49 |                     >
50 |                 </button>
51 |             </div>
52 |         </div>
53 |     }
54 | }
55 | 


--------------------------------------------------------------------------------
/train_tokenizer.py:
--------------------------------------------------------------------------------
 1 | import tqdm
 2 | from  tokenizers.normalizers import *
 3 | from tokenizers import Tokenizer, decoders, pre_tokenizers
 4 | from tokenizers.models import BPE
 5 | from tokenizers.pre_tokenizers import ByteLevel
 6 | from tokenizers.trainers import BpeTrainer
 7 | 
 8 | files = [f"data/tokenizer_data.txt"]
 9 | 
10 | def data_iterator():
11 |     # Removes all characters, that are only used once in the entire dataset
12 |     chars_used_once = set()
13 |     char_counter = {}
14 | 
15 |     for file in files:
16 |         with open(file, "r") as f:
17 |             for line in tqdm.tqdm(f.readlines(), desc=f"Counting characters {file}"):
18 |                 for char in line:
19 |                     if char in char_counter:
20 |                         char_counter[char] += 1
21 |                     else:
22 |                         char_counter[char] = 1
23 | 
24 |     for char, count in char_counter.items():
25 |         if count <= 2:
26 |             chars_used_once.add(char)
27 | 
28 |     print(f"Removing characters:")
29 |     print(chars_used_once)
30 | 
31 |     for file in files:
32 |         with open(file, "r") as f:
33 |             for line in tqdm.tqdm(f.readlines(), desc=f"Processing {file}"):
34 |                 line_without_chars = "".join([char for char in line if char not in chars_used_once])
35 |                 yield line_without_chars
36 | 
37 | tokenizer = Tokenizer(BPE(unk_token="[UNK]"))
38 | 
39 | trainer = BpeTrainer(special_tokens=["[UNK]"], vocab_size = 255, show_progress=True, end_of_word_suffix="</w>")
40 | tokenizer.pre_tokenizer = pre_tokenizers.Sequence( [pre_tokenizers.WhitespaceSplit() ])
41 | tokenizer.normalizer = Sequence([Lowercase(), NFD(), StripAccents()])
42 | tokenizer.train_from_iterator(data_iterator(), trainer=trainer)
43 | tokenizer.decoder = decoders.BPEDecoder()
44 | 
45 | tokenizer.save("tokenizer.json")
46 | 
47 | encoded = tokenizer.encode("This is a test!")
48 | print(encoded.ids)
49 | 
50 | decoded = tokenizer.decode(encoded.ids)
51 | print(decoded)
52 | 
53 | # Try to load the tokenizer
54 | tokenizer = Tokenizer.from_file("tokenizer.json")
55 | encoded = tokenizer.encode("This is á test!")
56 | print(encoded.ids)
57 | decoded = tokenizer.decode(encoded.ids)
58 | print(decoded)
59 | 


--------------------------------------------------------------------------------
/tune.py:
--------------------------------------------------------------------------------
 1 | # Used as an adapter between wandb and rust code
 2 | import argparse
 3 | import json
 4 | import subprocess
 5 | 
 6 | import wandb
 7 | 
 8 | sweep_config = {
 9 |     "method": "bayes",
10 |     "metric": {
11 |         "goal": "maximize",
12 |         "name": "train_inf_gain"
13 |     },
14 |     "parameters": {
15 |         "datasetSize": {
16 |             "distribution": "int_uniform",
17 |             "max": 180_000_000,
18 |             "min": 1_000_000,
19 |         },
20 |         "dictionarySizePercentage" : {
21 |             "distribution": "uniform",
22 |             "max": 1,
23 |             "min": 0
24 |         },
25 | 
26 |         "compressionLevel": {
27 |             "distribution": "int_uniform",
28 |             "max": 8,
29 |             "min": 1
30 |         },
31 |         "d": {
32 |             "values": [6, 8]
33 |         },
34 |         "f": {
35 |             "distribution": "int_uniform",
36 |             "max": 26,
37 |             "min": 5
38 |         },
39 |         "k": {
40 |             "distribution": "int_uniform",
41 |             "max": 2048,
42 |             "min": 16
43 |         },
44 |     }
45 | 
46 | }
47 | 
48 | def evaluate():
49 |     run = wandb.init()
50 | 
51 |     conf_dict = {}
52 |     for param in sweep_config["parameters"].keys():
53 |         conf_dict[param] = wandb.config[param]
54 | 
55 |     # Create a json object with the arguments
56 |     arguments = json.dumps(conf_dict)
57 |     print("Starting with arguments:")
58 |     print(arguments)
59 |     # Start the binary, write the arguments to its stdin and read the output
60 |     command = ["target/release/tuning"]
61 | 
62 |     process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
63 |     output, stderr = process.communicate(arguments)
64 |     print(output)
65 |     print(stderr)
66 | 
67 | 
68 | 
69 |     # Parse the last line as json
70 |     try:
71 |         output = json.loads(output.splitlines()[-1])
72 |         wandb.log(output)
73 |     except json.JSONDecodeError:
74 |         print("Error decoding json")
75 |         wandb.log({})
76 |     except IndexError:
77 |         print("Error parsing output")
78 |         wandb.log({})
79 | 
80 | 
81 | 
82 | 
83 | # Initialize a new sweep
84 | sweep_id = wandb.sweep(sweep_config)
85 | wandb.agent(sweep_id=sweep_id, function=evaluate)
86 | 
87 | 
88 | 
89 | 
90 | 
91 | 


--------------------------------------------------------------------------------
/src/component/prompt_section.rs:
--------------------------------------------------------------------------------
 1 | use crate::chat::ChatHistory;
 2 | use crate::component::prompt_input::PromptInput;
 3 | use crate::model::{cut_prompt, get_next_token};
 4 | use leptos::{
 5 |     component, spawn_local, view, Callback, IntoView, ReadSignal, SignalUpdate, WriteSignal,
 6 | };
 7 | 
 8 | #[component]
 9 | pub fn PromptSection(
10 |     set_chat: WriteSignal<ChatHistory>,
11 |     selected_model_index: ReadSignal<usize>,
12 | ) -> impl IntoView {
13 |     view! {
14 |         <section class="prompt">
15 |             <div class="prompt__wrapper">
16 |                 <PromptInput on_submit=Callback::new(move |prompt: String| {
17 |                     let model_idx = selected_model_index();
18 |                     set_chat
19 |                         .update(|chat| {
20 |                             chat.new_user_message(prompt.clone());
21 |                         });
22 |                     spawn_local(async move {
23 |                         set_chat
24 |                             .update(|chat| {
25 |                                 chat.new_server_message("thinking...".to_string());
26 |                             });
27 |                         if let Ok(Some(response)) = get_next_token(model_idx, prompt.clone()).await
28 |                         {
29 |                             set_chat
30 |                                 .update(|chat| {
31 |                                     chat.replace_last_server_message(
32 |                                         cut_prompt(&prompt, &response),
33 |                                     );
34 |                                 });
35 |                             let mut response = response;
36 |                             while let Ok(Some(stream)) = get_next_token(model_idx, response.clone())
37 |                                 .await
38 |                             {
39 |                                 set_chat
40 |                                     .update(|chat| {
41 |                                         chat.replace_last_server_message(
42 |                                             cut_prompt(&prompt, &stream),
43 |                                         );
44 |                                     });
45 |                                 response = stream;
46 |                             }
47 |                         }
48 |                     });
49 |                 })/>
50 | 
51 |                 <p class="prompt__disclaimer">ChatCLM can make mistakes. Check important info.</p>
52 |             </div>
53 |         </section>
54 |     }
55 | }
56 | 


--------------------------------------------------------------------------------
/src/error_template.rs:
--------------------------------------------------------------------------------
 1 | use http::status::StatusCode;
 2 | use leptos::*;
 3 | use thiserror::Error;
 4 | 
 5 | #[derive(Clone, Debug, Error)]
 6 | pub enum AppError {
 7 |     #[error("Not Found")]
 8 |     NotFound,
 9 | }
10 | 
11 | impl AppError {
12 |     pub fn status_code(&self) -> StatusCode {
13 |         match self {
14 |             AppError::NotFound => StatusCode::NOT_FOUND,
15 |         }
16 |     }
17 | }
18 | 
19 | // A basic function to display errors served by the error boundaries.
20 | // Feel free to do more complicated things here than just displaying the error.
21 | #[component]
22 | pub fn ErrorTemplate(
23 |     #[prop(optional)] outside_errors: Option<Errors>,
24 |     #[prop(optional)] errors: Option<RwSignal<Errors>>,
25 | ) -> impl IntoView {
26 |     let errors = match outside_errors {
27 |         Some(e) => create_rw_signal(e),
28 |         None => match errors {
29 |             Some(e) => e,
30 |             None => panic!("No Errors found and we expected errors!"),
31 |         },
32 |     };
33 |     // Get Errors from Signal
34 |     let errors = errors.get_untracked();
35 | 
36 |     // Downcast lets us take a type that implements `std::error::Error`
37 |     let errors: Vec<AppError> = errors
38 |         .into_iter()
39 |         .filter_map(|(_k, v)| v.downcast_ref::<AppError>().cloned())
40 |         .collect();
41 |     println!("Errors: {errors:#?}");
42 | 
43 |     // Only the response code for the first error is actually sent from the server
44 |     // this may be customized by the specific application
45 |     #[cfg(feature = "ssr")]
46 |     {
47 |         use leptos_axum::ResponseOptions;
48 |         let response = use_context::<ResponseOptions>();
49 |         if let Some(response) = response {
50 |             response.set_status(errors[0].status_code());
51 |         }
52 |     }
53 | 
54 |     view! {
55 |         <h1>{if errors.len() > 1 { "Errors" } else { "Error" }}</h1>
56 |         <For
57 |             // a function that returns the items we're iterating over; a signal is fine
58 |             each=move || { errors.clone().into_iter().enumerate() }
59 |             // a unique key for each item as a reference
60 |             key=|(index, _error)| *index
61 |             // renders each item to a view
62 |             children=move |error| {
63 |                 let error_string = error.1.to_string();
64 |                 let error_code = error.1.status_code();
65 |                 view! {
66 |                     <h2>{error_code.to_string()}</h2>
67 |                     <p>"Error: " {error_string}</p>
68 |                 }
69 |             }
70 |         />
71 |     }
72 | }
73 | 


--------------------------------------------------------------------------------
/src/app.rs:
--------------------------------------------------------------------------------
 1 | use crate::chat::ChatHistory;
 2 | use crate::component::chat::Chat;
 3 | use crate::component::navbar::NavBar;
 4 | use crate::component::prompt_section::PromptSection;
 5 | use crate::error_template::{AppError, ErrorTemplate};
 6 | use leptos::*;
 7 | use leptos_meta::*;
 8 | use leptos_router::*;
 9 | 
10 | #[component]
11 | pub fn App() -> impl IntoView {
12 |     // Provides context that manages stylesheets, titles, meta tags, etc.
13 |     provide_meta_context();
14 | 
15 |     // create chat as reactive signal object
16 |     let (chat, set_chat) = create_signal(ChatHistory::default());
17 |     let (selected_model_index, set_selected_model_index) = create_signal(0usize);
18 |     // fill with dummy data
19 |     set_chat.update(|chat| {
20 |         chat.new_server_message("Welcome to ChatCLM!".to_string());
21 |         chat.new_user_message("Hello!".to_string());
22 |         chat.new_server_message("Type a message and press Enter to chat.".to_string());
23 |     });
24 | 
25 |     view! {
26 |         <Stylesheet id="leptos" href="/pkg/chatclm.css"/>
27 | 
28 |         // sets the document title
29 |         <Title text="ChatCLM"/>
30 | 
31 |         // content for this welcome page
32 |         <Router fallback=|| {
33 |             let mut outside_errors = Errors::default();
34 |             outside_errors.insert_with_default_key(AppError::NotFound);
35 |             view! { <ErrorTemplate outside_errors/> }.into_view()
36 |         }>
37 |             <main>
38 |                 <Routes>
39 |                     <Route
40 |                         path=""
41 |                         view=move || {
42 |                             view! {
43 |                                 <HomePage
44 |                                     chat=chat
45 |                                     set_chat=set_chat
46 |                                     selected_model_index=selected_model_index
47 |                                     set_selected_model_index=set_selected_model_index
48 |                                 />
49 |                             }
50 |                         }
51 |                     />
52 |                 </Routes>
53 |             </main>
54 |         </Router>
55 |     }
56 | }
57 | 
58 | #[component]
59 | fn HomePage(
60 |     chat: ReadSignal<ChatHistory>,
61 |     set_chat: WriteSignal<ChatHistory>,
62 |     selected_model_index: ReadSignal<usize>,
63 |     set_selected_model_index: WriteSignal<usize>,
64 | ) -> impl IntoView {
65 |     view! {
66 |         <NavBar
67 |             selected_model_index=selected_model_index
68 |             set_selected_model_index=set_selected_model_index
69 |         />
70 | 
71 |         <Chat chat=chat/>
72 | 
73 |         <PromptSection set_chat=set_chat selected_model_index=selected_model_index/>
74 |     }
75 | }
76 | 


--------------------------------------------------------------------------------
/src/backend/tokenizer.rs:
--------------------------------------------------------------------------------
 1 | use itertools::Itertools;
 2 | use tiktoken_rs::{CoreBPE, p50k_base};
 3 | use tokenizers::tokenizer::Tokenizer;
 4 | 
 5 | use crate::backend::{MAX_TOKEN, Token};
 6 | 
 7 | static TOKENIZER_PATH: &str = "tokenizer.json";
 8 | 
 9 | #[derive(Clone)]
10 | pub enum ClmTokenizer {
11 |     GPT2(CoreBPE),
12 |     Custom(Tokenizer),
13 | }
14 | 
15 | impl ClmTokenizer {
16 |     #[allow(dead_code)]
17 |     fn new_gpt2() -> Self {
18 |         let tokenizer = p50k_base().unwrap();
19 |         ClmTokenizer::GPT2(tokenizer)
20 |     }
21 | 
22 |     pub fn get_max_token(&self) -> Token {
23 |         match self {
24 |             ClmTokenizer::GPT2(_) => MAX_TOKEN as Token,
25 |             ClmTokenizer::Custom(tokenizer) => (tokenizer.get_vocab_size(true) - 1) as Token,
26 |         }
27 |     }
28 | 
29 |     pub(crate) fn new_custom() -> Self {
30 |         let tokenizer = Tokenizer::from_file(TOKENIZER_PATH).unwrap();
31 |         ClmTokenizer::Custom(tokenizer)
32 |     }
33 | 
34 |     pub(crate) fn encode(&self, text: &str) -> Vec<Token> {
35 |         match self {
36 |             ClmTokenizer::GPT2(tokenizer) => tokenizer.encode_ordinary(text).iter().map(|&x| x as Token).collect(),
37 |             ClmTokenizer::Custom(tokenizer) => {
38 |                 let encoding = tokenizer.encode(text, false).unwrap();
39 |                 let ids = encoding.get_ids();
40 |                 ids.iter().map(|&x| x as Token).collect()
41 |             }
42 |         }
43 |     }
44 | 
45 |     pub(crate) fn decode(&self, tokens: Vec<Token>) -> String {
46 |         match self {
47 |             ClmTokenizer::GPT2(tokenizer) => tokenizer.decode(tokens.iter().map(|&x| x as usize).collect_vec()).unwrap(),
48 |             ClmTokenizer::Custom(tokenizer) => {
49 |                 tokenizer.decode(tokens.iter().map(|&x| x as u32).collect_vec().as_ref(), false).unwrap()
50 |             }
51 |         }
52 |     }
53 | }
54 | 
55 | #[cfg(test)]
56 | mod tests {
57 |     use crate::backend::tokenizer::ClmTokenizer;
58 | 
59 |     #[test]
60 |     fn custom_tokenizer_works() {
61 |         let tokenizer = ClmTokenizer::new_custom();
62 |         let encoding = tokenizer.encode("Hello, world!");
63 |         let decoded = tokenizer.decode(encoding);
64 |         assert_eq!(decoded, "hello, world!");
65 |     }
66 | 
67 |     #[test]
68 |     fn gpt2_tokenizer_works() {
69 |         let tokenizer = ClmTokenizer::new_gpt2();
70 |         let encoding = tokenizer.encode("Hello, world!");
71 |         let decoded = tokenizer.decode(encoding);
72 |         assert_eq!(decoded, "Hello, world!");
73 |     }
74 | 
75 |     #[test]
76 |     fn custom_tokenizer_special_characters() {
77 |         let tokenizer = ClmTokenizer::new_custom();
78 |         let encoding = tokenizer.encode("Hello, world! 🌍");
79 |         let decoded = tokenizer.decode(encoding);
80 |         assert_eq!(decoded, "hello, world! [UNK]");
81 |     }
82 | }
83 | 
84 | 


--------------------------------------------------------------------------------
/src/model.rs:
--------------------------------------------------------------------------------
  1 | use std::sync::LazyLock;
  2 | use leptos::{server, ServerFnError};
  3 | #[cfg(feature = "ssr")]
  4 | use crate::backend::clm_model::ClmModel;
  5 | #[derive(Copy, Clone)]
  6 | pub enum FrontendModel {
  7 |     ChatCLM1_0,
  8 |     ChatGPT3_5,
  9 |     ChatGPT4o,
 10 |     ChatRandom,
 11 | }
 12 | 
 13 | impl FrontendModel {
 14 |     pub fn name(&self) -> &'static str {
 15 |         match self {
 16 |             FrontendModel::ChatCLM1_0 => "ChatCLM 0.1-pre-alpha",
 17 |             FrontendModel::ChatGPT3_5 => "ChatGPT 3.5",
 18 |             FrontendModel::ChatGPT4o => "ChatGPT 4o",
 19 |             FrontendModel::ChatRandom => "ChatRandom",
 20 |         }
 21 |     }
 22 | 
 23 |     pub fn from_index(index: usize) -> Self {
 24 |         match index {
 25 |             0 => FrontendModel::ChatCLM1_0,
 26 |             1 => FrontendModel::ChatGPT3_5,
 27 |             2 => FrontendModel::ChatGPT4o,
 28 |             3 => FrontendModel::ChatRandom,
 29 |             _ => panic!("Invalid model index"),
 30 |         }
 31 |     }
 32 | 
 33 | 
 34 |     #[cfg(feature = "ssr")]
 35 |     pub async fn predict_next_token(model_idx: usize, prompt: String) -> Option<String> {
 36 |         match Self::from_index(model_idx) {
 37 |             FrontendModel::ChatCLM1_0 => chat_clm_next_token(prompt).await,
 38 |             FrontendModel::ChatGPT4o => gpt4o_next_token(prompt).await,
 39 |             FrontendModel::ChatRandom => random_next_token(prompt).await,
 40 |             _ => random_next_token(prompt).await,
 41 |         }
 42 |     }
 43 | }
 44 | 
 45 | pub async fn random_next_token(prompt: String) -> Option<String> {
 46 |     // sleep 200 ms
 47 |     let random_number = rand::random::<u8>() % 7 + 1;
 48 |     if random_number == 1 {
 49 |         None
 50 |     } else {
 51 |         Some(format!("{} next", prompt))
 52 |     }
 53 | }
 54 | 
 55 | #[cfg(feature = "ssr")]
 56 | static CLM: LazyLock<ClmModel> = LazyLock::new(|| ClmModel::from_checkpoint("clm_model.bin"));
 57 | 
 58 | #[cfg(feature = "ssr")]
 59 | pub async fn chat_clm_next_token(prompt: String ) -> Option<String> {
 60 | 
 61 |     if prompt.len() > 250 {
 62 |         return None;
 63 |     }
 64 | 
 65 |     Some(CLM.predict_next(prompt, 1, 3))
 66 | 
 67 | }
 68 | 
 69 | pub async fn gpt4o_next_token(_prompt: String) -> Option<String> {
 70 |     /*let client = Client::new();
 71 | 
 72 |     let request = CreateCompletionRequestArgs::default()
 73 |         .model("gpt-3.5-turbo-instruct")
 74 |         .prompt(prompt)
 75 |         .max_tokens(40_u16)
 76 |         .build().unwrap();
 77 | 
 78 |     let result = client.completions().create(request).await;
 79 |     if let Ok(response) = result {
 80 |         return Some(response.choices.clone()[0].clone().text);
 81 |     }*/
 82 |     None
 83 | }
 84 | 
 85 | #[server(GetNextToken, "/api")]
 86 | pub async fn get_next_token(
 87 |     model_idx: usize,
 88 |     prompt: String,
 89 | ) -> Result<Option<String>, ServerFnError> {
 90 |     Ok(FrontendModel::predict_next_token(model_idx, prompt).await)
 91 | }
 92 | 
 93 | pub fn cut_prompt(prompt: &String, response: &String) -> String {
 94 |     let prompt = prompt.trim();
 95 |     let response = response.trim();
 96 |     if response.starts_with(prompt) {
 97 |         response[prompt.len()..].trim().to_string()
 98 |     } else {
 99 |         response.to_string()
100 |     }
101 | }
102 | 


--------------------------------------------------------------------------------
/Cargo.toml:
--------------------------------------------------------------------------------
  1 | [package]
  2 | name = "chatclm"
  3 | version = "0.1.0"
  4 | edition = "2021"
  5 | 
  6 | 
  7 | [[bin]]
  8 | name = "tuning"
  9 | path = "src/tuning.rs"
 10 | 
 11 | [profile.release]
 12 | debug = true
 13 | 
 14 | [lib]
 15 | crate-type = ["cdylib", "rlib"]
 16 | 
 17 | [dependencies]
 18 | axum = { version = "0.7", optional = true }
 19 | console_error_panic_hook = "0.1"
 20 | leptos = { version = "0.6", features = ["nightly"] }
 21 | leptos_axum = { version = "0.6", optional = true }
 22 | leptos_meta = { version = "0.6", features = ["nightly"] }
 23 | leptos_router = { version = "0.6", features = ["nightly"] }
 24 | tokio = { version = "1", features = ["rt-multi-thread"], optional = true }
 25 | tower = { version = "0.4", optional = true }
 26 | tower-http = { version = "0.5", features = ["fs"], optional = true }
 27 | wasm-bindgen = "=0.2.92"
 28 | thiserror = "1"
 29 | tracing = { version = "0.1", optional = true }
 30 | http = "1"
 31 | chrono = "0.4.38"
 32 | rand = "0.8.5"
 33 | zstd-sys = { version = "2.0.10", features = ["experimental", "zstdmt"], optional = true}
 34 | zstd = { version = "0.13.1", optional = true }
 35 | 
 36 | tiktoken-rs = "0.5.9"
 37 | glob = "0.3.1"
 38 | rayon = "1.10.0"
 39 | regex = "1.10.4"
 40 | itertools = "0.13.0"
 41 | indicatif = "0.17.8"
 42 | serde = { version = "1.0.202", features = ["derive"] }
 43 | rmp-serde = "1.3.0"
 44 | serde_json = "1.0.117"
 45 | num = "0.4.3"
 46 | rusqlite = "0.31.0"
 47 | tokenizers = "0.19.1"
 48 | 
 49 | [features]
 50 | default = ["ssr"]
 51 | hydrate = ["leptos/hydrate", "leptos_meta/hydrate", "leptos_router/hydrate"]
 52 | ssr = [
 53 |     "dep:axum",
 54 |     "dep:tokio",
 55 |     "dep:tower",
 56 |     "dep:zstd-sys",
 57 |     "dep:zstd",
 58 |     "dep:tower-http",
 59 |     "dep:leptos_axum",
 60 |     "leptos/ssr",
 61 |     "leptos_meta/ssr",
 62 |     "leptos_router/ssr",
 63 |     "dep:tracing",
 64 | ]
 65 | 
 66 | # Defines a size-optimized profile for the WASM bundle in release mode
 67 | [profile.wasm-release]
 68 | inherits = "release"
 69 | opt-level = 'z'
 70 | lto = true
 71 | codegen-units = 1
 72 | panic = "abort"
 73 | 
 74 | [package.metadata.leptos]
 75 | # The name used by wasm-bindgen/cargo-leptos for the JS/WASM bundle. Defaults to the crate name
 76 | output-name = "chatclm"
 77 | 
 78 | # The site root folder is where cargo-leptos generate all output. WARNING: all content of this folder will be erased on a rebuild. Use it in your server setup.
 79 | site-root = "target/site"
 80 | 
 81 | # The site-root relative folder where all compiled output (JS, WASM and CSS) is written
 82 | # Defaults to pkg
 83 | site-pkg-dir = "pkg"
 84 | 
 85 | # [Optional] The source CSS file. If it ends with .sass or .scss then it will be compiled by dart-sass into CSS. The CSS is optimized by Lightning CSS before being written to <site-root>/<site-pkg>/app.css
 86 | style-file = "style/main.scss"
 87 | # Assets source dir. All files found here will be copied and synchronized to site-root.
 88 | # The assets-dir cannot have a sub directory with the same name/path as site-pkg-dir.
 89 | #
 90 | # Optional. Env: LEPTOS_ASSETS_DIR.
 91 | assets-dir = "public"
 92 | 
 93 | # The IP and port (ex: 127.0.0.1:3000) where the server serves the content. Use it in your server setup.
 94 | site-addr = "127.0.0.1:3000"
 95 | 
 96 | # The port to use for automatic reload monitoring
 97 | reload-port = 3001
 98 | 
 99 | # [Optional] Command to use when running end2end tests. It will run in the end2end dir.
100 | #   [Windows] for non-WSL use "npx.cmd playwright test"
101 | #   This binary name can be checked in Powershell with Get-Command npx
102 | end2end-cmd = "npx playwright test"
103 | end2end-dir = "end2end"
104 | 
105 | #  The browserlist query used for optimizing the CSS.
106 | browserquery = "defaults"
107 | 
108 | # The environment Leptos will run in, usually either "DEV" or "PROD"
109 | env = "DEV"
110 | 
111 | # The features to use when compiling the bin target
112 | #
113 | # Optional. Can be over-ridden with the command line parameter --bin-features
114 | bin-features = ["ssr"]
115 | 
116 | # If the --no-default-features flag should be used when compiling the bin target
117 | #
118 | # Optional. Defaults to false.
119 | bin-default-features = false
120 | 
121 | # The features to use when compiling the lib target
122 | #
123 | # Optional. Can be over-ridden with the command line parameter --lib-features
124 | lib-features = ["hydrate"]
125 | 
126 | # If the --no-default-features flag should be used when compiling the lib target
127 | #
128 | # Optional. Defaults to false.
129 | lib-default-features = false
130 | 
131 | # The profile to use for the lib target when compiling for release
132 | #
133 | # Optional. Defaults to "release".
134 | lib-profile-release = "wasm-release"
135 | 
136 | [[workspace.metadata.leptos]]
137 | # project name
138 | name = "chatclm"
139 | bin-target = "chatclm"
140 | #lib-package = "front"
141 | 
142 | 


--------------------------------------------------------------------------------
/src/backend/mod.rs:
--------------------------------------------------------------------------------
  1 | pub mod trainer;
  2 | pub mod training_options;
  3 | pub mod clm_model;
  4 | pub mod dataset;
  5 | pub mod evaluation;
  6 | pub mod ensemble_model;
  7 | mod tokenizer;
  8 | 
  9 | 
 10 | // https://wortschatz.uni-leipzig.de/en/download/English
 11 | const DATA_PATH: &str = "./data";
 12 | 
 13 | pub type Token = u8;
 14 | const BYTES_PER_TOKEN: usize = std::mem::size_of::<Token>();
 15 | pub const MAX_TOKEN: Token = 254; // Please update if u use another tokenizer!!!!
 16 | 
 17 | pub fn tokens_to_bytes(tokens: &Vec<Token>) -> Vec<u8> {
 18 |     tokens.iter().flat_map(|x| (*x).to_be_bytes()).collect()
 19 | }
 20 | 
 21 | 
 22 | 
 23 | const INFERENCE_COMPRESSION_LEVEL: i32 = 1;
 24 | 
 25 | #[cfg(test)]
 26 | pub mod tests {
 27 |     use itertools::Itertools;
 28 |     use rand::distributions::Uniform;
 29 |     use rand::Rng;
 30 |     use crate::backend::*;
 31 |     use crate::backend::clm_model::ClmModel;
 32 |     use crate::backend::dataset::Dataset;
 33 |     use crate::backend::trainer::train_model;
 34 |     use crate::backend::training_options::TrainingOptions;
 35 | 
 36 |     pub fn random_tokens(n: usize) -> Vec<Token> {
 37 |         rand::thread_rng().sample_iter(Uniform::from(0..MAX_TOKEN)).take(n).collect_vec()
 38 |     }
 39 | 
 40 |     pub fn predict_loop() {
 41 |         // create text for a test prompt
 42 |         //let mut prompt = "The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy".to_string();
 43 | 
 44 |         let mut prompt = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Proin tincidunt urna nisl, non molestie velit aliquam nec. In in erat id est porttitor efficitur ac eleifend ex. Nam auctor lacus urna, a sodales metus bibendum ut. Vestibulum vulputate facilisis ultrices. Vestibulum ut euismod erat. Maecenas pretium egestas nunc, non efficitur eros interdum eget. Suspendisse eleifend augue eu viverra rutrum. Phasellus non elementum erat, sit amet ultrices nunc. Sed facilisis at ipsum nec sagittis. Nulla non placerat purus. Pellentesque sed mollis enim. Praesent tincidunt purus id tellus tristique, ut ".to_string();
 45 | 
 46 |         let clm = ClmModel::from_checkpoint("model.zstd_dict");
 47 | 
 48 |         println!("{}", prompt);
 49 | 
 50 |         for _ in 0..100 {
 51 |             prompt = clm.predict_next(prompt.clone(), 0, 0);
 52 |             println!("{}", prompt)
 53 |         }
 54 |     }
 55 | 
 56 |     #[test]
 57 |     fn can_compress_and_decompress() {
 58 |         let clm = ClmModel::from_buffer(Vec::new());
 59 |         let start: Vec<Token> = clm.tokenizer.encode("Lorem ipsum dolor sit amet, consectetur adipiscing elit. Proin tincidunt urna nisl, non molestie velit aliquam nec. In in erat id est porttitor efficitur ac eleifend ex. Nam auctor lacus urna, a sodales metus bibendum ut. Vestibulum vulputate facilisis ultrices. Vestibulum ut euismod erat. Maecenas pretium egestas nunc, non efficitur eros interdum eget. Suspendisse eleifend augue eu viverra rutrum. Phasellus non elementum erat, sit amet ultrices nunc. Sed facilisis at ipsum nec sagittis. Nulla non placerat purus. Pellentesque sed mollis enim. Praesent tincidunt purus id tellus tristique, ut rhoncus justo fringilla. Suspendisse fermentum ultrices dolor, vel mollis enim. Aliquam eros.");
 60 |         let compressed = clm.compress(&start);
 61 |         let decompressed = clm.decompress_to_tokens(&compressed);
 62 | 
 63 |         assert_eq!(decompressed, start);
 64 |     }
 65 | 
 66 |     #[test]
 67 |     fn save_and_load_model() {
 68 |         let data: Vec<Token> = random_tokens(100);
 69 |         let training_data = (0usize..10).map(|_| data.clone()).collect_vec();
 70 | 
 71 |         let model = train_model(&training_data, &TrainingOptions::new());
 72 |         model.save_checkpoint("model_test.zstd_dict");
 73 |         let loaded_model = ClmModel::from_checkpoint("model_test.zstd_dict");
 74 | 
 75 |         let compressed = model.compress(&data);
 76 |         let compressed_loaded = loaded_model.compress(&data);
 77 | 
 78 |         assert_eq!(compressed, compressed_loaded);
 79 | 
 80 |         // cleanup
 81 |         std::fs::remove_file("model_test.zstd_dict").unwrap();
 82 |     }
 83 |     #[test]
 84 |     fn dictionary_helps_compression() {
 85 |         let data: Vec<Token> = random_tokens(50);
 86 |         
 87 |         let training_data = Dataset::from_data((0usize..10).map(|_| data.clone()).collect_vec());
 88 | 
 89 |         let trained_model = train_model(training_data.get_data(), &TrainingOptions::new());
 90 |         let untrained_model = train_model(&Vec::new(), &TrainingOptions::default());
 91 | 
 92 |         let compressed = trained_model.compress(&data);
 93 |         let compressed_no_dict = untrained_model.compress(&data);
 94 | 
 95 |         println!("Compressed size with dict: {}, without dict: {}", compressed.len(), compressed_no_dict.len());
 96 | 
 97 |         assert!(compressed.len() < compressed_no_dict.len());
 98 |     }
 99 | }
100 | 


--------------------------------------------------------------------------------
/src/backend/ensemble_model.rs:
--------------------------------------------------------------------------------
  1 | 
  2 | use rayon::prelude::*;
  3 | use rusqlite::Connection;
  4 | 
  5 | use crate::backend::clm_model::ClmModel;
  6 | use crate::backend::dataset::Dataset;
  7 | use crate::backend::Token;
  8 | use crate::backend::trainer::train_model;
  9 | use crate::backend::training_options::TrainingOptions;
 10 | 
 11 | pub struct EnsembleModel<'a> {
 12 |     models: Vec<ClmModel<'a>>,
 13 | }
 14 | 
 15 | impl EnsembleModel<'_> {
 16 |     pub fn train(data: Dataset, options: &TrainingOptions) -> Self {
 17 | 
 18 |         if data.get_data().is_empty() {
 19 |             let models : Vec<ClmModel> = (0..options.ensemble_size).map(|_| ClmModel::from_buffer(vec![])).collect();
 20 |             return EnsembleModel { models: models };
 21 |         }
 22 | 
 23 |         let chunks  =
 24 |             Box::new(data.split_into_chunks(options.ensemble_size));
 25 |         
 26 | 
 27 |         let progress_bar = indicatif::ProgressBar::new(options.ensemble_size as u64);
 28 | 
 29 | 
 30 |         // train a model for each chunk
 31 |         let models = chunks.par_bridge().map(|chunk| {
 32 |             let model = train_model(chunk.get_data(), options);
 33 |             progress_bar.inc(1);
 34 |             model
 35 |         }).collect();
 36 | 
 37 |         progress_bar.finish();
 38 | 
 39 |         EnsembleModel { models }
 40 |     }
 41 | 
 42 |     pub fn save_checkpoint(&self, path: &str) {
 43 |         // delete the file if it already exists
 44 | 
 45 |         if std::path::Path::new(path).exists() {
 46 |             println!("Model checkpoint already exists, deleting it");
 47 |             std::fs::remove_file(path).unwrap();
 48 |         }
 49 | 
 50 |         // write all models to a sqlite database
 51 |         let conn = Connection::open(path).unwrap();
 52 |         conn.execute("CREATE TABLE models (id INTEGER PRIMARY KEY, model BLOB)", []).unwrap();
 53 | 
 54 |         for (i, model) in self.models.iter().enumerate() {
 55 |             let mut stmt = conn.prepare("INSERT INTO models (id, model) VALUES (?, ?)").unwrap();
 56 |             let model_bytes = model.to_buffer();
 57 |             stmt.execute((&i, &model_bytes)).unwrap();
 58 |         }
 59 | 
 60 |         conn.close().unwrap();
 61 |     }
 62 | 
 63 |     pub fn from_checkpoint(path: &str) -> Self {
 64 |         // read all models from a sqlite database
 65 |         let  conn = Connection::open(path).unwrap();
 66 |         let models;
 67 |         {
 68 |             let mut stmt = conn.prepare("SELECT model FROM models").unwrap();
 69 |             models = stmt.query_map([], |row| {
 70 |                 let model_bytes: Vec<u8> = row.get(0)?;
 71 |                 Ok(ClmModel::from_buffer(model_bytes))
 72 |             }).unwrap().map(|x| x.unwrap()).collect();
 73 |         }
 74 |         conn.close().unwrap();
 75 |         EnsembleModel { models }
 76 |     }
 77 | 
 78 |     pub fn compressed_size(&self, tokens: &Vec<Token>) -> f64 {
 79 |         let mut total_size = 0.0;
 80 |         for model in &self.models {
 81 |             total_size += model.compress(tokens).len() as f64;
 82 |         }
 83 |         total_size / self.models.len() as f64
 84 |     }
 85 | }
 86 | 
 87 | #[cfg(test)]
 88 | mod tests {
 89 |     use crate::backend::dataset::Dataset;
 90 |     use crate::backend::ensemble_model::EnsembleModel;
 91 |     use crate::backend::training_options::TrainingOptions;
 92 | 
 93 |     #[test]
 94 |     fn training_works() {
 95 |         let dataset = Dataset::test_dataset();
 96 | 
 97 |         let mut options = TrainingOptions::default();
 98 |         options.ensemble_size = 2;
 99 |         let trained_model = EnsembleModel::train(dataset, &options);
100 | 
101 |         assert_eq!(trained_model.models.len(), 2);
102 |     }
103 | 
104 |     #[test]
105 |     fn compressed_size_improves() {
106 |         let dataset = Dataset::test_dataset();
107 | 
108 |         let mut options = TrainingOptions::default();
109 |         options.ensemble_size = 2;
110 |         let trained_model = EnsembleModel::train(dataset.clone(), &options);
111 |         let untrained_model = EnsembleModel::train(Dataset::empty(), &options);
112 | 
113 |         let trained_size = trained_model.compressed_size(&dataset.get_data()[0]);
114 |         let naive_size = untrained_model.compressed_size(&dataset.get_data()[0]);
115 | 
116 |         println!("Trained size: {}", trained_size);
117 |         println!("Naive size: {}", naive_size);
118 |         assert!(trained_size < naive_size);
119 |     }
120 | 
121 |     #[test]
122 |     fn save_and_load_ensemble_model() {
123 |         let dataset = Dataset::test_dataset();
124 | 
125 |         let mut options = TrainingOptions::default();
126 |         options.ensemble_size = 2;
127 |         let trained_model = EnsembleModel::train(dataset, &options);
128 | 
129 |         trained_model.save_checkpoint("test.ensemble");
130 | 
131 |         let loaded_model = EnsembleModel::from_checkpoint("test.ensemble");
132 | 
133 |         assert_eq!(trained_model.models.len(), loaded_model.models.len());
134 |         assert_eq!(trained_model.models[0].get_dictionary_size(), loaded_model.models[0].get_dictionary_size());
135 | 
136 |         // clean up
137 |         std::fs::remove_file("test.ensemble").unwrap();
138 |     }
139 | }
140 | 
141 | 


--------------------------------------------------------------------------------
/style/main.scss:
--------------------------------------------------------------------------------
  1 | * {
  2 |   box-sizing: border-box;
  3 |   margin: 0;
  4 |   padding: 0;
  5 |   font-family: Helvetica, sans-serif;
  6 |   font-size: 16px;
  7 |   -webkit-font-smoothing: antialiased;
  8 | }
  9 | 
 10 | :root {
 11 |   color-scheme: dark;
 12 | 
 13 |   --max-content-width: 75rem;
 14 |   --content-side-padding: 5rem;
 15 | 
 16 |   --color-background: #212121;
 17 |   --color-container: #2f2f2f;
 18 |   --color-container-hover: #424242;
 19 |   --color-container-border: #e3e3e3;
 20 |   --color-text: #ececec;
 21 |   --color-text-secondary: #b4b4b4;
 22 |   --color-button: #676767;
 23 | }
 24 | 
 25 | html {
 26 |   font-size: 10px;
 27 | }
 28 | 
 29 | body {
 30 |   background-color: var(--color-background);
 31 | }
 32 | 
 33 | main {
 34 |   display: grid;
 35 |   grid-template-rows: max-content 1fr max-content;
 36 |   height: 100vh;
 37 | }
 38 | 
 39 | .hidden {
 40 |   display: none;
 41 | }
 42 | 
 43 | .navbar {
 44 |   width: 100%;
 45 |   background-color: var(--color-background);
 46 |   display: block;
 47 |   padding: 1rem var(--content-side-padding);
 48 | }
 49 | 
 50 | .prompt {
 51 |   width: 100%;
 52 |   display: grid;
 53 |   justify-items: center;
 54 |   padding: 1rem var(--content-side-padding) 3rem var(--content-side-padding);
 55 |   background-color: var(--color-background);
 56 | 
 57 |   &__wrapper {
 58 |     width: 100%;
 59 |     max-width: var(--max-content-width);
 60 |   }
 61 | 
 62 |   &__disclaimer {
 63 |     display: block;
 64 |     text-align: center;
 65 |     padding-top: 1rem;
 66 |     color: var(--color-text-secondary);
 67 |     font-size: 1.2rem;
 68 |   }
 69 | }
 70 | 
 71 | .prompt_input {
 72 |   padding: 0 0 0 2rem;
 73 |   border-radius: 2.3rem;
 74 |   background-color: var(--color-container);
 75 | 
 76 |   &__textarea_wrapper {
 77 |     padding: 0.5rem;
 78 |     display: grid;
 79 |     grid-template-columns: 1fr 4rem;
 80 |     gap: 1rem;
 81 |     align-items: end;
 82 |     position: relative;
 83 |   }
 84 | 
 85 |   &__textarea {
 86 |     width: 100%;
 87 |     border: none;
 88 |     outline: none;
 89 |     background: none;
 90 |     resize: none;
 91 |     padding: 0.75rem 0;
 92 |     line-height: 25px;
 93 |     display: block;
 94 |     color: var(--color-text);
 95 |     overflow-x: hidden;
 96 |     overflow-y: auto;
 97 |     max-height: 30vh;
 98 |   }
 99 | 
100 |   &__textarea_placeholder {
101 |     position: absolute;
102 |     color: var(--color-text-secondary);
103 |     bottom: 1.6rem;
104 |     pointer-events: none;
105 |   }
106 | 
107 |   &__send_button {
108 |     width: 4rem;
109 |     height: 4rem;
110 |     border: none;
111 |     border-radius: 50%;
112 |     background-color: var(--color-button);
113 |     color: var(--color-container);
114 |     display: grid;
115 |     justify-items: center;
116 |     padding-top: 0.9rem;
117 |     cursor: pointer;
118 |     user-select: none;
119 |     font-weight: bold;
120 |     outline: none;
121 | 
122 |     &:hover, &:focus {
123 |       background-color: var(--color-text);
124 |     }
125 |   }
126 | }
127 | 
128 | .chat {
129 |   width: 100%;
130 |   padding: 4rem var(--content-side-padding);
131 |   display: flex;
132 |   flex-direction: column;
133 |   align-items: center;
134 |   gap: 2rem;
135 |   overflow-y: auto;
136 | }
137 | 
138 | .chat_message {
139 |   width: 100%;
140 |   max-width: var(--max-content-width);
141 |   display: grid;
142 | 
143 |   p {
144 |     line-height: 1.5;
145 |     color: var(--color-text);
146 |   }
147 | 
148 |   &__bubble {
149 |     padding: 1.5rem 2rem;
150 |     max-width: 70%;
151 |     justify-self: end;
152 |     border-radius: 2.5rem;
153 |     background-color: var(--color-container);
154 |   }
155 | 
156 |   &__icon {
157 |     width: 3.5rem;
158 |     height: 3.5rem;
159 |     border-radius: 50%;
160 |     border: 0.2rem solid var(--color-container);
161 |     padding: 0.5rem;
162 |     display: grid;
163 |     justify-items: center;
164 | 
165 |     & > div {
166 |       font-size: 4.5rem;
167 |       margin-top: -.5rem;
168 |       margin-left: 0.15rem;
169 |       text-align: center;
170 |       color: var(--color-text-secondary);
171 |     }
172 |   }
173 | }
174 | 
175 | .chat_message--machine {
176 |   grid-template-columns: 3.5rem 1fr;
177 |   gap: 2rem;
178 | 
179 |   & > p {
180 |     padding-top: 0.5rem;
181 |   }
182 | }
183 | 
184 | .dropdown {
185 |   position: relative;
186 | 
187 |   &__selected {
188 |     width: max-content;
189 |     padding: 1.5rem 2.5rem;
190 |     border-radius: 1rem;
191 |     background-color: var(--color-container);
192 |     color: var(--color-text);
193 |     user-select: none;
194 |     cursor: pointer;
195 |     display: flex;
196 |     gap: 3rem;
197 |   }
198 | 
199 |   &__icon {
200 |     color: var(--color-text-secondary);
201 |     transform: scaleX(1.5) rotate(90deg);
202 |   }
203 | 
204 |   &__options {
205 |     width: max-content;
206 |     min-width: 250px;
207 |     background-color: var(--color-container);
208 |     position: absolute;
209 |     z-index: 3;
210 |     top: 5.5rem;
211 |     color: var(--color-text);
212 |     border-radius: 1rem;
213 |     padding: 1rem;
214 |   }
215 | 
216 |   &__option {
217 |     padding: 1.5rem 2rem;
218 |     border-radius: 0.5rem;
219 |     cursor: pointer;
220 |     user-select: none;
221 | 
222 |     &:hover {
223 |       background-color: var(--color-container-hover);
224 |     }
225 |   }
226 | }
227 | 


--------------------------------------------------------------------------------
/src/backend/evaluation.rs:
--------------------------------------------------------------------------------
  1 | use num::Num;
  2 | use num::pow::Pow;
  3 | use rand::Rng;
  4 | 
  5 | use crate::backend::clm_model::ClmModel;
  6 | use crate::backend::dataset::Dataset;
  7 | use crate::backend::{MAX_TOKEN, Token};
  8 | 
  9 | const SAMPLES: usize = 20000;
 10 | 
 11 | 
 12 | fn average_with_error(values: Vec<f64>) -> (f64, f64){
 13 |     let average = values.iter().sum::<f64>() / values.len() as f64;
 14 |     let standard_deviation = values.iter().map(|x| (x - values.iter().sum::<f64>() / values.len() as f64).pow(2)).sum::<f64>() / values.len() as f64;
 15 |     let standard_error = standard_deviation / (values.len() as f64).sqrt();
 16 | 
 17 |     (average, standard_error)
 18 | }
 19 | 
 20 | fn fmax<T: Num + PartialOrd>(a: T, b: T) -> T {
 21 |     if a > b {
 22 |         a
 23 |     } else {
 24 |         b
 25 |     }
 26 | }
 27 | 
 28 | impl<'a> ClmModel<'a> {
 29 |     pub fn average_bytes_per_token(&self, test_data: &Dataset) -> (f64, f64) {
 30 |         let mut values = Vec::new();
 31 | 
 32 |         let mut rng = rand::thread_rng();
 33 |         for _ in 0..SAMPLES {
 34 |             let sentence_index = rng.gen_range(0..test_data.get_data().len());
 35 |             let sentence = &test_data.get_data()[sentence_index];
 36 |             if sentence.len() < 7 {
 37 |                 continue;
 38 |             }
 39 | 
 40 |             let pos = rng.gen_range(5..sentence.len());
 41 | 
 42 |             let prompt = sentence[..pos].to_vec();
 43 |             let next_token = sentence[pos];
 44 |             let compressed_prompt = self.compress(&prompt);
 45 |             let compressed = self.compress_together(&prompt, &vec![next_token]);
 46 |             values.push(compressed as f64 - compressed_prompt.len() as f64)
 47 |         }
 48 | 
 49 |         average_with_error(values)
 50 |     }
 51 | 
 52 |     pub fn average_information_gain(&self, test_data: &Dataset) -> (f64, f64){
 53 |         let mut values = Vec::new();
 54 | 
 55 |         let mut rng = rand::thread_rng();
 56 |         for _ in 0..SAMPLES {
 57 |             let sentence_index = rng.gen_range(0..test_data.get_data().len());
 58 |             let sentence = &test_data.get_data()[sentence_index];
 59 |             if sentence.len() < 7 {
 60 |                 continue;
 61 |             }
 62 | 
 63 |             let pos = rng.gen_range(5..sentence.len());
 64 | 
 65 |             let prompt = sentence[..pos].to_vec();
 66 |             let next_token = sentence[pos];
 67 |             let random_token = rng.gen_range(0..MAX_TOKEN) as Token;
 68 | 
 69 |             let compressed_prompt = self.compress(&prompt).len() as f64;
 70 |             let compressed_truth = self.compress_together(&prompt, &vec![next_token]) as f64;
 71 |             let compressed_random = self.compress_together(&prompt, &vec![random_token]) as f64;
 72 | 
 73 |             let truth_bytes_added = compressed_truth - compressed_prompt;
 74 |             let random_bytes_added = compressed_random - compressed_prompt;
 75 | 
 76 |             values.push(random_bytes_added/fmax(0.1, truth_bytes_added));
 77 |         }
 78 | 
 79 |         average_with_error(values)
 80 |     }
 81 | }
 82 | 
 83 | #[cfg(test)]
 84 | mod tests {
 85 |     use crate::backend::dataset::Dataset;
 86 |     use crate::backend::tests::random_tokens;
 87 |     use crate::backend::trainer::train_model;
 88 |     use crate::backend::training_options::TrainingOptions;
 89 | 
 90 |     #[test]
 91 |     fn average_bytes_per_token_improves() {
 92 |         let random_data = random_tokens(300);
 93 |         let training_data = Dataset::from_data((0..10).map(|_| random_data.clone()).collect());
 94 | 
 95 |         let model = train_model(&Vec::new(), &TrainingOptions::new());
 96 |         let (initial_avg,_) = model.average_bytes_per_token(&training_data);
 97 | 
 98 |         let trained_model = train_model(&training_data.get_data(), &TrainingOptions::new());
 99 |         let (trained_avg, _) = trained_model.average_bytes_per_token(&training_data);
100 | 
101 |         assert!(trained_avg < initial_avg);
102 | 
103 |         println!("Initial avg: {}", initial_avg);
104 |         println!("Trained avg: {}", trained_avg);
105 |     }
106 | 
107 |     #[test]
108 |     fn untrained_model_has_average_information_gain_of_one() {
109 |         let random_data = random_tokens(300);
110 |         let testing_data = Dataset::from_data((0..10).map(|_| random_data.clone()).collect());
111 | 
112 |         let model = train_model(&Vec::new(), &TrainingOptions::new());
113 |         let (initial_avg, initial_stderr) = model.average_information_gain(&testing_data);
114 | 
115 |         // 99%  confidence interval
116 |         let confidence_interval = 2.576 * initial_stderr;
117 |         println!("Initial avg: {}", initial_avg);
118 |         println!("Standard error: {}", initial_stderr);
119 |         println!("Confidence interval: [{} – {}]", initial_avg - confidence_interval, initial_avg + confidence_interval);
120 | 
121 |         assert!(0.99f64 <= initial_avg + confidence_interval);
122 |         assert!(1.01f64 >= initial_avg - confidence_interval);
123 | 
124 |     }
125 |     #[test]
126 |     fn average_information_gain_is_greater_than_one() {
127 |         let random_data = random_tokens(100);
128 |         let training_data = Dataset::from_data((0..8).map(|_| random_data.clone()).collect());
129 | 
130 |         let model = train_model(&Vec::new(), &TrainingOptions::new());
131 |         let (initial_avg,_) = model.average_information_gain(&training_data);
132 | 
133 |         let trained_model = train_model(&training_data.get_data(), &TrainingOptions::new());
134 |         let (trained_avg, trained_stderr) = trained_model.average_information_gain(&training_data);
135 | 
136 |         assert!(trained_avg > 1f64);
137 | 
138 |         println!("Initial avg: {}", initial_avg);
139 |         println!("Trained avg: {}", trained_avg);
140 |         println!("Trained stderr: {}", trained_stderr);
141 |     }
142 | }


--------------------------------------------------------------------------------
/src/backend/clm_model.rs:
--------------------------------------------------------------------------------
  1 | use std::fs::File;
  2 | use std::io::{Read, Write};
  3 | 
  4 | use itertools::Itertools;
  5 | use rand::prelude::SliceRandom;
  6 | use rand::thread_rng;
  7 | use rayon::iter::{ParallelBridge, ParallelIterator};
  8 | use zstd::dict::{DecoderDictionary, EncoderDictionary};
  9 | 
 10 | use crate::backend;
 11 | use crate::backend::{INFERENCE_COMPRESSION_LEVEL, MAX_TOKEN, Token};
 12 | use crate::backend::tokenizer::ClmTokenizer;
 13 | 
 14 | pub struct ClmModel<'a> {
 15 |     dict: EncoderDictionary<'a>,
 16 |     model_buffer: Vec<u8>,
 17 |     pub(crate) tokenizer: ClmTokenizer,
 18 | }
 19 | 
 20 | impl Clone for ClmModel<'_> {
 21 |     fn clone(&self) -> Self {
 22 |         let dict = EncoderDictionary::copy(&*self.model_buffer, INFERENCE_COMPRESSION_LEVEL);
 23 |         let tokenizer = self.tokenizer.clone();
 24 |         Self { dict, model_buffer: self.model_buffer.clone(), tokenizer }
 25 |     }
 26 | }
 27 | 
 28 | 
 29 | impl<'a> ClmModel<'a> {
 30 |     pub fn from_buffer(model_buffer: Vec<u8>) -> Self {
 31 |         let dict = EncoderDictionary::copy(&*model_buffer, INFERENCE_COMPRESSION_LEVEL);
 32 |         let tokenizer = ClmTokenizer::new_custom();
 33 |         Self { dict, model_buffer, tokenizer }
 34 |     }
 35 | 
 36 |     pub fn to_buffer(&self) -> Vec<u8> {
 37 |         self.model_buffer.clone()
 38 |     }
 39 | 
 40 |     pub fn compress(&self, tokens: &Vec<Token>) -> Vec<u8> {
 41 |         let raw_data = backend::tokens_to_bytes(tokens);
 42 | 
 43 |         // Actual compression
 44 |         let mut writer = zstd::stream::write::Encoder::with_prepared_dictionary(Vec::new(), &self.dict).unwrap();
 45 |         writer.write_all(&raw_data).unwrap();
 46 |         let compressed = writer.finish().unwrap();
 47 |         compressed
 48 |     }
 49 | 
 50 |     pub fn predict_next(&self, prompt: String, depth: usize, width: usize) -> String {
 51 |         let mut tokens = self.tokenizer.encode(&prompt);
 52 |         let (next_token, _size) = self.predict_tokens(&tokens, depth, width);
 53 |         tokens.push(next_token);
 54 |         self.tokenizer.decode(tokens)
 55 |     }
 56 | 
 57 |     fn get_next_token_sizes(&self, tokens: &Vec<Token>) -> Vec<(Token, usize)> {
 58 |         (1..MAX_TOKEN)
 59 |             .par_bridge()
 60 |             .map(|x| x as Token)
 61 |             .map(|x| {
 62 |                 let mut prompt = tokens.clone();
 63 |                 prompt.push(x);
 64 |                 (x, self.compress(&prompt).len())
 65 |             }).collect()
 66 |     }
 67 | 
 68 |     fn predict_tokens(&self, tokens: &Vec<Token>, depth: usize, width: usize) -> (Token, usize) {
 69 |         let mut c = self.get_next_token_sizes(tokens);
 70 | 
 71 |         c.shuffle(&mut thread_rng());
 72 | 
 73 |         c.sort_by(|a, b| a.1.cmp(&b.1));
 74 | 
 75 |         let mut hist = c.iter().map(|x| x.1).counts().iter().map(|(k, v)| (*k, *v)).collect::<Vec<_>>();
 76 |         hist.sort_by(|a, b| a.1.cmp(&b.1));
 77 | 
 78 |         println!("Predicting tokens: {:?}", hist);
 79 | 
 80 |         if depth == 0 {
 81 |             return c[0];
 82 |         }
 83 | 
 84 |         let mut best = (0, std::usize::MAX);
 85 |         for (token, _) in c.iter().take(width) {
 86 |             let (_next_token, next_compression) = self.predict_tokens(tokens, depth - 1, width / 2);
 87 |             if next_compression < best.1 {
 88 |                 best = (*token, next_compression);
 89 |             }
 90 |         }
 91 |         best
 92 |     }
 93 | 
 94 |     pub fn decompress_to_tokens(&self, compressed: &[u8]) -> Vec<Token> {
 95 |         let dict = DecoderDictionary::copy(self.model_buffer.as_slice());
 96 |         let mut reader = zstd::stream::read::Decoder::with_prepared_dictionary(compressed, &dict).unwrap();
 97 | 
 98 |         let mut decompressed = Vec::new();
 99 |         reader.read_to_end(&mut decompressed).unwrap();
100 | 
101 |         let mut tokens = Vec::new();
102 |         for i in decompressed.chunks(backend::BYTES_PER_TOKEN) {
103 |             if backend::BYTES_PER_TOKEN == 1 {
104 |                 tokens.push(Token::from_be_bytes([i[0]]));
105 |             } else if backend::BYTES_PER_TOKEN == 8 {
106 |                 todo!("Implement 8 byte tokens")
107 |                 //tokens.push(Token::from_be_bytes([i[0], i[1], i[2], i[3], i[4], i[5], i[6], i[7]]));
108 |             }
109 |         }
110 | 
111 |         tokens
112 |     }
113 | 
114 |     pub fn evaluate(&self, test_data: &Vec<Vec<Token>>) -> f64 {
115 |         let mut total = 0;
116 |         let mut correct = 0;
117 |         for tokens in test_data {
118 |             let compressed = self.compress(tokens);
119 |             let decompressed = self.decompress_to_tokens(&compressed);
120 |             total += tokens.len();
121 |             correct += tokens.iter().zip(decompressed.iter()).filter(|(a, b)| a == b).count();
122 |         }
123 |         correct as f64 / total as f64
124 |     }
125 | 
126 |     pub fn get_dictionary_size(&self) -> usize {
127 |         self.model_buffer.len()
128 |     }
129 | 
130 |     #[allow(dead_code)]
131 |     pub(crate) fn save_checkpoint(&self, path: &str) {
132 |         // write the buffer as Vec<u8> to a flat file
133 |         let mut file = File::create(path).unwrap();
134 |         file.write_all(&self.model_buffer).unwrap();
135 |         file.flush().unwrap();
136 |     }
137 | 
138 |     pub(crate) fn from_checkpoint(path: &str) -> ClmModel<'a> {
139 |         // read the buffer from a flat file and return a new ClmModel
140 |         let mut file = File::open(path).unwrap();
141 |         let mut buffer = Vec::new();
142 |         file.read_to_end(&mut buffer).unwrap();
143 |         ClmModel::from_buffer(buffer)
144 |     }
145 | 
146 |     pub(crate) fn compress_together(&self, prompt: &Vec<Token>, next: &Vec<Token>) -> usize {
147 |         let mut prompt = prompt.clone();
148 |         prompt.extend(next);
149 |         self.compress(&prompt).len()
150 |     }
151 | }
152 | 


--------------------------------------------------------------------------------
/src/tuning.rs:
--------------------------------------------------------------------------------
  1 | use indicatif::ProgressIterator;
  2 | use rand::seq::SliceRandom;
  3 | use rayon::prelude::*;
  4 | use serde::{Deserialize, Serialize};
  5 | 
  6 | use chatclm::backend::dataset::Dataset;
  7 | use chatclm::backend::ensemble_model::EnsembleModel;
  8 | use chatclm::backend::MAX_TOKEN;
  9 | use chatclm::backend::Token;
 10 | use chatclm::backend::training_options::TrainingOptions;
 11 | 
 12 | #[derive(Serialize, Deserialize)]
 13 | #[serde(rename_all = "camelCase")]
 14 | struct TuningParameters {
 15 |     d: u32,
 16 |     f: u32,
 17 |     k: f64,
 18 |     compression_level: u32,
 19 |     dataset_size: u32,
 20 |     dictionary_size_percentage: f64,
 21 | }
 22 | 
 23 | #[derive(Serialize, Deserialize)]
 24 | struct TuningMetrics {
 25 |     val_bpt: f64,
 26 |     val_bpt_stderr: f64,
 27 |     train_bpt: f64,
 28 |     train_pbt_stderr: f64,
 29 |     val_inf_gain: f64,
 30 |     val_inf_gain_stderr: f64,
 31 |     train_inf_gain: f64,
 32 |     train_inf_gain_stderr: f64,
 33 |     training_time: f64,
 34 |     dictionary_size: usize,
 35 | }
 36 | /*
 37 | fn main() {
 38 | 
 39 |     // read the dataset
 40 |     let dataset = Dataset::load_or_compute("dataset.checkpoint");
 41 | 
 42 |     // read the parameters from stdin
 43 |     let mut input = String::new();
 44 |     std::io::stdin().read_line(&mut input).unwrap();
 45 |     let params: TuningParameters = serde_json::from_str(&input).unwrap();
 46 | 
 47 | 
 48 |     let (train, test) = dataset.split_train_test(0.9);
 49 |     
 50 |     let shrunk_train = dataset.shrink_to_size(params.dataset_size as usize);
 51 |     
 52 |     
 53 |     // train the model
 54 |     let start_time = std::time::Instant::now();
 55 |     let default_params = TrainingOptions::default();
 56 |     let training_paramters = TrainingOptions {
 57 |         accel: default_params.accel,
 58 |         d: params.d,
 59 |         f: params.f,
 60 |         k: params.k as u32,
 61 |         shrink_dict: default_params.shrink_dict,
 62 |         shrink_dict_max_regression: default_params.shrink_dict_max_regression,
 63 |         split_point: default_params.split_point,
 64 |         steps: default_params.steps,
 65 |         nb_threads: 8,
 66 |         compression_level: params.compression_level,
 67 |         dictionary_size_percentage: params.dictionary_size_percentage, 
 68 |         ensemble_size: 1
 69 |     };
 70 | 
 71 |     let model = train_model(train.get_data(), &training_paramters);
 72 | 
 73 |     let time = start_time.elapsed().as_secs_f64();
 74 | 
 75 |     // evaluate the model
 76 |     let val_accuracy = model.average_bytes_per_token(&test);
 77 |     let train_accuracy = model.average_bytes_per_token(&shrunk_train);
 78 | 
 79 |     let val_inf_gain = model.average_information_gain(&test);
 80 |     let train_inf_gain = model.average_information_gain(&shrunk_train);
 81 | 
 82 |     // write the metrics to stdout
 83 |     let metrics = TuningMetrics {
 84 |         val_bpt: val_accuracy.0,
 85 |         val_bpt_stderr: val_accuracy.1,
 86 |         train_bpt: train_accuracy.0,
 87 |         train_pbt_stderr: train_accuracy.1,
 88 |         val_inf_gain: val_inf_gain.0,
 89 |         val_inf_gain_stderr: val_inf_gain.1,
 90 |         train_inf_gain: train_inf_gain.0,
 91 |         train_inf_gain_stderr: train_inf_gain.1,
 92 |         dictionary_size: model.get_dictionary_size(),
 93 |         training_time: time
 94 |     };
 95 |     
 96 |     // flush stdout
 97 |     io::stdout().flush().unwrap();
 98 |     let output = serde_json::to_string(&metrics).unwrap();
 99 |     println!("{}", output);
100 |     io::stdout().flush().unwrap();
101 | }*/
102 | 
103 | fn main() {
104 |     rayon::ThreadPoolBuilder::new().num_threads(8).build_global().unwrap();
105 | 
106 | 
107 |     let retrain = false;
108 |     let trained_model = if retrain {
109 |         println!("Reading dataset");
110 |         let dataset = Dataset::load_or_compute("dataset.checkpoint");
111 |         println!("Training models");
112 | 
113 |         let mut options = TrainingOptions::default();
114 |         options.ensemble_size = 20;
115 |         let trained_model = EnsembleModel::train(dataset, &options);
116 |         trained_model.save_checkpoint("ensemble.checkpoint");
117 |         trained_model
118 |     } else {
119 |         EnsembleModel::from_checkpoint("ensemble.checkpoint")
120 |     };
121 | 
122 | 
123 |     // Tokenize "The quick brown fox jumps over the lazy dog"
124 |     let prompt = "The quick brown fox jumps over the lazy";
125 |     let mut prompt_tokens = Dataset::tokenize(prompt);
126 |     let mut size_before = trained_model.compressed_size(&prompt_tokens);
127 | 
128 |     for _ in 0..50 {
129 |         println!("Prompt: {} ", Dataset::detokenize(prompt_tokens.clone()).as_str());
130 | 
131 |         let mut sizes: Vec<(Token, f64)> = (0..(Dataset::get_tokenizer().get_max_token()+1)).progress().par_bridge().map(|next_token| {
132 |             let mut tokens = prompt_tokens.clone();
133 |             tokens.push(next_token as Token);
134 |             let size = trained_model.compressed_size(&tokens);
135 |             println!("Size: {}", size);
136 |             (next_token as Token, 1.0 / (size - size_before))
137 |         }).collect();
138 | 
139 |         //shuffle
140 |         sizes.shuffle(&mut rand::thread_rng());
141 | 
142 |         // sort descending
143 |         sizes.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap());
144 | 
145 |         // take only the first 200 tokens
146 |         sizes.truncate(10);
147 | 
148 |         // subtract the mimimum likelihood
149 |         let min_size = sizes.iter().map(|(_token, size)| *size).min_by(|a, b| a.partial_cmp(b).unwrap()).unwrap();
150 |         sizes.iter_mut().for_each(|(_token, size)| *size -= min_size);
151 | 
152 |         // normalize the distribution
153 |         let sum: f64 = sizes.iter().map(|(_token, size)| *size).sum();
154 |         sizes.iter_mut().for_each(|(_token, size)| *size /= sum);
155 | 
156 |         // print the sum of all sizes
157 |         let sum: f64 = sizes.iter().map(|(_token, size)| *size).sum();
158 |         println!("Sum: {}", sum);
159 | 
160 |         // print the ten smallest sizes
161 |         for (token, size) in sizes.iter().take(10) {
162 |             println!("Token: '{}' ({}), Size: {}", Dataset::detokenize(vec![*token]), *token as u64,  size);
163 |         }
164 | 
165 | 
166 |         // choose a token with probability proportional to the size
167 |         let mut rng = rand::thread_rng();
168 |         let (next_token, next_size) = sizes.choose_weighted(&mut rng, |(_token, size)| *size).unwrap();
169 |         prompt_tokens.push(*next_token);
170 |         size_before = *next_size;
171 |     }
172 | }


--------------------------------------------------------------------------------
/tokenizer.json:
--------------------------------------------------------------------------------
  1 | {
  2 |   "version": "1.0",
  3 |   "truncation": null,
  4 |   "padding": null,
  5 |   "added_tokens": [
  6 |     {
  7 |       "id": 0,
  8 |       "content": "[UNK]",
  9 |       "single_word": false,
 10 |       "lstrip": false,
 11 |       "rstrip": false,
 12 |       "normalized": false,
 13 |       "special": true
 14 |     }
 15 |   ],
 16 |   "normalizer": {
 17 |     "type": "Sequence",
 18 |     "normalizers": [
 19 |       {
 20 |         "type": "Lowercase"
 21 |       },
 22 |       {
 23 |         "type": "NFD"
 24 |       },
 25 |       {
 26 |         "type": "StripAccents"
 27 |       }
 28 |     ]
 29 |   },
 30 |   "pre_tokenizer": {
 31 |     "type": "Sequence",
 32 |     "pretokenizers": [
 33 |       {
 34 |         "type": "WhitespaceSplit"
 35 |       }
 36 |     ]
 37 |   },
 38 |   "post_processor": null,
 39 |   "decoder": {
 40 |     "type": "BPEDecoder",
 41 |     "suffix": "</w>"
 42 |   },
 43 |   "model": {
 44 |     "type": "BPE",
 45 |     "dropout": null,
 46 |     "unk_token": "[UNK]",
 47 |     "continuing_subword_prefix": null,
 48 |     "end_of_word_suffix": "</w>",
 49 |     "fuse_unk": false,
 50 |     "byte_fallback": false,
 51 |     "ignore_merges": false,
 52 |     "vocab": {
 53 |       "[UNK]": 0,
 54 |       "!": 1,
 55 |       "\"": 2,
 56 |       "$": 3,
 57 |       "%": 4,
 58 |       "&": 5,
 59 |       "'": 6,
 60 |       "(": 7,
 61 |       ")": 8,
 62 |       "*": 9,
 63 |       "+": 10,
 64 |       ",": 11,
 65 |       "-": 12,
 66 |       ".": 13,
 67 |       "/": 14,
 68 |       "0": 15,
 69 |       "1": 16,
 70 |       "2": 17,
 71 |       "3": 18,
 72 |       "4": 19,
 73 |       "5": 20,
 74 |       "6": 21,
 75 |       "7": 22,
 76 |       "8": 23,
 77 |       "9": 24,
 78 |       ":": 25,
 79 |       ";": 26,
 80 |       "<": 27,
 81 |       ">": 28,
 82 |       "?": 29,
 83 |       "`": 30,
 84 |       "a": 31,
 85 |       "b": 32,
 86 |       "c": 33,
 87 |       "d": 34,
 88 |       "e": 35,
 89 |       "f": 36,
 90 |       "g": 37,
 91 |       "h": 38,
 92 |       "i": 39,
 93 |       "j": 40,
 94 |       "k": 41,
 95 |       "l": 42,
 96 |       "m": 43,
 97 |       "n": 44,
 98 |       "o": 45,
 99 |       "p": 46,
100 |       "q": 47,
101 |       "r": 48,
102 |       "s": 49,
103 |       "t": 50,
104 |       "u": 51,
105 |       "v": 52,
106 |       "w": 53,
107 |       "x": 54,
108 |       "y": 55,
109 |       "z": 56,
110 |       "£": 57,
111 |       "¦": 58,
112 |       "«": 59,
113 |       "®": 60,
114 |       "°": 61,
115 |       "²": 62,
116 |       "´": 63,
117 |       "·": 64,
118 |       "»": 65,
119 |       "½": 66,
120 |       "¾": 67,
121 |       "×": 68,
122 |       "ð": 69,
123 |       "ø": 70,
124 |       "˜": 71,
125 |       "​": 72,
126 |       "‐": 73,
127 |       "‑": 74,
128 |       "–": 75,
129 |       "—": 76,
130 |       "‘": 77,
131 |       "’": 78,
132 |       "‚": 79,
133 |       "“": 80,
134 |       "”": 81,
135 |       "•": 82,
136 |       "…": 83,
137 |       "€": 84,
138 |       "™": 85,
139 |       "": 86,
140 |       "e</w>": 87,
141 |       ".</w>": 88,
142 |       "y</w>": 89,
143 |       "!</w>": 90,
144 |       "4</w>": 91,
145 |       "a</w>": 92,
146 |       ",</w>": 93,
147 |       "”</w>": 94,
148 |       "8</w>": 95,
149 |       "d</w>": 96,
150 |       "n</w>": 97,
151 |       "r</w>": 98,
152 |       "h</w>": 99,
153 |       "s</w>": 100,
154 |       "k</w>": 101,
155 |       "?</w>": 102,
156 |       "g</w>": 103,
157 |       "i</w>": 104,
158 |       "%</w>": 105,
159 |       "q</w>": 106,
160 |       ")</w>": 107,
161 |       "c</w>": 108,
162 |       "’</w>": 109,
163 |       "t</w>": 110,
164 |       "p</w>": 111,
165 |       "0</w>": 112,
166 |       "o</w>": 113,
167 |       ";</w>": 114,
168 |       "\"</w>": 115,
169 |       "m</w>": 116,
170 |       "w</w>": 117,
171 |       "l</w>": 118,
172 |       ":</w>": 119,
173 |       "v</w>": 120,
174 |       "-</w>": 121,
175 |       "1</w>": 122,
176 |       "5</w>": 123,
177 |       "6</w>": 124,
178 |       "…</w>": 125,
179 |       "z</w>": 126,
180 |       "7</w>": 127,
181 |       "'</w>": 128,
182 |       "u</w>": 129,
183 |       "b</w>": 130,
184 |       "</w>": 131,
185 |       "x</w>": 132,
186 |       "2</w>": 133,
187 |       "f</w>": 134,
188 |       "3</w>": 135,
189 |       "9</w>": 136,
190 |       "j</w>": 137,
191 |       "®</w>": 138,
192 |       "+</w>": 139,
193 |       "></w>": 140,
194 |       "—</w>": 141,
195 |       "™</w>": 142,
196 |       "/</w>": 143,
197 |       "½</w>": 144,
198 |       "–</w>": 145,
199 |       "*</w>": 146,
200 |       "«</w>": 147,
201 |       "“</w>": 148,
202 |       "»</w>": 149,
203 |       "&</w>": 150,
204 |       "°</w>": 151,
205 |       "$</w>": 152,
206 |       "¦</w>": 153,
207 |       "•</w>": 154,
208 |       "(</w>": 155,
209 |       "​</w>": 156,
210 |       "¾</w>": 157,
211 |       "<</w>": 158,
212 |       "`</w>": 159,
213 |       "‚</w>": 160,
214 |       "‘</w>": 161,
215 |       "·</w>": 162,
216 |       "²</w>": 163,
217 |       "£</w>": 164,
218 |       "˜</w>": 165,
219 |       "×</w>": 166,
220 |       "th": 167,
221 |       "in": 168,
222 |       "an": 169,
223 |       "the</w>": 170,
224 |       "er": 171,
225 |       "re": 172,
226 |       "ou": 173,
227 |       "en": 174,
228 |       "ar": 175,
229 |       "ti": 176,
230 |       "on": 177,
231 |       "and</w>": 178,
232 |       "to</w>": 179,
233 |       "ing</w>": 180,
234 |       "or": 181,
235 |       "it": 182,
236 |       "ed</w>": 183,
237 |       "al": 184,
238 |       "st": 185,
239 |       "of</w>": 186,
240 |       "ro": 187,
241 |       "on</w>": 188,
242 |       "er</w>": 189,
243 |       "in</w>": 190,
244 |       "es": 191,
245 |       "is</w>": 192,
246 |       "or</w>": 193,
247 |       "at": 194,
248 |       "il": 195,
249 |       "es</w>": 196,
250 |       "ac": 197,
251 |       "ic": 198,
252 |       "el": 199,
253 |       "ha": 200,
254 |       "om": 201,
255 |       "is": 202,
256 |       "at</w>": 203,
257 |       "al</w>": 204,
258 |       "ec": 205,
259 |       "pl": 206,
260 |       "as": 207,
261 |       "for</w>": 208,
262 |       "an</w>": 209,
263 |       "ati": 210,
264 |       "as</w>": 211,
265 |       "ve</w>": 212,
266 |       "ur": 213,
267 |       "us": 214,
268 |       "lo": 215,
269 |       "ch": 216,
270 |       "wh": 217,
271 |       "our</w>": 218,
272 |       "id": 219,
273 |       "en</w>": 220,
274 |       "pro": 221,
275 |       "le": 222,
276 |       "ol": 223,
277 |       "un": 224,
278 |       "ad": 225,
279 |       "ent": 226,
280 |       "wit": 227,
281 |       "ig": 228,
282 |       "with</w>": 229,
283 |       "ab": 230,
284 |       "em": 231,
285 |       "ly</w>": 232,
286 |       "yo": 233,
287 |       "you</w>": 234,
288 |       "com": 235,
289 |       "are</w>": 236,
290 |       "et": 237,
291 |       "su": 238,
292 |       "no": 239,
293 |       "ent</w>": 240,
294 |       "ag": 241,
295 |       "am": 242,
296 |       "it</w>": 243,
297 |       "con": 244,
298 |       "that</w>": 245,
299 |       "ver": 246,
300 |       "ri": 247,
301 |       "the": 248,
302 |       "ex": 249,
303 |       "ed": 250,
304 |       "li": 251,
305 |       "s.</w>": 252,
306 |       "sh": 253,
307 |       "be</w>": 254
308 |     },
309 |     "merges": [
310 |       "t h",
311 |       "i n",
312 |       "a n",
313 |       "th e</w>",
314 |       "e r",
315 |       "r e",
316 |       "o u",
317 |       "e n",
318 |       "a r",
319 |       "t i",
320 |       "o n",
321 |       "an d</w>",
322 |       "t o</w>",
323 |       "in g</w>",
324 |       "o r",
325 |       "i t",
326 |       "e d</w>",
327 |       "a l",
328 |       "s t",
329 |       "o f</w>",
330 |       "r o",
331 |       "o n</w>",
332 |       "e r</w>",
333 |       "i n</w>",
334 |       "e s",
335 |       "i s</w>",
336 |       "o r</w>",
337 |       "a t",
338 |       "i l",
339 |       "e s</w>",
340 |       "a c",
341 |       "i c",
342 |       "e l",
343 |       "h a",
344 |       "o m",
345 |       "i s",
346 |       "a t</w>",
347 |       "a l</w>",
348 |       "e c",
349 |       "p l",
350 |       "a s",
351 |       "f or</w>",
352 |       "a n</w>",
353 |       "a ti",
354 |       "a s</w>",
355 |       "v e</w>",
356 |       "u r",
357 |       "u s",
358 |       "l o",
359 |       "c h",
360 |       "w h",
361 |       "ou r</w>",
362 |       "i d",
363 |       "e n</w>",
364 |       "p ro",
365 |       "l e",
366 |       "o l",
367 |       "u n",
368 |       "a d",
369 |       "en t",
370 |       "w it",
371 |       "i g",
372 |       "wit h</w>",
373 |       "a b",
374 |       "e m",
375 |       "l y</w>",
376 |       "y o",
377 |       "yo u</w>",
378 |       "c om",
379 |       "ar e</w>",
380 |       "e t",
381 |       "s u",
382 |       "n o",
383 |       "en t</w>",
384 |       "a g",
385 |       "a m",
386 |       "i t</w>",
387 |       "c on",
388 |       "th at</w>",
389 |       "v er",
390 |       "r i",
391 |       "th e",
392 |       "e x",
393 |       "e d",
394 |       "l i",
395 |       "s .</w>",
396 |       "s h",
397 |       "b e</w>"
398 |     ]
399 |   }
400 | }


--------------------------------------------------------------------------------
/src/backend/dataset.rs:
--------------------------------------------------------------------------------
  1 | use std::fs::File;
  2 | use std::io::BufWriter;
  3 | use std::io::{BufRead, BufReader};
  4 | 
  5 | use glob::glob;
  6 | use rand::seq::SliceRandom;
  7 | use rayon::iter::*;
  8 | 
  9 | use regex::Regex;
 10 | 
 11 | use serde::{Deserialize, Serialize};
 12 | use rmp_serde::{Deserializer, Serializer};
 13 | 
 14 | use crate::backend::{DATA_PATH, Token};
 15 | use crate::backend::tokenizer::ClmTokenizer;
 16 | 
 17 | #[derive(Serialize, Deserialize, Clone)]
 18 | pub struct Dataset {
 19 |     data: Vec<Vec<Token>>,
 20 | }
 21 | 
 22 | 
 23 | 
 24 | impl Dataset {
 25 |     fn locate_data_files() -> Vec<String> {
 26 |         glob(&format!("{}/**/*-sentences.txt", DATA_PATH))
 27 |             .expect("Failed to read glob pattern")
 28 |             .filter_map(Result::ok)
 29 |             .map(|x| x.display().to_string())
 30 |             .collect()
 31 |     }
 32 | 
 33 |     fn compute_from_files(files: Vec<String>) -> Dataset {
 34 |         let tokenizer = Self::get_tokenizer();
 35 |         let re = Regex::new(r"^\d+\s").unwrap();
 36 | 
 37 |         let pb = indicatif::ProgressBar::new(0);
 38 |         pb.set_style(indicatif::ProgressStyle::default_bar().template("{msg} {bar:60.cyan/blue} {pos}/{len} {per_sec}").unwrap());
 39 |         pb.set_message("Tokenizing");
 40 |         pb.set_length(files.iter().map(|file|
 41 |               File::open(file).unwrap().metadata().unwrap().len()
 42 |         ).sum());
 43 | 
 44 | 
 45 |         let tokens = files.iter()
 46 |             .flat_map(|filename| {
 47 |                 let file = File::open(filename).unwrap();
 48 |                 BufReader::new(file).lines()
 49 |             })
 50 |             .par_bridge()
 51 |             .map(|x| x.expect("Failed to read line"))
 52 |             .map(|x| {pb.inc(x.len() as u64); x})
 53 |             .map(|x| re.replace_all(&x, "").to_string())
 54 |             .map(|x| tokenizer.encode(&x))
 55 |             .collect();
 56 | 
 57 |         pb.finish();
 58 | 
 59 |         println!("Shuffling dataset");
 60 |         let mut dataset = Dataset { data: tokens };
 61 |         dataset.shuffle();
 62 |         dataset
 63 | 
 64 |     }
 65 | 
 66 |     pub fn split_train_test(&self, train_size: f32) -> (Dataset, Dataset) {
 67 |         let train_size = (self.data.len() as f32 * train_size).round() as usize;
 68 |         let (train, test) = self.data.split_at(train_size);
 69 |         (Dataset { data: train.to_vec() }, Dataset { data: test.to_vec() })
 70 |     }
 71 | 
 72 |     fn save_to_file(&self, filename: &str) {
 73 |         let file = File::create(filename).unwrap();
 74 |         let mut writer = BufWriter::new(file);
 75 |         let mut serializer = Serializer::new(&mut writer);
 76 |         self.serialize(&mut serializer).unwrap();
 77 |     }
 78 | 
 79 |     pub fn load_from_file(filename: &str) -> Dataset {
 80 |         let file = File::open(filename).unwrap();
 81 |         let mut reader = BufReader::new(file);
 82 |         let mut deserializer = Deserializer::new(&mut reader);
 83 |         Dataset::deserialize(&mut deserializer).unwrap()
 84 |     }
 85 | 
 86 |     pub fn shuffle(&mut self) {
 87 |         let mut rng = rand::thread_rng();
 88 |         self.data.shuffle(&mut rng);
 89 |     }
 90 | 
 91 |     pub fn load_or_compute(filename: &str) -> Dataset {
 92 |         if std::path::Path::new(filename).exists() {
 93 |             Dataset::load_from_file(filename)
 94 |         } else {
 95 |             let dataset = Dataset::compute_from_files(Dataset::locate_data_files());
 96 |             dataset.save_to_file(filename);
 97 |             dataset
 98 |         }
 99 |     }
100 | 
101 |     pub fn get_data(&self) -> &Vec<Vec<Token>> {
102 |         &self.data
103 |     }
104 | 
105 |     pub fn get_data_out(self) -> Vec<Vec<Token>> {
106 |         self.data
107 |     }
108 | 
109 |     pub fn from_data(data: Vec<Vec<Token>>) -> Dataset {
110 |         Dataset { data }
111 |     }
112 | 
113 |     pub fn shrink_to_size(&self, tokens: usize) -> Dataset {
114 |         let mut total_tokens  = 0;
115 |         let mut new_dataset = Vec::new();
116 | 
117 |         for line in self.data.iter() {
118 |             if total_tokens + line.len() <= tokens {
119 |                 new_dataset.push(line.clone());
120 |                 total_tokens += line.len();
121 |             } else if total_tokens > tokens {
122 |                 continue;
123 |             } else if total_tokens + line.len() > tokens {
124 |                 let partial_vec = line[0..(tokens-total_tokens)].to_vec();
125 |                 total_tokens += partial_vec.len();
126 |                 new_dataset.push(partial_vec);
127 |             }
128 |         }
129 | 
130 |         Self::from_data(new_dataset)
131 |     }
132 | 
133 |     // Join adjacent lines until they reach a certain size
134 |     pub fn join_lines(self, chunk_size: usize) -> Dataset {
135 |         let new_data = self.data.into_iter().fold(Vec::new(), |mut acc, line| {
136 |             if acc.is_empty() {
137 |                 acc.push(line);
138 |             } else if acc.last().unwrap().len() < chunk_size {
139 |                 acc.last_mut().unwrap().extend(line);
140 |             } else {
141 |                 acc.push(line);
142 |             }
143 |             acc
144 |         });
145 |         
146 |         Self::from_data(new_data)
147 |     }
148 | 
149 |     pub fn split_into_chunks(&self, chunk_count: usize) -> impl Iterator<Item=Dataset>  + '_{
150 | 
151 |         if self.data.is_empty() {
152 |            panic!("Cannot split empty dataset into chunks");
153 |         }
154 | 
155 |         let chunk_size = (self.data.len() as f64 / chunk_count as f64).ceil() as usize;
156 |         // split into n chunks, each containing chunk_size sentences
157 |         self.data.chunks(chunk_size).map(|x| Dataset::from_data(x.to_vec()))
158 |     }
159 |     pub fn empty() -> Dataset {
160 |         Dataset { data: Vec::new() }
161 |     }
162 | 
163 |     pub fn get_tokenizer() -> ClmTokenizer {
164 |         ClmTokenizer::new_custom()
165 |     }
166 |     pub fn tokenize(text: &str) -> Vec<Token> {
167 |         Self::get_tokenizer().encode(text)
168 |     }
169 |     
170 |     pub fn detokenize(tokens: Vec<Token>) -> String {
171 |         Self::get_tokenizer().decode(tokens)
172 |     }
173 | 
174 |     #[cfg(test)]
175 |     pub fn test_dataset() -> Dataset {
176 |         let mut data = Vec::new();
177 |         for _ in 0..20 {
178 |             let mut sentence = Vec::new();
179 |             for _ in 0..100 {
180 |                 sentence.push(rand::random());
181 |             }
182 |             data.push(sentence);
183 |         }
184 |         Dataset { data }
185 |     }
186 | }
187 | 
188 | #[cfg(test)]
189 | mod tests {
190 |     use itertools::Itertools;
191 |     use crate::backend::dataset::Dataset;
192 | 
193 |     #[test]
194 |     fn test_locate_data_files() {
195 |         let files = Dataset::locate_data_files();
196 |         assert_ne!(files.len(), 0);
197 |     }
198 | 
199 |     #[test]
200 |     fn save_and_load_dataset() {
201 |         let start_time = std::time::Instant::now();
202 |         let dataset = Dataset::compute_from_files(vec!["./data/tests.txt".to_string()]);
203 |         println!("Dataset computed in {:?}", start_time.elapsed());
204 |         dataset.save_to_file("dataset.msgpack");
205 | 
206 |         let load_time = std::time::Instant::now();
207 |         let loaded_dataset = Dataset::load_from_file("dataset.msgpack");
208 |         println!("Dataset loaded in {:?}", load_time.elapsed());
209 |         println!("Computed file size: {:?}", std::fs::metadata("dataset.msgpack").unwrap().len());
210 |         println!("Original file size: {:?}", std::fs::metadata("./data/tests.txt").unwrap().len());
211 | 
212 |         assert_eq!(dataset.data.len(), loaded_dataset.data.len());
213 | 
214 |         // deep comparison
215 |         for (a, b) in dataset.data.iter().zip(loaded_dataset.data.iter()) {
216 |             assert_eq!(a, b);
217 |         }
218 | 
219 |         // cleanup
220 |         std::fs::remove_file("dataset.msgpack").unwrap();
221 |     }
222 | 
223 | 
224 |     #[test]
225 |     fn test_compute_from_files() {
226 |         let dataset = Dataset::compute_from_files(vec!["data/tests.txt".to_string()]);
227 |         assert_eq!(dataset.data.len(), 1000);
228 | 
229 |         // no sentence should be empty
230 |         assert_eq!(dataset.data.iter().filter(|x| x.is_empty()).count(), 0);
231 |     }
232 | 
233 |     #[test]
234 |     fn shrink_dataset() {
235 |         let dataset = Dataset::compute_from_files(vec!["data/tests.txt".to_string()]);
236 |         let shrunk = dataset.shrink_to_size(1000);
237 |         assert_eq!(shrunk.data.iter().map(|x| x.len()).sum::<usize>(), 1000);
238 |     }
239 | 
240 | 
241 |     #[test]
242 |     fn split_into_chunks() {
243 |         let dataset = Dataset::compute_from_files(vec!["data/tests.txt".to_string()]);
244 |         let chunks = dataset.split_into_chunks(10).collect_vec();
245 |         assert_eq!(chunks.len(), 10);
246 |         assert_eq!(chunks.iter().map(|x| x.data.len()).sum::<usize>(), dataset.data.len());
247 |     }
248 |     
249 |     #[test]
250 |     fn join_lines() {
251 |         let dataset = Dataset::compute_from_files(vec!["data/tests.txt".to_string()]);
252 |         let joined = dataset.join_lines(100);
253 |         // the last line can be shorter than 100, it doesn't matter
254 |         let min_len = joined.data.iter().rev().skip(1).map(|x| x.len()).min().unwrap();
255 |         println!("Min len: {}", min_len);
256 |         assert!(min_len >= 100);
257 |     }
258 | }


--------------------------------------------------------------------------------
/Cargo.lock:
--------------------------------------------------------------------------------
   1 | # This file is automatically @generated by Cargo.
   2 | # It is not intended for manual editing.
   3 | version = 3
   4 | 
   5 | [[package]]
   6 | name = "addr2line"
   7 | version = "0.21.0"
   8 | source = "registry+https://github.com/rust-lang/crates.io-index"
   9 | checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb"
  10 | dependencies = [
  11 |  "gimli",
  12 | ]
  13 | 
  14 | [[package]]
  15 | name = "adler"
  16 | version = "1.0.2"
  17 | source = "registry+https://github.com/rust-lang/crates.io-index"
  18 | checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
  19 | 
  20 | [[package]]
  21 | name = "ahash"
  22 | version = "0.8.11"
  23 | source = "registry+https://github.com/rust-lang/crates.io-index"
  24 | checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
  25 | dependencies = [
  26 |  "cfg-if",
  27 |  "once_cell",
  28 |  "version_check",
  29 |  "zerocopy",
  30 | ]
  31 | 
  32 | [[package]]
  33 | name = "aho-corasick"
  34 | version = "1.1.3"
  35 | source = "registry+https://github.com/rust-lang/crates.io-index"
  36 | checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
  37 | dependencies = [
  38 |  "memchr",
  39 | ]
  40 | 
  41 | [[package]]
  42 | name = "allocator-api2"
  43 | version = "0.2.18"
  44 | source = "registry+https://github.com/rust-lang/crates.io-index"
  45 | checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f"
  46 | 
  47 | [[package]]
  48 | name = "android-tzdata"
  49 | version = "0.1.1"
  50 | source = "registry+https://github.com/rust-lang/crates.io-index"
  51 | checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"
  52 | 
  53 | [[package]]
  54 | name = "android_system_properties"
  55 | version = "0.1.5"
  56 | source = "registry+https://github.com/rust-lang/crates.io-index"
  57 | checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
  58 | dependencies = [
  59 |  "libc",
  60 | ]
  61 | 
  62 | [[package]]
  63 | name = "anyhow"
  64 | version = "1.0.86"
  65 | source = "registry+https://github.com/rust-lang/crates.io-index"
  66 | checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da"
  67 | 
  68 | [[package]]
  69 | name = "async-recursion"
  70 | version = "1.1.1"
  71 | source = "registry+https://github.com/rust-lang/crates.io-index"
  72 | checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11"
  73 | dependencies = [
  74 |  "proc-macro2",
  75 |  "quote",
  76 |  "syn 2.0.66",
  77 | ]
  78 | 
  79 | [[package]]
  80 | name = "async-trait"
  81 | version = "0.1.80"
  82 | source = "registry+https://github.com/rust-lang/crates.io-index"
  83 | checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca"
  84 | dependencies = [
  85 |  "proc-macro2",
  86 |  "quote",
  87 |  "syn 2.0.66",
  88 | ]
  89 | 
  90 | [[package]]
  91 | name = "attribute-derive"
  92 | version = "0.9.1"
  93 | source = "registry+https://github.com/rust-lang/crates.io-index"
  94 | checksum = "8b48808b337d6b74c15ff9becfc0e139fe2b4e2b224d670a0ecdb46b0b2d3d9b"
  95 | dependencies = [
  96 |  "attribute-derive-macro",
  97 |  "derive-where",
  98 |  "manyhow",
  99 |  "proc-macro2",
 100 |  "quote",
 101 |  "syn 2.0.66",
 102 | ]
 103 | 
 104 | [[package]]
 105 | name = "attribute-derive-macro"
 106 | version = "0.9.1"
 107 | source = "registry+https://github.com/rust-lang/crates.io-index"
 108 | checksum = "5b19cbd63850ecff821c413e12846a67ec9f4ce7309c70959b94ecf9b2575ee2"
 109 | dependencies = [
 110 |  "collection_literals",
 111 |  "interpolator",
 112 |  "manyhow",
 113 |  "proc-macro-utils",
 114 |  "proc-macro2",
 115 |  "quote",
 116 |  "quote-use",
 117 |  "syn 2.0.66",
 118 | ]
 119 | 
 120 | [[package]]
 121 | name = "autocfg"
 122 | version = "1.3.0"
 123 | source = "registry+https://github.com/rust-lang/crates.io-index"
 124 | checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0"
 125 | 
 126 | [[package]]
 127 | name = "axum"
 128 | version = "0.7.5"
 129 | source = "registry+https://github.com/rust-lang/crates.io-index"
 130 | checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf"
 131 | dependencies = [
 132 |  "async-trait",
 133 |  "axum-core",
 134 |  "bytes",
 135 |  "futures-util",
 136 |  "http 1.1.0",
 137 |  "http-body",
 138 |  "http-body-util",
 139 |  "hyper",
 140 |  "hyper-util",
 141 |  "itoa",
 142 |  "matchit",
 143 |  "memchr",
 144 |  "mime",
 145 |  "multer",
 146 |  "percent-encoding",
 147 |  "pin-project-lite",
 148 |  "rustversion",
 149 |  "serde",
 150 |  "serde_json",
 151 |  "serde_path_to_error",
 152 |  "serde_urlencoded",
 153 |  "sync_wrapper 1.0.1",
 154 |  "tokio",
 155 |  "tower",
 156 |  "tower-layer",
 157 |  "tower-service",
 158 |  "tracing",
 159 | ]
 160 | 
 161 | [[package]]
 162 | name = "axum-core"
 163 | version = "0.4.3"
 164 | source = "registry+https://github.com/rust-lang/crates.io-index"
 165 | checksum = "a15c63fd72d41492dc4f497196f5da1fb04fb7529e631d73630d1b491e47a2e3"
 166 | dependencies = [
 167 |  "async-trait",
 168 |  "bytes",
 169 |  "futures-util",
 170 |  "http 1.1.0",
 171 |  "http-body",
 172 |  "http-body-util",
 173 |  "mime",
 174 |  "pin-project-lite",
 175 |  "rustversion",
 176 |  "sync_wrapper 0.1.2",
 177 |  "tower-layer",
 178 |  "tower-service",
 179 |  "tracing",
 180 | ]
 181 | 
 182 | [[package]]
 183 | name = "backtrace"
 184 | version = "0.3.71"
 185 | source = "registry+https://github.com/rust-lang/crates.io-index"
 186 | checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d"
 187 | dependencies = [
 188 |  "addr2line",
 189 |  "cc",
 190 |  "cfg-if",
 191 |  "libc",
 192 |  "miniz_oxide",
 193 |  "object",
 194 |  "rustc-demangle",
 195 | ]
 196 | 
 197 | [[package]]
 198 | name = "base64"
 199 | version = "0.13.1"
 200 | source = "registry+https://github.com/rust-lang/crates.io-index"
 201 | checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
 202 | 
 203 | [[package]]
 204 | name = "base64"
 205 | version = "0.21.7"
 206 | source = "registry+https://github.com/rust-lang/crates.io-index"
 207 | checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
 208 | 
 209 | [[package]]
 210 | name = "base64"
 211 | version = "0.22.1"
 212 | source = "registry+https://github.com/rust-lang/crates.io-index"
 213 | checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
 214 | 
 215 | [[package]]
 216 | name = "bit-set"
 217 | version = "0.5.3"
 218 | source = "registry+https://github.com/rust-lang/crates.io-index"
 219 | checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1"
 220 | dependencies = [
 221 |  "bit-vec",
 222 | ]
 223 | 
 224 | [[package]]
 225 | name = "bit-vec"
 226 | version = "0.6.3"
 227 | source = "registry+https://github.com/rust-lang/crates.io-index"
 228 | checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb"
 229 | 
 230 | [[package]]
 231 | name = "bitflags"
 232 | version = "1.3.2"
 233 | source = "registry+https://github.com/rust-lang/crates.io-index"
 234 | checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 235 | 
 236 | [[package]]
 237 | name = "bitflags"
 238 | version = "2.5.0"
 239 | source = "registry+https://github.com/rust-lang/crates.io-index"
 240 | checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1"
 241 | 
 242 | [[package]]
 243 | name = "bstr"
 244 | version = "1.9.1"
 245 | source = "registry+https://github.com/rust-lang/crates.io-index"
 246 | checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706"
 247 | dependencies = [
 248 |  "memchr",
 249 |  "regex-automata",
 250 |  "serde",
 251 | ]
 252 | 
 253 | [[package]]
 254 | name = "bumpalo"
 255 | version = "3.16.0"
 256 | source = "registry+https://github.com/rust-lang/crates.io-index"
 257 | checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c"
 258 | 
 259 | [[package]]
 260 | name = "byteorder"
 261 | version = "1.5.0"
 262 | source = "registry+https://github.com/rust-lang/crates.io-index"
 263 | checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
 264 | 
 265 | [[package]]
 266 | name = "bytes"
 267 | version = "1.6.0"
 268 | source = "registry+https://github.com/rust-lang/crates.io-index"
 269 | checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9"
 270 | 
 271 | [[package]]
 272 | name = "cached"
 273 | version = "0.45.1"
 274 | source = "registry+https://github.com/rust-lang/crates.io-index"
 275 | checksum = "90eb5776f28a149524d1d8623035760b4454ec881e8cf3838fa8d7e1b11254b3"
 276 | dependencies = [
 277 |  "cached_proc_macro",
 278 |  "cached_proc_macro_types",
 279 |  "hashbrown 0.13.2",
 280 |  "instant",
 281 |  "once_cell",
 282 |  "thiserror",
 283 | ]
 284 | 
 285 | [[package]]
 286 | name = "cached_proc_macro"
 287 | version = "0.18.1"
 288 | source = "registry+https://github.com/rust-lang/crates.io-index"
 289 | checksum = "c878c71c2821aa2058722038a59a67583a4240524687c6028571c9b395ded61f"
 290 | dependencies = [
 291 |  "darling 0.14.4",
 292 |  "proc-macro2",
 293 |  "quote",
 294 |  "syn 1.0.109",
 295 | ]
 296 | 
 297 | [[package]]
 298 | name = "cached_proc_macro_types"
 299 | version = "0.1.1"
 300 | source = "registry+https://github.com/rust-lang/crates.io-index"
 301 | checksum = "ade8366b8bd5ba243f0a58f036cc0ca8a2f069cff1a2351ef1cac6b083e16fc0"
 302 | 
 303 | [[package]]
 304 | name = "camino"
 305 | version = "1.1.7"
 306 | source = "registry+https://github.com/rust-lang/crates.io-index"
 307 | checksum = "e0ec6b951b160caa93cc0c7b209e5a3bff7aae9062213451ac99493cd844c239"
 308 | 
 309 | [[package]]
 310 | name = "cc"
 311 | version = "1.0.98"
 312 | source = "registry+https://github.com/rust-lang/crates.io-index"
 313 | checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f"
 314 | dependencies = [
 315 |  "jobserver",
 316 |  "libc",
 317 |  "once_cell",
 318 | ]
 319 | 
 320 | [[package]]
 321 | name = "cfg-if"
 322 | version = "1.0.0"
 323 | source = "registry+https://github.com/rust-lang/crates.io-index"
 324 | checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 325 | 
 326 | [[package]]
 327 | name = "chatclm"
 328 | version = "0.1.0"
 329 | dependencies = [
 330 |  "axum",
 331 |  "chrono",
 332 |  "console_error_panic_hook",
 333 |  "glob",
 334 |  "http 1.1.0",
 335 |  "indicatif",
 336 |  "itertools 0.13.0",
 337 |  "leptos",
 338 |  "leptos_axum",
 339 |  "leptos_meta",
 340 |  "leptos_router",
 341 |  "num",
 342 |  "rand",
 343 |  "rayon",
 344 |  "regex",
 345 |  "rmp-serde",
 346 |  "rusqlite",
 347 |  "serde",
 348 |  "serde_json",
 349 |  "thiserror",
 350 |  "tiktoken-rs",
 351 |  "tokenizers",
 352 |  "tokio",
 353 |  "tower",
 354 |  "tower-http",
 355 |  "tracing",
 356 |  "wasm-bindgen",
 357 |  "zstd",
 358 |  "zstd-sys",
 359 | ]
 360 | 
 361 | [[package]]
 362 | name = "chrono"
 363 | version = "0.4.38"
 364 | source = "registry+https://github.com/rust-lang/crates.io-index"
 365 | checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401"
 366 | dependencies = [
 367 |  "android-tzdata",
 368 |  "iana-time-zone",
 369 |  "js-sys",
 370 |  "num-traits",
 371 |  "wasm-bindgen",
 372 |  "windows-targets 0.52.5",
 373 | ]
 374 | 
 375 | [[package]]
 376 | name = "ciborium"
 377 | version = "0.2.2"
 378 | source = "registry+https://github.com/rust-lang/crates.io-index"
 379 | checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e"
 380 | dependencies = [
 381 |  "ciborium-io",
 382 |  "ciborium-ll",
 383 |  "serde",
 384 | ]
 385 | 
 386 | [[package]]
 387 | name = "ciborium-io"
 388 | version = "0.2.2"
 389 | source = "registry+https://github.com/rust-lang/crates.io-index"
 390 | checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757"
 391 | 
 392 | [[package]]
 393 | name = "ciborium-ll"
 394 | version = "0.2.2"
 395 | source = "registry+https://github.com/rust-lang/crates.io-index"
 396 | checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9"
 397 | dependencies = [
 398 |  "ciborium-io",
 399 |  "half",
 400 | ]
 401 | 
 402 | [[package]]
 403 | name = "collection_literals"
 404 | version = "1.0.1"
 405 | source = "registry+https://github.com/rust-lang/crates.io-index"
 406 | checksum = "186dce98367766de751c42c4f03970fc60fc012296e706ccbb9d5df9b6c1e271"
 407 | 
 408 | [[package]]
 409 | name = "config"
 410 | version = "0.14.0"
 411 | source = "registry+https://github.com/rust-lang/crates.io-index"
 412 | checksum = "7328b20597b53c2454f0b1919720c25c7339051c02b72b7e05409e00b14132be"
 413 | dependencies = [
 414 |  "convert_case",
 415 |  "lazy_static",
 416 |  "nom",
 417 |  "pathdiff",
 418 |  "serde",
 419 |  "toml",
 420 | ]
 421 | 
 422 | [[package]]
 423 | name = "console"
 424 | version = "0.15.8"
 425 | source = "registry+https://github.com/rust-lang/crates.io-index"
 426 | checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb"
 427 | dependencies = [
 428 |  "encode_unicode",
 429 |  "lazy_static",
 430 |  "libc",
 431 |  "unicode-width",
 432 |  "windows-sys 0.52.0",
 433 | ]
 434 | 
 435 | [[package]]
 436 | name = "console_error_panic_hook"
 437 | version = "0.1.7"
 438 | source = "registry+https://github.com/rust-lang/crates.io-index"
 439 | checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc"
 440 | dependencies = [
 441 |  "cfg-if",
 442 |  "wasm-bindgen",
 443 | ]
 444 | 
 445 | [[package]]
 446 | name = "const_format"
 447 | version = "0.2.32"
 448 | source = "registry+https://github.com/rust-lang/crates.io-index"
 449 | checksum = "e3a214c7af3d04997541b18d432afaff4c455e79e2029079647e72fc2bd27673"
 450 | dependencies = [
 451 |  "const_format_proc_macros",
 452 | ]
 453 | 
 454 | [[package]]
 455 | name = "const_format_proc_macros"
 456 | version = "0.2.32"
 457 | source = "registry+https://github.com/rust-lang/crates.io-index"
 458 | checksum = "c7f6ff08fd20f4f299298a28e2dfa8a8ba1036e6cd2460ac1de7b425d76f2500"
 459 | dependencies = [
 460 |  "proc-macro2",
 461 |  "quote",
 462 |  "unicode-xid",
 463 | ]
 464 | 
 465 | [[package]]
 466 | name = "convert_case"
 467 | version = "0.6.0"
 468 | source = "registry+https://github.com/rust-lang/crates.io-index"
 469 | checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca"
 470 | dependencies = [
 471 |  "unicode-segmentation",
 472 | ]
 473 | 
 474 | [[package]]
 475 | name = "core-foundation-sys"
 476 | version = "0.8.6"
 477 | source = "registry+https://github.com/rust-lang/crates.io-index"
 478 | checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f"
 479 | 
 480 | [[package]]
 481 | name = "crossbeam-deque"
 482 | version = "0.8.5"
 483 | source = "registry+https://github.com/rust-lang/crates.io-index"
 484 | checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d"
 485 | dependencies = [
 486 |  "crossbeam-epoch",
 487 |  "crossbeam-utils",
 488 | ]
 489 | 
 490 | [[package]]
 491 | name = "crossbeam-epoch"
 492 | version = "0.9.18"
 493 | source = "registry+https://github.com/rust-lang/crates.io-index"
 494 | checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
 495 | dependencies = [
 496 |  "crossbeam-utils",
 497 | ]
 498 | 
 499 | [[package]]
 500 | name = "crossbeam-utils"
 501 | version = "0.8.20"
 502 | source = "registry+https://github.com/rust-lang/crates.io-index"
 503 | checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
 504 | 
 505 | [[package]]
 506 | name = "crunchy"
 507 | version = "0.2.2"
 508 | source = "registry+https://github.com/rust-lang/crates.io-index"
 509 | checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7"
 510 | 
 511 | [[package]]
 512 | name = "darling"
 513 | version = "0.14.4"
 514 | source = "registry+https://github.com/rust-lang/crates.io-index"
 515 | checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850"
 516 | dependencies = [
 517 |  "darling_core 0.14.4",
 518 |  "darling_macro 0.14.4",
 519 | ]
 520 | 
 521 | [[package]]
 522 | name = "darling"
 523 | version = "0.20.9"
 524 | source = "registry+https://github.com/rust-lang/crates.io-index"
 525 | checksum = "83b2eb4d90d12bdda5ed17de686c2acb4c57914f8f921b8da7e112b5a36f3fe1"
 526 | dependencies = [
 527 |  "darling_core 0.20.9",
 528 |  "darling_macro 0.20.9",
 529 | ]
 530 | 
 531 | [[package]]
 532 | name = "darling_core"
 533 | version = "0.14.4"
 534 | source = "registry+https://github.com/rust-lang/crates.io-index"
 535 | checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0"
 536 | dependencies = [
 537 |  "fnv",
 538 |  "ident_case",
 539 |  "proc-macro2",
 540 |  "quote",
 541 |  "strsim 0.10.0",
 542 |  "syn 1.0.109",
 543 | ]
 544 | 
 545 | [[package]]
 546 | name = "darling_core"
 547 | version = "0.20.9"
 548 | source = "registry+https://github.com/rust-lang/crates.io-index"
 549 | checksum = "622687fe0bac72a04e5599029151f5796111b90f1baaa9b544d807a5e31cd120"
 550 | dependencies = [
 551 |  "fnv",
 552 |  "ident_case",
 553 |  "proc-macro2",
 554 |  "quote",
 555 |  "strsim 0.11.1",
 556 |  "syn 2.0.66",
 557 | ]
 558 | 
 559 | [[package]]
 560 | name = "darling_macro"
 561 | version = "0.14.4"
 562 | source = "registry+https://github.com/rust-lang/crates.io-index"
 563 | checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e"
 564 | dependencies = [
 565 |  "darling_core 0.14.4",
 566 |  "quote",
 567 |  "syn 1.0.109",
 568 | ]
 569 | 
 570 | [[package]]
 571 | name = "darling_macro"
 572 | version = "0.20.9"
 573 | source = "registry+https://github.com/rust-lang/crates.io-index"
 574 | checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178"
 575 | dependencies = [
 576 |  "darling_core 0.20.9",
 577 |  "quote",
 578 |  "syn 2.0.66",
 579 | ]
 580 | 
 581 | [[package]]
 582 | name = "dashmap"
 583 | version = "5.5.3"
 584 | source = "registry+https://github.com/rust-lang/crates.io-index"
 585 | checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856"
 586 | dependencies = [
 587 |  "cfg-if",
 588 |  "hashbrown 0.14.5",
 589 |  "lock_api",
 590 |  "once_cell",
 591 |  "parking_lot_core",
 592 | ]
 593 | 
 594 | [[package]]
 595 | name = "derive-where"
 596 | version = "1.2.7"
 597 | source = "registry+https://github.com/rust-lang/crates.io-index"
 598 | checksum = "62d671cc41a825ebabc75757b62d3d168c577f9149b2d49ece1dad1f72119d25"
 599 | dependencies = [
 600 |  "proc-macro2",
 601 |  "quote",
 602 |  "syn 2.0.66",
 603 | ]
 604 | 
 605 | [[package]]
 606 | name = "derive_builder"
 607 | version = "0.20.0"
 608 | source = "registry+https://github.com/rust-lang/crates.io-index"
 609 | checksum = "0350b5cb0331628a5916d6c5c0b72e97393b8b6b03b47a9284f4e7f5a405ffd7"
 610 | dependencies = [
 611 |  "derive_builder_macro",
 612 | ]
 613 | 
 614 | [[package]]
 615 | name = "derive_builder_core"
 616 | version = "0.20.0"
 617 | source = "registry+https://github.com/rust-lang/crates.io-index"
 618 | checksum = "d48cda787f839151732d396ac69e3473923d54312c070ee21e9effcaa8ca0b1d"
 619 | dependencies = [
 620 |  "darling 0.20.9",
 621 |  "proc-macro2",
 622 |  "quote",
 623 |  "syn 2.0.66",
 624 | ]
 625 | 
 626 | [[package]]
 627 | name = "derive_builder_macro"
 628 | version = "0.20.0"
 629 | source = "registry+https://github.com/rust-lang/crates.io-index"
 630 | checksum = "206868b8242f27cecce124c19fd88157fbd0dd334df2587f36417bafbc85097b"
 631 | dependencies = [
 632 |  "derive_builder_core",
 633 |  "syn 2.0.66",
 634 | ]
 635 | 
 636 | [[package]]
 637 | name = "drain_filter_polyfill"
 638 | version = "0.1.3"
 639 | source = "registry+https://github.com/rust-lang/crates.io-index"
 640 | checksum = "669a445ee724c5c69b1b06fe0b63e70a1c84bc9bb7d9696cd4f4e3ec45050408"
 641 | 
 642 | [[package]]
 643 | name = "either"
 644 | version = "1.12.0"
 645 | source = "registry+https://github.com/rust-lang/crates.io-index"
 646 | checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b"
 647 | 
 648 | [[package]]
 649 | name = "encode_unicode"
 650 | version = "0.3.6"
 651 | source = "registry+https://github.com/rust-lang/crates.io-index"
 652 | checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
 653 | 
 654 | [[package]]
 655 | name = "encoding_rs"
 656 | version = "0.8.34"
 657 | source = "registry+https://github.com/rust-lang/crates.io-index"
 658 | checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59"
 659 | dependencies = [
 660 |  "cfg-if",
 661 | ]
 662 | 
 663 | [[package]]
 664 | name = "equivalent"
 665 | version = "1.0.1"
 666 | source = "registry+https://github.com/rust-lang/crates.io-index"
 667 | checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
 668 | 
 669 | [[package]]
 670 | name = "esaxx-rs"
 671 | version = "0.1.10"
 672 | source = "registry+https://github.com/rust-lang/crates.io-index"
 673 | checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6"
 674 | dependencies = [
 675 |  "cc",
 676 | ]
 677 | 
 678 | [[package]]
 679 | name = "fallible-iterator"
 680 | version = "0.3.0"
 681 | source = "registry+https://github.com/rust-lang/crates.io-index"
 682 | checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649"
 683 | 
 684 | [[package]]
 685 | name = "fallible-streaming-iterator"
 686 | version = "0.1.9"
 687 | source = "registry+https://github.com/rust-lang/crates.io-index"
 688 | checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
 689 | 
 690 | [[package]]
 691 | name = "fancy-regex"
 692 | version = "0.12.0"
 693 | source = "registry+https://github.com/rust-lang/crates.io-index"
 694 | checksum = "7493d4c459da9f84325ad297371a6b2b8a162800873a22e3b6b6512e61d18c05"
 695 | dependencies = [
 696 |  "bit-set",
 697 |  "regex",
 698 | ]
 699 | 
 700 | [[package]]
 701 | name = "fnv"
 702 | version = "1.0.7"
 703 | source = "registry+https://github.com/rust-lang/crates.io-index"
 704 | checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 705 | 
 706 | [[package]]
 707 | name = "form_urlencoded"
 708 | version = "1.2.1"
 709 | source = "registry+https://github.com/rust-lang/crates.io-index"
 710 | checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456"
 711 | dependencies = [
 712 |  "percent-encoding",
 713 | ]
 714 | 
 715 | [[package]]
 716 | name = "futures"
 717 | version = "0.3.30"
 718 | source = "registry+https://github.com/rust-lang/crates.io-index"
 719 | checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0"
 720 | dependencies = [
 721 |  "futures-channel",
 722 |  "futures-core",
 723 |  "futures-executor",
 724 |  "futures-io",
 725 |  "futures-sink",
 726 |  "futures-task",
 727 |  "futures-util",
 728 | ]
 729 | 
 730 | [[package]]
 731 | name = "futures-channel"
 732 | version = "0.3.30"
 733 | source = "registry+https://github.com/rust-lang/crates.io-index"
 734 | checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78"
 735 | dependencies = [
 736 |  "futures-core",
 737 |  "futures-sink",
 738 | ]
 739 | 
 740 | [[package]]
 741 | name = "futures-core"
 742 | version = "0.3.30"
 743 | source = "registry+https://github.com/rust-lang/crates.io-index"
 744 | checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d"
 745 | 
 746 | [[package]]
 747 | name = "futures-executor"
 748 | version = "0.3.30"
 749 | source = "registry+https://github.com/rust-lang/crates.io-index"
 750 | checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d"
 751 | dependencies = [
 752 |  "futures-core",
 753 |  "futures-task",
 754 |  "futures-util",
 755 | ]
 756 | 
 757 | [[package]]
 758 | name = "futures-io"
 759 | version = "0.3.30"
 760 | source = "registry+https://github.com/rust-lang/crates.io-index"
 761 | checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1"
 762 | 
 763 | [[package]]
 764 | name = "futures-macro"
 765 | version = "0.3.30"
 766 | source = "registry+https://github.com/rust-lang/crates.io-index"
 767 | checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac"
 768 | dependencies = [
 769 |  "proc-macro2",
 770 |  "quote",
 771 |  "syn 2.0.66",
 772 | ]
 773 | 
 774 | [[package]]
 775 | name = "futures-sink"
 776 | version = "0.3.30"
 777 | source = "registry+https://github.com/rust-lang/crates.io-index"
 778 | checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5"
 779 | 
 780 | [[package]]
 781 | name = "futures-task"
 782 | version = "0.3.30"
 783 | source = "registry+https://github.com/rust-lang/crates.io-index"
 784 | checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004"
 785 | 
 786 | [[package]]
 787 | name = "futures-util"
 788 | version = "0.3.30"
 789 | source = "registry+https://github.com/rust-lang/crates.io-index"
 790 | checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48"
 791 | dependencies = [
 792 |  "futures-channel",
 793 |  "futures-core",
 794 |  "futures-io",
 795 |  "futures-macro",
 796 |  "futures-sink",
 797 |  "futures-task",
 798 |  "memchr",
 799 |  "pin-project-lite",
 800 |  "pin-utils",
 801 |  "slab",
 802 | ]
 803 | 
 804 | [[package]]
 805 | name = "getrandom"
 806 | version = "0.2.15"
 807 | source = "registry+https://github.com/rust-lang/crates.io-index"
 808 | checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
 809 | dependencies = [
 810 |  "cfg-if",
 811 |  "js-sys",
 812 |  "libc",
 813 |  "wasi",
 814 |  "wasm-bindgen",
 815 | ]
 816 | 
 817 | [[package]]
 818 | name = "gimli"
 819 | version = "0.28.1"
 820 | source = "registry+https://github.com/rust-lang/crates.io-index"
 821 | checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253"
 822 | 
 823 | [[package]]
 824 | name = "glob"
 825 | version = "0.3.1"
 826 | source = "registry+https://github.com/rust-lang/crates.io-index"
 827 | checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
 828 | 
 829 | [[package]]
 830 | name = "gloo-net"
 831 | version = "0.5.0"
 832 | source = "registry+https://github.com/rust-lang/crates.io-index"
 833 | checksum = "43aaa242d1239a8822c15c645f02166398da4f8b5c4bae795c1f5b44e9eee173"
 834 | dependencies = [
 835 |  "futures-channel",
 836 |  "futures-core",
 837 |  "futures-sink",
 838 |  "gloo-utils",
 839 |  "http 0.2.12",
 840 |  "js-sys",
 841 |  "pin-project",
 842 |  "serde",
 843 |  "serde_json",
 844 |  "thiserror",
 845 |  "wasm-bindgen",
 846 |  "wasm-bindgen-futures",
 847 |  "web-sys",
 848 | ]
 849 | 
 850 | [[package]]
 851 | name = "gloo-utils"
 852 | version = "0.2.0"
 853 | source = "registry+https://github.com/rust-lang/crates.io-index"
 854 | checksum = "0b5555354113b18c547c1d3a98fbf7fb32a9ff4f6fa112ce823a21641a0ba3aa"
 855 | dependencies = [
 856 |  "js-sys",
 857 |  "serde",
 858 |  "serde_json",
 859 |  "wasm-bindgen",
 860 |  "web-sys",
 861 | ]
 862 | 
 863 | [[package]]
 864 | name = "half"
 865 | version = "2.4.1"
 866 | source = "registry+https://github.com/rust-lang/crates.io-index"
 867 | checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888"
 868 | dependencies = [
 869 |  "cfg-if",
 870 |  "crunchy",
 871 | ]
 872 | 
 873 | [[package]]
 874 | name = "hashbrown"
 875 | version = "0.13.2"
 876 | source = "registry+https://github.com/rust-lang/crates.io-index"
 877 | checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e"
 878 | 
 879 | [[package]]
 880 | name = "hashbrown"
 881 | version = "0.14.5"
 882 | source = "registry+https://github.com/rust-lang/crates.io-index"
 883 | checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
 884 | dependencies = [
 885 |  "ahash",
 886 |  "allocator-api2",
 887 | ]
 888 | 
 889 | [[package]]
 890 | name = "hashlink"
 891 | version = "0.9.1"
 892 | source = "registry+https://github.com/rust-lang/crates.io-index"
 893 | checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af"
 894 | dependencies = [
 895 |  "hashbrown 0.14.5",
 896 | ]
 897 | 
 898 | [[package]]
 899 | name = "hermit-abi"
 900 | version = "0.3.9"
 901 | source = "registry+https://github.com/rust-lang/crates.io-index"
 902 | checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024"
 903 | 
 904 | [[package]]
 905 | name = "html-escape"
 906 | version = "0.2.13"
 907 | source = "registry+https://github.com/rust-lang/crates.io-index"
 908 | checksum = "6d1ad449764d627e22bfd7cd5e8868264fc9236e07c752972b4080cd351cb476"
 909 | dependencies = [
 910 |  "utf8-width",
 911 | ]
 912 | 
 913 | [[package]]
 914 | name = "http"
 915 | version = "0.2.12"
 916 | source = "registry+https://github.com/rust-lang/crates.io-index"
 917 | checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1"
 918 | dependencies = [
 919 |  "bytes",
 920 |  "fnv",
 921 |  "itoa",
 922 | ]
 923 | 
 924 | [[package]]
 925 | name = "http"
 926 | version = "1.1.0"
 927 | source = "registry+https://github.com/rust-lang/crates.io-index"
 928 | checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258"
 929 | dependencies = [
 930 |  "bytes",
 931 |  "fnv",
 932 |  "itoa",
 933 | ]
 934 | 
 935 | [[package]]
 936 | name = "http-body"
 937 | version = "1.0.0"
 938 | source = "registry+https://github.com/rust-lang/crates.io-index"
 939 | checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643"
 940 | dependencies = [
 941 |  "bytes",
 942 |  "http 1.1.0",
 943 | ]
 944 | 
 945 | [[package]]
 946 | name = "http-body-util"
 947 | version = "0.1.1"
 948 | source = "registry+https://github.com/rust-lang/crates.io-index"
 949 | checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d"
 950 | dependencies = [
 951 |  "bytes",
 952 |  "futures-core",
 953 |  "http 1.1.0",
 954 |  "http-body",
 955 |  "pin-project-lite",
 956 | ]
 957 | 
 958 | [[package]]
 959 | name = "http-range-header"
 960 | version = "0.4.1"
 961 | source = "registry+https://github.com/rust-lang/crates.io-index"
 962 | checksum = "08a397c49fec283e3d6211adbe480be95aae5f304cfb923e9970e08956d5168a"
 963 | 
 964 | [[package]]
 965 | name = "httparse"
 966 | version = "1.8.0"
 967 | source = "registry+https://github.com/rust-lang/crates.io-index"
 968 | checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904"
 969 | 
 970 | [[package]]
 971 | name = "httpdate"
 972 | version = "1.0.3"
 973 | source = "registry+https://github.com/rust-lang/crates.io-index"
 974 | checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
 975 | 
 976 | [[package]]
 977 | name = "hyper"
 978 | version = "1.3.1"
 979 | source = "registry+https://github.com/rust-lang/crates.io-index"
 980 | checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d"
 981 | dependencies = [
 982 |  "bytes",
 983 |  "futures-channel",
 984 |  "futures-util",
 985 |  "http 1.1.0",
 986 |  "http-body",
 987 |  "httparse",
 988 |  "httpdate",
 989 |  "itoa",
 990 |  "pin-project-lite",
 991 |  "smallvec",
 992 |  "tokio",
 993 | ]
 994 | 
 995 | [[package]]
 996 | name = "hyper-util"
 997 | version = "0.1.3"
 998 | source = "registry+https://github.com/rust-lang/crates.io-index"
 999 | checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa"
1000 | dependencies = [
1001 |  "bytes",
1002 |  "futures-util",
1003 |  "http 1.1.0",
1004 |  "http-body",
1005 |  "hyper",
1006 |  "pin-project-lite",
1007 |  "socket2",
1008 |  "tokio",
1009 | ]
1010 | 
1011 | [[package]]
1012 | name = "iana-time-zone"
1013 | version = "0.1.60"
1014 | source = "registry+https://github.com/rust-lang/crates.io-index"
1015 | checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141"
1016 | dependencies = [
1017 |  "android_system_properties",
1018 |  "core-foundation-sys",
1019 |  "iana-time-zone-haiku",
1020 |  "js-sys",
1021 |  "wasm-bindgen",
1022 |  "windows-core",
1023 | ]
1024 | 
1025 | [[package]]
1026 | name = "iana-time-zone-haiku"
1027 | version = "0.1.2"
1028 | source = "registry+https://github.com/rust-lang/crates.io-index"
1029 | checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
1030 | dependencies = [
1031 |  "cc",
1032 | ]
1033 | 
1034 | [[package]]
1035 | name = "ident_case"
1036 | version = "1.0.1"
1037 | source = "registry+https://github.com/rust-lang/crates.io-index"
1038 | checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
1039 | 
1040 | [[package]]
1041 | name = "idna"
1042 | version = "0.5.0"
1043 | source = "registry+https://github.com/rust-lang/crates.io-index"
1044 | checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6"
1045 | dependencies = [
1046 |  "unicode-bidi",
1047 |  "unicode-normalization",
1048 | ]
1049 | 
1050 | [[package]]
1051 | name = "indexmap"
1052 | version = "2.2.6"
1053 | source = "registry+https://github.com/rust-lang/crates.io-index"
1054 | checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26"
1055 | dependencies = [
1056 |  "equivalent",
1057 |  "hashbrown 0.14.5",
1058 | ]
1059 | 
1060 | [[package]]
1061 | name = "indicatif"
1062 | version = "0.17.8"
1063 | source = "registry+https://github.com/rust-lang/crates.io-index"
1064 | checksum = "763a5a8f45087d6bcea4222e7b72c291a054edf80e4ef6efd2a4979878c7bea3"
1065 | dependencies = [
1066 |  "console",
1067 |  "instant",
1068 |  "number_prefix",
1069 |  "portable-atomic",
1070 |  "unicode-width",
1071 | ]
1072 | 
1073 | [[package]]
1074 | name = "instant"
1075 | version = "0.1.13"
1076 | source = "registry+https://github.com/rust-lang/crates.io-index"
1077 | checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222"
1078 | dependencies = [
1079 |  "cfg-if",
1080 | ]
1081 | 
1082 | [[package]]
1083 | name = "interpolator"
1084 | version = "0.5.0"
1085 | source = "registry+https://github.com/rust-lang/crates.io-index"
1086 | checksum = "71dd52191aae121e8611f1e8dc3e324dd0dd1dee1e6dd91d10ee07a3cfb4d9d8"
1087 | 
1088 | [[package]]
1089 | name = "inventory"
1090 | version = "0.3.15"
1091 | source = "registry+https://github.com/rust-lang/crates.io-index"
1092 | checksum = "f958d3d68f4167080a18141e10381e7634563984a537f2a49a30fd8e53ac5767"
1093 | 
1094 | [[package]]
1095 | name = "itertools"
1096 | version = "0.11.0"
1097 | source = "registry+https://github.com/rust-lang/crates.io-index"
1098 | checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57"
1099 | dependencies = [
1100 |  "either",
1101 | ]
1102 | 
1103 | [[package]]
1104 | name = "itertools"
1105 | version = "0.12.1"
1106 | source = "registry+https://github.com/rust-lang/crates.io-index"
1107 | checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
1108 | dependencies = [
1109 |  "either",
1110 | ]
1111 | 
1112 | [[package]]
1113 | name = "itertools"
1114 | version = "0.13.0"
1115 | source = "registry+https://github.com/rust-lang/crates.io-index"
1116 | checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
1117 | dependencies = [
1118 |  "either",
1119 | ]
1120 | 
1121 | [[package]]
1122 | name = "itoa"
1123 | version = "1.0.11"
1124 | source = "registry+https://github.com/rust-lang/crates.io-index"
1125 | checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
1126 | 
1127 | [[package]]
1128 | name = "jobserver"
1129 | version = "0.1.31"
1130 | source = "registry+https://github.com/rust-lang/crates.io-index"
1131 | checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e"
1132 | dependencies = [
1133 |  "libc",
1134 | ]
1135 | 
1136 | [[package]]
1137 | name = "js-sys"
1138 | version = "0.3.69"
1139 | source = "registry+https://github.com/rust-lang/crates.io-index"
1140 | checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d"
1141 | dependencies = [
1142 |  "wasm-bindgen",
1143 | ]
1144 | 
1145 | [[package]]
1146 | name = "lazy_static"
1147 | version = "1.4.0"
1148 | source = "registry+https://github.com/rust-lang/crates.io-index"
1149 | checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
1150 | 
1151 | [[package]]
1152 | name = "leptos"
1153 | version = "0.6.11"
1154 | source = "registry+https://github.com/rust-lang/crates.io-index"
1155 | checksum = "20f79fe71c41f5a0506c273f6698a1971bb994ef52a88aeaf4eccb159fcd1e11"
1156 | dependencies = [
1157 |  "cfg-if",
1158 |  "leptos_config",
1159 |  "leptos_dom",
1160 |  "leptos_macro",
1161 |  "leptos_reactive",
1162 |  "leptos_server",
1163 |  "server_fn",
1164 |  "tracing",
1165 |  "typed-builder",
1166 |  "typed-builder-macro",
1167 |  "wasm-bindgen",
1168 |  "web-sys",
1169 | ]
1170 | 
1171 | [[package]]
1172 | name = "leptos_axum"
1173 | version = "0.6.11"
1174 | source = "registry+https://github.com/rust-lang/crates.io-index"
1175 | checksum = "590b945e92fe5790820f348f82c0937ea9b58ba5976459377134946eb8ed449b"
1176 | dependencies = [
1177 |  "axum",
1178 |  "cfg-if",
1179 |  "futures",
1180 |  "http-body-util",
1181 |  "leptos",
1182 |  "leptos_integration_utils",
1183 |  "leptos_macro",
1184 |  "leptos_meta",
1185 |  "leptos_router",
1186 |  "once_cell",
1187 |  "parking_lot",
1188 |  "serde_json",
1189 |  "server_fn",
1190 |  "tokio",
1191 |  "tokio-util",
1192 |  "tracing",
1193 | ]
1194 | 
1195 | [[package]]
1196 | name = "leptos_config"
1197 | version = "0.6.11"
1198 | source = "registry+https://github.com/rust-lang/crates.io-index"
1199 | checksum = "a3caa62f62e8e575051305ed6ac5648dc695f202c7220a98aca21cf4e9a978cf"
1200 | dependencies = [
1201 |  "config",
1202 |  "regex",
1203 |  "serde",
1204 |  "thiserror",
1205 |  "typed-builder",
1206 | ]
1207 | 
1208 | [[package]]
1209 | name = "leptos_dom"
1210 | version = "0.6.11"
1211 | source = "registry+https://github.com/rust-lang/crates.io-index"
1212 | checksum = "96e84abb02efd711f0842ff3e444292bfa9963811c37e7be3980a052628ed63b"
1213 | dependencies = [
1214 |  "async-recursion",
1215 |  "cfg-if",
1216 |  "drain_filter_polyfill",
1217 |  "futures",
1218 |  "getrandom",
1219 |  "html-escape",
1220 |  "indexmap",
1221 |  "itertools 0.12.1",
1222 |  "js-sys",
1223 |  "leptos_reactive",
1224 |  "once_cell",
1225 |  "pad-adapter",
1226 |  "paste",
1227 |  "rustc-hash",
1228 |  "serde",
1229 |  "serde_json",
1230 |  "server_fn",
1231 |  "smallvec",
1232 |  "tracing",
1233 |  "wasm-bindgen",
1234 |  "wasm-bindgen-futures",
1235 |  "web-sys",
1236 | ]
1237 | 
1238 | [[package]]
1239 | name = "leptos_hot_reload"
1240 | version = "0.6.11"
1241 | source = "registry+https://github.com/rust-lang/crates.io-index"
1242 | checksum = "c4ee917deba2522a7f22ca826df84a8800d66ac918e58b489875e1f4fb8bc6b8"
1243 | dependencies = [
1244 |  "anyhow",
1245 |  "camino",
1246 |  "indexmap",
1247 |  "parking_lot",
1248 |  "proc-macro2",
1249 |  "quote",
1250 |  "rstml",
1251 |  "serde",
1252 |  "syn 2.0.66",
1253 |  "walkdir",
1254 | ]
1255 | 
1256 | [[package]]
1257 | name = "leptos_integration_utils"
1258 | version = "0.6.11"
1259 | source = "registry+https://github.com/rust-lang/crates.io-index"
1260 | checksum = "a1f504afe3e2ac30ca15ba9b74d27243e8919e93d1f78bad32e5e8ec23eaca4b"
1261 | dependencies = [
1262 |  "futures",
1263 |  "leptos",
1264 |  "leptos_config",
1265 |  "leptos_hot_reload",
1266 |  "leptos_meta",
1267 |  "tracing",
1268 | ]
1269 | 
1270 | [[package]]
1271 | name = "leptos_macro"
1272 | version = "0.6.11"
1273 | source = "registry+https://github.com/rust-lang/crates.io-index"
1274 | checksum = "31197c2c624c405bec5f1dc8dd5d903a6030d1f0b8e362a01a3a215fcbad5051"
1275 | dependencies = [
1276 |  "attribute-derive",
1277 |  "cfg-if",
1278 |  "convert_case",
1279 |  "html-escape",
1280 |  "itertools 0.12.1",
1281 |  "leptos_hot_reload",
1282 |  "prettyplease",
1283 |  "proc-macro-error",
1284 |  "proc-macro2",
1285 |  "quote",
1286 |  "rstml",
1287 |  "server_fn_macro",
1288 |  "syn 2.0.66",
1289 |  "tracing",
1290 |  "uuid",
1291 | ]
1292 | 
1293 | [[package]]
1294 | name = "leptos_meta"
1295 | version = "0.6.11"
1296 | source = "registry+https://github.com/rust-lang/crates.io-index"
1297 | checksum = "a00900e82a4ca892828db93fce1d4c009480ff3959406e6965aa937c8bab7403"
1298 | dependencies = [
1299 |  "cfg-if",
1300 |  "indexmap",
1301 |  "leptos",
1302 |  "tracing",
1303 |  "wasm-bindgen",
1304 |  "web-sys",
1305 | ]
1306 | 
1307 | [[package]]
1308 | name = "leptos_reactive"
1309 | version = "0.6.11"
1310 | source = "registry+https://github.com/rust-lang/crates.io-index"
1311 | checksum = "057de706568ce8f1f223ae69f796c10ad0563ad270d10717e70c2b2d22eefa60"
1312 | dependencies = [
1313 |  "base64 0.22.1",
1314 |  "cfg-if",
1315 |  "futures",
1316 |  "indexmap",
1317 |  "js-sys",
1318 |  "paste",
1319 |  "pin-project",
1320 |  "rustc-hash",
1321 |  "self_cell",
1322 |  "serde",
1323 |  "serde-wasm-bindgen",
1324 |  "serde_json",
1325 |  "slotmap",
1326 |  "thiserror",
1327 |  "tokio",
1328 |  "tracing",
1329 |  "wasm-bindgen",
1330 |  "wasm-bindgen-futures",
1331 |  "web-sys",
1332 | ]
1333 | 
1334 | [[package]]
1335 | name = "leptos_router"
1336 | version = "0.6.11"
1337 | source = "registry+https://github.com/rust-lang/crates.io-index"
1338 | checksum = "d7fcc2a95a20c8f41adb39770e65c48ffe33cd9503b83669c54edd9b33ba8aa8"
1339 | dependencies = [
1340 |  "cached",
1341 |  "cfg-if",
1342 |  "gloo-net",
1343 |  "itertools 0.12.1",
1344 |  "js-sys",
1345 |  "lazy_static",
1346 |  "leptos",
1347 |  "leptos_integration_utils",
1348 |  "leptos_meta",
1349 |  "linear-map",
1350 |  "lru",
1351 |  "once_cell",
1352 |  "percent-encoding",
1353 |  "regex",
1354 |  "send_wrapper",
1355 |  "serde",
1356 |  "serde_json",
1357 |  "serde_qs",
1358 |  "thiserror",
1359 |  "tracing",
1360 |  "url",
1361 |  "wasm-bindgen",
1362 |  "wasm-bindgen-futures",
1363 |  "web-sys",
1364 | ]
1365 | 
1366 | [[package]]
1367 | name = "leptos_server"
1368 | version = "0.6.11"
1369 | source = "registry+https://github.com/rust-lang/crates.io-index"
1370 | checksum = "8f197d9cbf7db3a09a5d6c561ad0547ad6bf4326bc6bc454171d5f6ee94f745a"
1371 | dependencies = [
1372 |  "inventory",
1373 |  "lazy_static",
1374 |  "leptos_macro",
1375 |  "leptos_reactive",
1376 |  "serde",
1377 |  "server_fn",
1378 |  "thiserror",
1379 |  "tracing",
1380 | ]
1381 | 
1382 | [[package]]
1383 | name = "libc"
1384 | version = "0.2.155"
1385 | source = "registry+https://github.com/rust-lang/crates.io-index"
1386 | checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
1387 | 
1388 | [[package]]
1389 | name = "libsqlite3-sys"
1390 | version = "0.28.0"
1391 | source = "registry+https://github.com/rust-lang/crates.io-index"
1392 | checksum = "0c10584274047cb335c23d3e61bcef8e323adae7c5c8c760540f73610177fc3f"
1393 | dependencies = [
1394 |  "pkg-config",
1395 |  "vcpkg",
1396 | ]
1397 | 
1398 | [[package]]
1399 | name = "linear-map"
1400 | version = "1.2.0"
1401 | source = "registry+https://github.com/rust-lang/crates.io-index"
1402 | checksum = "bfae20f6b19ad527b550c223fddc3077a547fc70cda94b9b566575423fd303ee"
1403 | dependencies = [
1404 |  "serde",
1405 |  "serde_test",
1406 | ]
1407 | 
1408 | [[package]]
1409 | name = "lock_api"
1410 | version = "0.4.12"
1411 | source = "registry+https://github.com/rust-lang/crates.io-index"
1412 | checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17"
1413 | dependencies = [
1414 |  "autocfg",
1415 |  "scopeguard",
1416 | ]
1417 | 
1418 | [[package]]
1419 | name = "log"
1420 | version = "0.4.21"
1421 | source = "registry+https://github.com/rust-lang/crates.io-index"
1422 | checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c"
1423 | 
1424 | [[package]]
1425 | name = "lru"
1426 | version = "0.11.1"
1427 | source = "registry+https://github.com/rust-lang/crates.io-index"
1428 | checksum = "a4a83fb7698b3643a0e34f9ae6f2e8f0178c0fd42f8b59d493aa271ff3a5bf21"
1429 | dependencies = [
1430 |  "hashbrown 0.14.5",
1431 | ]
1432 | 
1433 | [[package]]
1434 | name = "macro_rules_attribute"
1435 | version = "0.2.0"
1436 | source = "registry+https://github.com/rust-lang/crates.io-index"
1437 | checksum = "8a82271f7bc033d84bbca59a3ce3e4159938cb08a9c3aebbe54d215131518a13"
1438 | dependencies = [
1439 |  "macro_rules_attribute-proc_macro",
1440 |  "paste",
1441 | ]
1442 | 
1443 | [[package]]
1444 | name = "macro_rules_attribute-proc_macro"
1445 | version = "0.2.0"
1446 | source = "registry+https://github.com/rust-lang/crates.io-index"
1447 | checksum = "b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568"
1448 | 
1449 | [[package]]
1450 | name = "manyhow"
1451 | version = "0.10.4"
1452 | source = "registry+https://github.com/rust-lang/crates.io-index"
1453 | checksum = "f91ea592d76c0b6471965708ccff7e6a5d277f676b90ab31f4d3f3fc77fade64"
1454 | dependencies = [
1455 |  "manyhow-macros",
1456 |  "proc-macro2",
1457 |  "quote",
1458 |  "syn 2.0.66",
1459 | ]
1460 | 
1461 | [[package]]
1462 | name = "manyhow-macros"
1463 | version = "0.10.4"
1464 | source = "registry+https://github.com/rust-lang/crates.io-index"
1465 | checksum = "c64621e2c08f2576e4194ea8be11daf24ac01249a4f53cd8befcbb7077120ead"
1466 | dependencies = [
1467 |  "proc-macro-utils",
1468 |  "proc-macro2",
1469 |  "quote",
1470 | ]
1471 | 
1472 | [[package]]
1473 | name = "matchit"
1474 | version = "0.7.3"
1475 | source = "registry+https://github.com/rust-lang/crates.io-index"
1476 | checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94"
1477 | 
1478 | [[package]]
1479 | name = "memchr"
1480 | version = "2.7.2"
1481 | source = "registry+https://github.com/rust-lang/crates.io-index"
1482 | checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d"
1483 | 
1484 | [[package]]
1485 | name = "mime"
1486 | version = "0.3.17"
1487 | source = "registry+https://github.com/rust-lang/crates.io-index"
1488 | checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
1489 | 
1490 | [[package]]
1491 | name = "mime_guess"
1492 | version = "2.0.4"
1493 | source = "registry+https://github.com/rust-lang/crates.io-index"
1494 | checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef"
1495 | dependencies = [
1496 |  "mime",
1497 |  "unicase",
1498 | ]
1499 | 
1500 | [[package]]
1501 | name = "minimal-lexical"
1502 | version = "0.2.1"
1503 | source = "registry+https://github.com/rust-lang/crates.io-index"
1504 | checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
1505 | 
1506 | [[package]]
1507 | name = "miniz_oxide"
1508 | version = "0.7.3"
1509 | source = "registry+https://github.com/rust-lang/crates.io-index"
1510 | checksum = "87dfd01fe195c66b572b37921ad8803d010623c0aca821bea2302239d155cdae"
1511 | dependencies = [
1512 |  "adler",
1513 | ]
1514 | 
1515 | [[package]]
1516 | name = "mio"
1517 | version = "0.8.11"
1518 | source = "registry+https://github.com/rust-lang/crates.io-index"
1519 | checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c"
1520 | dependencies = [
1521 |  "libc",
1522 |  "wasi",
1523 |  "windows-sys 0.48.0",
1524 | ]
1525 | 
1526 | [[package]]
1527 | name = "monostate"
1528 | version = "0.1.13"
1529 | source = "registry+https://github.com/rust-lang/crates.io-index"
1530 | checksum = "0d208407d7552cd041d8cdb69a1bc3303e029c598738177a3d87082004dc0e1e"
1531 | dependencies = [
1532 |  "monostate-impl",
1533 |  "serde",
1534 | ]
1535 | 
1536 | [[package]]
1537 | name = "monostate-impl"
1538 | version = "0.1.13"
1539 | source = "registry+https://github.com/rust-lang/crates.io-index"
1540 | checksum = "a7ce64b975ed4f123575d11afd9491f2e37bbd5813fbfbc0f09ae1fbddea74e0"
1541 | dependencies = [
1542 |  "proc-macro2",
1543 |  "quote",
1544 |  "syn 2.0.66",
1545 | ]
1546 | 
1547 | [[package]]
1548 | name = "multer"
1549 | version = "3.1.0"
1550 | source = "registry+https://github.com/rust-lang/crates.io-index"
1551 | checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b"
1552 | dependencies = [
1553 |  "bytes",
1554 |  "encoding_rs",
1555 |  "futures-util",
1556 |  "http 1.1.0",
1557 |  "httparse",
1558 |  "memchr",
1559 |  "mime",
1560 |  "spin",
1561 |  "version_check",
1562 | ]
1563 | 
1564 | [[package]]
1565 | name = "nom"
1566 | version = "7.1.3"
1567 | source = "registry+https://github.com/rust-lang/crates.io-index"
1568 | checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
1569 | dependencies = [
1570 |  "memchr",
1571 |  "minimal-lexical",
1572 | ]
1573 | 
1574 | [[package]]
1575 | name = "num"
1576 | version = "0.4.3"
1577 | source = "registry+https://github.com/rust-lang/crates.io-index"
1578 | checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23"
1579 | dependencies = [
1580 |  "num-bigint",
1581 |  "num-complex",
1582 |  "num-integer",
1583 |  "num-iter",
1584 |  "num-rational",
1585 |  "num-traits",
1586 | ]
1587 | 
1588 | [[package]]
1589 | name = "num-bigint"
1590 | version = "0.4.5"
1591 | source = "registry+https://github.com/rust-lang/crates.io-index"
1592 | checksum = "c165a9ab64cf766f73521c0dd2cfdff64f488b8f0b3e621face3462d3db536d7"
1593 | dependencies = [
1594 |  "num-integer",
1595 |  "num-traits",
1596 | ]
1597 | 
1598 | [[package]]
1599 | name = "num-complex"
1600 | version = "0.4.6"
1601 | source = "registry+https://github.com/rust-lang/crates.io-index"
1602 | checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"
1603 | dependencies = [
1604 |  "num-traits",
1605 | ]
1606 | 
1607 | [[package]]
1608 | name = "num-integer"
1609 | version = "0.1.46"
1610 | source = "registry+https://github.com/rust-lang/crates.io-index"
1611 | checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
1612 | dependencies = [
1613 |  "num-traits",
1614 | ]
1615 | 
1616 | [[package]]
1617 | name = "num-iter"
1618 | version = "0.1.45"
1619 | source = "registry+https://github.com/rust-lang/crates.io-index"
1620 | checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf"
1621 | dependencies = [
1622 |  "autocfg",
1623 |  "num-integer",
1624 |  "num-traits",
1625 | ]
1626 | 
1627 | [[package]]
1628 | name = "num-rational"
1629 | version = "0.4.2"
1630 | source = "registry+https://github.com/rust-lang/crates.io-index"
1631 | checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824"
1632 | dependencies = [
1633 |  "num-bigint",
1634 |  "num-integer",
1635 |  "num-traits",
1636 | ]
1637 | 
1638 | [[package]]
1639 | name = "num-traits"
1640 | version = "0.2.19"
1641 | source = "registry+https://github.com/rust-lang/crates.io-index"
1642 | checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
1643 | dependencies = [
1644 |  "autocfg",
1645 | ]
1646 | 
1647 | [[package]]
1648 | name = "num_cpus"
1649 | version = "1.16.0"
1650 | source = "registry+https://github.com/rust-lang/crates.io-index"
1651 | checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
1652 | dependencies = [
1653 |  "hermit-abi",
1654 |  "libc",
1655 | ]
1656 | 
1657 | [[package]]
1658 | name = "number_prefix"
1659 | version = "0.4.0"
1660 | source = "registry+https://github.com/rust-lang/crates.io-index"
1661 | checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
1662 | 
1663 | [[package]]
1664 | name = "object"
1665 | version = "0.32.2"
1666 | source = "registry+https://github.com/rust-lang/crates.io-index"
1667 | checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441"
1668 | dependencies = [
1669 |  "memchr",
1670 | ]
1671 | 
1672 | [[package]]
1673 | name = "once_cell"
1674 | version = "1.19.0"
1675 | source = "registry+https://github.com/rust-lang/crates.io-index"
1676 | checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
1677 | 
1678 | [[package]]
1679 | name = "onig"
1680 | version = "6.4.0"
1681 | source = "registry+https://github.com/rust-lang/crates.io-index"
1682 | checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f"
1683 | dependencies = [
1684 |  "bitflags 1.3.2",
1685 |  "libc",
1686 |  "once_cell",
1687 |  "onig_sys",
1688 | ]
1689 | 
1690 | [[package]]
1691 | name = "onig_sys"
1692 | version = "69.8.1"
1693 | source = "registry+https://github.com/rust-lang/crates.io-index"
1694 | checksum = "7b829e3d7e9cc74c7e315ee8edb185bf4190da5acde74afd7fc59c35b1f086e7"
1695 | dependencies = [
1696 |  "cc",
1697 |  "pkg-config",
1698 | ]
1699 | 
1700 | [[package]]
1701 | name = "pad-adapter"
1702 | version = "0.1.1"
1703 | source = "registry+https://github.com/rust-lang/crates.io-index"
1704 | checksum = "56d80efc4b6721e8be2a10a5df21a30fa0b470f1539e53d8b4e6e75faf938b63"
1705 | 
1706 | [[package]]
1707 | name = "parking_lot"
1708 | version = "0.12.2"
1709 | source = "registry+https://github.com/rust-lang/crates.io-index"
1710 | checksum = "7e4af0ca4f6caed20e900d564c242b8e5d4903fdacf31d3daf527b66fe6f42fb"
1711 | dependencies = [
1712 |  "lock_api",
1713 |  "parking_lot_core",
1714 | ]
1715 | 
1716 | [[package]]
1717 | name = "parking_lot_core"
1718 | version = "0.9.10"
1719 | source = "registry+https://github.com/rust-lang/crates.io-index"
1720 | checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
1721 | dependencies = [
1722 |  "cfg-if",
1723 |  "libc",
1724 |  "redox_syscall",
1725 |  "smallvec",
1726 |  "windows-targets 0.52.5",
1727 | ]
1728 | 
1729 | [[package]]
1730 | name = "paste"
1731 | version = "1.0.15"
1732 | source = "registry+https://github.com/rust-lang/crates.io-index"
1733 | checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
1734 | 
1735 | [[package]]
1736 | name = "pathdiff"
1737 | version = "0.2.1"
1738 | source = "registry+https://github.com/rust-lang/crates.io-index"
1739 | checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd"
1740 | 
1741 | [[package]]
1742 | name = "percent-encoding"
1743 | version = "2.3.1"
1744 | source = "registry+https://github.com/rust-lang/crates.io-index"
1745 | checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
1746 | 
1747 | [[package]]
1748 | name = "pin-project"
1749 | version = "1.1.5"
1750 | source = "registry+https://github.com/rust-lang/crates.io-index"
1751 | checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3"
1752 | dependencies = [
1753 |  "pin-project-internal",
1754 | ]
1755 | 
1756 | [[package]]
1757 | name = "pin-project-internal"
1758 | version = "1.1.5"
1759 | source = "registry+https://github.com/rust-lang/crates.io-index"
1760 | checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965"
1761 | dependencies = [
1762 |  "proc-macro2",
1763 |  "quote",
1764 |  "syn 2.0.66",
1765 | ]
1766 | 
1767 | [[package]]
1768 | name = "pin-project-lite"
1769 | version = "0.2.14"
1770 | source = "registry+https://github.com/rust-lang/crates.io-index"
1771 | checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02"
1772 | 
1773 | [[package]]
1774 | name = "pin-utils"
1775 | version = "0.1.0"
1776 | source = "registry+https://github.com/rust-lang/crates.io-index"
1777 | checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
1778 | 
1779 | [[package]]
1780 | name = "pkg-config"
1781 | version = "0.3.30"
1782 | source = "registry+https://github.com/rust-lang/crates.io-index"
1783 | checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"
1784 | 
1785 | [[package]]
1786 | name = "portable-atomic"
1787 | version = "1.6.0"
1788 | source = "registry+https://github.com/rust-lang/crates.io-index"
1789 | checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0"
1790 | 
1791 | [[package]]
1792 | name = "ppv-lite86"
1793 | version = "0.2.17"
1794 | source = "registry+https://github.com/rust-lang/crates.io-index"
1795 | checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
1796 | 
1797 | [[package]]
1798 | name = "prettyplease"
1799 | version = "0.2.20"
1800 | source = "registry+https://github.com/rust-lang/crates.io-index"
1801 | checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e"
1802 | dependencies = [
1803 |  "proc-macro2",
1804 |  "syn 2.0.66",
1805 | ]
1806 | 
1807 | [[package]]
1808 | name = "proc-macro-error"
1809 | version = "1.0.4"
1810 | source = "registry+https://github.com/rust-lang/crates.io-index"
1811 | checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
1812 | dependencies = [
1813 |  "proc-macro-error-attr",
1814 |  "proc-macro2",
1815 |  "quote",
1816 |  "version_check",
1817 | ]
1818 | 
1819 | [[package]]
1820 | name = "proc-macro-error-attr"
1821 | version = "1.0.4"
1822 | source = "registry+https://github.com/rust-lang/crates.io-index"
1823 | checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
1824 | dependencies = [
1825 |  "proc-macro2",
1826 |  "quote",
1827 |  "version_check",
1828 | ]
1829 | 
1830 | [[package]]
1831 | name = "proc-macro-utils"
1832 | version = "0.8.0"
1833 | source = "registry+https://github.com/rust-lang/crates.io-index"
1834 | checksum = "3f59e109e2f795a5070e69578c4dc101068139f74616778025ae1011d4cd41a8"
1835 | dependencies = [
1836 |  "proc-macro2",
1837 |  "quote",
1838 |  "smallvec",
1839 | ]
1840 | 
1841 | [[package]]
1842 | name = "proc-macro2"
1843 | version = "1.0.83"
1844 | source = "registry+https://github.com/rust-lang/crates.io-index"
1845 | checksum = "0b33eb56c327dec362a9e55b3ad14f9d2f0904fb5a5b03b513ab5465399e9f43"
1846 | dependencies = [
1847 |  "unicode-ident",
1848 | ]
1849 | 
1850 | [[package]]
1851 | name = "proc-macro2-diagnostics"
1852 | version = "0.10.1"
1853 | source = "registry+https://github.com/rust-lang/crates.io-index"
1854 | checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8"
1855 | dependencies = [
1856 |  "proc-macro2",
1857 |  "quote",
1858 |  "syn 2.0.66",
1859 |  "version_check",
1860 |  "yansi",
1861 | ]
1862 | 
1863 | [[package]]
1864 | name = "quote"
1865 | version = "1.0.36"
1866 | source = "registry+https://github.com/rust-lang/crates.io-index"
1867 | checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
1868 | dependencies = [
1869 |  "proc-macro2",
1870 | ]
1871 | 
1872 | [[package]]
1873 | name = "quote-use"
1874 | version = "0.8.0"
1875 | source = "registry+https://github.com/rust-lang/crates.io-index"
1876 | checksum = "b393938dcaab992375d7b3df7887fa98cc91c2f3590598251e7c609e2b788139"
1877 | dependencies = [
1878 |  "quote",
1879 |  "quote-use-macros",
1880 | ]
1881 | 
1882 | [[package]]
1883 | name = "quote-use-macros"
1884 | version = "0.8.0"
1885 | source = "registry+https://github.com/rust-lang/crates.io-index"
1886 | checksum = "71d8772387900c205780e2c240cfe4dd01355ab4f96a503d99bdf34ad73180ef"
1887 | dependencies = [
1888 |  "derive-where",
1889 |  "proc-macro-utils",
1890 |  "proc-macro2",
1891 |  "quote",
1892 |  "syn 2.0.66",
1893 | ]
1894 | 
1895 | [[package]]
1896 | name = "rand"
1897 | version = "0.8.5"
1898 | source = "registry+https://github.com/rust-lang/crates.io-index"
1899 | checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
1900 | dependencies = [
1901 |  "libc",
1902 |  "rand_chacha",
1903 |  "rand_core",
1904 | ]
1905 | 
1906 | [[package]]
1907 | name = "rand_chacha"
1908 | version = "0.3.1"
1909 | source = "registry+https://github.com/rust-lang/crates.io-index"
1910 | checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
1911 | dependencies = [
1912 |  "ppv-lite86",
1913 |  "rand_core",
1914 | ]
1915 | 
1916 | [[package]]
1917 | name = "rand_core"
1918 | version = "0.6.4"
1919 | source = "registry+https://github.com/rust-lang/crates.io-index"
1920 | checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
1921 | dependencies = [
1922 |  "getrandom",
1923 | ]
1924 | 
1925 | [[package]]
1926 | name = "rayon"
1927 | version = "1.10.0"
1928 | source = "registry+https://github.com/rust-lang/crates.io-index"
1929 | checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa"
1930 | dependencies = [
1931 |  "either",
1932 |  "rayon-core",
1933 | ]
1934 | 
1935 | [[package]]
1936 | name = "rayon-cond"
1937 | version = "0.3.0"
1938 | source = "registry+https://github.com/rust-lang/crates.io-index"
1939 | checksum = "059f538b55efd2309c9794130bc149c6a553db90e9d99c2030785c82f0bd7df9"
1940 | dependencies = [
1941 |  "either",
1942 |  "itertools 0.11.0",
1943 |  "rayon",
1944 | ]
1945 | 
1946 | [[package]]
1947 | name = "rayon-core"
1948 | version = "1.12.1"
1949 | source = "registry+https://github.com/rust-lang/crates.io-index"
1950 | checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"
1951 | dependencies = [
1952 |  "crossbeam-deque",
1953 |  "crossbeam-utils",
1954 | ]
1955 | 
1956 | [[package]]
1957 | name = "redox_syscall"
1958 | version = "0.5.1"
1959 | source = "registry+https://github.com/rust-lang/crates.io-index"
1960 | checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e"
1961 | dependencies = [
1962 |  "bitflags 2.5.0",
1963 | ]
1964 | 
1965 | [[package]]
1966 | name = "regex"
1967 | version = "1.10.4"
1968 | source = "registry+https://github.com/rust-lang/crates.io-index"
1969 | checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c"
1970 | dependencies = [
1971 |  "aho-corasick",
1972 |  "memchr",
1973 |  "regex-automata",
1974 |  "regex-syntax",
1975 | ]
1976 | 
1977 | [[package]]
1978 | name = "regex-automata"
1979 | version = "0.4.6"
1980 | source = "registry+https://github.com/rust-lang/crates.io-index"
1981 | checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea"
1982 | dependencies = [
1983 |  "aho-corasick",
1984 |  "memchr",
1985 |  "regex-syntax",
1986 | ]
1987 | 
1988 | [[package]]
1989 | name = "regex-syntax"
1990 | version = "0.8.3"
1991 | source = "registry+https://github.com/rust-lang/crates.io-index"
1992 | checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56"
1993 | 
1994 | [[package]]
1995 | name = "rmp"
1996 | version = "0.8.14"
1997 | source = "registry+https://github.com/rust-lang/crates.io-index"
1998 | checksum = "228ed7c16fa39782c3b3468e974aec2795e9089153cd08ee2e9aefb3613334c4"
1999 | dependencies = [
2000 |  "byteorder",
2001 |  "num-traits",
2002 |  "paste",
2003 | ]
2004 | 
2005 | [[package]]
2006 | name = "rmp-serde"
2007 | version = "1.3.0"
2008 | source = "registry+https://github.com/rust-lang/crates.io-index"
2009 | checksum = "52e599a477cf9840e92f2cde9a7189e67b42c57532749bf90aea6ec10facd4db"
2010 | dependencies = [
2011 |  "byteorder",
2012 |  "rmp",
2013 |  "serde",
2014 | ]
2015 | 
2016 | [[package]]
2017 | name = "rstml"
2018 | version = "0.11.2"
2019 | source = "registry+https://github.com/rust-lang/crates.io-index"
2020 | checksum = "fe542870b8f59dd45ad11d382e5339c9a1047cde059be136a7016095bbdefa77"
2021 | dependencies = [
2022 |  "proc-macro2",
2023 |  "proc-macro2-diagnostics",
2024 |  "quote",
2025 |  "syn 2.0.66",
2026 |  "syn_derive",
2027 |  "thiserror",
2028 | ]
2029 | 
2030 | [[package]]
2031 | name = "rusqlite"
2032 | version = "0.31.0"
2033 | source = "registry+https://github.com/rust-lang/crates.io-index"
2034 | checksum = "b838eba278d213a8beaf485bd313fd580ca4505a00d5871caeb1457c55322cae"
2035 | dependencies = [
2036 |  "bitflags 2.5.0",
2037 |  "fallible-iterator",
2038 |  "fallible-streaming-iterator",
2039 |  "hashlink",
2040 |  "libsqlite3-sys",
2041 |  "smallvec",
2042 | ]
2043 | 
2044 | [[package]]
2045 | name = "rustc-demangle"
2046 | version = "0.1.24"
2047 | source = "registry+https://github.com/rust-lang/crates.io-index"
2048 | checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
2049 | 
2050 | [[package]]
2051 | name = "rustc-hash"
2052 | version = "1.1.0"
2053 | source = "registry+https://github.com/rust-lang/crates.io-index"
2054 | checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
2055 | 
2056 | [[package]]
2057 | name = "rustversion"
2058 | version = "1.0.17"
2059 | source = "registry+https://github.com/rust-lang/crates.io-index"
2060 | checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6"
2061 | 
2062 | [[package]]
2063 | name = "ryu"
2064 | version = "1.0.18"
2065 | source = "registry+https://github.com/rust-lang/crates.io-index"
2066 | checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
2067 | 
2068 | [[package]]
2069 | name = "same-file"
2070 | version = "1.0.6"
2071 | source = "registry+https://github.com/rust-lang/crates.io-index"
2072 | checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
2073 | dependencies = [
2074 |  "winapi-util",
2075 | ]
2076 | 
2077 | [[package]]
2078 | name = "scopeguard"
2079 | version = "1.2.0"
2080 | source = "registry+https://github.com/rust-lang/crates.io-index"
2081 | checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
2082 | 
2083 | [[package]]
2084 | name = "self_cell"
2085 | version = "1.0.4"
2086 | source = "registry+https://github.com/rust-lang/crates.io-index"
2087 | checksum = "d369a96f978623eb3dc28807c4852d6cc617fed53da5d3c400feff1ef34a714a"
2088 | 
2089 | [[package]]
2090 | name = "send_wrapper"
2091 | version = "0.6.0"
2092 | source = "registry+https://github.com/rust-lang/crates.io-index"
2093 | checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73"
2094 | dependencies = [
2095 |  "futures-core",
2096 | ]
2097 | 
2098 | [[package]]
2099 | name = "serde"
2100 | version = "1.0.202"
2101 | source = "registry+https://github.com/rust-lang/crates.io-index"
2102 | checksum = "226b61a0d411b2ba5ff6d7f73a476ac4f8bb900373459cd00fab8512828ba395"
2103 | dependencies = [
2104 |  "serde_derive",
2105 | ]
2106 | 
2107 | [[package]]
2108 | name = "serde-wasm-bindgen"
2109 | version = "0.6.5"
2110 | source = "registry+https://github.com/rust-lang/crates.io-index"
2111 | checksum = "8302e169f0eddcc139c70f139d19d6467353af16f9fce27e8c30158036a1e16b"
2112 | dependencies = [
2113 |  "js-sys",
2114 |  "serde",
2115 |  "wasm-bindgen",
2116 | ]
2117 | 
2118 | [[package]]
2119 | name = "serde_derive"
2120 | version = "1.0.202"
2121 | source = "registry+https://github.com/rust-lang/crates.io-index"
2122 | checksum = "6048858004bcff69094cd972ed40a32500f153bd3be9f716b2eed2e8217c4838"
2123 | dependencies = [
2124 |  "proc-macro2",
2125 |  "quote",
2126 |  "syn 2.0.66",
2127 | ]
2128 | 
2129 | [[package]]
2130 | name = "serde_json"
2131 | version = "1.0.117"
2132 | source = "registry+https://github.com/rust-lang/crates.io-index"
2133 | checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3"
2134 | dependencies = [
2135 |  "itoa",
2136 |  "ryu",
2137 |  "serde",
2138 | ]
2139 | 
2140 | [[package]]
2141 | name = "serde_path_to_error"
2142 | version = "0.1.16"
2143 | source = "registry+https://github.com/rust-lang/crates.io-index"
2144 | checksum = "af99884400da37c88f5e9146b7f1fd0fbcae8f6eec4e9da38b67d05486f814a6"
2145 | dependencies = [
2146 |  "itoa",
2147 |  "serde",
2148 | ]
2149 | 
2150 | [[package]]
2151 | name = "serde_qs"
2152 | version = "0.12.0"
2153 | source = "registry+https://github.com/rust-lang/crates.io-index"
2154 | checksum = "0431a35568651e363364210c91983c1da5eb29404d9f0928b67d4ebcfa7d330c"
2155 | dependencies = [
2156 |  "percent-encoding",
2157 |  "serde",
2158 |  "thiserror",
2159 | ]
2160 | 
2161 | [[package]]
2162 | name = "serde_spanned"
2163 | version = "0.6.6"
2164 | source = "registry+https://github.com/rust-lang/crates.io-index"
2165 | checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0"
2166 | dependencies = [
2167 |  "serde",
2168 | ]
2169 | 
2170 | [[package]]
2171 | name = "serde_test"
2172 | version = "1.0.176"
2173 | source = "registry+https://github.com/rust-lang/crates.io-index"
2174 | checksum = "5a2f49ace1498612d14f7e0b8245519584db8299541dfe31a06374a828d620ab"
2175 | dependencies = [
2176 |  "serde",
2177 | ]
2178 | 
2179 | [[package]]
2180 | name = "serde_urlencoded"
2181 | version = "0.7.1"
2182 | source = "registry+https://github.com/rust-lang/crates.io-index"
2183 | checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"
2184 | dependencies = [
2185 |  "form_urlencoded",
2186 |  "itoa",
2187 |  "ryu",
2188 |  "serde",
2189 | ]
2190 | 
2191 | [[package]]
2192 | name = "server_fn"
2193 | version = "0.6.11"
2194 | source = "registry+https://github.com/rust-lang/crates.io-index"
2195 | checksum = "536a5b959673643ee01e59ae41bf01425482c8070dee95d7061ee2d45296b59c"
2196 | dependencies = [
2197 |  "axum",
2198 |  "bytes",
2199 |  "ciborium",
2200 |  "const_format",
2201 |  "dashmap",
2202 |  "futures",
2203 |  "gloo-net",
2204 |  "http 1.1.0",
2205 |  "http-body-util",
2206 |  "hyper",
2207 |  "inventory",
2208 |  "js-sys",
2209 |  "once_cell",
2210 |  "send_wrapper",
2211 |  "serde",
2212 |  "serde_json",
2213 |  "serde_qs",
2214 |  "server_fn_macro_default",
2215 |  "thiserror",
2216 |  "tower",
2217 |  "tower-layer",
2218 |  "url",
2219 |  "wasm-bindgen",
2220 |  "wasm-bindgen-futures",
2221 |  "wasm-streams",
2222 |  "web-sys",
2223 |  "xxhash-rust",
2224 | ]
2225 | 
2226 | [[package]]
2227 | name = "server_fn_macro"
2228 | version = "0.6.11"
2229 | source = "registry+https://github.com/rust-lang/crates.io-index"
2230 | checksum = "064dd9b256e78bf2886774f265cc34d2aefdd05b430c58c78a69eceef21b5e60"
2231 | dependencies = [
2232 |  "const_format",
2233 |  "convert_case",
2234 |  "proc-macro2",
2235 |  "quote",
2236 |  "syn 2.0.66",
2237 |  "xxhash-rust",
2238 | ]
2239 | 
2240 | [[package]]
2241 | name = "server_fn_macro_default"
2242 | version = "0.6.11"
2243 | source = "registry+https://github.com/rust-lang/crates.io-index"
2244 | checksum = "f4ad11700cbccdbd313703916eb8c97301ee423c4a06e5421b77956fdcb36a9f"
2245 | dependencies = [
2246 |  "server_fn_macro",
2247 |  "syn 2.0.66",
2248 | ]
2249 | 
2250 | [[package]]
2251 | name = "slab"
2252 | version = "0.4.9"
2253 | source = "registry+https://github.com/rust-lang/crates.io-index"
2254 | checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67"
2255 | dependencies = [
2256 |  "autocfg",
2257 | ]
2258 | 
2259 | [[package]]
2260 | name = "slotmap"
2261 | version = "1.0.7"
2262 | source = "registry+https://github.com/rust-lang/crates.io-index"
2263 | checksum = "dbff4acf519f630b3a3ddcfaea6c06b42174d9a44bc70c620e9ed1649d58b82a"
2264 | dependencies = [
2265 |  "serde",
2266 |  "version_check",
2267 | ]
2268 | 
2269 | [[package]]
2270 | name = "smallvec"
2271 | version = "1.13.2"
2272 | source = "registry+https://github.com/rust-lang/crates.io-index"
2273 | checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
2274 | 
2275 | [[package]]
2276 | name = "socket2"
2277 | version = "0.5.7"
2278 | source = "registry+https://github.com/rust-lang/crates.io-index"
2279 | checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c"
2280 | dependencies = [
2281 |  "libc",
2282 |  "windows-sys 0.52.0",
2283 | ]
2284 | 
2285 | [[package]]
2286 | name = "spin"
2287 | version = "0.9.8"
2288 | source = "registry+https://github.com/rust-lang/crates.io-index"
2289 | checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
2290 | 
2291 | [[package]]
2292 | name = "spm_precompiled"
2293 | version = "0.1.4"
2294 | source = "registry+https://github.com/rust-lang/crates.io-index"
2295 | checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326"
2296 | dependencies = [
2297 |  "base64 0.13.1",
2298 |  "nom",
2299 |  "serde",
2300 |  "unicode-segmentation",
2301 | ]
2302 | 
2303 | [[package]]
2304 | name = "strsim"
2305 | version = "0.10.0"
2306 | source = "registry+https://github.com/rust-lang/crates.io-index"
2307 | checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
2308 | 
2309 | [[package]]
2310 | name = "strsim"
2311 | version = "0.11.1"
2312 | source = "registry+https://github.com/rust-lang/crates.io-index"
2313 | checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
2314 | 
2315 | [[package]]
2316 | name = "syn"
2317 | version = "1.0.109"
2318 | source = "registry+https://github.com/rust-lang/crates.io-index"
2319 | checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
2320 | dependencies = [
2321 |  "proc-macro2",
2322 |  "quote",
2323 |  "unicode-ident",
2324 | ]
2325 | 
2326 | [[package]]
2327 | name = "syn"
2328 | version = "2.0.66"
2329 | source = "registry+https://github.com/rust-lang/crates.io-index"
2330 | checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5"
2331 | dependencies = [
2332 |  "proc-macro2",
2333 |  "quote",
2334 |  "unicode-ident",
2335 | ]
2336 | 
2337 | [[package]]
2338 | name = "syn_derive"
2339 | version = "0.1.8"
2340 | source = "registry+https://github.com/rust-lang/crates.io-index"
2341 | checksum = "1329189c02ff984e9736652b1631330da25eaa6bc639089ed4915d25446cbe7b"
2342 | dependencies = [
2343 |  "proc-macro-error",
2344 |  "proc-macro2",
2345 |  "quote",
2346 |  "syn 2.0.66",
2347 | ]
2348 | 
2349 | [[package]]
2350 | name = "sync_wrapper"
2351 | version = "0.1.2"
2352 | source = "registry+https://github.com/rust-lang/crates.io-index"
2353 | checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
2354 | 
2355 | [[package]]
2356 | name = "sync_wrapper"
2357 | version = "1.0.1"
2358 | source = "registry+https://github.com/rust-lang/crates.io-index"
2359 | checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394"
2360 | 
2361 | [[package]]
2362 | name = "thiserror"
2363 | version = "1.0.61"
2364 | source = "registry+https://github.com/rust-lang/crates.io-index"
2365 | checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709"
2366 | dependencies = [
2367 |  "thiserror-impl",
2368 | ]
2369 | 
2370 | [[package]]
2371 | name = "thiserror-impl"
2372 | version = "1.0.61"
2373 | source = "registry+https://github.com/rust-lang/crates.io-index"
2374 | checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533"
2375 | dependencies = [
2376 |  "proc-macro2",
2377 |  "quote",
2378 |  "syn 2.0.66",
2379 | ]
2380 | 
2381 | [[package]]
2382 | name = "tiktoken-rs"
2383 | version = "0.5.9"
2384 | source = "registry+https://github.com/rust-lang/crates.io-index"
2385 | checksum = "c314e7ce51440f9e8f5a497394682a57b7c323d0f4d0a6b1b13c429056e0e234"
2386 | dependencies = [
2387 |  "anyhow",
2388 |  "base64 0.21.7",
2389 |  "bstr",
2390 |  "fancy-regex",
2391 |  "lazy_static",
2392 |  "parking_lot",
2393 |  "rustc-hash",
2394 | ]
2395 | 
2396 | [[package]]
2397 | name = "tinyvec"
2398 | version = "1.6.0"
2399 | source = "registry+https://github.com/rust-lang/crates.io-index"
2400 | checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50"
2401 | dependencies = [
2402 |  "tinyvec_macros",
2403 | ]
2404 | 
2405 | [[package]]
2406 | name = "tinyvec_macros"
2407 | version = "0.1.1"
2408 | source = "registry+https://github.com/rust-lang/crates.io-index"
2409 | checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
2410 | 
2411 | [[package]]
2412 | name = "tokenizers"
2413 | version = "0.19.1"
2414 | source = "registry+https://github.com/rust-lang/crates.io-index"
2415 | checksum = "e500fad1dd3af3d626327e6a3fe5050e664a6eaa4708b8ca92f1794aaf73e6fd"
2416 | dependencies = [
2417 |  "aho-corasick",
2418 |  "derive_builder",
2419 |  "esaxx-rs",
2420 |  "getrandom",
2421 |  "indicatif",
2422 |  "itertools 0.12.1",
2423 |  "lazy_static",
2424 |  "log",
2425 |  "macro_rules_attribute",
2426 |  "monostate",
2427 |  "onig",
2428 |  "paste",
2429 |  "rand",
2430 |  "rayon",
2431 |  "rayon-cond",
2432 |  "regex",
2433 |  "regex-syntax",
2434 |  "serde",
2435 |  "serde_json",
2436 |  "spm_precompiled",
2437 |  "thiserror",
2438 |  "unicode-normalization-alignments",
2439 |  "unicode-segmentation",
2440 |  "unicode_categories",
2441 | ]
2442 | 
2443 | [[package]]
2444 | name = "tokio"
2445 | version = "1.37.0"
2446 | source = "registry+https://github.com/rust-lang/crates.io-index"
2447 | checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787"
2448 | dependencies = [
2449 |  "backtrace",
2450 |  "bytes",
2451 |  "libc",
2452 |  "mio",
2453 |  "num_cpus",
2454 |  "pin-project-lite",
2455 |  "socket2",
2456 |  "tokio-macros",
2457 |  "windows-sys 0.48.0",
2458 | ]
2459 | 
2460 | [[package]]
2461 | name = "tokio-macros"
2462 | version = "2.2.0"
2463 | source = "registry+https://github.com/rust-lang/crates.io-index"
2464 | checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b"
2465 | dependencies = [
2466 |  "proc-macro2",
2467 |  "quote",
2468 |  "syn 2.0.66",
2469 | ]
2470 | 
2471 | [[package]]
2472 | name = "tokio-util"
2473 | version = "0.7.11"
2474 | source = "registry+https://github.com/rust-lang/crates.io-index"
2475 | checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1"
2476 | dependencies = [
2477 |  "bytes",
2478 |  "futures-core",
2479 |  "futures-sink",
2480 |  "futures-util",
2481 |  "hashbrown 0.14.5",
2482 |  "pin-project-lite",
2483 |  "tokio",
2484 | ]
2485 | 
2486 | [[package]]
2487 | name = "toml"
2488 | version = "0.8.13"
2489 | source = "registry+https://github.com/rust-lang/crates.io-index"
2490 | checksum = "a4e43f8cc456c9704c851ae29c67e17ef65d2c30017c17a9765b89c382dc8bba"
2491 | dependencies = [
2492 |  "serde",
2493 |  "serde_spanned",
2494 |  "toml_datetime",
2495 |  "toml_edit",
2496 | ]
2497 | 
2498 | [[package]]
2499 | name = "toml_datetime"
2500 | version = "0.6.6"
2501 | source = "registry+https://github.com/rust-lang/crates.io-index"
2502 | checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf"
2503 | dependencies = [
2504 |  "serde",
2505 | ]
2506 | 
2507 | [[package]]
2508 | name = "toml_edit"
2509 | version = "0.22.13"
2510 | source = "registry+https://github.com/rust-lang/crates.io-index"
2511 | checksum = "c127785850e8c20836d49732ae6abfa47616e60bf9d9f57c43c250361a9db96c"
2512 | dependencies = [
2513 |  "indexmap",
2514 |  "serde",
2515 |  "serde_spanned",
2516 |  "toml_datetime",
2517 |  "winnow",
2518 | ]
2519 | 
2520 | [[package]]
2521 | name = "tower"
2522 | version = "0.4.13"
2523 | source = "registry+https://github.com/rust-lang/crates.io-index"
2524 | checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c"
2525 | dependencies = [
2526 |  "futures-core",
2527 |  "futures-util",
2528 |  "pin-project",
2529 |  "pin-project-lite",
2530 |  "tokio",
2531 |  "tower-layer",
2532 |  "tower-service",
2533 |  "tracing",
2534 | ]
2535 | 
2536 | [[package]]
2537 | name = "tower-http"
2538 | version = "0.5.2"
2539 | source = "registry+https://github.com/rust-lang/crates.io-index"
2540 | checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5"
2541 | dependencies = [
2542 |  "bitflags 2.5.0",
2543 |  "bytes",
2544 |  "futures-util",
2545 |  "http 1.1.0",
2546 |  "http-body",
2547 |  "http-body-util",
2548 |  "http-range-header",
2549 |  "httpdate",
2550 |  "mime",
2551 |  "mime_guess",
2552 |  "percent-encoding",
2553 |  "pin-project-lite",
2554 |  "tokio",
2555 |  "tokio-util",
2556 |  "tower-layer",
2557 |  "tower-service",
2558 |  "tracing",
2559 | ]
2560 | 
2561 | [[package]]
2562 | name = "tower-layer"
2563 | version = "0.3.2"
2564 | source = "registry+https://github.com/rust-lang/crates.io-index"
2565 | checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0"
2566 | 
2567 | [[package]]
2568 | name = "tower-service"
2569 | version = "0.3.2"
2570 | source = "registry+https://github.com/rust-lang/crates.io-index"
2571 | checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
2572 | 
2573 | [[package]]
2574 | name = "tracing"
2575 | version = "0.1.40"
2576 | source = "registry+https://github.com/rust-lang/crates.io-index"
2577 | checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef"
2578 | dependencies = [
2579 |  "log",
2580 |  "pin-project-lite",
2581 |  "tracing-attributes",
2582 |  "tracing-core",
2583 | ]
2584 | 
2585 | [[package]]
2586 | name = "tracing-attributes"
2587 | version = "0.1.27"
2588 | source = "registry+https://github.com/rust-lang/crates.io-index"
2589 | checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
2590 | dependencies = [
2591 |  "proc-macro2",
2592 |  "quote",
2593 |  "syn 2.0.66",
2594 | ]
2595 | 
2596 | [[package]]
2597 | name = "tracing-core"
2598 | version = "0.1.32"
2599 | source = "registry+https://github.com/rust-lang/crates.io-index"
2600 | checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54"
2601 | dependencies = [
2602 |  "once_cell",
2603 | ]
2604 | 
2605 | [[package]]
2606 | name = "typed-builder"
2607 | version = "0.18.2"
2608 | source = "registry+https://github.com/rust-lang/crates.io-index"
2609 | checksum = "77739c880e00693faef3d65ea3aad725f196da38b22fdc7ea6ded6e1ce4d3add"
2610 | dependencies = [
2611 |  "typed-builder-macro",
2612 | ]
2613 | 
2614 | [[package]]
2615 | name = "typed-builder-macro"
2616 | version = "0.18.2"
2617 | source = "registry+https://github.com/rust-lang/crates.io-index"
2618 | checksum = "1f718dfaf347dcb5b983bfc87608144b0bad87970aebcbea5ce44d2a30c08e63"
2619 | dependencies = [
2620 |  "proc-macro2",
2621 |  "quote",
2622 |  "syn 2.0.66",
2623 | ]
2624 | 
2625 | [[package]]
2626 | name = "unicase"
2627 | version = "2.7.0"
2628 | source = "registry+https://github.com/rust-lang/crates.io-index"
2629 | checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89"
2630 | dependencies = [
2631 |  "version_check",
2632 | ]
2633 | 
2634 | [[package]]
2635 | name = "unicode-bidi"
2636 | version = "0.3.15"
2637 | source = "registry+https://github.com/rust-lang/crates.io-index"
2638 | checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75"
2639 | 
2640 | [[package]]
2641 | name = "unicode-ident"
2642 | version = "1.0.12"
2643 | source = "registry+https://github.com/rust-lang/crates.io-index"
2644 | checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
2645 | 
2646 | [[package]]
2647 | name = "unicode-normalization"
2648 | version = "0.1.23"
2649 | source = "registry+https://github.com/rust-lang/crates.io-index"
2650 | checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5"
2651 | dependencies = [
2652 |  "tinyvec",
2653 | ]
2654 | 
2655 | [[package]]
2656 | name = "unicode-normalization-alignments"
2657 | version = "0.1.12"
2658 | source = "registry+https://github.com/rust-lang/crates.io-index"
2659 | checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de"
2660 | dependencies = [
2661 |  "smallvec",
2662 | ]
2663 | 
2664 | [[package]]
2665 | name = "unicode-segmentation"
2666 | version = "1.11.0"
2667 | source = "registry+https://github.com/rust-lang/crates.io-index"
2668 | checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202"
2669 | 
2670 | [[package]]
2671 | name = "unicode-width"
2672 | version = "0.1.12"
2673 | source = "registry+https://github.com/rust-lang/crates.io-index"
2674 | checksum = "68f5e5f3158ecfd4b8ff6fe086db7c8467a2dfdac97fe420f2b7c4aa97af66d6"
2675 | 
2676 | [[package]]
2677 | name = "unicode-xid"
2678 | version = "0.2.4"
2679 | source = "registry+https://github.com/rust-lang/crates.io-index"
2680 | checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c"
2681 | 
2682 | [[package]]
2683 | name = "unicode_categories"
2684 | version = "0.1.1"
2685 | source = "registry+https://github.com/rust-lang/crates.io-index"
2686 | checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
2687 | 
2688 | [[package]]
2689 | name = "url"
2690 | version = "2.5.0"
2691 | source = "registry+https://github.com/rust-lang/crates.io-index"
2692 | checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633"
2693 | dependencies = [
2694 |  "form_urlencoded",
2695 |  "idna",
2696 |  "percent-encoding",
2697 | ]
2698 | 
2699 | [[package]]
2700 | name = "utf8-width"
2701 | version = "0.1.7"
2702 | source = "registry+https://github.com/rust-lang/crates.io-index"
2703 | checksum = "86bd8d4e895da8537e5315b8254664e6b769c4ff3db18321b297a1e7004392e3"
2704 | 
2705 | [[package]]
2706 | name = "uuid"
2707 | version = "1.8.0"
2708 | source = "registry+https://github.com/rust-lang/crates.io-index"
2709 | checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0"
2710 | dependencies = [
2711 |  "getrandom",
2712 | ]
2713 | 
2714 | [[package]]
2715 | name = "vcpkg"
2716 | version = "0.2.15"
2717 | source = "registry+https://github.com/rust-lang/crates.io-index"
2718 | checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
2719 | 
2720 | [[package]]
2721 | name = "version_check"
2722 | version = "0.9.4"
2723 | source = "registry+https://github.com/rust-lang/crates.io-index"
2724 | checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
2725 | 
2726 | [[package]]
2727 | name = "walkdir"
2728 | version = "2.5.0"
2729 | source = "registry+https://github.com/rust-lang/crates.io-index"
2730 | checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
2731 | dependencies = [
2732 |  "same-file",
2733 |  "winapi-util",
2734 | ]
2735 | 
2736 | [[package]]
2737 | name = "wasi"
2738 | version = "0.11.0+wasi-snapshot-preview1"
2739 | source = "registry+https://github.com/rust-lang/crates.io-index"
2740 | checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
2741 | 
2742 | [[package]]
2743 | name = "wasm-bindgen"
2744 | version = "0.2.92"
2745 | source = "registry+https://github.com/rust-lang/crates.io-index"
2746 | checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8"
2747 | dependencies = [
2748 |  "cfg-if",
2749 |  "wasm-bindgen-macro",
2750 | ]
2751 | 
2752 | [[package]]
2753 | name = "wasm-bindgen-backend"
2754 | version = "0.2.92"
2755 | source = "registry+https://github.com/rust-lang/crates.io-index"
2756 | checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da"
2757 | dependencies = [
2758 |  "bumpalo",
2759 |  "log",
2760 |  "once_cell",
2761 |  "proc-macro2",
2762 |  "quote",
2763 |  "syn 2.0.66",
2764 |  "wasm-bindgen-shared",
2765 | ]
2766 | 
2767 | [[package]]
2768 | name = "wasm-bindgen-futures"
2769 | version = "0.4.42"
2770 | source = "registry+https://github.com/rust-lang/crates.io-index"
2771 | checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0"
2772 | dependencies = [
2773 |  "cfg-if",
2774 |  "js-sys",
2775 |  "wasm-bindgen",
2776 |  "web-sys",
2777 | ]
2778 | 
2779 | [[package]]
2780 | name = "wasm-bindgen-macro"
2781 | version = "0.2.92"
2782 | source = "registry+https://github.com/rust-lang/crates.io-index"
2783 | checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726"
2784 | dependencies = [
2785 |  "quote",
2786 |  "wasm-bindgen-macro-support",
2787 | ]
2788 | 
2789 | [[package]]
2790 | name = "wasm-bindgen-macro-support"
2791 | version = "0.2.92"
2792 | source = "registry+https://github.com/rust-lang/crates.io-index"
2793 | checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7"
2794 | dependencies = [
2795 |  "proc-macro2",
2796 |  "quote",
2797 |  "syn 2.0.66",
2798 |  "wasm-bindgen-backend",
2799 |  "wasm-bindgen-shared",
2800 | ]
2801 | 
2802 | [[package]]
2803 | name = "wasm-bindgen-shared"
2804 | version = "0.2.92"
2805 | source = "registry+https://github.com/rust-lang/crates.io-index"
2806 | checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96"
2807 | 
2808 | [[package]]
2809 | name = "wasm-streams"
2810 | version = "0.4.0"
2811 | source = "registry+https://github.com/rust-lang/crates.io-index"
2812 | checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129"
2813 | dependencies = [
2814 |  "futures-util",
2815 |  "js-sys",
2816 |  "wasm-bindgen",
2817 |  "wasm-bindgen-futures",
2818 |  "web-sys",
2819 | ]
2820 | 
2821 | [[package]]
2822 | name = "web-sys"
2823 | version = "0.3.69"
2824 | source = "registry+https://github.com/rust-lang/crates.io-index"
2825 | checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef"
2826 | dependencies = [
2827 |  "js-sys",
2828 |  "wasm-bindgen",
2829 | ]
2830 | 
2831 | [[package]]
2832 | name = "winapi-util"
2833 | version = "0.1.8"
2834 | source = "registry+https://github.com/rust-lang/crates.io-index"
2835 | checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b"
2836 | dependencies = [
2837 |  "windows-sys 0.52.0",
2838 | ]
2839 | 
2840 | [[package]]
2841 | name = "windows-core"
2842 | version = "0.52.0"
2843 | source = "registry+https://github.com/rust-lang/crates.io-index"
2844 | checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
2845 | dependencies = [
2846 |  "windows-targets 0.52.5",
2847 | ]
2848 | 
2849 | [[package]]
2850 | name = "windows-sys"
2851 | version = "0.48.0"
2852 | source = "registry+https://github.com/rust-lang/crates.io-index"
2853 | checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
2854 | dependencies = [
2855 |  "windows-targets 0.48.5",
2856 | ]
2857 | 
2858 | [[package]]
2859 | name = "windows-sys"
2860 | version = "0.52.0"
2861 | source = "registry+https://github.com/rust-lang/crates.io-index"
2862 | checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
2863 | dependencies = [
2864 |  "windows-targets 0.52.5",
2865 | ]
2866 | 
2867 | [[package]]
2868 | name = "windows-targets"
2869 | version = "0.48.5"
2870 | source = "registry+https://github.com/rust-lang/crates.io-index"
2871 | checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"
2872 | dependencies = [
2873 |  "windows_aarch64_gnullvm 0.48.5",
2874 |  "windows_aarch64_msvc 0.48.5",
2875 |  "windows_i686_gnu 0.48.5",
2876 |  "windows_i686_msvc 0.48.5",
2877 |  "windows_x86_64_gnu 0.48.5",
2878 |  "windows_x86_64_gnullvm 0.48.5",
2879 |  "windows_x86_64_msvc 0.48.5",
2880 | ]
2881 | 
2882 | [[package]]
2883 | name = "windows-targets"
2884 | version = "0.52.5"
2885 | source = "registry+https://github.com/rust-lang/crates.io-index"
2886 | checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb"
2887 | dependencies = [
2888 |  "windows_aarch64_gnullvm 0.52.5",
2889 |  "windows_aarch64_msvc 0.52.5",
2890 |  "windows_i686_gnu 0.52.5",
2891 |  "windows_i686_gnullvm",
2892 |  "windows_i686_msvc 0.52.5",
2893 |  "windows_x86_64_gnu 0.52.5",
2894 |  "windows_x86_64_gnullvm 0.52.5",
2895 |  "windows_x86_64_msvc 0.52.5",
2896 | ]
2897 | 
2898 | [[package]]
2899 | name = "windows_aarch64_gnullvm"
2900 | version = "0.48.5"
2901 | source = "registry+https://github.com/rust-lang/crates.io-index"
2902 | checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
2903 | 
2904 | [[package]]
2905 | name = "windows_aarch64_gnullvm"
2906 | version = "0.52.5"
2907 | source = "registry+https://github.com/rust-lang/crates.io-index"
2908 | checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263"
2909 | 
2910 | [[package]]
2911 | name = "windows_aarch64_msvc"
2912 | version = "0.48.5"
2913 | source = "registry+https://github.com/rust-lang/crates.io-index"
2914 | checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
2915 | 
2916 | [[package]]
2917 | name = "windows_aarch64_msvc"
2918 | version = "0.52.5"
2919 | source = "registry+https://github.com/rust-lang/crates.io-index"
2920 | checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6"
2921 | 
2922 | [[package]]
2923 | name = "windows_i686_gnu"
2924 | version = "0.48.5"
2925 | source = "registry+https://github.com/rust-lang/crates.io-index"
2926 | checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
2927 | 
2928 | [[package]]
2929 | name = "windows_i686_gnu"
2930 | version = "0.52.5"
2931 | source = "registry+https://github.com/rust-lang/crates.io-index"
2932 | checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670"
2933 | 
2934 | [[package]]
2935 | name = "windows_i686_gnullvm"
2936 | version = "0.52.5"
2937 | source = "registry+https://github.com/rust-lang/crates.io-index"
2938 | checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9"
2939 | 
2940 | [[package]]
2941 | name = "windows_i686_msvc"
2942 | version = "0.48.5"
2943 | source = "registry+https://github.com/rust-lang/crates.io-index"
2944 | checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
2945 | 
2946 | [[package]]
2947 | name = "windows_i686_msvc"
2948 | version = "0.52.5"
2949 | source = "registry+https://github.com/rust-lang/crates.io-index"
2950 | checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf"
2951 | 
2952 | [[package]]
2953 | name = "windows_x86_64_gnu"
2954 | version = "0.48.5"
2955 | source = "registry+https://github.com/rust-lang/crates.io-index"
2956 | checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
2957 | 
2958 | [[package]]
2959 | name = "windows_x86_64_gnu"
2960 | version = "0.52.5"
2961 | source = "registry+https://github.com/rust-lang/crates.io-index"
2962 | checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9"
2963 | 
2964 | [[package]]
2965 | name = "windows_x86_64_gnullvm"
2966 | version = "0.48.5"
2967 | source = "registry+https://github.com/rust-lang/crates.io-index"
2968 | checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
2969 | 
2970 | [[package]]
2971 | name = "windows_x86_64_gnullvm"
2972 | version = "0.52.5"
2973 | source = "registry+https://github.com/rust-lang/crates.io-index"
2974 | checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596"
2975 | 
2976 | [[package]]
2977 | name = "windows_x86_64_msvc"
2978 | version = "0.48.5"
2979 | source = "registry+https://github.com/rust-lang/crates.io-index"
2980 | checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
2981 | 
2982 | [[package]]
2983 | name = "windows_x86_64_msvc"
2984 | version = "0.52.5"
2985 | source = "registry+https://github.com/rust-lang/crates.io-index"
2986 | checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0"
2987 | 
2988 | [[package]]
2989 | name = "winnow"
2990 | version = "0.6.8"
2991 | source = "registry+https://github.com/rust-lang/crates.io-index"
2992 | checksum = "c3c52e9c97a68071b23e836c9380edae937f17b9c4667bd021973efc689f618d"
2993 | dependencies = [
2994 |  "memchr",
2995 | ]
2996 | 
2997 | [[package]]
2998 | name = "xxhash-rust"
2999 | version = "0.8.10"
3000 | source = "registry+https://github.com/rust-lang/crates.io-index"
3001 | checksum = "927da81e25be1e1a2901d59b81b37dd2efd1fc9c9345a55007f09bf5a2d3ee03"
3002 | 
3003 | [[package]]
3004 | name = "yansi"
3005 | version = "1.0.1"
3006 | source = "registry+https://github.com/rust-lang/crates.io-index"
3007 | checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049"
3008 | 
3009 | [[package]]
3010 | name = "zerocopy"
3011 | version = "0.7.34"
3012 | source = "registry+https://github.com/rust-lang/crates.io-index"
3013 | checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087"
3014 | dependencies = [
3015 |  "zerocopy-derive",
3016 | ]
3017 | 
3018 | [[package]]
3019 | name = "zerocopy-derive"
3020 | version = "0.7.34"
3021 | source = "registry+https://github.com/rust-lang/crates.io-index"
3022 | checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b"
3023 | dependencies = [
3024 |  "proc-macro2",
3025 |  "quote",
3026 |  "syn 2.0.66",
3027 | ]
3028 | 
3029 | [[package]]
3030 | name = "zstd"
3031 | version = "0.13.1"
3032 | source = "registry+https://github.com/rust-lang/crates.io-index"
3033 | checksum = "2d789b1514203a1120ad2429eae43a7bd32b90976a7bb8a05f7ec02fa88cc23a"
3034 | dependencies = [
3035 |  "zstd-safe",
3036 | ]
3037 | 
3038 | [[package]]
3039 | name = "zstd-safe"
3040 | version = "7.1.0"
3041 | source = "registry+https://github.com/rust-lang/crates.io-index"
3042 | checksum = "1cd99b45c6bc03a018c8b8a86025678c87e55526064e38f9df301989dce7ec0a"
3043 | dependencies = [
3044 |  "zstd-sys",
3045 | ]
3046 | 
3047 | [[package]]
3048 | name = "zstd-sys"
3049 | version = "2.0.10+zstd.1.5.6"
3050 | source = "registry+https://github.com/rust-lang/crates.io-index"
3051 | checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa"
3052 | dependencies = [
3053 |  "cc",
3054 |  "pkg-config",
3055 | ]
3056 | 


--------------------------------------------------------------------------------