├── .devcontainer ├── devcontainer.json └── docker-compose.yml ├── .github └── workflows │ ├── ci.yml │ └── release.yml ├── .gitignore ├── Cargo.lock ├── Cargo.toml ├── Cross.toml ├── LICENSE ├── README.md ├── migrations ├── 20230420060200_setup.down.sql ├── 20230420060200_setup.up.sql ├── 20230425135000_alter_text_type.down.sql ├── 20230425135000_alter_text_type.up.sql ├── 20230511100521_image_title.down.sql └── 20230511100521_image_title.up.sql ├── summarizer ├── Cargo.toml ├── Readme.md └── src │ ├── api │ ├── mod.rs │ └── v1 │ │ ├── mod.rs │ │ └── summary.rs │ ├── bin │ ├── controller.rs │ ├── job.rs │ └── server.rs │ ├── database │ ├── database.rs │ ├── mod.rs │ └── models.rs │ ├── default.rs │ ├── error.rs │ ├── image.rs │ ├── lib.rs │ ├── scheduler.rs │ ├── summarize.rs │ ├── tokenize │ ├── mod.rs │ ├── token.rs │ └── tokenizer.rs │ ├── utils.rs │ └── youtube.rs ├── ui ├── .gitignore ├── index.html ├── package-lock.json ├── package.json ├── src │ ├── App.svelte │ ├── app.css │ ├── default.ts │ ├── lib │ │ └── card.svelte │ ├── main.ts │ ├── model │ │ ├── card.ts │ │ └── input.ts │ ├── parser.ts │ └── vite-env.d.ts ├── svelte.config.js ├── tsconfig.json ├── tsconfig.node.json └── vite.config.ts └── youtube-transcript ├── Cargo.toml ├── Readme.md └── src ├── config.rs ├── error.rs ├── lib.rs ├── main.rs ├── parser.rs ├── utils.rs └── youtube.rs /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | // For format details, see https://aka.ms/devcontainer.json. For config options, see the 2 | // README at: https://github.com/devcontainers/templates/tree/main/src/ubuntu 3 | { 4 | "name": "summarizer", 5 | // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile 6 | "dockerComposeFile": "docker-compose.yml", 7 | "workspaceFolder": "/workspace/${localWorkspaceFolderBasename}", 8 | "service": "devcontainer", 9 | "features": { 10 | "ghcr.io/devcontainers/features/node:1": { 11 | "nodeGypDependencies": true, 12 | "version": "lts" 13 | }, 14 | "ghcr.io/devcontainers/features/python:1": { 15 | "installTools": false, 16 | "installJupyterlab": false 17 | }, 18 | "ghcr.io/devcontainers/features/rust:1": { 19 | "version": "latest", 20 | "profile": "default" 21 | }, 22 | "ghcr.io/akhildevelops/devcontainer-features/apt": { 23 | "PACKAGES": "postgresql-client,pkg-config" 24 | }, 25 | "ghcr.io/devcontainers-contrib/features/bash-command": { 26 | "command": "cargo install sqlx-cli cross" 27 | }, 28 | "ghcr.io/devcontainers/features/docker-in-docker": {} 29 | }, 30 | "overrideFeatureInstallOrder": [ 31 | "ghcr.io/akhildevelops/devcontainer-features/apt", 32 | "ghcr.io/devcontainers/features/rust:1", 33 | "ghcr.io/devcontainers-contrib/features/bash-command" 34 | ], 35 | // Chowning is a workaround. 36 | "postAttachCommand": "cat .env | xargs -n 1 echo export $1 >> ~/.bashrc && sudo chown -R vscode:rustlang /usr/local/cargo", 37 | // Features to add to the dev container. More info: https://containers.dev/features. 38 | // "features": {}, 39 | // Use 'forwardPorts' to make a list of ports inside the container available locally. 40 | // "forwardPorts": [], 41 | // Use 'postCreateCommand' to run commands after the container is created. 42 | // "postCreateCommand": "uname -a", 43 | // Configure tool-specific properties. 44 | "customizations": { 45 | "vscode": { 46 | "extensions": [ 47 | "mhutchie.git-graph", 48 | "eamodio.gitlens", 49 | "cweijan.vscode-database-client2", 50 | "svelte.svelte-vscode", 51 | "rangav.vscode-thunder-client" 52 | ] 53 | } 54 | }, 55 | "containerUser": "vscode", 56 | // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. 57 | // "remoteUser": "vscode" 58 | } -------------------------------------------------------------------------------- /.devcontainer/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.8' 2 | services: 3 | devcontainer: 4 | image: mcr.microsoft.com/devcontainers/base:jammy 5 | volumes: 6 | - ../../:/workspace:cached 7 | network_mode: service:db 8 | command: sleep infinity 9 | 10 | db: 11 | image: postgres:latest 12 | restart: unless-stopped 13 | volumes: 14 | - postgres-data:/var/lib/postgresql/data 15 | environment: 16 | POSTGRES_PASSWORD: postgres 17 | POSTGRES_USER: postgres 18 | POSTGRES_DB: postgres 19 | volumes: 20 | postgres-data: 21 | 22 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - main 7 | paths-ignore: 8 | - '.github/workflows/publish.yml' 9 | - '.github/workflows/release.yml' 10 | - '**README.md' 11 | - '.vscode/**' 12 | - '.gitignore' 13 | - 'LICENSE' 14 | 15 | merge_group: 16 | 17 | jobs: 18 | tests-package: 19 | runs-on: ubuntu-latest 20 | steps: 21 | - name: Checkout 22 | uses: actions/checkout@v3 23 | - name: Rust tooling 24 | uses: dtolnay/rust-toolchain@stable 25 | - name: Cache rust 26 | uses: Swatinem/rust-cache@v2 27 | - name: Test Rust 28 | run: cargo test 29 | 30 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | tags: 6 | - "v*.*.*" 7 | permissions: 8 | contents: write 9 | jobs: 10 | github-artifact: 11 | runs-on: ubuntu-latest 12 | strategy: 13 | matrix: 14 | package: [youtube-transcript] 15 | steps: 16 | - name: Checkout 17 | uses: actions/checkout@v3 18 | - name: Rust tooling 19 | uses: dtolnay/rust-toolchain@stable 20 | - name: Cache rust 21 | uses: Swatinem/rust-cache@v2 22 | - name: generate artifacts 23 | run: cargo build --release 24 | - name: Release 25 | uses: softprops/action-gh-release@v1 26 | with: 27 | files: target/release/youtube-transcript 28 | 29 | publish-crate: 30 | runs-on: ubuntu-latest 31 | environment: production 32 | env: 33 | CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} 34 | needs: [github-artifact] 35 | strategy: 36 | matrix: 37 | package: [youtube-transcript] 38 | steps: 39 | - name: Checkout 40 | uses: actions/checkout@v3 41 | - name: Rust tooling 42 | uses: dtolnay/rust-toolchain@stable 43 | - name: Cache rust 44 | uses: Swatinem/rust-cache@v2 45 | - name: publish 46 | run: cargo publish -p ${{ matrix.package }} 47 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | 3 | # Skip scrap 4 | scrap/ 5 | 6 | #Env 7 | .env 8 | 9 | #Editor 10 | .vscode/ 11 | 12 | # book 13 | book.txt 14 | 15 | # data 16 | data/ -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | 2 | [workspace] 3 | members = ["youtube-transcript", "summarizer"] 4 | resolver = "2" 5 | 6 | 7 | [workspace.package] 8 | edition = "2021" 9 | version = "0.3.2" 10 | repository = "https://github.com/akhildevelops/summarizer" 11 | 12 | [workspace.dependencies] 13 | async-openai = { version = "^0.10" } 14 | tokio = { version = "^1.27", features = ["macros"] } 15 | serde_json = { version = "~1.0" } 16 | serde = { version = "~1.0", features = ["derive"] } 17 | strum = "^0.26" 18 | strum_macros = "^0.26" 19 | roxmltree = { version = "~0.18" } 20 | reqwest = { version = "~0.11" } 21 | clap = { version = "4.2.1", features = ["string"] } 22 | once_cell = { version = "^1.17" } 23 | apalis = { version = "^0.3.6" } 24 | anyhow = { version = "~1.0" } 25 | chrono = { version = "~0.4" } 26 | futures = { version = "~0.3" } 27 | env_logger = { version = "~0.10.0" } 28 | log = { version = "~0.4.17" } 29 | axum = { version = "~0.6" } 30 | regex = { version = "1" } 31 | tower = { version = "0.4", features = ["util"] } 32 | tower-http = { version = "0.4", features = ["fs"] } 33 | tracing = "0.1" 34 | tracing-subscriber = { version = "0.3", features = ["env-filter"] } 35 | 36 | [profile.release] 37 | panic = "abort" 38 | lto = "fat" 39 | debug-assertions = false 40 | strip = true 41 | -------------------------------------------------------------------------------- /Cross.toml: -------------------------------------------------------------------------------- 1 | [target.arm-unknown-linux-gnueabihf] 2 | pre-build = [ 3 | "dpkg --add-architecture $CROSS_DEB_ARCH", 4 | "apt-get update && apt-get install --assume-yes libssl-dev:amd64 libssl-dev:$CROSS_DEB_ARCH", 5 | ] 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Akhil 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Summarizer 2 | Podcast and Youtube 3 | 4 | # Setup 5 | - Install rust from https://www.rust-lang.org/tools/install if not present in the system. 6 | - Clone the repo 7 | - Install sqlx-cli - `cargo install sqlx-cli --no-default-features --features native-tls,postgres` 8 | - Point existing postgresql (or install) through environment variable in the terminal - `export DATABASE_URL=postgresql://:@:/` 9 | - Get OpenAI Key and export as a env variable: `export OPENAI_API_KEY=**********` 10 | 11 | # Start the service 12 | ### Hub 13 | - This is the hub that accepts jobs - `cargo run --bin controller` 14 | ### Client 15 | - Open a new terminal. 16 | - This will submit jobs to hub - `sqlx migrate run --ignore-missing && cargo run --bin jobs` 17 | ### Server 18 | - A restapi to interact with the service - `cargo run --bin server` 19 | ### UI 20 | - A small piece of UI to see all the summaries - `cd ui && npm install && VITE_SUMMARIZER_URL=http://localhost:3001/api/v1 npm run dev` 21 | 22 | ## Cross-compilation to raspberrypi 23 | - Install podman or docker 24 | - Install cross cli - `cargo install cross --git https://github.com/cross-rs/cross` 25 | - Run `cross build --release --features=vendored-ssl --target=arm-unknown-linux-gnueabihf` 26 | - Find binaries in `target/arm-unknown-linux-gnueabihf/release/` 27 | - Port them to raspberry pi device and follow above steps of starting of service 28 | 29 | -------------------------------------------------------------------------------- /migrations/20230420060200_setup.down.sql: -------------------------------------------------------------------------------- 1 | -- Add down migration script here 2 | DROP TABLE IF EXISTS TRANSCRIPTSUMMARY; 3 | DROP TYPE IF EXISTS summary_agent; 4 | DROP TABLE IF EXISTS TRANSCRIPT; 5 | DROP TABLE IF EXISTS REMOTEURL; -------------------------------------------------------------------------------- /migrations/20230420060200_setup.up.sql: -------------------------------------------------------------------------------- 1 | -- Add up migration script here 2 | 3 | CREATE TABLE 4 | IF NOT EXISTS REMOTEURL( 5 | id INT PRIMARY KEY GENERATED ALWAYS AS IDENTITY, 6 | created_at TIMESTAMP NOT NULL DEFAULT NOW(), 7 | link VARCHAR 8 | ); 9 | 10 | CREATE TABLE 11 | IF NOT EXISTS TRANSCRIPT( 12 | id INT PRIMARY KEY GENERATED ALWAYS AS IDENTITY, 13 | created_at TIMESTAMP NOT NULL DEFAULT NOW(), 14 | remote_id INT REFERENCES REMOTEURL ON DELETE RESTRICT, 15 | -- Doesn't delete YOUTUBE(ID) if it still has a reference to youtube_id 16 | content VARCHAR 17 | ); 18 | 19 | CREATE TABLE 20 | IF NOT EXISTS TRANSCRIPTSUMMARY( 21 | id INT PRIMARY KEY GENERATED ALWAYS AS IDENTITY, 22 | created_at TIMESTAMP NOT NULL DEFAULT NOW(), 23 | transcript_id INT REFERENCES TRANSCRIPT ON DELETE RESTRICT, 24 | -- Doesn't deletes Transcript(ID) if it has a reference to TranscriptSummary table 25 | content VARCHAR 26 | ); -------------------------------------------------------------------------------- /migrations/20230425135000_alter_text_type.down.sql: -------------------------------------------------------------------------------- 1 | -- Add down migration script here 2 | 3 | ALTER TABLE transcript ALTER COLUMN content TYPE VARCHAR; 4 | 5 | ALTER TABLE transcriptsummary ALTER COLUMN content TYPE VARCHAR; -------------------------------------------------------------------------------- /migrations/20230425135000_alter_text_type.up.sql: -------------------------------------------------------------------------------- 1 | -- Add up migration script here 2 | 3 | ALTER TABLE transcript ALTER COLUMN content TYPE TEXT; 4 | 5 | ALTER TABLE transcriptsummary ALTER COLUMN content TYPE TEXT; -------------------------------------------------------------------------------- /migrations/20230511100521_image_title.down.sql: -------------------------------------------------------------------------------- 1 | -- Add down migration script here 2 | 3 | ALTER TABLE remoteurl DROP COLUMN title; 4 | 5 | ALTER TABLE remoteurl DROP COLUMN image_id; -------------------------------------------------------------------------------- /migrations/20230511100521_image_title.up.sql: -------------------------------------------------------------------------------- 1 | -- Add up migration script here 2 | 3 | ALTER TABLE remoteurl ADD COLUMN title VARCHAR; 4 | 5 | ALTER TABLE remoteurl ADD COLUMN image_id VARCHAR; -------------------------------------------------------------------------------- /summarizer/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "summarizer" 3 | version.workspace = true 4 | edition.workspace = true 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | axum = { workspace = true } 10 | async-openai = { workspace = true } 11 | youtube-transcript = { path = "../youtube-transcript" } 12 | tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } 13 | apalis = { workspace = true, features = ["postgres", "extensions"] } 14 | serde = { workspace = true, features = ["derive"] } 15 | sqlx = { version = "^0.5", features = ["postgres"] } 16 | openssl = { version = "0.10", features = ["vendored"], optional = true } 17 | anyhow = { workspace = true } 18 | chrono = { workspace = true } 19 | tiktoken-rs = { version = "0.5" } 20 | futures = { workspace = true, features = ["executor"] } 21 | env_logger = { workspace = true } 22 | log = { workspace = true } 23 | axum-core = { version = "~0.3.4" } 24 | serde_json = { workspace = true } 25 | reqwest = { workspace = true } 26 | regex = { workspace = true } 27 | tower = { workspace = true } 28 | tower-http = { workspace = true, features = ["trace", "cors"] } 29 | tracing = { workspace = true } 30 | tracing-subscriber = { workspace = true } 31 | axum-extra = { version = "0.7" } 32 | 33 | [features] 34 | vendored-ssl = ["dep:openssl"] 35 | -------------------------------------------------------------------------------- /summarizer/Readme.md: -------------------------------------------------------------------------------- 1 | Hidsadf sadf -------------------------------------------------------------------------------- /summarizer/src/api/mod.rs: -------------------------------------------------------------------------------- 1 | mod v1; 2 | use axum::{ 3 | routing::{get, post}, 4 | Router, 5 | }; 6 | use sqlx::postgres::PgPool; 7 | use tower_http::services::{ServeDir, ServeFile}; 8 | use v1::{summaries, summarize}; 9 | pub fn get_router(pgpool: PgPool) -> Router { 10 | Router::new() 11 | .route("/", get(|| async { "Summarizer" })) 12 | .nest( 13 | "/api", 14 | Router::new().nest( 15 | "/v1", 16 | Router::new() 17 | .route("/summarize", post(summarize)) 18 | .route("/summaries", get(summaries)) 19 | .nest_service( 20 | "/thumbnails", 21 | ServeDir::new("./data").fallback(ServeFile::new("./data/notfound.jpg")), 22 | ) 23 | .with_state(pgpool), 24 | ), 25 | ) 26 | } 27 | -------------------------------------------------------------------------------- /summarizer/src/api/v1/mod.rs: -------------------------------------------------------------------------------- 1 | mod summary; 2 | 3 | pub(crate) use summary::{summaries, summarize}; 4 | -------------------------------------------------------------------------------- /summarizer/src/api/v1/summary.rs: -------------------------------------------------------------------------------- 1 | use crate::database::Postgresmethods; 2 | use crate::error::Serror; 3 | use crate::scheduler::Youtubelink; 4 | use apalis::postgres::PostgresStorage; 5 | use apalis::prelude::Storage; 6 | use axum::extract::{Json, State}; 7 | use serde::Deserialize; 8 | use serde_json; 9 | use sqlx::postgres::PgPool; 10 | pub(crate) async fn summaries(State(pg): State) -> Result, Serror> { 11 | let pg = Postgresmethods::new(&pg); 12 | let summaries = pg.get_summaries().await?; 13 | Ok(Json(serde_json::to_value(summaries)?)) 14 | } 15 | 16 | #[derive(Deserialize)] 17 | pub struct Link { 18 | link: String, 19 | } 20 | 21 | pub(crate) async fn summarize( 22 | State(pg): State, 23 | Json(link): Json, 24 | ) -> Result, Serror> { 25 | let mut storage: PostgresStorage = PostgresStorage::new(pg); 26 | let you = Youtubelink(link.link.clone()); 27 | let _job = storage 28 | .push(you) 29 | .await 30 | .map_err(|_| Serror::Database(format!("Cannot create a job for the link: {}", link.link))); 31 | let resp_json = format!("{{\"registerd_link\":\"{}\"}}", link.link); 32 | Ok(Json(serde_json::from_str(&resp_json)?)) 33 | } 34 | -------------------------------------------------------------------------------- /summarizer/src/bin/controller.rs: -------------------------------------------------------------------------------- 1 | use summarizer::error::Serror; 2 | use summarizer::scheduler::setup_youtube_data_workers; 3 | use summarizer::utils::env_var; 4 | use tokio; 5 | 6 | #[tokio::main] 7 | async fn main() -> Result<(), Serror> { 8 | let postgres_url = env_var("DATABASE_URL")?; 9 | setup_youtube_data_workers(&postgres_url).await 10 | } 11 | -------------------------------------------------------------------------------- /summarizer/src/bin/job.rs: -------------------------------------------------------------------------------- 1 | use apalis::postgres::PostgresStorage; 2 | use apalis::prelude::Storage; 3 | use summarizer::error::Serror; 4 | use summarizer::scheduler::Youtubelink; 5 | use summarizer::utils::env_var; 6 | use tokio; 7 | 8 | #[tokio::main] 9 | async fn main() -> Result<(), Serror> { 10 | let postgres_url = env_var("DATABASE_URL")?; 11 | let links = [ 12 | "https://www.youtube.com/watch?v=sBH-ngpL0zo", 13 | "https://www.youtube.com/watch?v=WYNRt-AwoUg", 14 | "https://www.youtube.com/watch?v=fPWzeFYtjfc", 15 | "https://www.youtube.com/watch?v=e64JTo7LMmY", 16 | "https://www.youtube.com/watch?v=vcTtYZX7Ahk", 17 | "https://www.youtube.com/watch?v=FzUeQ7AZ3O0", 18 | "https://www.youtube.com/watch?v=sWSgQFmWMxw", 19 | "https://www.youtube.com/watch?v=vjfypJq5pK8", 20 | ]; 21 | let mut storage: PostgresStorage = PostgresStorage::connect(postgres_url).await?; 22 | for link in links { 23 | let you = Youtubelink(link.to_string()); 24 | let _job = storage 25 | .push(you) 26 | .await 27 | .map_err(|_| Serror::Database("asdf".to_string()))?; 28 | } 29 | Ok(()) 30 | } 31 | -------------------------------------------------------------------------------- /summarizer/src/bin/server.rs: -------------------------------------------------------------------------------- 1 | use axum; 2 | use sqlx::postgres::PgPool; 3 | use std::net::SocketAddr; 4 | use summarizer::api; 5 | use summarizer::utils::env_var; 6 | use tokio; 7 | use tower_http::cors::{Any, CorsLayer}; 8 | use tower_http::trace::TraceLayer; 9 | #[tokio::main] 10 | async fn main() -> Result<(), String> { 11 | // setup tracing for debugging 12 | tracing_subscriber::fmt() 13 | .with_max_level(tracing::Level::DEBUG) 14 | .init(); 15 | 16 | // connect to postgres 17 | let postgres_url = env_var("DATABASE_URL").map_err(|x| x.to_string())?; 18 | let pg = PgPool::connect(&postgres_url) 19 | .await 20 | .map_err(|x| x.to_string())?; 21 | 22 | // Setup server 23 | let app = api::get_router(pg); 24 | let cors = CorsLayer::permissive(); 25 | let addr = SocketAddr::from(([0, 0, 0, 0], 3001)); 26 | tracing::debug!("listening on {}", addr); 27 | axum::Server::bind(&addr) 28 | .serve( 29 | app.layer(TraceLayer::new_for_http()) 30 | .layer(cors) 31 | .into_make_service(), 32 | ) 33 | .await 34 | .unwrap(); 35 | Ok(()) 36 | } 37 | -------------------------------------------------------------------------------- /summarizer/src/database/database.rs: -------------------------------------------------------------------------------- 1 | use super::models::{Remoteurl, Summary, Transcript, TranscriptSummary}; 2 | use sqlx::postgres::PgPool; 3 | use sqlx::Error; 4 | pub(crate) struct Postgresmethods<'a> { 5 | client: &'a PgPool, 6 | } 7 | 8 | impl<'a> Postgresmethods<'a> { 9 | pub(crate) fn new(pool: &'a PgPool) -> Self { 10 | Self { client: pool } 11 | } 12 | } 13 | 14 | impl<'a> Postgresmethods<'a> { 15 | pub(crate) async fn get_summaries(&self) -> Result, Error> { 16 | let query = 17 | "SELECT ts.created_at, ts.content, remoteurl.link,remoteurl.image_id,remoteurl.title 18 | FROM transcriptsummary ts 19 | JOIN transcript ON ts.transcript_id=transcript.id 20 | JOIN remoteurl ON transcript.remote_id=remoteurl.id"; 21 | sqlx::query_as::<_, Summary>(query) 22 | .fetch_all(self.client) 23 | .await 24 | } 25 | 26 | pub(crate) async fn insert_remoteurl( 27 | &self, 28 | url: &str, 29 | image_id: &str, 30 | title: &str, 31 | ) -> Result { 32 | let insert_query = format!( 33 | "INSERT INTO remoteurl (link,image_id,title) VALUES ('{}','{}','{}') RETURNING *", 34 | url, image_id, title 35 | ); 36 | sqlx::query_as::<_, Remoteurl>(&insert_query) 37 | .fetch_one(self.client) 38 | .await 39 | } 40 | pub(crate) async fn insert_transcript( 41 | &self, 42 | transcript: &str, 43 | remote_url: &Remoteurl, 44 | ) -> Result { 45 | let transcript = transcript.replace("'", "''"); // Escape single quotes 46 | let insert_query = format!( 47 | "INSERT INTO transcript (remote_id,content) VALUES ({},'{}') RETURNING id, created_at, remote_id", 48 | remote_url.id, transcript 49 | ); 50 | sqlx::query_as::<_, Transcript>(&insert_query) 51 | .fetch_one(self.client) 52 | .await 53 | } 54 | pub(crate) async fn insert_transcriptsummary( 55 | &self, 56 | summary: &str, 57 | transcript: &Transcript, 58 | ) -> Result { 59 | let summary = summary.replace("'", "''"); // Escape single quotes 60 | let insert_query = format!( 61 | "INSERT INTO transcriptsummary (transcript_id,content) VALUES ({},'{}') RETURNING id, created_at, transcript_id", 62 | transcript.id, summary 63 | ); 64 | sqlx::query_as::<_, TranscriptSummary>(&insert_query) 65 | .fetch_one(self.client) 66 | .await 67 | } 68 | } 69 | 70 | #[cfg(test)] 71 | mod test { 72 | use super::*; 73 | use chrono::NaiveDateTime; 74 | #[tokio::test] 75 | #[ignore = "Will activate after mocking db"] 76 | async fn insert_trans() { 77 | let p_url = "postgres://postgres:postgres@db/summarizer"; 78 | let pool = PgPool::connect(p_url).await.unwrap(); 79 | let pmethods = Postgresmethods::new(&pool); 80 | let r_url = Remoteurl { 81 | id: 1, 82 | created_at: NaiveDateTime::MIN, 83 | link: "sadf".to_string(), 84 | image_id: "asfd".to_string(), 85 | title: "asdf".to_string(), 86 | }; 87 | pmethods 88 | .insert_transcript("asdfdsaf", &r_url) 89 | .await 90 | .unwrap(); 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /summarizer/src/database/mod.rs: -------------------------------------------------------------------------------- 1 | mod database; 2 | mod models; 3 | 4 | pub(crate) use database::Postgresmethods; 5 | -------------------------------------------------------------------------------- /summarizer/src/database/models.rs: -------------------------------------------------------------------------------- 1 | use chrono::NaiveDateTime; 2 | use serde::Serialize; 3 | use sqlx::FromRow; 4 | #[derive(FromRow)] 5 | pub(crate) struct Remoteurl { 6 | pub id: i32, 7 | pub created_at: NaiveDateTime, 8 | pub link: String, 9 | pub image_id: String, 10 | pub title: String, 11 | } 12 | 13 | #[derive(FromRow)] 14 | pub(crate) struct Transcript { 15 | pub id: i32, 16 | pub created_at: NaiveDateTime, 17 | pub remote_id: i32, 18 | } 19 | 20 | #[derive(FromRow)] 21 | pub(crate) struct TranscriptSummary { 22 | pub id: i32, 23 | pub created_at: NaiveDateTime, 24 | pub transcript_id: i32, 25 | } 26 | 27 | #[derive(FromRow, Serialize)] 28 | pub(crate) struct Summary { 29 | pub created_at: NaiveDateTime, 30 | pub content: String, 31 | pub link: String, 32 | pub image_id: Option, 33 | pub title: Option, 34 | } 35 | -------------------------------------------------------------------------------- /summarizer/src/default.rs: -------------------------------------------------------------------------------- 1 | pub(crate) const GPT_MODEL: &str = "gpt-3.5-turbo"; 2 | pub(crate) const SYSTEM_PROMPT: &str = "You will summarize the text, that can be readable in one to two minutes. Treat every input I type as a big text and help to summarize"; 3 | -------------------------------------------------------------------------------- /summarizer/src/error.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Error as anyhowError; 2 | use async_openai::error::OpenAIError; 3 | use axum::extract::Json; 4 | use axum_core::response::IntoResponse; 5 | use regex::Error as regexError; 6 | use reqwest::Error as reqError; 7 | use serde::Serialize; 8 | use serde_json::Error as serde_jsonError; 9 | use sqlx::Error as Sqlxerror; 10 | use std::env::VarError; 11 | use std::io::Error as Ioerror; 12 | use std::{error::Error, fmt::Display}; 13 | #[derive(Debug, Serialize)] 14 | pub enum Serror { 15 | Youtubefetch(String), 16 | Scheduler(String), 17 | Database(String), 18 | Environment(String), 19 | Other(String), 20 | OpenAIError(String), 21 | Tokenize(String), 22 | Communication(String), 23 | } 24 | 25 | impl IntoResponse for Serror { 26 | fn into_response(self) -> axum_core::response::Response { 27 | Json(self).into_response() 28 | } 29 | } 30 | 31 | impl Display for Serror { 32 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 33 | match self { 34 | Self::Youtubefetch(x) => write!(f, "Youtubefetch: {}", x), 35 | Self::Scheduler(x) => write!(f, "Scheduler: {}", x), 36 | Self::Database(x) => write!(f, "Database: {}", x), 37 | Self::Environment(x) => write!(f, "Environment {}", x), 38 | Self::Other(x) => write!(f, "Other: {}", x), 39 | Self::OpenAIError(x) => write!(f, "Openai: {}", x), 40 | Self::Tokenize(x) => writeln!(f, "Tokenize: {}", x), 41 | Self::Communication(x) => writeln!(f, "Request: {}", x), 42 | } 43 | } 44 | } 45 | 46 | impl Error for Serror {} 47 | 48 | impl From for Serror { 49 | fn from(value: Ioerror) -> Self { 50 | Self::Scheduler(value.to_string()) 51 | } 52 | } 53 | 54 | impl From for Serror { 55 | fn from(value: Sqlxerror) -> Self { 56 | Self::Database(value.to_string()) 57 | } 58 | } 59 | 60 | impl From for Serror { 61 | fn from(value: VarError) -> Self { 62 | Self::Environment(value.to_string()) 63 | } 64 | } 65 | 66 | impl From for Serror { 67 | fn from(value: anyhowError) -> Self { 68 | Self::Other(value.to_string()) 69 | } 70 | } 71 | 72 | impl From for Serror { 73 | fn from(value: OpenAIError) -> Self { 74 | Self::OpenAIError(value.to_string()) 75 | } 76 | } 77 | 78 | impl From for Serror { 79 | fn from(value: serde_jsonError) -> Self { 80 | Self::Other(value.to_string()) 81 | } 82 | } 83 | 84 | impl From for Serror { 85 | fn from(value: reqError) -> Self { 86 | Self::Communication(value.to_string()) 87 | } 88 | } 89 | 90 | impl From for Serror { 91 | fn from(value: regexError) -> Self { 92 | Self::Other(value.to_string()) 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /summarizer/src/image.rs: -------------------------------------------------------------------------------- 1 | use std::path::PathBuf; 2 | 3 | #[derive(Debug)] 4 | pub(crate) struct Image { 5 | filepath: PathBuf, 6 | } 7 | -------------------------------------------------------------------------------- /summarizer/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod api; 2 | mod database; 3 | mod default; 4 | pub mod error; 5 | pub mod scheduler; 6 | mod summarize; 7 | mod tokenize; 8 | pub mod utils; 9 | mod youtube; 10 | pub use summarize::Summarizer; 11 | pub use youtube::{Youtube, YoutubeContent}; 12 | mod image; 13 | -------------------------------------------------------------------------------- /summarizer/src/scheduler.rs: -------------------------------------------------------------------------------- 1 | use crate::database::Postgresmethods; 2 | use crate::error::Serror; 3 | use crate::youtube::Youtube; 4 | use crate::Summarizer; 5 | use apalis::layers::Extension; 6 | use apalis::postgres::PostgresStorage; 7 | use apalis::prelude::*; 8 | use reqwest; 9 | use serde::{Deserialize, Serialize}; 10 | use sqlx::postgres::PgPool; 11 | use std::{fs, io::Write}; 12 | #[derive(Deserialize, Serialize)] 13 | pub struct Youtubelink(pub String); 14 | 15 | impl Job for Youtubelink { 16 | const NAME: &'static str = "youtube-transcript"; 17 | } 18 | 19 | pub async fn transcript_summary( 20 | job: impl Into, 21 | ctx: JobContext, 22 | ) -> Result<(), Serror> { 23 | let pgpool = ctx 24 | .data_opt::() 25 | .ok_or_else(|| Serror::Other("Cannot observe Pgpool connection".to_string()))?; 26 | let summarizer = ctx 27 | .data_opt::() 28 | .ok_or(Serror::Other("Summarizer cannot be found".to_string()))?; 29 | let pm = Postgresmethods::new(&pgpool); 30 | let youtube_link: Youtubelink = job.into(); 31 | let youtube_content = Youtube::link(&youtube_link.0)?.content().await?; 32 | 33 | // description 34 | let description = youtube_content.transcript_text().await?; 35 | 36 | // title 37 | let title = youtube_content 38 | .title()? 39 | .unwrap_or("[Title Not Found]".to_string()); 40 | 41 | // image 42 | let image = youtube_content.image_link(); 43 | let response = reqwest::get(image).await?; 44 | let image_raw = response.bytes().await?; 45 | let path = "./data"; 46 | std::fs::create_dir_all(path)?; 47 | let mut file = fs::OpenOptions::new() 48 | .write(true) 49 | .create(true) 50 | .open(format!("{}/{}.jpg", path, youtube_content.video_id))?; 51 | file.write_all(&image_raw)?; 52 | 53 | let remote_url = pm 54 | .insert_remoteurl(&youtube_link.0, &youtube_content.video_id, &title) 55 | .await?; 56 | 57 | let ts = pm.insert_transcript(&description, &remote_url).await?; 58 | let _summary = summarizer.summarize(&description).await?; 59 | pm.insert_transcriptsummary(&_summary, &ts).await?; 60 | Ok(()) 61 | } 62 | 63 | pub async fn setup_youtube_data_workers(postgres_url: &str) -> Result<(), Serror> { 64 | let pgpool = PgPool::connect(&postgres_url).await?; 65 | let ps_client = PostgresStorage::::new(pgpool.clone()); 66 | ps_client.setup().await?; 67 | let summarizer = Summarizer::default_params()?; 68 | ps_client.setup().await?; 69 | Monitor::new() 70 | .register_with_count(1, move |_| { 71 | WorkerBuilder::new(ps_client.clone()) 72 | .layer(Extension(pgpool.clone())) 73 | .layer(Extension(summarizer.clone())) 74 | .build_fn(transcript_summary) 75 | }) 76 | .run() 77 | .await?; 78 | Ok(()) 79 | } 80 | 81 | #[cfg(test)] 82 | mod test { 83 | use super::*; 84 | #[test] 85 | fn test_write_file() { 86 | let path = "./data"; 87 | std::fs::create_dir_all(path).unwrap(); 88 | let mut file = fs::OpenOptions::new() 89 | .write(true) 90 | .create(true) 91 | .open(format!("{}/{}.jpg", path, "asdf")) 92 | .unwrap(); 93 | file.write_all(&vec![12, 23, 45]).unwrap(); 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /summarizer/src/summarize.rs: -------------------------------------------------------------------------------- 1 | use crate::default; 2 | use crate::error::Serror; 3 | use crate::tokenize::{OpenAI, Tokenizer}; 4 | use async_openai::{ 5 | types::{ChatCompletionRequestMessageArgs, CreateChatCompletionRequestArgs, Role}, 6 | Client, 7 | }; 8 | use futures::future::join_all; 9 | use log; 10 | 11 | pub trait Summarize { 12 | fn description(&self) -> &str; 13 | } 14 | 15 | impl Summarize for String { 16 | fn description(&self) -> &str { 17 | &self 18 | } 19 | } 20 | 21 | #[derive(Clone)] 22 | pub struct Summarizer { 23 | openai: OpenAI, 24 | client: Client, 25 | } 26 | 27 | impl Summarizer { 28 | pub fn default_params() -> Result { 29 | log::info!("Initializing Summarizer from default params"); 30 | Ok(Self { 31 | openai: OpenAI::default_params()?, 32 | client: Client::new(), 33 | }) 34 | } 35 | } 36 | 37 | impl Summarizer { 38 | pub async fn summarize(&self, x: &impl Summarize) -> Result { 39 | log::info!("Summarizing the Content"); 40 | let client = &self.client; 41 | let chat = client.chat(); 42 | let mut interm: String; 43 | let mut content = x.description(); 44 | log::debug!("Length of content: {}", content.len()); 45 | loop { 46 | log::debug!( 47 | "Breaking the content into segments with a max token limit of {}", 48 | OpenAI::MAX_N_TOKENS 49 | ); 50 | let segments = self 51 | .openai 52 | .tokenize_in_max_tokenlimit(content)? 53 | .detokenize_inarray()?; 54 | log::debug!("Got {} segments", segments.len()); 55 | 56 | log::debug!("Creating Chat request for each segment"); 57 | let all_requests: Vec> = segments 58 | .iter() 59 | .map(|x| { 60 | let request = CreateChatCompletionRequestArgs::default() 61 | .model(default::GPT_MODEL) 62 | .messages([ 63 | ChatCompletionRequestMessageArgs::default() 64 | .role(Role::System) 65 | .content(default::SYSTEM_PROMPT) 66 | .build()?, 67 | ChatCompletionRequestMessageArgs::default() 68 | .role(Role::User) 69 | .content(x.description()) 70 | .build()?, 71 | ]) 72 | .build()?; 73 | Ok(chat.create(request)) 74 | }) 75 | .collect::>(); 76 | if all_requests.len() == 1 { 77 | log::debug!("Got only one segment!, summarizing for that content"); 78 | return Ok(all_requests 79 | .into_iter() 80 | .next() 81 | .ok_or(Serror::Other("cannot find first request".to_string()))?? 82 | .await? 83 | .choices 84 | .into_iter() 85 | .next() 86 | .ok_or(Serror::Other("cannot find summary".to_string()))? 87 | .message 88 | .content); 89 | } else { 90 | log::debug!( 91 | "For the {} segments, summarizing individually and finally merging them to a single string.", 92 | all_requests.len() 93 | ); 94 | let futures = all_requests.into_iter().filter_map(|x| x.ok()); 95 | interm = join_all(futures) 96 | .await 97 | .into_iter() 98 | .filter_map(|x| { 99 | let x = x.unwrap().choices.into_iter().next()?.message.content; 100 | Some(x) 101 | }) 102 | .collect::(); 103 | content = &interm; 104 | log::debug!("Looping again with the content to be this summarized content") 105 | } 106 | } 107 | } 108 | } 109 | 110 | #[cfg(test)] 111 | mod test { 112 | use super::*; 113 | use crate::Youtube; 114 | use env_logger; 115 | fn log_init() { 116 | env_logger::builder().is_test(true).try_init().unwrap(); 117 | } 118 | struct DUMMY; 119 | impl Summarize for DUMMY { 120 | fn description(&self) -> &str { 121 | "The thread initiating the shutdown blocks until all spawned work has been stopped. This can take an indefinite amount of time. The Drop implementation waits forever for this. 122 | 123 | shutdown_background and shutdown_timeout can be used if waiting forever is undesired. When the timeout is reached, spawned work that did not stop in time and threads running it are leaked. The work continues to run until one of the stopping conditions is fulfilled, but the thread initiating the shutdown is unblocked. 124 | 125 | Once the runtime has been dropped, any outstanding I/O resources bound to it will no longer function. Calling any method on them will result in an error." 126 | } 127 | } 128 | #[tokio::test] 129 | #[ignore = "Requires mocking openai response"] 130 | async fn summarize() { 131 | log_init(); 132 | let summarizer = Summarizer::default_params().unwrap(); 133 | let resp = summarizer.summarize(&DUMMY).await.unwrap(); 134 | println!("{resp}") 135 | } 136 | 137 | #[tokio::test] 138 | #[ignore = "Requires mocking openai response"] 139 | async fn summarize_youtube_small() { 140 | log_init(); 141 | let content = Youtube::link("https://www.youtube.com/watch?v=WYNRt-AwoUg") 142 | .unwrap() 143 | .content() 144 | .await 145 | .unwrap() 146 | .transcript_text() 147 | .await 148 | .unwrap(); 149 | let summarizer = Summarizer::default_params().unwrap(); 150 | let resp = summarizer.summarize(&content).await.unwrap(); 151 | println!("{resp}") 152 | } 153 | 154 | #[tokio::test] 155 | #[ignore = "Requires mocking openai response"] 156 | async fn summarize_youtube_1hr() { 157 | log_init(); 158 | let content = Youtube::link("https://www.youtube.com/watch?v=sBH-ngpL0zo") 159 | .unwrap() 160 | .content() 161 | .await 162 | .unwrap() 163 | .transcript_text() 164 | .await 165 | .unwrap(); 166 | let summarizer = Summarizer::default_params().unwrap(); 167 | let resp = summarizer.summarize(&content).await.unwrap(); 168 | println!("{resp}") 169 | } 170 | } 171 | -------------------------------------------------------------------------------- /summarizer/src/tokenize/mod.rs: -------------------------------------------------------------------------------- 1 | mod token; 2 | mod tokenizer; 3 | pub use token::Tokenizer; 4 | pub use tokenizer::OpenAI; 5 | -------------------------------------------------------------------------------- /summarizer/src/tokenize/token.rs: -------------------------------------------------------------------------------- 1 | use crate::error::Serror; 2 | use tiktoken_rs::CoreBPE; 3 | pub struct Tokens<'a> { 4 | tokens: Vec, 5 | tokenizer: &'a CoreBPE, 6 | max_tokens: usize, 7 | } 8 | 9 | impl<'a> Tokens<'a> { 10 | pub fn detokenize(&self) -> Result { 11 | Ok(self.tokenizer.decode(self.tokens.clone())?) 12 | } 13 | } 14 | 15 | pub struct MultiTokens<'a> { 16 | tokens: Vec>, 17 | tokenizer: &'a CoreBPE, 18 | max_tokens: usize, 19 | } 20 | 21 | impl<'a> MultiTokens<'a> { 22 | pub fn detokenize(&self) -> Result { 23 | let mapper = self.tokens.iter().map(|x| self.tokenizer.decode(x.clone())); 24 | let mut whole_str: String = "".to_string(); 25 | for each_string in mapper { 26 | let content = each_string?; 27 | whole_str += &content 28 | } 29 | Ok(whole_str) 30 | } 31 | 32 | pub fn detokenize_inarray(&self) -> Result, Serror> { 33 | let mapper = self.tokens.iter().map(|x| self.tokenizer.decode(x.clone())); 34 | let mut contents: Vec = vec![]; 35 | for each_string in mapper { 36 | let content = each_string?; 37 | contents.push(content) 38 | } 39 | Ok(contents) 40 | } 41 | } 42 | 43 | pub trait Tokenizer { 44 | const MAX_N_TOKENS: usize; 45 | fn bpe(&self) -> &CoreBPE; 46 | fn tokenize(&self, text: &str) -> Tokens { 47 | let tokens = self.bpe().encode_with_special_tokens(text); 48 | Tokens { 49 | tokens, 50 | tokenizer: self.bpe(), 51 | max_tokens: Self::MAX_N_TOKENS, 52 | } 53 | } 54 | fn tokenize_in_max_tokenlimit(&self, text: &str) -> Result { 55 | let encoded_text = self.bpe().encode_with_special_tokens(text); 56 | let mut n_tokens = self.bpe().encode_with_special_tokens(text).len(); 57 | let mut tokens: Vec> = vec![]; 58 | let mut start = 0; 59 | loop { 60 | if n_tokens == 0 { 61 | break; 62 | } 63 | let max_lim = std::cmp::min(n_tokens, Self::MAX_N_TOKENS); 64 | tokens.push(encoded_text[start..start + max_lim].to_vec()); 65 | n_tokens -= max_lim; 66 | start += max_lim; 67 | } 68 | Ok(MultiTokens { 69 | tokenizer: self.bpe(), 70 | tokens: tokens, 71 | max_tokens: Self::MAX_N_TOKENS, 72 | }) 73 | } 74 | } 75 | 76 | #[cfg(test)] 77 | mod test { 78 | 79 | use super::*; 80 | use tiktoken_rs::{cl100k_base, CoreBPE}; 81 | struct DummToken(CoreBPE); 82 | 83 | impl Tokenizer for DummToken { 84 | const MAX_N_TOKENS: usize = 3; 85 | fn bpe(&self) -> &CoreBPE { 86 | &self.0 87 | } 88 | } 89 | 90 | #[test] 91 | fn test_a_tokenize() { 92 | let dt = DummToken(cl100k_base().unwrap()); 93 | assert_eq!(dt.tokenize("a").tokens, [64]); 94 | } 95 | #[test] 96 | fn test_a_multitokenize() { 97 | let dt = DummToken(cl100k_base().unwrap()); 98 | assert_eq!(dt.tokenize_in_max_tokenlimit("a").unwrap().tokens, [[64]]); 99 | } 100 | #[test] 101 | fn test_abcd_tokenize() { 102 | let dt = DummToken(cl100k_base().unwrap()); 103 | assert_eq!(dt.tokenize("abcd").tokens, [69744]); 104 | } 105 | 106 | #[test] 107 | fn test_complex_tokenize() { 108 | let dt = DummToken(cl100k_base().unwrap()); 109 | assert_eq!( 110 | dt.tokenize("aurwfhdhbasdfiawifujbads").tokens, 111 | [4202, 44183, 16373, 71, 18275, 3013, 72, 675, 333, 9832, 65, 7819] 112 | ); 113 | } 114 | #[test] 115 | fn test_max_tokenlimit_tokenize() { 116 | let dt = DummToken(cl100k_base().unwrap()); 117 | let some_vec = [ 118 | [4202, 44183, 16373], 119 | [71, 18275, 3013], 120 | [72, 675, 333], 121 | [9832, 65, 7819], 122 | ]; 123 | assert_eq!( 124 | dt.tokenize_in_max_tokenlimit("aurwfhdhbasdfiawifujbads") 125 | .unwrap() 126 | .tokens, 127 | some_vec 128 | ); 129 | } 130 | #[test] 131 | fn test_max_tokenlimit_diff_tokenize() { 132 | let dt = DummToken(cl100k_base().unwrap()); 133 | let text = "asdfiawifujbads"; 134 | assert_eq!( 135 | dt.tokenize_in_max_tokenlimit(text).unwrap().tokens, 136 | [ 137 | [77715, 72, 675].to_vec(), 138 | [333, 9832, 65].to_vec(), 139 | [7819].to_vec() 140 | ] 141 | ); 142 | assert_eq!(dt.tokenize(text).tokens.len(), 7); 143 | } 144 | #[test] 145 | fn test_detokenize() { 146 | let dt = cl100k_base().unwrap(); 147 | let tokens = Tokens { 148 | tokenizer: &dt, 149 | tokens: [64].to_vec(), 150 | max_tokens: 10, 151 | }; 152 | assert_eq!(tokens.detokenize().unwrap(), "a"); 153 | } 154 | #[test] 155 | fn test_detokenize_sequence() { 156 | let dt = cl100k_base().unwrap(); 157 | let tokens = Tokens { 158 | tokenizer: &dt, 159 | tokens: [69744].to_vec(), 160 | max_tokens: 10, 161 | }; 162 | assert_eq!(tokens.detokenize().unwrap(), "abcd"); 163 | } 164 | #[test] 165 | fn test_detokenize_complex() { 166 | let dt = cl100k_base().unwrap(); 167 | let tokens = Tokens { 168 | tokenizer: &dt, 169 | tokens: [ 170 | 4202, 44183, 16373, 71, 18275, 3013, 72, 675, 333, 9832, 65, 7819, 171 | ] 172 | .to_vec(), 173 | max_tokens: 10, 174 | }; 175 | assert_eq!(tokens.detokenize().unwrap(), "aurwfhdhbasdfiawifujbads"); 176 | } 177 | #[test] 178 | fn test_detokenize_complex_multi() { 179 | let dt = cl100k_base().unwrap(); 180 | let tokens = MultiTokens { 181 | tokenizer: &dt, 182 | tokens: [ 183 | [4202, 44183, 16373].to_vec(), 184 | [71, 18275, 3013].to_vec(), 185 | [72, 675, 333].to_vec(), 186 | [9832, 65, 7819].to_vec(), 187 | ] 188 | .to_vec(), 189 | max_tokens: 10, 190 | }; 191 | assert_eq!(tokens.detokenize().unwrap(), "aurwfhdhbasdfiawifujbads"); 192 | } 193 | #[test] 194 | fn test_detokenize_complex_multi_different_lengths() { 195 | let dt = cl100k_base().unwrap(); 196 | let tokens = MultiTokens { 197 | tokenizer: &dt, 198 | tokens: [ 199 | [77715, 72, 675].to_vec(), 200 | [333, 9832, 65].to_vec(), 201 | [7819].to_vec(), 202 | ] 203 | .to_vec(), 204 | max_tokens: 10, 205 | }; 206 | assert_eq!(tokens.detokenize().unwrap(), "asdfiawifujbads"); 207 | } 208 | } 209 | -------------------------------------------------------------------------------- /summarizer/src/tokenize/tokenizer.rs: -------------------------------------------------------------------------------- 1 | use super::token::Tokenizer; 2 | use crate::error::Serror; 3 | use log; 4 | use tiktoken_rs::{cl100k_base, CoreBPE}; 5 | #[derive(Clone)] 6 | pub struct OpenAI { 7 | tokenizer: CoreBPE, 8 | } 9 | 10 | impl OpenAI { 11 | pub fn default_params() -> Result { 12 | log::info!("Initializing OpenAI from default params"); 13 | log::debug!("Using cl100k_base as the tokenizer"); 14 | let bpe = cl100k_base()?; 15 | Ok(Self { tokenizer: bpe }) 16 | } 17 | } 18 | 19 | impl Tokenizer for OpenAI { 20 | const MAX_N_TOKENS: usize = 2000; 21 | fn bpe(&self) -> &CoreBPE { 22 | &self.tokenizer 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /summarizer/src/utils.rs: -------------------------------------------------------------------------------- 1 | use crate::error::Serror; 2 | use std::ffi::OsStr; 3 | pub fn env_var(var: T) -> Result 4 | where 5 | T: AsRef, 6 | { 7 | std::env::var(&var).map_err(|_| { 8 | Serror::Environment({ 9 | if let Some(x) = var.as_ref().to_str() { 10 | format!("environment variable: {} not found", x) 11 | } else { 12 | "environment variable not found".to_string() 13 | } 14 | }) 15 | }) 16 | } 17 | -------------------------------------------------------------------------------- /summarizer/src/youtube.rs: -------------------------------------------------------------------------------- 1 | use crate::error::Serror; 2 | use regex::Regex; 3 | use reqwest::Client; 4 | use youtube_transcript::{Transcript, YoutubeBuilder}; 5 | pub struct Youtube<'a> { 6 | link: &'a str, 7 | video_id_extractor: Regex, 8 | } 9 | impl<'a> Youtube<'a> { 10 | pub fn link(link: &'a str) -> Result { 11 | Ok(Self { 12 | link, 13 | video_id_extractor: Regex::new(r"v=([^&]+)")?, 14 | }) 15 | } 16 | 17 | fn video_id(&self) -> Result<&str, Serror> { 18 | let capture = self 19 | .video_id_extractor 20 | .captures(self.link) 21 | .ok_or(Serror::Youtubefetch(format!( 22 | "Cannot capture the video id in the url: {}", 23 | self.link 24 | )))? 25 | .get(1) 26 | .ok_or(Serror::Youtubefetch(format!( 27 | "Cannot capture the video id in the url: {}", 28 | self.link 29 | )))?; 30 | 31 | Ok(capture.as_str()) 32 | } 33 | 34 | pub async fn content(&self) -> Result { 35 | // Get Youtube Body 36 | let client = Client::default(); 37 | let response = client.get(self.link).send().await?; 38 | let content = response.text().await?; 39 | 40 | Ok(YoutubeContent { 41 | content: content, 42 | video_id: self.video_id()?.to_string(), 43 | }) 44 | } 45 | } 46 | 47 | #[derive(Debug)] 48 | pub struct YoutubeContent { 49 | content: String, 50 | pub(crate) video_id: String, 51 | } 52 | 53 | impl YoutubeContent { 54 | /// Get transcript 55 | pub async fn transcirpt(&self) -> Result { 56 | YoutubeBuilder::default() 57 | .build() 58 | .transcript_from_text(&self.content) 59 | .await 60 | .map_err(|x| Serror::Youtubefetch(x.to_string())) 61 | } 62 | 63 | /// Get image link 64 | pub fn image_link(&self) -> String { 65 | format!("https://img.youtube.com/vi/{}/0.jpg", self.video_id) 66 | } 67 | 68 | /// Get title 69 | pub fn title(&self) -> Result, Serror> { 70 | let c = Regex::new(r"(.+)")?.captures(&self.content); 71 | if let Some(capture) = c { 72 | if let Some(match_str) = capture.get(1) { 73 | return Ok(Some(match_str.as_str().to_string())); 74 | } 75 | } 76 | Ok(None) 77 | } 78 | 79 | /// Get transcript in text 80 | pub async fn transcript_text(&self) -> Result { 81 | let transcript = self.transcirpt().await?; 82 | Ok(transcript 83 | .transcripts 84 | .into_iter() 85 | .map(|x| format!("{} ", x.text)) 86 | .collect::()) 87 | } 88 | } 89 | 90 | #[cfg(test)] 91 | mod test { 92 | use super::*; 93 | #[tokio::test] 94 | #[ignore = "Requires mocking youtube response"] 95 | async fn test_content() { 96 | let content = Youtube::link("https://www.youtube.com/watch?v=GJLlxj_dtq8") 97 | .unwrap() 98 | .content() 99 | .await 100 | .unwrap() 101 | .transcript_text() 102 | .await 103 | .unwrap(); 104 | println!("{}", content); 105 | } 106 | 107 | #[tokio::test] 108 | #[ignore = "Requires mocking youtube response"] 109 | async fn test_title() { 110 | let title = Youtube::link("https://www.youtube.com/watch?v=GJLlxj_dtq8") 111 | .unwrap() 112 | .content() 113 | .await 114 | .unwrap() 115 | .title() 116 | .unwrap() 117 | .expect("No title found"); 118 | println!("{}", title); 119 | } 120 | 121 | #[tokio::test] 122 | #[ignore = "Requires mocking youtube response"] 123 | async fn test_image() { 124 | let image_link = Youtube::link("https://www.youtube.com/watch?v=GJLlxj_dtq8") 125 | .unwrap() 126 | .content() 127 | .await 128 | .unwrap() 129 | .image_link(); 130 | println!("{}", image_link); 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /ui/.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | pnpm-debug.log* 8 | lerna-debug.log* 9 | 10 | node_modules 11 | dist 12 | dist-ssr 13 | *.local 14 | 15 | # Editor directories and files 16 | .vscode/* 17 | !.vscode/extensions.json 18 | .idea 19 | .DS_Store 20 | *.suo 21 | *.ntvs* 22 | *.njsproj 23 | *.sln 24 | *.sw? 25 | 26 | # Sample Data 27 | sample.json -------------------------------------------------------------------------------- /ui/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Podcast summarizer 10 | 11 | 12 | 13 |
14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /ui/package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ui", 3 | "version": "0.0.0", 4 | "lockfileVersion": 3, 5 | "requires": true, 6 | "packages": { 7 | "": { 8 | "name": "ui", 9 | "version": "0.0.0", 10 | "devDependencies": { 11 | "@sveltejs/vite-plugin-svelte": "^2.0.3", 12 | "@tsconfig/svelte": "^4.0.1", 13 | "svelte": "^3.57.0", 14 | "svelte-check": "^2.10.3", 15 | "tslib": "^2.5.0", 16 | "typescript": "^5.0.2", 17 | "vite": "^4.3.2" 18 | } 19 | }, 20 | "node_modules/@esbuild/android-arm": { 21 | "version": "0.17.18", 22 | "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.17.18.tgz", 23 | "integrity": "sha512-EmwL+vUBZJ7mhFCs5lA4ZimpUH3WMAoqvOIYhVQwdIgSpHC8ImHdsRyhHAVxpDYUSm0lWvd63z0XH1IlImS2Qw==", 24 | "cpu": [ 25 | "arm" 26 | ], 27 | "dev": true, 28 | "optional": true, 29 | "os": [ 30 | "android" 31 | ], 32 | "engines": { 33 | "node": ">=12" 34 | } 35 | }, 36 | "node_modules/@esbuild/android-arm64": { 37 | "version": "0.17.18", 38 | "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.17.18.tgz", 39 | "integrity": "sha512-/iq0aK0eeHgSC3z55ucMAHO05OIqmQehiGay8eP5l/5l+iEr4EIbh4/MI8xD9qRFjqzgkc0JkX0LculNC9mXBw==", 40 | "cpu": [ 41 | "arm64" 42 | ], 43 | "dev": true, 44 | "optional": true, 45 | "os": [ 46 | "android" 47 | ], 48 | "engines": { 49 | "node": ">=12" 50 | } 51 | }, 52 | "node_modules/@esbuild/android-x64": { 53 | "version": "0.17.18", 54 | "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.17.18.tgz", 55 | "integrity": "sha512-x+0efYNBF3NPW2Xc5bFOSFW7tTXdAcpfEg2nXmxegm4mJuVeS+i109m/7HMiOQ6M12aVGGFlqJX3RhNdYM2lWg==", 56 | "cpu": [ 57 | "x64" 58 | ], 59 | "dev": true, 60 | "optional": true, 61 | "os": [ 62 | "android" 63 | ], 64 | "engines": { 65 | "node": ">=12" 66 | } 67 | }, 68 | "node_modules/@esbuild/darwin-arm64": { 69 | "version": "0.17.18", 70 | "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.17.18.tgz", 71 | "integrity": "sha512-6tY+djEAdF48M1ONWnQb1C+6LiXrKjmqjzPNPWXhu/GzOHTHX2nh8Mo2ZAmBFg0kIodHhciEgUBtcYCAIjGbjQ==", 72 | "cpu": [ 73 | "arm64" 74 | ], 75 | "dev": true, 76 | "optional": true, 77 | "os": [ 78 | "darwin" 79 | ], 80 | "engines": { 81 | "node": ">=12" 82 | } 83 | }, 84 | "node_modules/@esbuild/darwin-x64": { 85 | "version": "0.17.18", 86 | "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.17.18.tgz", 87 | "integrity": "sha512-Qq84ykvLvya3dO49wVC9FFCNUfSrQJLbxhoQk/TE1r6MjHo3sFF2tlJCwMjhkBVq3/ahUisj7+EpRSz0/+8+9A==", 88 | "cpu": [ 89 | "x64" 90 | ], 91 | "dev": true, 92 | "optional": true, 93 | "os": [ 94 | "darwin" 95 | ], 96 | "engines": { 97 | "node": ">=12" 98 | } 99 | }, 100 | "node_modules/@esbuild/freebsd-arm64": { 101 | "version": "0.17.18", 102 | "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.17.18.tgz", 103 | "integrity": "sha512-fw/ZfxfAzuHfaQeMDhbzxp9mc+mHn1Y94VDHFHjGvt2Uxl10mT4CDavHm+/L9KG441t1QdABqkVYwakMUeyLRA==", 104 | "cpu": [ 105 | "arm64" 106 | ], 107 | "dev": true, 108 | "optional": true, 109 | "os": [ 110 | "freebsd" 111 | ], 112 | "engines": { 113 | "node": ">=12" 114 | } 115 | }, 116 | "node_modules/@esbuild/freebsd-x64": { 117 | "version": "0.17.18", 118 | "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.17.18.tgz", 119 | "integrity": "sha512-FQFbRtTaEi8ZBi/A6kxOC0V0E9B/97vPdYjY9NdawyLd4Qk5VD5g2pbWN2VR1c0xhzcJm74HWpObPszWC+qTew==", 120 | "cpu": [ 121 | "x64" 122 | ], 123 | "dev": true, 124 | "optional": true, 125 | "os": [ 126 | "freebsd" 127 | ], 128 | "engines": { 129 | "node": ">=12" 130 | } 131 | }, 132 | "node_modules/@esbuild/linux-arm": { 133 | "version": "0.17.18", 134 | "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.17.18.tgz", 135 | "integrity": "sha512-jW+UCM40LzHcouIaqv3e/oRs0JM76JfhHjCavPxMUti7VAPh8CaGSlS7cmyrdpzSk7A+8f0hiedHqr/LMnfijg==", 136 | "cpu": [ 137 | "arm" 138 | ], 139 | "dev": true, 140 | "optional": true, 141 | "os": [ 142 | "linux" 143 | ], 144 | "engines": { 145 | "node": ">=12" 146 | } 147 | }, 148 | "node_modules/@esbuild/linux-arm64": { 149 | "version": "0.17.18", 150 | "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.17.18.tgz", 151 | "integrity": "sha512-R7pZvQZFOY2sxUG8P6A21eq6q+eBv7JPQYIybHVf1XkQYC+lT7nDBdC7wWKTrbvMXKRaGudp/dzZCwL/863mZQ==", 152 | "cpu": [ 153 | "arm64" 154 | ], 155 | "dev": true, 156 | "optional": true, 157 | "os": [ 158 | "linux" 159 | ], 160 | "engines": { 161 | "node": ">=12" 162 | } 163 | }, 164 | "node_modules/@esbuild/linux-ia32": { 165 | "version": "0.17.18", 166 | "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.17.18.tgz", 167 | "integrity": "sha512-ygIMc3I7wxgXIxk6j3V00VlABIjq260i967Cp9BNAk5pOOpIXmd1RFQJQX9Io7KRsthDrQYrtcx7QCof4o3ZoQ==", 168 | "cpu": [ 169 | "ia32" 170 | ], 171 | "dev": true, 172 | "optional": true, 173 | "os": [ 174 | "linux" 175 | ], 176 | "engines": { 177 | "node": ">=12" 178 | } 179 | }, 180 | "node_modules/@esbuild/linux-loong64": { 181 | "version": "0.17.18", 182 | "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.17.18.tgz", 183 | "integrity": "sha512-bvPG+MyFs5ZlwYclCG1D744oHk1Pv7j8psF5TfYx7otCVmcJsEXgFEhQkbhNW8otDHL1a2KDINW20cfCgnzgMQ==", 184 | "cpu": [ 185 | "loong64" 186 | ], 187 | "dev": true, 188 | "optional": true, 189 | "os": [ 190 | "linux" 191 | ], 192 | "engines": { 193 | "node": ">=12" 194 | } 195 | }, 196 | "node_modules/@esbuild/linux-mips64el": { 197 | "version": "0.17.18", 198 | "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.17.18.tgz", 199 | "integrity": "sha512-oVqckATOAGuiUOa6wr8TXaVPSa+6IwVJrGidmNZS1cZVx0HqkTMkqFGD2HIx9H1RvOwFeWYdaYbdY6B89KUMxA==", 200 | "cpu": [ 201 | "mips64el" 202 | ], 203 | "dev": true, 204 | "optional": true, 205 | "os": [ 206 | "linux" 207 | ], 208 | "engines": { 209 | "node": ">=12" 210 | } 211 | }, 212 | "node_modules/@esbuild/linux-ppc64": { 213 | "version": "0.17.18", 214 | "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.17.18.tgz", 215 | "integrity": "sha512-3dLlQO+b/LnQNxgH4l9rqa2/IwRJVN9u/bK63FhOPB4xqiRqlQAU0qDU3JJuf0BmaH0yytTBdoSBHrb2jqc5qQ==", 216 | "cpu": [ 217 | "ppc64" 218 | ], 219 | "dev": true, 220 | "optional": true, 221 | "os": [ 222 | "linux" 223 | ], 224 | "engines": { 225 | "node": ">=12" 226 | } 227 | }, 228 | "node_modules/@esbuild/linux-riscv64": { 229 | "version": "0.17.18", 230 | "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.17.18.tgz", 231 | "integrity": "sha512-/x7leOyDPjZV3TcsdfrSI107zItVnsX1q2nho7hbbQoKnmoeUWjs+08rKKt4AUXju7+3aRZSsKrJtaRmsdL1xA==", 232 | "cpu": [ 233 | "riscv64" 234 | ], 235 | "dev": true, 236 | "optional": true, 237 | "os": [ 238 | "linux" 239 | ], 240 | "engines": { 241 | "node": ">=12" 242 | } 243 | }, 244 | "node_modules/@esbuild/linux-s390x": { 245 | "version": "0.17.18", 246 | "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.17.18.tgz", 247 | "integrity": "sha512-cX0I8Q9xQkL/6F5zWdYmVf5JSQt+ZfZD2bJudZrWD+4mnUvoZ3TDDXtDX2mUaq6upMFv9FlfIh4Gfun0tbGzuw==", 248 | "cpu": [ 249 | "s390x" 250 | ], 251 | "dev": true, 252 | "optional": true, 253 | "os": [ 254 | "linux" 255 | ], 256 | "engines": { 257 | "node": ">=12" 258 | } 259 | }, 260 | "node_modules/@esbuild/linux-x64": { 261 | "version": "0.17.18", 262 | "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.17.18.tgz", 263 | "integrity": "sha512-66RmRsPlYy4jFl0vG80GcNRdirx4nVWAzJmXkevgphP1qf4dsLQCpSKGM3DUQCojwU1hnepI63gNZdrr02wHUA==", 264 | "cpu": [ 265 | "x64" 266 | ], 267 | "dev": true, 268 | "optional": true, 269 | "os": [ 270 | "linux" 271 | ], 272 | "engines": { 273 | "node": ">=12" 274 | } 275 | }, 276 | "node_modules/@esbuild/netbsd-x64": { 277 | "version": "0.17.18", 278 | "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.17.18.tgz", 279 | "integrity": "sha512-95IRY7mI2yrkLlTLb1gpDxdC5WLC5mZDi+kA9dmM5XAGxCME0F8i4bYH4jZreaJ6lIZ0B8hTrweqG1fUyW7jbg==", 280 | "cpu": [ 281 | "x64" 282 | ], 283 | "dev": true, 284 | "optional": true, 285 | "os": [ 286 | "netbsd" 287 | ], 288 | "engines": { 289 | "node": ">=12" 290 | } 291 | }, 292 | "node_modules/@esbuild/openbsd-x64": { 293 | "version": "0.17.18", 294 | "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.17.18.tgz", 295 | "integrity": "sha512-WevVOgcng+8hSZ4Q3BKL3n1xTv5H6Nb53cBrtzzEjDbbnOmucEVcZeGCsCOi9bAOcDYEeBZbD2SJNBxlfP3qiA==", 296 | "cpu": [ 297 | "x64" 298 | ], 299 | "dev": true, 300 | "optional": true, 301 | "os": [ 302 | "openbsd" 303 | ], 304 | "engines": { 305 | "node": ">=12" 306 | } 307 | }, 308 | "node_modules/@esbuild/sunos-x64": { 309 | "version": "0.17.18", 310 | "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.17.18.tgz", 311 | "integrity": "sha512-Rzf4QfQagnwhQXVBS3BYUlxmEbcV7MY+BH5vfDZekU5eYpcffHSyjU8T0xucKVuOcdCsMo+Ur5wmgQJH2GfNrg==", 312 | "cpu": [ 313 | "x64" 314 | ], 315 | "dev": true, 316 | "optional": true, 317 | "os": [ 318 | "sunos" 319 | ], 320 | "engines": { 321 | "node": ">=12" 322 | } 323 | }, 324 | "node_modules/@esbuild/win32-arm64": { 325 | "version": "0.17.18", 326 | "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.17.18.tgz", 327 | "integrity": "sha512-Kb3Ko/KKaWhjeAm2YoT/cNZaHaD1Yk/pa3FTsmqo9uFh1D1Rfco7BBLIPdDOozrObj2sahslFuAQGvWbgWldAg==", 328 | "cpu": [ 329 | "arm64" 330 | ], 331 | "dev": true, 332 | "optional": true, 333 | "os": [ 334 | "win32" 335 | ], 336 | "engines": { 337 | "node": ">=12" 338 | } 339 | }, 340 | "node_modules/@esbuild/win32-ia32": { 341 | "version": "0.17.18", 342 | "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.17.18.tgz", 343 | "integrity": "sha512-0/xUMIdkVHwkvxfbd5+lfG7mHOf2FRrxNbPiKWg9C4fFrB8H0guClmaM3BFiRUYrznVoyxTIyC/Ou2B7QQSwmw==", 344 | "cpu": [ 345 | "ia32" 346 | ], 347 | "dev": true, 348 | "optional": true, 349 | "os": [ 350 | "win32" 351 | ], 352 | "engines": { 353 | "node": ">=12" 354 | } 355 | }, 356 | "node_modules/@esbuild/win32-x64": { 357 | "version": "0.17.18", 358 | "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.17.18.tgz", 359 | "integrity": "sha512-qU25Ma1I3NqTSHJUOKi9sAH1/Mzuvlke0ioMJRthLXKm7JiSKVwFghlGbDLOO2sARECGhja4xYfRAZNPAkooYg==", 360 | "cpu": [ 361 | "x64" 362 | ], 363 | "dev": true, 364 | "optional": true, 365 | "os": [ 366 | "win32" 367 | ], 368 | "engines": { 369 | "node": ">=12" 370 | } 371 | }, 372 | "node_modules/@jridgewell/resolve-uri": { 373 | "version": "3.1.0", 374 | "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz", 375 | "integrity": "sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==", 376 | "dev": true, 377 | "engines": { 378 | "node": ">=6.0.0" 379 | } 380 | }, 381 | "node_modules/@jridgewell/sourcemap-codec": { 382 | "version": "1.4.15", 383 | "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz", 384 | "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==", 385 | "dev": true 386 | }, 387 | "node_modules/@jridgewell/trace-mapping": { 388 | "version": "0.3.18", 389 | "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.18.tgz", 390 | "integrity": "sha512-w+niJYzMHdd7USdiH2U6869nqhD2nbfZXND5Yp93qIbEmnDNk7PD48o+YchRVpzMU7M6jVCbenTR7PA1FLQ9pA==", 391 | "dev": true, 392 | "dependencies": { 393 | "@jridgewell/resolve-uri": "3.1.0", 394 | "@jridgewell/sourcemap-codec": "1.4.14" 395 | } 396 | }, 397 | "node_modules/@jridgewell/trace-mapping/node_modules/@jridgewell/sourcemap-codec": { 398 | "version": "1.4.14", 399 | "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz", 400 | "integrity": "sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==", 401 | "dev": true 402 | }, 403 | "node_modules/@nodelib/fs.scandir": { 404 | "version": "2.1.5", 405 | "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", 406 | "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", 407 | "dev": true, 408 | "dependencies": { 409 | "@nodelib/fs.stat": "2.0.5", 410 | "run-parallel": "^1.1.9" 411 | }, 412 | "engines": { 413 | "node": ">= 8" 414 | } 415 | }, 416 | "node_modules/@nodelib/fs.stat": { 417 | "version": "2.0.5", 418 | "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", 419 | "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", 420 | "dev": true, 421 | "engines": { 422 | "node": ">= 8" 423 | } 424 | }, 425 | "node_modules/@nodelib/fs.walk": { 426 | "version": "1.2.8", 427 | "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", 428 | "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", 429 | "dev": true, 430 | "dependencies": { 431 | "@nodelib/fs.scandir": "2.1.5", 432 | "fastq": "^1.6.0" 433 | }, 434 | "engines": { 435 | "node": ">= 8" 436 | } 437 | }, 438 | "node_modules/@sveltejs/vite-plugin-svelte": { 439 | "version": "2.1.1", 440 | "resolved": "https://registry.npmjs.org/@sveltejs/vite-plugin-svelte/-/vite-plugin-svelte-2.1.1.tgz", 441 | "integrity": "sha512-7YeBDt4us0FiIMNsVXxyaP4Hwyn2/v9x3oqStkHU3ZdIc5O22pGwUwH33wUqYo+7Itdmo8zxJ45Qvfm3H7UUjQ==", 442 | "dev": true, 443 | "dependencies": { 444 | "debug": "^4.3.4", 445 | "deepmerge": "^4.3.1", 446 | "kleur": "^4.1.5", 447 | "magic-string": "^0.30.0", 448 | "svelte-hmr": "^0.15.1", 449 | "vitefu": "^0.2.4" 450 | }, 451 | "engines": { 452 | "node": "^14.18.0 || >= 16" 453 | }, 454 | "peerDependencies": { 455 | "svelte": "^3.54.0", 456 | "vite": "^4.0.0" 457 | } 458 | }, 459 | "node_modules/@tsconfig/svelte": { 460 | "version": "4.0.1", 461 | "resolved": "https://registry.npmjs.org/@tsconfig/svelte/-/svelte-4.0.1.tgz", 462 | "integrity": "sha512-B+XlGpmuAQzJqDoBATNCvEPqQg0HkO7S8pM14QDI5NsmtymzRexQ1N+nX2H6RTtFbuFgaZD4I8AAi8voGg0GLg==", 463 | "dev": true 464 | }, 465 | "node_modules/@types/pug": { 466 | "version": "2.0.6", 467 | "resolved": "https://registry.npmjs.org/@types/pug/-/pug-2.0.6.tgz", 468 | "integrity": "sha512-SnHmG9wN1UVmagJOnyo/qkk0Z7gejYxOYYmaAwr5u2yFYfsupN3sg10kyzN8Hep/2zbHxCnsumxOoRIRMBwKCg==", 469 | "dev": true 470 | }, 471 | "node_modules/@types/sass": { 472 | "version": "1.45.0", 473 | "resolved": "https://registry.npmjs.org/@types/sass/-/sass-1.45.0.tgz", 474 | "integrity": "sha512-jn7qwGFmJHwUSphV8zZneO3GmtlgLsmhs/LQyVvQbIIa+fzGMUiHI4HXJZL3FT8MJmgXWbLGiVVY7ElvHq6vDA==", 475 | "deprecated": "This is a stub types definition. sass provides its own type definitions, so you do not need this installed.", 476 | "dev": true, 477 | "dependencies": { 478 | "sass": "*" 479 | } 480 | }, 481 | "node_modules/anymatch": { 482 | "version": "3.1.3", 483 | "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", 484 | "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", 485 | "dev": true, 486 | "dependencies": { 487 | "normalize-path": "^3.0.0", 488 | "picomatch": "^2.0.4" 489 | }, 490 | "engines": { 491 | "node": ">= 8" 492 | } 493 | }, 494 | "node_modules/balanced-match": { 495 | "version": "1.0.2", 496 | "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", 497 | "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", 498 | "dev": true 499 | }, 500 | "node_modules/binary-extensions": { 501 | "version": "2.2.0", 502 | "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", 503 | "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", 504 | "dev": true, 505 | "engines": { 506 | "node": ">=8" 507 | } 508 | }, 509 | "node_modules/brace-expansion": { 510 | "version": "1.1.11", 511 | "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", 512 | "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", 513 | "dev": true, 514 | "dependencies": { 515 | "balanced-match": "^1.0.0", 516 | "concat-map": "0.0.1" 517 | } 518 | }, 519 | "node_modules/braces": { 520 | "version": "3.0.2", 521 | "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", 522 | "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", 523 | "dev": true, 524 | "dependencies": { 525 | "fill-range": "^7.0.1" 526 | }, 527 | "engines": { 528 | "node": ">=8" 529 | } 530 | }, 531 | "node_modules/buffer-crc32": { 532 | "version": "0.2.13", 533 | "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", 534 | "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", 535 | "dev": true, 536 | "engines": { 537 | "node": "*" 538 | } 539 | }, 540 | "node_modules/callsites": { 541 | "version": "3.1.0", 542 | "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", 543 | "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", 544 | "dev": true, 545 | "engines": { 546 | "node": ">=6" 547 | } 548 | }, 549 | "node_modules/chokidar": { 550 | "version": "3.5.3", 551 | "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", 552 | "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", 553 | "dev": true, 554 | "funding": [ 555 | { 556 | "type": "individual", 557 | "url": "https://paulmillr.com/funding/" 558 | } 559 | ], 560 | "dependencies": { 561 | "anymatch": "~3.1.2", 562 | "braces": "~3.0.2", 563 | "glob-parent": "~5.1.2", 564 | "is-binary-path": "~2.1.0", 565 | "is-glob": "~4.0.1", 566 | "normalize-path": "~3.0.0", 567 | "readdirp": "~3.6.0" 568 | }, 569 | "engines": { 570 | "node": ">= 8.10.0" 571 | }, 572 | "optionalDependencies": { 573 | "fsevents": "~2.3.2" 574 | } 575 | }, 576 | "node_modules/concat-map": { 577 | "version": "0.0.1", 578 | "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", 579 | "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", 580 | "dev": true 581 | }, 582 | "node_modules/debug": { 583 | "version": "4.3.4", 584 | "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", 585 | "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", 586 | "dev": true, 587 | "dependencies": { 588 | "ms": "2.1.2" 589 | }, 590 | "engines": { 591 | "node": ">=6.0" 592 | }, 593 | "peerDependenciesMeta": { 594 | "supports-color": { 595 | "optional": true 596 | } 597 | } 598 | }, 599 | "node_modules/deepmerge": { 600 | "version": "4.3.1", 601 | "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", 602 | "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", 603 | "dev": true, 604 | "engines": { 605 | "node": ">=0.10.0" 606 | } 607 | }, 608 | "node_modules/detect-indent": { 609 | "version": "6.1.0", 610 | "resolved": "https://registry.npmjs.org/detect-indent/-/detect-indent-6.1.0.tgz", 611 | "integrity": "sha512-reYkTUJAZb9gUuZ2RvVCNhVHdg62RHnJ7WJl8ftMi4diZ6NWlciOzQN88pUhSELEwflJht4oQDv0F0BMlwaYtA==", 612 | "dev": true, 613 | "engines": { 614 | "node": ">=8" 615 | } 616 | }, 617 | "node_modules/es6-promise": { 618 | "version": "3.3.1", 619 | "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-3.3.1.tgz", 620 | "integrity": "sha512-SOp9Phqvqn7jtEUxPWdWfWoLmyt2VaJ6MpvP9Comy1MceMXqE6bxvaTu4iaxpYYPzhny28Lc+M87/c2cPK6lDg==", 621 | "dev": true 622 | }, 623 | "node_modules/esbuild": { 624 | "version": "0.17.18", 625 | "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.17.18.tgz", 626 | "integrity": "sha512-z1lix43jBs6UKjcZVKOw2xx69ffE2aG0PygLL5qJ9OS/gy0Ewd1gW/PUQIOIQGXBHWNywSc0floSKoMFF8aK2w==", 627 | "dev": true, 628 | "hasInstallScript": true, 629 | "bin": { 630 | "esbuild": "bin/esbuild" 631 | }, 632 | "engines": { 633 | "node": ">=12" 634 | }, 635 | "optionalDependencies": { 636 | "@esbuild/android-arm": "0.17.18", 637 | "@esbuild/android-arm64": "0.17.18", 638 | "@esbuild/android-x64": "0.17.18", 639 | "@esbuild/darwin-arm64": "0.17.18", 640 | "@esbuild/darwin-x64": "0.17.18", 641 | "@esbuild/freebsd-arm64": "0.17.18", 642 | "@esbuild/freebsd-x64": "0.17.18", 643 | "@esbuild/linux-arm": "0.17.18", 644 | "@esbuild/linux-arm64": "0.17.18", 645 | "@esbuild/linux-ia32": "0.17.18", 646 | "@esbuild/linux-loong64": "0.17.18", 647 | "@esbuild/linux-mips64el": "0.17.18", 648 | "@esbuild/linux-ppc64": "0.17.18", 649 | "@esbuild/linux-riscv64": "0.17.18", 650 | "@esbuild/linux-s390x": "0.17.18", 651 | "@esbuild/linux-x64": "0.17.18", 652 | "@esbuild/netbsd-x64": "0.17.18", 653 | "@esbuild/openbsd-x64": "0.17.18", 654 | "@esbuild/sunos-x64": "0.17.18", 655 | "@esbuild/win32-arm64": "0.17.18", 656 | "@esbuild/win32-ia32": "0.17.18", 657 | "@esbuild/win32-x64": "0.17.18" 658 | } 659 | }, 660 | "node_modules/fast-glob": { 661 | "version": "3.2.12", 662 | "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.12.tgz", 663 | "integrity": "sha512-DVj4CQIYYow0BlaelwK1pHl5n5cRSJfM60UA0zK891sVInoPri2Ekj7+e1CT3/3qxXenpI+nBBmQAcJPJgaj4w==", 664 | "dev": true, 665 | "dependencies": { 666 | "@nodelib/fs.stat": "^2.0.2", 667 | "@nodelib/fs.walk": "^1.2.3", 668 | "glob-parent": "^5.1.2", 669 | "merge2": "^1.3.0", 670 | "micromatch": "^4.0.4" 671 | }, 672 | "engines": { 673 | "node": ">=8.6.0" 674 | } 675 | }, 676 | "node_modules/fastq": { 677 | "version": "1.15.0", 678 | "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz", 679 | "integrity": "sha512-wBrocU2LCXXa+lWBt8RoIRD89Fi8OdABODa/kEnyeyjS5aZO5/GNvI5sEINADqP/h8M29UHTHUb53sUu5Ihqdw==", 680 | "dev": true, 681 | "dependencies": { 682 | "reusify": "^1.0.4" 683 | } 684 | }, 685 | "node_modules/fill-range": { 686 | "version": "7.0.1", 687 | "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", 688 | "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", 689 | "dev": true, 690 | "dependencies": { 691 | "to-regex-range": "^5.0.1" 692 | }, 693 | "engines": { 694 | "node": ">=8" 695 | } 696 | }, 697 | "node_modules/fs.realpath": { 698 | "version": "1.0.0", 699 | "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", 700 | "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", 701 | "dev": true 702 | }, 703 | "node_modules/fsevents": { 704 | "version": "2.3.2", 705 | "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", 706 | "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", 707 | "dev": true, 708 | "hasInstallScript": true, 709 | "optional": true, 710 | "os": [ 711 | "darwin" 712 | ], 713 | "engines": { 714 | "node": "^8.16.0 || ^10.6.0 || >=11.0.0" 715 | } 716 | }, 717 | "node_modules/glob": { 718 | "version": "7.2.3", 719 | "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", 720 | "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", 721 | "dev": true, 722 | "dependencies": { 723 | "fs.realpath": "^1.0.0", 724 | "inflight": "^1.0.4", 725 | "inherits": "2", 726 | "minimatch": "^3.1.1", 727 | "once": "^1.3.0", 728 | "path-is-absolute": "^1.0.0" 729 | }, 730 | "engines": { 731 | "node": "*" 732 | }, 733 | "funding": { 734 | "url": "https://github.com/sponsors/isaacs" 735 | } 736 | }, 737 | "node_modules/glob-parent": { 738 | "version": "5.1.2", 739 | "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", 740 | "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", 741 | "dev": true, 742 | "dependencies": { 743 | "is-glob": "^4.0.1" 744 | }, 745 | "engines": { 746 | "node": ">= 6" 747 | } 748 | }, 749 | "node_modules/graceful-fs": { 750 | "version": "4.2.11", 751 | "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", 752 | "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", 753 | "dev": true 754 | }, 755 | "node_modules/immutable": { 756 | "version": "4.3.0", 757 | "resolved": "https://registry.npmjs.org/immutable/-/immutable-4.3.0.tgz", 758 | "integrity": "sha512-0AOCmOip+xgJwEVTQj1EfiDDOkPmuyllDuTuEX+DDXUgapLAsBIfkg3sxCYyCEA8mQqZrrxPUGjcOQ2JS3WLkg==", 759 | "dev": true 760 | }, 761 | "node_modules/import-fresh": { 762 | "version": "3.3.0", 763 | "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", 764 | "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==", 765 | "dev": true, 766 | "dependencies": { 767 | "parent-module": "^1.0.0", 768 | "resolve-from": "^4.0.0" 769 | }, 770 | "engines": { 771 | "node": ">=6" 772 | }, 773 | "funding": { 774 | "url": "https://github.com/sponsors/sindresorhus" 775 | } 776 | }, 777 | "node_modules/inflight": { 778 | "version": "1.0.6", 779 | "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", 780 | "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", 781 | "dev": true, 782 | "dependencies": { 783 | "once": "^1.3.0", 784 | "wrappy": "1" 785 | } 786 | }, 787 | "node_modules/inherits": { 788 | "version": "2.0.4", 789 | "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", 790 | "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", 791 | "dev": true 792 | }, 793 | "node_modules/is-binary-path": { 794 | "version": "2.1.0", 795 | "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", 796 | "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", 797 | "dev": true, 798 | "dependencies": { 799 | "binary-extensions": "^2.0.0" 800 | }, 801 | "engines": { 802 | "node": ">=8" 803 | } 804 | }, 805 | "node_modules/is-extglob": { 806 | "version": "2.1.1", 807 | "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", 808 | "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", 809 | "dev": true, 810 | "engines": { 811 | "node": ">=0.10.0" 812 | } 813 | }, 814 | "node_modules/is-glob": { 815 | "version": "4.0.3", 816 | "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", 817 | "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", 818 | "dev": true, 819 | "dependencies": { 820 | "is-extglob": "^2.1.1" 821 | }, 822 | "engines": { 823 | "node": ">=0.10.0" 824 | } 825 | }, 826 | "node_modules/is-number": { 827 | "version": "7.0.0", 828 | "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", 829 | "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", 830 | "dev": true, 831 | "engines": { 832 | "node": ">=0.12.0" 833 | } 834 | }, 835 | "node_modules/kleur": { 836 | "version": "4.1.5", 837 | "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", 838 | "integrity": "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==", 839 | "dev": true, 840 | "engines": { 841 | "node": ">=6" 842 | } 843 | }, 844 | "node_modules/magic-string": { 845 | "version": "0.30.0", 846 | "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.0.tgz", 847 | "integrity": "sha512-LA+31JYDJLs82r2ScLrlz1GjSgu66ZV518eyWT+S8VhyQn/JL0u9MeBOvQMGYiPk1DBiSN9DDMOcXvigJZaViQ==", 848 | "dev": true, 849 | "dependencies": { 850 | "@jridgewell/sourcemap-codec": "^1.4.13" 851 | }, 852 | "engines": { 853 | "node": ">=12" 854 | } 855 | }, 856 | "node_modules/merge2": { 857 | "version": "1.4.1", 858 | "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", 859 | "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", 860 | "dev": true, 861 | "engines": { 862 | "node": ">= 8" 863 | } 864 | }, 865 | "node_modules/micromatch": { 866 | "version": "4.0.5", 867 | "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", 868 | "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", 869 | "dev": true, 870 | "dependencies": { 871 | "braces": "^3.0.2", 872 | "picomatch": "^2.3.1" 873 | }, 874 | "engines": { 875 | "node": ">=8.6" 876 | } 877 | }, 878 | "node_modules/min-indent": { 879 | "version": "1.0.1", 880 | "resolved": "https://registry.npmjs.org/min-indent/-/min-indent-1.0.1.tgz", 881 | "integrity": "sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==", 882 | "dev": true, 883 | "engines": { 884 | "node": ">=4" 885 | } 886 | }, 887 | "node_modules/minimatch": { 888 | "version": "3.1.2", 889 | "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", 890 | "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", 891 | "dev": true, 892 | "dependencies": { 893 | "brace-expansion": "^1.1.7" 894 | }, 895 | "engines": { 896 | "node": "*" 897 | } 898 | }, 899 | "node_modules/minimist": { 900 | "version": "1.2.8", 901 | "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", 902 | "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", 903 | "dev": true, 904 | "funding": { 905 | "url": "https://github.com/sponsors/ljharb" 906 | } 907 | }, 908 | "node_modules/mkdirp": { 909 | "version": "0.5.6", 910 | "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", 911 | "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", 912 | "dev": true, 913 | "dependencies": { 914 | "minimist": "^1.2.6" 915 | }, 916 | "bin": { 917 | "mkdirp": "bin/cmd.js" 918 | } 919 | }, 920 | "node_modules/mri": { 921 | "version": "1.2.0", 922 | "resolved": "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz", 923 | "integrity": "sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==", 924 | "dev": true, 925 | "engines": { 926 | "node": ">=4" 927 | } 928 | }, 929 | "node_modules/ms": { 930 | "version": "2.1.2", 931 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", 932 | "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", 933 | "dev": true 934 | }, 935 | "node_modules/nanoid": { 936 | "version": "3.3.6", 937 | "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz", 938 | "integrity": "sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==", 939 | "dev": true, 940 | "funding": [ 941 | { 942 | "type": "github", 943 | "url": "https://github.com/sponsors/ai" 944 | } 945 | ], 946 | "bin": { 947 | "nanoid": "bin/nanoid.cjs" 948 | }, 949 | "engines": { 950 | "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" 951 | } 952 | }, 953 | "node_modules/normalize-path": { 954 | "version": "3.0.0", 955 | "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", 956 | "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", 957 | "dev": true, 958 | "engines": { 959 | "node": ">=0.10.0" 960 | } 961 | }, 962 | "node_modules/once": { 963 | "version": "1.4.0", 964 | "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", 965 | "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", 966 | "dev": true, 967 | "dependencies": { 968 | "wrappy": "1" 969 | } 970 | }, 971 | "node_modules/parent-module": { 972 | "version": "1.0.1", 973 | "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", 974 | "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", 975 | "dev": true, 976 | "dependencies": { 977 | "callsites": "^3.0.0" 978 | }, 979 | "engines": { 980 | "node": ">=6" 981 | } 982 | }, 983 | "node_modules/path-is-absolute": { 984 | "version": "1.0.1", 985 | "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", 986 | "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", 987 | "dev": true, 988 | "engines": { 989 | "node": ">=0.10.0" 990 | } 991 | }, 992 | "node_modules/picocolors": { 993 | "version": "1.0.0", 994 | "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", 995 | "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==", 996 | "dev": true 997 | }, 998 | "node_modules/picomatch": { 999 | "version": "2.3.1", 1000 | "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", 1001 | "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", 1002 | "dev": true, 1003 | "engines": { 1004 | "node": ">=8.6" 1005 | }, 1006 | "funding": { 1007 | "url": "https://github.com/sponsors/jonschlinkert" 1008 | } 1009 | }, 1010 | "node_modules/postcss": { 1011 | "version": "8.4.23", 1012 | "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.23.tgz", 1013 | "integrity": "sha512-bQ3qMcpF6A/YjR55xtoTr0jGOlnPOKAIMdOWiv0EIT6HVPEaJiJB4NLljSbiHoC2RX7DN5Uvjtpbg1NPdwv1oA==", 1014 | "dev": true, 1015 | "funding": [ 1016 | { 1017 | "type": "opencollective", 1018 | "url": "https://opencollective.com/postcss/" 1019 | }, 1020 | { 1021 | "type": "tidelift", 1022 | "url": "https://tidelift.com/funding/github/npm/postcss" 1023 | }, 1024 | { 1025 | "type": "github", 1026 | "url": "https://github.com/sponsors/ai" 1027 | } 1028 | ], 1029 | "dependencies": { 1030 | "nanoid": "^3.3.6", 1031 | "picocolors": "^1.0.0", 1032 | "source-map-js": "^1.0.2" 1033 | }, 1034 | "engines": { 1035 | "node": "^10 || ^12 || >=14" 1036 | } 1037 | }, 1038 | "node_modules/queue-microtask": { 1039 | "version": "1.2.3", 1040 | "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", 1041 | "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", 1042 | "dev": true, 1043 | "funding": [ 1044 | { 1045 | "type": "github", 1046 | "url": "https://github.com/sponsors/feross" 1047 | }, 1048 | { 1049 | "type": "patreon", 1050 | "url": "https://www.patreon.com/feross" 1051 | }, 1052 | { 1053 | "type": "consulting", 1054 | "url": "https://feross.org/support" 1055 | } 1056 | ] 1057 | }, 1058 | "node_modules/readdirp": { 1059 | "version": "3.6.0", 1060 | "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", 1061 | "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", 1062 | "dev": true, 1063 | "dependencies": { 1064 | "picomatch": "^2.2.1" 1065 | }, 1066 | "engines": { 1067 | "node": ">=8.10.0" 1068 | } 1069 | }, 1070 | "node_modules/resolve-from": { 1071 | "version": "4.0.0", 1072 | "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", 1073 | "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", 1074 | "dev": true, 1075 | "engines": { 1076 | "node": ">=4" 1077 | } 1078 | }, 1079 | "node_modules/reusify": { 1080 | "version": "1.0.4", 1081 | "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", 1082 | "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", 1083 | "dev": true, 1084 | "engines": { 1085 | "iojs": ">=1.0.0", 1086 | "node": ">=0.10.0" 1087 | } 1088 | }, 1089 | "node_modules/rimraf": { 1090 | "version": "2.7.1", 1091 | "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", 1092 | "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", 1093 | "dev": true, 1094 | "dependencies": { 1095 | "glob": "^7.1.3" 1096 | }, 1097 | "bin": { 1098 | "rimraf": "bin.js" 1099 | } 1100 | }, 1101 | "node_modules/rollup": { 1102 | "version": "3.21.2", 1103 | "resolved": "https://registry.npmjs.org/rollup/-/rollup-3.21.2.tgz", 1104 | "integrity": "sha512-c4vC+JZ3bbF4Kqq2TtM7zSKtSyMybFOjqmomFax3xpfYaPZDZ4iz8NMIuBRMjnXOcKYozw7bC6vhJjiWD6JpzQ==", 1105 | "dev": true, 1106 | "bin": { 1107 | "rollup": "dist/bin/rollup" 1108 | }, 1109 | "engines": { 1110 | "node": ">=14.18.0", 1111 | "npm": ">=8.0.0" 1112 | }, 1113 | "optionalDependencies": { 1114 | "fsevents": "~2.3.2" 1115 | } 1116 | }, 1117 | "node_modules/run-parallel": { 1118 | "version": "1.2.0", 1119 | "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", 1120 | "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", 1121 | "dev": true, 1122 | "funding": [ 1123 | { 1124 | "type": "github", 1125 | "url": "https://github.com/sponsors/feross" 1126 | }, 1127 | { 1128 | "type": "patreon", 1129 | "url": "https://www.patreon.com/feross" 1130 | }, 1131 | { 1132 | "type": "consulting", 1133 | "url": "https://feross.org/support" 1134 | } 1135 | ], 1136 | "dependencies": { 1137 | "queue-microtask": "^1.2.2" 1138 | } 1139 | }, 1140 | "node_modules/sade": { 1141 | "version": "1.8.1", 1142 | "resolved": "https://registry.npmjs.org/sade/-/sade-1.8.1.tgz", 1143 | "integrity": "sha512-xal3CZX1Xlo/k4ApwCFrHVACi9fBqJ7V+mwhBsuf/1IOKbBy098Fex+Wa/5QMubw09pSZ/u8EY8PWgevJsXp1A==", 1144 | "dev": true, 1145 | "dependencies": { 1146 | "mri": "^1.1.0" 1147 | }, 1148 | "engines": { 1149 | "node": ">=6" 1150 | } 1151 | }, 1152 | "node_modules/sander": { 1153 | "version": "0.5.1", 1154 | "resolved": "https://registry.npmjs.org/sander/-/sander-0.5.1.tgz", 1155 | "integrity": "sha512-3lVqBir7WuKDHGrKRDn/1Ye3kwpXaDOMsiRP1wd6wpZW56gJhsbp5RqQpA6JG/P+pkXizygnr1dKR8vzWaVsfA==", 1156 | "dev": true, 1157 | "dependencies": { 1158 | "es6-promise": "^3.1.2", 1159 | "graceful-fs": "^4.1.3", 1160 | "mkdirp": "^0.5.1", 1161 | "rimraf": "^2.5.2" 1162 | } 1163 | }, 1164 | "node_modules/sass": { 1165 | "version": "1.62.1", 1166 | "resolved": "https://registry.npmjs.org/sass/-/sass-1.62.1.tgz", 1167 | "integrity": "sha512-NHpxIzN29MXvWiuswfc1W3I0N8SXBd8UR26WntmDlRYf0bSADnwnOjsyMZ3lMezSlArD33Vs3YFhp7dWvL770A==", 1168 | "dev": true, 1169 | "dependencies": { 1170 | "chokidar": ">=3.0.0 <4.0.0", 1171 | "immutable": "^4.0.0", 1172 | "source-map-js": ">=0.6.2 <2.0.0" 1173 | }, 1174 | "bin": { 1175 | "sass": "sass.js" 1176 | }, 1177 | "engines": { 1178 | "node": ">=14.0.0" 1179 | } 1180 | }, 1181 | "node_modules/sorcery": { 1182 | "version": "0.10.0", 1183 | "resolved": "https://registry.npmjs.org/sorcery/-/sorcery-0.10.0.tgz", 1184 | "integrity": "sha512-R5ocFmKZQFfSTstfOtHjJuAwbpGyf9qjQa1egyhvXSbM7emjrtLXtGdZsDJDABC85YBfVvrOiGWKSYXPKdvP1g==", 1185 | "dev": true, 1186 | "dependencies": { 1187 | "buffer-crc32": "^0.2.5", 1188 | "minimist": "^1.2.0", 1189 | "sander": "^0.5.0", 1190 | "sourcemap-codec": "^1.3.0" 1191 | }, 1192 | "bin": { 1193 | "sorcery": "bin/index.js" 1194 | } 1195 | }, 1196 | "node_modules/source-map-js": { 1197 | "version": "1.0.2", 1198 | "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.2.tgz", 1199 | "integrity": "sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==", 1200 | "dev": true, 1201 | "engines": { 1202 | "node": ">=0.10.0" 1203 | } 1204 | }, 1205 | "node_modules/sourcemap-codec": { 1206 | "version": "1.4.8", 1207 | "resolved": "https://registry.npmjs.org/sourcemap-codec/-/sourcemap-codec-1.4.8.tgz", 1208 | "integrity": "sha512-9NykojV5Uih4lgo5So5dtw+f0JgJX30KCNI8gwhz2J9A15wD0Ml6tjHKwf6fTSa6fAdVBdZeNOs9eJ71qCk8vA==", 1209 | "deprecated": "Please use @jridgewell/sourcemap-codec instead", 1210 | "dev": true 1211 | }, 1212 | "node_modules/strip-indent": { 1213 | "version": "3.0.0", 1214 | "resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-3.0.0.tgz", 1215 | "integrity": "sha512-laJTa3Jb+VQpaC6DseHhF7dXVqHTfJPCRDaEbid/drOhgitgYku/letMUqOXFoWV0zIIUbjpdH2t+tYj4bQMRQ==", 1216 | "dev": true, 1217 | "dependencies": { 1218 | "min-indent": "^1.0.0" 1219 | }, 1220 | "engines": { 1221 | "node": ">=8" 1222 | } 1223 | }, 1224 | "node_modules/svelte": { 1225 | "version": "3.58.0", 1226 | "resolved": "https://registry.npmjs.org/svelte/-/svelte-3.58.0.tgz", 1227 | "integrity": "sha512-brIBNNB76mXFmU/Kerm4wFnkskBbluBDCjx/8TcpYRb298Yh2dztS2kQ6bhtjMcvUhd5ynClfwpz5h2gnzdQ1A==", 1228 | "dev": true, 1229 | "engines": { 1230 | "node": ">= 8" 1231 | } 1232 | }, 1233 | "node_modules/svelte-check": { 1234 | "version": "2.10.3", 1235 | "resolved": "https://registry.npmjs.org/svelte-check/-/svelte-check-2.10.3.tgz", 1236 | "integrity": "sha512-Nt1aWHTOKFReBpmJ1vPug0aGysqPwJh2seM1OvICfM2oeyaA62mOiy5EvkXhltGfhCcIQcq2LoE0l1CwcWPjlw==", 1237 | "dev": true, 1238 | "dependencies": { 1239 | "@jridgewell/trace-mapping": "^0.3.9", 1240 | "chokidar": "^3.4.1", 1241 | "fast-glob": "^3.2.7", 1242 | "import-fresh": "^3.2.1", 1243 | "picocolors": "^1.0.0", 1244 | "sade": "^1.7.4", 1245 | "svelte-preprocess": "^4.0.0", 1246 | "typescript": "*" 1247 | }, 1248 | "bin": { 1249 | "svelte-check": "bin/svelte-check" 1250 | }, 1251 | "peerDependencies": { 1252 | "svelte": "^3.24.0" 1253 | } 1254 | }, 1255 | "node_modules/svelte-check/node_modules/magic-string": { 1256 | "version": "0.25.9", 1257 | "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.25.9.tgz", 1258 | "integrity": "sha512-RmF0AsMzgt25qzqqLc1+MbHmhdx0ojF2Fvs4XnOqz2ZOBXzzkEwc/dJQZCYHAn7v1jbVOjAZfK8msRn4BxO4VQ==", 1259 | "dev": true, 1260 | "dependencies": { 1261 | "sourcemap-codec": "^1.4.8" 1262 | } 1263 | }, 1264 | "node_modules/svelte-check/node_modules/svelte-preprocess": { 1265 | "version": "4.10.7", 1266 | "resolved": "https://registry.npmjs.org/svelte-preprocess/-/svelte-preprocess-4.10.7.tgz", 1267 | "integrity": "sha512-sNPBnqYD6FnmdBrUmBCaqS00RyCsCpj2BG58A1JBswNF7b0OKviwxqVrOL/CKyJrLSClrSeqQv5BXNg2RUbPOw==", 1268 | "dev": true, 1269 | "hasInstallScript": true, 1270 | "dependencies": { 1271 | "@types/pug": "^2.0.4", 1272 | "@types/sass": "^1.16.0", 1273 | "detect-indent": "^6.0.0", 1274 | "magic-string": "^0.25.7", 1275 | "sorcery": "^0.10.0", 1276 | "strip-indent": "^3.0.0" 1277 | }, 1278 | "engines": { 1279 | "node": ">= 9.11.2" 1280 | }, 1281 | "peerDependencies": { 1282 | "@babel/core": "^7.10.2", 1283 | "coffeescript": "^2.5.1", 1284 | "less": "^3.11.3 || ^4.0.0", 1285 | "postcss": "^7 || ^8", 1286 | "postcss-load-config": "^2.1.0 || ^3.0.0 || ^4.0.0", 1287 | "pug": "^3.0.0", 1288 | "sass": "^1.26.8", 1289 | "stylus": "^0.55.0", 1290 | "sugarss": "^2.0.0", 1291 | "svelte": "^3.23.0", 1292 | "typescript": "^3.9.5 || ^4.0.0" 1293 | }, 1294 | "peerDependenciesMeta": { 1295 | "@babel/core": { 1296 | "optional": true 1297 | }, 1298 | "coffeescript": { 1299 | "optional": true 1300 | }, 1301 | "less": { 1302 | "optional": true 1303 | }, 1304 | "node-sass": { 1305 | "optional": true 1306 | }, 1307 | "postcss": { 1308 | "optional": true 1309 | }, 1310 | "postcss-load-config": { 1311 | "optional": true 1312 | }, 1313 | "pug": { 1314 | "optional": true 1315 | }, 1316 | "sass": { 1317 | "optional": true 1318 | }, 1319 | "stylus": { 1320 | "optional": true 1321 | }, 1322 | "sugarss": { 1323 | "optional": true 1324 | }, 1325 | "typescript": { 1326 | "optional": true 1327 | } 1328 | } 1329 | }, 1330 | "node_modules/svelte-check/node_modules/typescript": { 1331 | "version": "4.9.5", 1332 | "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz", 1333 | "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==", 1334 | "dev": true, 1335 | "bin": { 1336 | "tsc": "bin/tsc", 1337 | "tsserver": "bin/tsserver" 1338 | }, 1339 | "engines": { 1340 | "node": ">=4.2.0" 1341 | } 1342 | }, 1343 | "node_modules/svelte-hmr": { 1344 | "version": "0.15.1", 1345 | "resolved": "https://registry.npmjs.org/svelte-hmr/-/svelte-hmr-0.15.1.tgz", 1346 | "integrity": "sha512-BiKB4RZ8YSwRKCNVdNxK/GfY+r4Kjgp9jCLEy0DuqAKfmQtpL38cQK3afdpjw4sqSs4PLi3jIPJIFp259NkZtA==", 1347 | "dev": true, 1348 | "engines": { 1349 | "node": "^12.20 || ^14.13.1 || >= 16" 1350 | }, 1351 | "peerDependencies": { 1352 | "svelte": ">=3.19.0" 1353 | } 1354 | }, 1355 | "node_modules/to-regex-range": { 1356 | "version": "5.0.1", 1357 | "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", 1358 | "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", 1359 | "dev": true, 1360 | "dependencies": { 1361 | "is-number": "^7.0.0" 1362 | }, 1363 | "engines": { 1364 | "node": ">=8.0" 1365 | } 1366 | }, 1367 | "node_modules/tslib": { 1368 | "version": "2.5.0", 1369 | "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz", 1370 | "integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg==", 1371 | "dev": true 1372 | }, 1373 | "node_modules/typescript": { 1374 | "version": "5.0.4", 1375 | "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.0.4.tgz", 1376 | "integrity": "sha512-cW9T5W9xY37cc+jfEnaUvX91foxtHkza3Nw3wkoF4sSlKn0MONdkdEndig/qPBWXNkmplh3NzayQzCiHM4/hqw==", 1377 | "dev": true, 1378 | "bin": { 1379 | "tsc": "bin/tsc", 1380 | "tsserver": "bin/tsserver" 1381 | }, 1382 | "engines": { 1383 | "node": ">=12.20" 1384 | } 1385 | }, 1386 | "node_modules/vite": { 1387 | "version": "4.3.3", 1388 | "resolved": "https://registry.npmjs.org/vite/-/vite-4.3.3.tgz", 1389 | "integrity": "sha512-MwFlLBO4udZXd+VBcezo3u8mC77YQk+ik+fbc0GZWGgzfbPP+8Kf0fldhARqvSYmtIWoAJ5BXPClUbMTlqFxrA==", 1390 | "dev": true, 1391 | "dependencies": { 1392 | "esbuild": "^0.17.5", 1393 | "postcss": "^8.4.23", 1394 | "rollup": "^3.21.0" 1395 | }, 1396 | "bin": { 1397 | "vite": "bin/vite.js" 1398 | }, 1399 | "engines": { 1400 | "node": "^14.18.0 || >=16.0.0" 1401 | }, 1402 | "optionalDependencies": { 1403 | "fsevents": "~2.3.2" 1404 | }, 1405 | "peerDependencies": { 1406 | "@types/node": ">= 14", 1407 | "less": "*", 1408 | "sass": "*", 1409 | "stylus": "*", 1410 | "sugarss": "*", 1411 | "terser": "^5.4.0" 1412 | }, 1413 | "peerDependenciesMeta": { 1414 | "@types/node": { 1415 | "optional": true 1416 | }, 1417 | "less": { 1418 | "optional": true 1419 | }, 1420 | "sass": { 1421 | "optional": true 1422 | }, 1423 | "stylus": { 1424 | "optional": true 1425 | }, 1426 | "sugarss": { 1427 | "optional": true 1428 | }, 1429 | "terser": { 1430 | "optional": true 1431 | } 1432 | } 1433 | }, 1434 | "node_modules/vitefu": { 1435 | "version": "0.2.4", 1436 | "resolved": "https://registry.npmjs.org/vitefu/-/vitefu-0.2.4.tgz", 1437 | "integrity": "sha512-fanAXjSaf9xXtOOeno8wZXIhgia+CZury481LsDaV++lSvcU2R9Ch2bPh3PYFyoHW+w9LqAeYRISVQjUIew14g==", 1438 | "dev": true, 1439 | "peerDependencies": { 1440 | "vite": "^3.0.0 || ^4.0.0" 1441 | }, 1442 | "peerDependenciesMeta": { 1443 | "vite": { 1444 | "optional": true 1445 | } 1446 | } 1447 | }, 1448 | "node_modules/wrappy": { 1449 | "version": "1.0.2", 1450 | "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", 1451 | "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", 1452 | "dev": true 1453 | } 1454 | } 1455 | } 1456 | -------------------------------------------------------------------------------- /ui/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ui", 3 | "private": true, 4 | "version": "0.0.0", 5 | "type": "module", 6 | "scripts": { 7 | "dev": "vite", 8 | "build": "vite build", 9 | "preview": "vite preview", 10 | "check": "svelte-check --tsconfig ./tsconfig.json" 11 | }, 12 | "devDependencies": { 13 | "@sveltejs/vite-plugin-svelte": "^2.0.3", 14 | "@tsconfig/svelte": "^4.0.1", 15 | "svelte": "^3.57.0", 16 | "svelte-check": "^2.10.3", 17 | "tslib": "^2.5.0", 18 | "typescript": "^5.0.2", 19 | "vite": "^4.3.2" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /ui/src/App.svelte: -------------------------------------------------------------------------------- 1 | 19 | 20 |
21 | 22 |
23 | 24 | 41 | -------------------------------------------------------------------------------- /ui/src/app.css: -------------------------------------------------------------------------------- 1 | :root { 2 | font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif; 3 | line-height: 1.5; 4 | font-weight: 400; 5 | 6 | color-scheme: light dark; 7 | color: rgba(255, 255, 255, 0.87); 8 | background-color: #242424; 9 | 10 | font-synthesis: none; 11 | text-rendering: optimizeLegibility; 12 | -webkit-font-smoothing: antialiased; 13 | -moz-osx-font-smoothing: grayscale; 14 | -webkit-text-size-adjust: 100%; 15 | } 16 | 17 | a { 18 | font-weight: 500; 19 | color: #646cff; 20 | text-decoration: inherit; 21 | } 22 | 23 | a:hover { 24 | color: #535bf2; 25 | } 26 | 27 | body { 28 | margin: 0; 29 | display: flex; 30 | place-items: center; 31 | min-width: 320px; 32 | min-height: 100vh; 33 | } 34 | 35 | h1 { 36 | font-size: 3.2em; 37 | line-height: 1.1; 38 | } 39 | 40 | #app { 41 | max-width: 1280px; 42 | margin: 0 auto; 43 | padding: 2rem; 44 | text-align: center; 45 | } 46 | 47 | button { 48 | border-radius: 8px; 49 | border: 1px solid transparent; 50 | padding: 0.6em 1.2em; 51 | font-size: 1em; 52 | font-weight: 500; 53 | font-family: inherit; 54 | background-color: #1a1a1a; 55 | cursor: pointer; 56 | transition: border-color 0.25s; 57 | } 58 | 59 | button:hover { 60 | border-color: #646cff; 61 | } 62 | 63 | button:focus, 64 | button:focus-visible { 65 | outline: 4px auto -webkit-focus-ring-color; 66 | } 67 | 68 | @media (prefers-color-scheme: light) { 69 | :root { 70 | color: #213547; 71 | background-color: #ffffff; 72 | } 73 | 74 | a:hover { 75 | color: #747bff; 76 | } 77 | 78 | button { 79 | background-color: #f9f9f9; 80 | } 81 | } -------------------------------------------------------------------------------- /ui/src/default.ts: -------------------------------------------------------------------------------- 1 | export const SUMMARIZER_URL: string = import.meta.env.VITE_SUMMARIZER_URL; -------------------------------------------------------------------------------- /ui/src/lib/card.svelte: -------------------------------------------------------------------------------- 1 | 7 | 8 |
9 | {#each cards as card} 10 | {#if card.title != "no-title"} 11 |
{ 14 | pop_card = card; 15 | visible = !visible; 16 | }} 17 | on:keypress={() => { 18 | console.log("Does nothing"); // To ignore the ay11 warning 19 | }} 20 | > 21 | Podcast Summary 22 |

{card.title}

23 |
24 | {/if} 25 | {/each} 26 | {#if visible} 27 |
28 |
29 |
30 | {pop_card.title} 31 |
32 |
(visible = !visible)} 35 | on:keypress={() => { 36 | console.log("Does Nothing"); 37 | }} 38 | > 39 | ❌ 40 |
41 |
42 |
43 | {pop_card.content} 44 |
45 |
46 | {/if} 47 |
48 | 49 | 92 | -------------------------------------------------------------------------------- /ui/src/main.ts: -------------------------------------------------------------------------------- 1 | import './app.css' 2 | import App from './App.svelte' 3 | 4 | const app = new App({ 5 | target: document.getElementById('app'), 6 | }) 7 | 8 | export default app 9 | -------------------------------------------------------------------------------- /ui/src/model/card.ts: -------------------------------------------------------------------------------- 1 | import type { Podcast, Podcasts } from "./input"; 2 | export interface Card { 3 | image: URL, 4 | title: string, 5 | short?: string, 6 | content: string 7 | } 8 | 9 | export class Card implements Card { 10 | image: URL; 11 | title: string; 12 | content: string; 13 | short?: string; 14 | 15 | constructor(image: URL, title: string, content: string, short: string | null = null) { 16 | this.image = image 17 | this.title = title 18 | this.content = content 19 | if (short !== null) { 20 | this.short = short 21 | } 22 | 23 | } 24 | 25 | static from(source: Podcast): Card { 26 | return new Card(new URL(source.image), source.title, source.text) 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /ui/src/model/input.ts: -------------------------------------------------------------------------------- 1 | export interface Podcasts { 2 | version: string, 3 | podcasts: Array 4 | } 5 | 6 | export interface Podcast { 7 | title: string, 8 | image: string, 9 | url: string, 10 | text: string 11 | } -------------------------------------------------------------------------------- /ui/src/parser.ts: -------------------------------------------------------------------------------- 1 | import type { Podcast, Podcasts } from "./model/input"; 2 | import { SUMMARIZER_URL } from "./default"; 3 | export interface api_response { 4 | content: string, 5 | created_at: Date, 6 | image_id: string | null, 7 | link: string, 8 | title: string | null 9 | } 10 | 11 | 12 | 13 | export function parse_api(data: api_response): Podcast { 14 | let image_link: string 15 | if (data.image_id === null) { 16 | image_link = `${SUMMARIZER_URL}/thumbnails/notfound.jpg` 17 | } else { 18 | image_link = `${SUMMARIZER_URL}/thumbnails/${data.image_id}.jpg` 19 | } 20 | return { 21 | title: data.title || "no-title", 22 | image: image_link, 23 | url: data.link, 24 | text: data.content 25 | } 26 | }; 27 | export function parse_api_array(data: Array): Podcasts { 28 | return { 29 | version: "0.1", 30 | podcasts: data.map((x) => parse_api(x)) 31 | } 32 | }; 33 | -------------------------------------------------------------------------------- /ui/src/vite-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | /// 3 | -------------------------------------------------------------------------------- /ui/svelte.config.js: -------------------------------------------------------------------------------- 1 | import { vitePreprocess } from '@sveltejs/vite-plugin-svelte' 2 | 3 | export default { 4 | // Consult https://svelte.dev/docs#compile-time-svelte-preprocess 5 | // for more information about preprocessors 6 | preprocess: vitePreprocess(), 7 | } 8 | -------------------------------------------------------------------------------- /ui/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "@tsconfig/svelte/tsconfig.json", 3 | "compilerOptions": { 4 | "target": "ESNext", 5 | "useDefineForClassFields": true, 6 | "module": "ESNext", 7 | "resolveJsonModule": true, 8 | /** 9 | * Typecheck JS in `.svelte` and `.js` files by default. 10 | * Disable checkJs if you'd like to use dynamic types in JS. 11 | * Note that setting allowJs false does not prevent the use 12 | * of JS in `.svelte` files. 13 | */ 14 | "allowJs": true, 15 | "checkJs": true, 16 | "isolatedModules": true, 17 | "strict": true 18 | }, 19 | "include": [ 20 | "src/**/*.d.ts", 21 | "src/**/*.ts", 22 | "src/**/*.js", 23 | "src/**/*.svelte" 24 | ], 25 | "references": [ 26 | { 27 | "path": "./tsconfig.node.json" 28 | } 29 | ] 30 | } -------------------------------------------------------------------------------- /ui/tsconfig.node.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "composite": true, 4 | "skipLibCheck": true, 5 | "module": "ESNext", 6 | "moduleResolution": "bundler" 7 | }, 8 | "include": ["vite.config.ts"] 9 | } 10 | -------------------------------------------------------------------------------- /ui/vite.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'vite' 2 | import { svelte } from '@sveltejs/vite-plugin-svelte' 3 | 4 | // https://vitejs.dev/config/ 5 | export default defineConfig({ 6 | plugins: [svelte()], 7 | }) 8 | -------------------------------------------------------------------------------- /youtube-transcript/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "youtube-transcript" 3 | description = "Transcripts Youtube" 4 | version.workspace = true 5 | edition.workspace = true 6 | license = "MIT" 7 | readme = "Readme.md" 8 | repository = "https://github.com/akhildevelops/summarizer/tree/main/youtube-transcript" 9 | 10 | 11 | [dependencies] 12 | tokio = { workspace = true } 13 | serde_json = { workspace = true } 14 | serde = { workspace = true } 15 | roxmltree = { workspace = true } 16 | reqwest = { workspace = true } 17 | clap = { workspace = true } 18 | once_cell = { workspace = true } 19 | strum = { workspace = true } 20 | strum_macros = { workspace = true } 21 | 22 | [lib] 23 | doctest = false 24 | -------------------------------------------------------------------------------- /youtube-transcript/Readme.md: -------------------------------------------------------------------------------- 1 | # Youtube Transcript 2 | Retrieve transcript of any youtube video. 3 | 4 | ### [Documentation](https://docs.rs/youtube-transcript) 5 | 6 | ## Install 7 | `cargo install youtube-transcript` 8 | 9 | ## Usage: 10 | 11 | 12 | ### as Commandline 13 | 14 | renders transcript in text format / json format. By default it's text. 15 | 16 | ``` bash 17 | youtube-transcript https://www.youtube.com/watch?v=RcYjXbSJBN8 18 | 19 | start at: 639ms for duration 2s 20 | welcome back 21 | ========== 22 | 23 | 24 | start at: 2s for duration 4s 25 | here we go again great to see you and 26 | ========== 27 | ... 28 | ... 29 | ``` 30 | 31 | For json 32 | ``` bash 33 | youtube-transcript --format json https://www.youtube.com/watch?v=RcYjXbSJBN8 34 | 35 | { 36 | "transcripts": [ 37 | { 38 | "text": "Hey, how's it going Dave 2d here?", 39 | "start": { 40 | "secs": 0, 41 | "nanos": 0 42 | }, 43 | "duration": { 44 | "secs": 1, 45 | "nanos": 539999962 46 | } 47 | }, 48 | { 49 | "text": "This is a Microsoft Surface go and when they first announced it I was interested in it", 50 | "start": { 51 | "secs": 1, 52 | "nanos": 539999962 53 | }, 54 | "duration": { 55 | "secs": 4, 56 | "nanos": 159999847 57 | } 58 | } 59 | ... 60 | ... 61 | ] 62 | } 63 | ... 64 | ... 65 | ``` 66 | 67 | ### as Library 68 | youtube-transcript is an async library and below is the example to use in an applicatio: 69 | ``` rust 70 | let link:&str="https://www.youtube.com/watch?v=RcYjXbSJBN8"; 71 | 72 | # Create a youtube instance from builder. 73 | let youtube_loader:Youtube = YoutubeBuilder::default().build(); 74 | 75 | # Get the transcript by loading youtube url. 76 | let transcript:Transcript=youtube_loader.transcript(link).await?; 77 | ``` 78 | 79 | 80 | ### Other tools 81 | Inspired from: [youtube-transcript-api](https://github.com/jdepoix/youtube-transcript-api) 82 | -------------------------------------------------------------------------------- /youtube-transcript/src/config.rs: -------------------------------------------------------------------------------- 1 | use once_cell::sync::Lazy; 2 | use strum_macros::{EnumString, IntoStaticStr}; 3 | 4 | pub struct HTMLParserConfig { 5 | pub from: &'static str, 6 | pub to: &'static str, 7 | } 8 | 9 | impl Default for HTMLParserConfig { 10 | fn default() -> Self { 11 | Self { 12 | from: "playerCaptionsTracklistRenderer\":", 13 | to: "},\"videoDetails\"", 14 | } 15 | } 16 | } 17 | 18 | #[allow(non_camel_case_types)] 19 | #[derive(Debug, Clone, Copy, Default, EnumString, IntoStaticStr)] 20 | pub enum LangCode { 21 | /// Arabic 22 | ar, 23 | /// Bengali 24 | bn, 25 | /// Bulgarian 26 | bg, 27 | /// Catalan 28 | ca, 29 | /// CN Chinese, Simplified 30 | zh, 31 | /// Croatian 32 | hr, 33 | /// Czech 34 | cs, 35 | /// Danish 36 | da, 37 | /// Dutch 38 | nl, 39 | /// English 40 | #[default] 41 | en, 42 | /// GB English, UK 43 | fil, 44 | /// Finnish 45 | fi, 46 | /// French 47 | fr, 48 | /// German 49 | de, 50 | /// Greek 51 | el, 52 | /// Gujarati 53 | gu, 54 | /// Hebrew 55 | iw, 56 | /// Hindi 57 | hi, 58 | /// Hungarian 59 | hu, 60 | /// Indonesian 61 | id, 62 | /// Italian 63 | it, 64 | /// Japanese 65 | ja, 66 | /// Kannada 67 | kn, 68 | /// Korean 69 | ko, 70 | /// Latvian 71 | lv, 72 | /// Lithuanian 73 | lt, 74 | /// Malay 75 | ms, 76 | /// Malayalam 77 | ml, 78 | /// Marathi 79 | mr, 80 | /// Norwegian 81 | no, 82 | /// Polish 83 | pl, 84 | /// BR Portuguese, Brazil 85 | pt, 86 | /// PT Portuguese, Portugal 87 | ro, 88 | /// Russian 89 | ru, 90 | /// Serbian 91 | sr, 92 | /// Slovak 93 | sk, 94 | /// Slovenian 95 | sl, 96 | /// Spanish 97 | es, 98 | /// Swahili 99 | sw, 100 | /// Swedish 101 | sv, 102 | /// Tamil 103 | ta, 104 | /// Telugu 105 | te, 106 | /// Thai 107 | th, 108 | /// Turkish 109 | tr, 110 | /// Ukrainian 111 | uk, 112 | /// Urdu 113 | ur, 114 | /// Vietnamese 115 | vi, 116 | } 117 | 118 | /// configuration that contains anchor points for identifying captions from youtube's html webpage. 119 | pub struct Config { 120 | pub(crate) parser: HTMLParserConfig, 121 | pub(crate) lang_code: LangCode, 122 | } 123 | 124 | impl Default for Config { 125 | fn default() -> Self { 126 | Self { 127 | parser: HTMLParserConfig::default(), 128 | lang_code: LangCode::default(), 129 | } 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /youtube-transcript/src/error.rs: -------------------------------------------------------------------------------- 1 | use std::{error::Error as StdError, fmt::Display}; 2 | 3 | #[derive(Debug)] 4 | pub enum Error { 5 | #[allow(dead_code)] 6 | ParseError(String), 7 | } 8 | 9 | impl Display for Error { 10 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 11 | write!(f, "{:?}", self) 12 | } 13 | } 14 | 15 | impl StdError for Error {} 16 | -------------------------------------------------------------------------------- /youtube-transcript/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![warn(missing_docs)] 2 | //! # Asynchronous Youtube Transcript library 3 | //! Get transcripts / captions of videos. 4 | //! ### Downloading transcript from Youtube: 5 | //! ```rust 6 | //! let link:&str = "https://www.youtube.com/watch?v=RcYjXbSJBN8"; 7 | //! 8 | //! # Create a youtube instance from builder. 9 | //! let youtube_loader:Youtube = YoutubeBuilder::default().build(); 10 | //! 11 | //! # Get the transcript by loading youtube url. 12 | //! let transcript:Transcript = youtube_loader.transcript(link).await?; 13 | //! ``` 14 | //! 15 | mod config; 16 | mod utils; 17 | 18 | mod error; 19 | mod parser; 20 | mod youtube; 21 | pub use config::{LangCode, Config}; 22 | pub use parser::{Transcript, TranscriptCore}; 23 | pub use youtube::{Youtube, YoutubeBuilder}; 24 | -------------------------------------------------------------------------------- /youtube-transcript/src/main.rs: -------------------------------------------------------------------------------- 1 | use std::str::FromStr; 2 | 3 | use clap::{ 4 | builder::{self, IntoResettable}, 5 | Arg, Command, 6 | }; 7 | use serde_json; 8 | use youtube_transcript::{LangCode, YoutubeBuilder}; 9 | 10 | #[derive(Clone)] 11 | enum Format { 12 | Json, 13 | Text, 14 | } 15 | impl IntoResettable for Format { 16 | fn into_resettable(self) -> builder::Resettable { 17 | let format_str = self.to_string(); 18 | >::into(format_str).into() 19 | } 20 | } 21 | 22 | impl ToString for Format { 23 | fn to_string(&self) -> String { 24 | match self { 25 | Format::Json => "json".to_string(), 26 | Format::Text => "text".to_string(), 27 | } 28 | } 29 | } 30 | impl TryFrom<&str> for Format { 31 | type Error = String; 32 | fn try_from(value: &str) -> Result { 33 | match value { 34 | "json" => Ok(Format::Json), 35 | "text" => Ok(Format::Text), 36 | _ => Err(format!( 37 | "Cannot find json / text as format definition. Recieved {}", 38 | value 39 | )), 40 | } 41 | } 42 | } 43 | 44 | fn format_parser(arg: &str) -> Result { 45 | arg.try_into() 46 | } 47 | 48 | fn format_lang_code(arg: &str) -> Result { 49 | LangCode::from_str(arg).map_err(|err| format!("{err:?}")) 50 | } 51 | 52 | #[tokio::main] 53 | async fn main() { 54 | let app = Command::new("yts") 55 | .arg( 56 | Arg::new("format") 57 | .help("ouput format") 58 | .long("format") 59 | .value_parser(builder::ValueParser::new(format_parser)) 60 | .default_value(Format::Text), 61 | ) 62 | .arg( 63 | Arg::new("lang_code") 64 | .help("language code") 65 | .long("lang-code") 66 | .value_parser(builder::ValueParser::new(format_lang_code)) 67 | .default_value(<&'static str>::from(LangCode::default())), 68 | ) 69 | .arg(Arg::new("link").help("Youtube-link")) 70 | .get_matches(); 71 | let format = app.get_one::("format").unwrap_or(&Format::Json); 72 | let link = app 73 | .get_one::("link") 74 | .expect("Youtube Link not provided"); 75 | let transcript = YoutubeBuilder::default() 76 | .lang_code( 77 | app.get_one::("lang_code") 78 | .copied() 79 | .unwrap_or(LangCode::default()), 80 | ) 81 | .build() 82 | .transcript(link) 83 | .await 84 | .unwrap(); 85 | 86 | let data = match format { 87 | Format::Json => serde_json::to_string(&transcript).unwrap(), 88 | Format::Text => String::from(transcript), 89 | }; 90 | println!("{}", data); 91 | } 92 | -------------------------------------------------------------------------------- /youtube-transcript/src/parser.rs: -------------------------------------------------------------------------------- 1 | use crate::error; 2 | use crate::utils::to_human_readable; 3 | use roxmltree::Document; 4 | use serde; 5 | use serde::Deserialize; 6 | use serde::Serialize; 7 | use serde_json; 8 | use std::error::Error; 9 | use std::time::Duration; 10 | #[derive(Deserialize)] 11 | pub(crate) struct Caption { 12 | #[serde(rename(deserialize = "baseUrl"))] 13 | pub base_url: String, 14 | #[serde(rename(deserialize = "languageCode"))] 15 | pub lang_code: String, 16 | } 17 | 18 | #[derive(Deserialize)] 19 | struct Captions { 20 | #[serde(rename(deserialize = "captionTracks"))] 21 | caption_tracks: Vec, 22 | } 23 | 24 | pub(crate) trait HTMLParser<'a> { 25 | fn html_string(&'a self) -> &'a str; 26 | 27 | fn caption(&'a self, from: &str, to: &str, lang_code: &str) -> Result { 28 | let html = self.html_string(); 29 | let start = html 30 | .split_once(from) 31 | .ok_or_else(|| error::Error::ParseError(format!("Cannot parse html for: {}", from)))? 32 | .1; 33 | let actual_json = start 34 | .split_once(to) 35 | .ok_or_else(|| error::Error::ParseError(format!("Cannot parse html to: {}", to)))? 36 | .0; 37 | let value: Captions = serde_json::from_str(actual_json) 38 | .map_err(|x| error::Error::ParseError(format!("{}", x)))?; 39 | let caption = value 40 | .caption_tracks 41 | .into_iter() 42 | .filter(|x| x.lang_code == lang_code) 43 | .next() 44 | .ok_or(error::Error::ParseError(format!( 45 | "Cannot find lang {lang_code}" 46 | )))?; 47 | Ok(caption) 48 | } 49 | } 50 | 51 | impl<'a> HTMLParser<'a> for String { 52 | fn html_string(&'a self) -> &'a str { 53 | self.as_str() 54 | } 55 | } 56 | 57 | impl<'a> HTMLParser<'a> for str { 58 | fn html_string(&'a self) -> &'a str { 59 | self 60 | } 61 | } 62 | 63 | /// Struct that contains data about transcirpt text along with start and duration in the whole video. 64 | #[derive(PartialEq, Debug, Serialize)] 65 | pub struct TranscriptCore { 66 | /// transcript text. Ex: "Hi How are you" 67 | pub text: String, 68 | /// starting time of the text in the whole video. Ex: "0 sec" 69 | pub start: Duration, 70 | /// duration of the text Ex: "0.8 sec" 71 | pub duration: Duration, 72 | } 73 | 74 | /// Struct containing youtube's transcript data as a Vec<[`TranscriptCore`]> 75 | #[derive(Serialize)] 76 | pub struct Transcript { 77 | /// List of transcript texts in [`TranscriptCore`] format 78 | pub transcripts: Vec, 79 | } 80 | 81 | impl IntoIterator for Transcript { 82 | type IntoIter = as IntoIterator>::IntoIter; 83 | type Item = TranscriptCore; 84 | 85 | fn into_iter(self) -> Self::IntoIter { 86 | self.transcripts.into_iter() 87 | } 88 | } 89 | 90 | impl From for String { 91 | fn from(value: Transcript) -> Self { 92 | { 93 | value 94 | .transcripts 95 | .into_iter() 96 | .map(|x| { 97 | let start_h = to_human_readable(&x.start); 98 | let dur_h = to_human_readable(&x.duration); 99 | format!( 100 | "\nstart at: {} for duration {}\n{}\n==========\n\n", 101 | start_h, dur_h, x.text 102 | ) 103 | }) 104 | .collect::() 105 | } 106 | } 107 | } 108 | 109 | pub(crate) struct TranscriptParser; 110 | 111 | impl TranscriptParser { 112 | pub fn parse<'input>( 113 | transcript: &'input Document<'input>, 114 | ) -> Result> { 115 | let mut transcripts = Vec::new(); 116 | let nodes = transcript 117 | .descendants() 118 | .filter(|x| x.tag_name() == "text".into()); 119 | for node in nodes { 120 | let start = node 121 | .attribute("start") 122 | .ok_or(error::Error::ParseError("transcript parse error".into()))? 123 | .parse::()?; 124 | let duration = node 125 | .attribute("dur") 126 | .ok_or(error::Error::ParseError("transcript parse error".into()))? 127 | .parse::()?; 128 | let node = node 129 | .last_child() 130 | .ok_or(error::Error::ParseError("transcript parse error".into()))?; 131 | let text = node 132 | .text() 133 | .ok_or(error::Error::ParseError("transcript error".into()))?; 134 | 135 | transcripts.push(TranscriptCore { 136 | text: text.into(), 137 | start: Duration::from_secs_f32(start), 138 | duration: Duration::from_secs_f32(duration), 139 | }) 140 | } 141 | Ok(Transcript { transcripts }) 142 | } 143 | } 144 | 145 | #[cfg(test)] 146 | mod test { 147 | use super::*; 148 | use crate::Config; 149 | const TRANSCRIPT: &str = r#"Hey, how&#39;s it going Dave 2d here?This is a Microsoft Surface go and when they first announced it I was interested in itIt seemed like a pretty compelling device having used it for a little whileI really think this is seriously the best product that Microsoft has put out in a very long time this thing starts at $400I don&#39;t think that base configuration is where you want to spend the money though. They have a mid tier one550 quite a bit more but you&#39;re getting double the RAM double the storage but significantly faster storageThat is the model that I think most people should pick up if you can afford that price bumpso this unit here, is that mid tier model the$550 unit and IReally like it. Ok, let&#39;s go around. This thing build quality is great. It&#39;s a surface productIt has a magnesium enclosure fit and finish on this is really well donetheTop surface has these new rounded edges and it actually makes the device a lot more comfortable to holdNot that the original surface products are like uncomfortableBut this small detail just makes it that much more ergonomic and that much more inviting to useIt&#39;s a nice touch and I think Microsoft should put this kind ofRounded edge on all of their products because it does make a difference. The screen is a 10 inch screenI thought I&#39;d be a little bit small for what I doBut it actually isn&#39;t it is noticeably smaller compared to like a 12 or 13 inch screen, but it doesn&#39;t feel particularly crampedIt&#39;s still a very usable surface area the bezels around that screen though are thick now visuallyIt&#39;s not attractive right having thick bezels. Like this doesn&#39;t look goodBut when you&#39;re actually using it, you won&#39;t notice it you&#39;ll be focused on your contentit&#39;s just that when this devices off or it&#39;s just sitting there and you&#39;re kind ofexamining it visually the bezels are thick the panel itself is nice its sharp great colors and brightness andHitting a price point like this with this kind of screenIt could not have been easy. Like we see four or five hundred dollar devices out there that have terrible screensThis thing looks really good. There is pen support as usual and feels relatively lag free to meI&#39;m not an artistBut the surface area feels reasonably sized for people that want to use it for any kind of digital creative workNow on the side are two speakers and they sound really good for this kind of device sizenice body to the soundExcellent stereo separation just from the positioning and you just get really clean audio that gets to a decent volumeYou also get a killer killer webcam $400 gets your webcam of this qualityit&#39;s actually one of the best kans I&#39;ve seen on any laptop period but when you compareThis webcam to something like a 12-inch MacBook. It just blows my mindI mean if you can stick a webcam like this into a $400 device, there&#39;s no excuse for other peopleThey should be using really good webcams and no one else is doing it, but surface does so good for themthis device though is not complete without the keyboard and the keyboard is aHundred bucks, which is crazy expensive you think about it. That&#39;s like at the base model. That&#39;s 20% of the cost, butThat&#39;s what we have. Okay, when it&#39;s connected up and it connects magnetically. It is an awesome. Awesomeproductivity deviceSo I was concerned that this keyboard would be really small and cramped and just kind of weird feeling because it is a lot smallerThan the regular service devices. It&#39;s not cramped. It&#39;s excellent. It does take a little bit of time to get used to itBut it is a really comfortable keyboard the trackpad feels good. It&#39;s a surface productSo tracking is accurate and gestures work nicely. But the pad is a little small. Maybe it&#39;s a visual thingI just wish there&#39;s a little bit more surface area to this trackpad. Okay performance on this device isGood, it&#39;s not amazing. It&#39;s a Pentium Gold chip and most productivity stuffLike emails web browsing or any kind of work-related stuff runs really smoothly on thisSo the drive feeds on the mid-tier model actually really good fast read speed but on the slower drive of the base model the wholeSystem is gonna feel a bit more sluggish and that reason alone makes it worth it to upgrade to the mid-tier modelBattery life is also pretty good getting around seven hours of battery life and to charge itYou can either use the included surface connect adapter or you can use the USB C portI really wish that the included adapter was USB C but its surface connect because well, that&#39;s Microsoft&#39;s for youOkay gaming performance. I was actually surprised by thisYou&#39;re not gonna be able to play some killer triple-a titlesBut light games are pretty good on this thing if you want to pick it up for some casual light gamesIt&#39;ll do the trick nowThe surface go is still a surface product through and through so if they&#39;re issues you had with surface products in the pastYou may have those same issues with this one. Like if you need more ports, there&#39;s still only one portIt&#39;s use BC this yearBut it&#39;s still only one port if you don&#39;t like the kickstand on your lapLike it&#39;s not an ideal situation for lap use, but it does work reasonablyWell plus infinite positions up to a certain degree, but I don&#39;t knowThis makes it fairly usable for most people I think but if you&#39;ve had issues in the past same issues nowOverall though really good product. I think for studentsThis is such a good option you get so much versatility on this thingYou get a great keyboard for taking notesYou can pull up course material and stuff in classGood option great for me to consumption for like a secondary device if you want it for thatI think you can&#39;t go wrong with this it is however notCheap once you add everything up together like the keyboard and like the mid tier unitIt&#39;s not the $400 device that they&#39;re kind of marketingSo you kind of have to take that into consideration, but overall I like this thing. Ok. Hope you guys enjoyed this video thumbsWe liked it subs we loved it. See you guys next time"#; 150 | struct HTML; 151 | 152 | impl HTMLParser<'_> for HTML { 153 | fn html_string(&self) -> &'static str { 154 | r#""elapsedMediaTimeSeconds": 0 } }, "captions": { "playerCaptionsTracklistRenderer": { "captionTracks": [{ "baseUrl": "https://www.youtube.com/api/timedtext?v=GJLlxj_dtq8\u0026caps=asr\u0026xoaf=5\u0026hl=en-GB\u0026ip=0.0.0.0\u0026ipbits=0\u0026expire=1681082354\u0026sparams=ip,ipbits,expire,v,caps,xoaf\u0026signature=13D068D838F3B1262B96D29751914C9E75100C4C.A99B64907A100E2E5F74ACE0BA586FB82F865CE3\u0026key=yt8\u0026lang=zh", "name": { "simpleText": "Chinese" }, "vssId": ".zh", "languageCode": "zh", "isTranslatable": true }, { "baseUrl": "https://www.youtube.com/api/timedtext?v=GJLlxj_dtq8\u0026caps=asr\u0026xoaf=5\u0026hl=en-GB\u0026ip=0.0.0.0\u0026ipbits=0\u0026expire=1681082354\u0026sparams=ip,ipbits,expire,v,caps,xoaf\u0026signature=13D068D838F3B1262B96D29751914C9E75100C4C.A99B64907A100E2E5F74ACE0BA586FB82F865CE3\u0026key=yt8\u0026lang=cs", "name": { "simpleText": "Czech" }, "vssId": ".cs", "languageCode": "cs", "isTranslatable": true }, { "baseUrl": "https://www.youtube.com/api/timedtext?v=GJLlxj_dtq8\u0026caps=asr\u0026xoaf=5\u0026hl=en-GB\u0026ip=0.0.0.0\u0026ipbits=0\u0026expire=1681082354\u0026sparams=ip,ipbits,expire,v,caps,xoaf\u0026signature=13D068D838F3B1262B96D29751914C9E75100C4C.A99B64907A100E2E5F74ACE0BA586FB82F865CE3\u0026key=yt8\u0026lang=en", "name": { "simpleText": "English" }, "vssId": ".en", "languageCode": "en", "isTranslatable": true }, { "baseUrl": "https://www.youtube.com/api/timedtext?v=GJLlxj_dtq8\u0026caps=asr\u0026xoaf=5\u0026hl=en-GB\u0026ip=0.0.0.0\u0026ipbits=0\u0026expire=1681082354\u0026sparams=ip,ipbits,expire,v,caps,xoaf\u0026signature=13D068D838F3B1262B96D29751914C9E75100C4C.A99B64907A100E2E5F74ACE0BA586FB82F865CE3\u0026key=yt8\u0026kind=asr\u0026lang=en", "name": { "simpleText": "English (auto-generated)" }, "vssId": "a.en", "languageCode": "en", "kind": "asr", "isTranslatable": true }, { "baseUrl": "https://www.youtube.com/api/timedtext?v=GJLlxj_dtq8\u0026caps=asr\u0026xoaf=5\u0026hl=en-GB\u0026ip=0.0.0.0\u0026ipbits=0\u0026expire=1681082354\u0026sparams=ip,ipbits,expire,v,caps,xoaf\u0026signature=13D068D838F3B1262B96D29751914C9E75100C4C.A99B64907A100E2E5F74ACE0BA586FB82F865CE3\u0026key=yt8\u0026lang=de", "name": { "simpleText": "German" }, "vssId": ".de", "languageCode": "de", "isTranslatable": true }, { "baseUrl": "https://www.youtube.com/api/timedtext?v=GJLlxj_dtq8\u0026caps=asr\u0026xoaf=5\u0026hl=en-GB\u0026ip=0.0.0.0\u0026ipbits=0\u0026expire=1681082354\u0026sparams=ip,ipbits,expire,v,caps,xoaf\u0026signature=13D068D838F3B1262B96D29751914C9E75100C4C.A99B64907A100E2E5F74ACE0BA586FB82F865CE3\u0026key=yt8\u0026lang=hi", "name": { "simpleText": "Hindi" }, "vssId": ".hi", "languageCode": "hi", "isTranslatable": true }, { "baseUrl": "https://www.youtube.com/api/timedtext?v=GJLlxj_dtq8\u0026caps=asr\u0026xoaf=5\u0026hl=en-GB\u0026ip=0.0.0.0\u0026ipbits=0\u0026expire=1681082354\u0026sparams=ip,ipbits,expire,v,caps,xoaf\u0026signature=13D068D838F3B1262B96D29751914C9E75100C4C.A99B64907A100E2E5F74ACE0BA586FB82F865CE3\u0026key=yt8\u0026lang=ja", "name": { "simpleText": "Japanese" }, "vssId": ".ja", "languageCode": "ja", "isTranslatable": true }, { "baseUrl": "https://www.youtube.com/api/timedtext?v=GJLlxj_dtq8\u0026caps=asr\u0026xoaf=5\u0026hl=en-GB\u0026ip=0.0.0.0\u0026ipbits=0\u0026expire=1681082354\u0026sparams=ip,ipbits,expire,v,caps,xoaf\u0026signature=13D068D838F3B1262B96D29751914C9E75100C4C.A99B64907A100E2E5F74ACE0BA586FB82F865CE3\u0026key=yt8\u0026lang=ko", "name": { "simpleText": "Korean" }, "vssId": ".ko", "languageCode": "ko", "isTranslatable": true }, { "baseUrl": "https://www.youtube.com/api/timedtext?v=GJLlxj_dtq8\u0026caps=asr\u0026xoaf=5\u0026hl=en-GB\u0026ip=0.0.0.0\u0026ipbits=0\u0026expire=1681082354\u0026sparams=ip,ipbits,expire,v,caps,xoaf\u0026signature=13D068D838F3B1262B96D29751914C9E75100C4C.A99B64907A100E2E5F74ACE0BA586FB82F865CE3\u0026key=yt8\u0026lang=es", "name": { "simpleText": "Spanish" }, "vssId": ".es", "languageCode": "es", "isTranslatable": true }], "audioTracks": [{ "captionTrackIndices": [0, 1, 2, 4, 5, 6, 7, 8, 3], "defaultCaptionTrackIndex": 2, "visibility": "UNKNOWN", "hasDefaultTrack": true, "captionsInitialState": "CAPTIONS_INITIAL_STATE_OFF_RECOMMENDED" }], "translationLanguages": [{ "languageCode": "af", "languageName": { "simpleText": "Afrikaans" } }, { "languageCode": "ak", "languageName": { "simpleText": "Akan" } }, { "languageCode": "sq", "languageName": { "simpleText": "Albanian" } }, { "languageCode": "am", "languageName": { "simpleText": "Amharic" } }, { "languageCode": "ar", "languageName": { "simpleText": "Arabic" } }, { "languageCode": "hy", "languageName": { "simpleText": "Armenian" } }, { "languageCode": "as", "languageName": { "simpleText": "Assamese" } }, { "languageCode": "ay", "languageName": { "simpleText": "Aymara" } }, { "languageCode": "az", "languageName": { "simpleText": "Azerbaijani" } }, { "languageCode": "bn", "languageName": { "simpleText": "Bangla" } }, { "languageCode": "eu", "languageName": { "simpleText": "Basque" } }, { "languageCode": "be", "languageName": { "simpleText": "Belarusian" } }, { "languageCode": "bho", "languageName": { "simpleText": "Bhojpuri" } }, { "languageCode": "bs", "languageName": { "simpleText": "Bosnian" } }, { "languageCode": "bg", "languageName": { "simpleText": "Bulgarian" } }, { "languageCode": "my", "languageName": { "simpleText": "Burmese" } }, { "languageCode": "ca", "languageName": { "simpleText": "Catalan" } }, { "languageCode": "ceb", "languageName": { "simpleText": "Cebuano" } }, { "languageCode": "zh-Hans", "languageName": { "simpleText": "Chinese (Simplified)" } }, { "languageCode": "zh-Hant", "languageName": { "simpleText": "Chinese (Traditional)" } }, { "languageCode": "co", "languageName": { "simpleText": "Corsican" } }, { "languageCode": "hr", "languageName": { "simpleText": "Croatian" } }, { "languageCode": "cs", "languageName": { "simpleText": "Czech" } }, { "languageCode": "da", "languageName": { "simpleText": "Danish" } }, { "languageCode": "dv", "languageName": { "simpleText": "Divehi" } }, { "languageCode": "nl", "languageName": { "simpleText": "Dutch" } }, { "languageCode": "en", "languageName": { "simpleText": "English" } }, { "languageCode": "eo", "languageName": { "simpleText": "Esperanto" } }, { "languageCode": "et", "languageName": { "simpleText": "Estonian" } }, { "languageCode": "ee", "languageName": { "simpleText": "Ewe" } }, { "languageCode": "fil", "languageName": { "simpleText": "Filipino" } }, { "languageCode": "fi", "languageName": { "simpleText": "Finnish" } }, { "languageCode": "fr", "languageName": { "simpleText": "French" } }, { "languageCode": "gl", "languageName": { "simpleText": "Galician" } }, { "languageCode": "lg", "languageName": { "simpleText": "Ganda" } }, { "languageCode": "ka", "languageName": { "simpleText": "Georgian" } }, { "languageCode": "de", "languageName": { "simpleText": "German" } }, { "languageCode": "el", "languageName": { "simpleText": "Greek" } }, { "languageCode": "gn", "languageName": { "simpleText": "Guarani" } }, { "languageCode": "gu", "languageName": { "simpleText": "Gujarati" } }, { "languageCode": "ht", "languageName": { "simpleText": "Haitian Creole" } }, { "languageCode": "ha", "languageName": { "simpleText": "Hausa" } }, { "languageCode": "haw", "languageName": { "simpleText": "Hawaiian" } }, { "languageCode": "iw", "languageName": { "simpleText": "Hebrew" } }, { "languageCode": "hi", "languageName": { "simpleText": "Hindi" } }, { "languageCode": "hmn", "languageName": { "simpleText": "Hmong" } }, { "languageCode": "hu", "languageName": { "simpleText": "Hungarian" } }, { "languageCode": "is", "languageName": { "simpleText": "Icelandic" } }, { "languageCode": "ig", "languageName": { "simpleText": "Igbo" } }, { "languageCode": "id", "languageName": { "simpleText": "Indonesian" } }, { "languageCode": "ga", "languageName": { "simpleText": "Irish" } }, { "languageCode": "it", "languageName": { "simpleText": "Italian" } }, { "languageCode": "ja", "languageName": { "simpleText": "Japanese" } }, { "languageCode": "jv", "languageName": { "simpleText": "Javanese" } }, { "languageCode": "kn", "languageName": { "simpleText": "Kannada" } }, { "languageCode": "kk", "languageName": { "simpleText": "Kazakh" } }, { "languageCode": "km", "languageName": { "simpleText": "Khmer" } }, { "languageCode": "rw", "languageName": { "simpleText": "Kinyarwanda" } }, { "languageCode": "ko", "languageName": { "simpleText": "Korean" } }, { "languageCode": "kri", "languageName": { "simpleText": "Krio" } }, { "languageCode": "ku", "languageName": { "simpleText": "Kurdish" } }, { "languageCode": "ky", "languageName": { "simpleText": "Kyrgyz" } }, { "languageCode": "lo", "languageName": { "simpleText": "Lao" } }, { "languageCode": "la", "languageName": { "simpleText": "Latin" } }, { "languageCode": "lv", "languageName": { "simpleText": "Latvian" } }, { "languageCode": "ln", "languageName": { "simpleText": "Lingala" } }, { "languageCode": "lt", "languageName": { "simpleText": "Lithuanian" } }, { "languageCode": "lb", "languageName": { "simpleText": "Luxembourgish" } }, { "languageCode": "mk", "languageName": { "simpleText": "Macedonian" } }, { "languageCode": "mg", "languageName": { "simpleText": "Malagasy" } }, { "languageCode": "ms", "languageName": { "simpleText": "Malay" } }, { "languageCode": "ml", "languageName": { "simpleText": "Malayalam" } }, { "languageCode": "mt", "languageName": { "simpleText": "Maltese" } }, { "languageCode": "mi", "languageName": { "simpleText": "Māori" } }, { "languageCode": "mr", "languageName": { "simpleText": "Marathi" } }, { "languageCode": "mn", "languageName": { "simpleText": "Mongolian" } }, { "languageCode": "ne", "languageName": { "simpleText": "Nepali" } }, { "languageCode": "nso", "languageName": { "simpleText": "Northern Sotho" } }, { "languageCode": "no", "languageName": { "simpleText": "Norwegian" } }, { "languageCode": "ny", "languageName": { "simpleText": "Nyanja" } }, { "languageCode": "or", "languageName": { "simpleText": "Odia" } }, { "languageCode": "om", "languageName": { "simpleText": "Oromo" } }, { "languageCode": "ps", "languageName": { "simpleText": "Pashto" } }, { "languageCode": "fa", "languageName": { "simpleText": "Persian" } }, { "languageCode": "pl", "languageName": { "simpleText": "Polish" } }, { "languageCode": "pt", "languageName": { "simpleText": "Portuguese" } }, { "languageCode": "pa", "languageName": { "simpleText": "Punjabi" } }, { "languageCode": "qu", "languageName": { "simpleText": "Quechua" } }, { "languageCode": "ro", "languageName": { "simpleText": "Romanian" } }, { "languageCode": "ru", "languageName": { "simpleText": "Russian" } }, { "languageCode": "sm", "languageName": { "simpleText": "Samoan" } }, { "languageCode": "sa", "languageName": { "simpleText": "Sanskrit" } }, { "languageCode": "gd", "languageName": { "simpleText": "Scottish Gaelic" } }, { "languageCode": "sr", "languageName": { "simpleText": "Serbian" } }, { "languageCode": "sn", "languageName": { "simpleText": "Shona" } }, { "languageCode": "sd", "languageName": { "simpleText": "Sindhi" } }, { "languageCode": "si", "languageName": { "simpleText": "Sinhala" } }, { "languageCode": "sk", "languageName": { "simpleText": "Slovak" } }, { "languageCode": "sl", "languageName": { "simpleText": "Slovenian" } }, { "languageCode": "so", "languageName": { "simpleText": "Somali" } }, { "languageCode": "st", "languageName": { "simpleText": "Southern Sotho" } }, { "languageCode": "es", "languageName": { "simpleText": "Spanish" } }, { "languageCode": "su", "languageName": { "simpleText": "Sundanese" } }, { "languageCode": "sw", "languageName": { "simpleText": "Swahili" } }, { "languageCode": "sv", "languageName": { "simpleText": "Swedish" } }, { "languageCode": "tg", "languageName": { "simpleText": "Tajik" } }, { "languageCode": "ta", "languageName": { "simpleText": "Tamil" } }, { "languageCode": "tt", "languageName": { "simpleText": "Tatar" } }, { "languageCode": "te", "languageName": { "simpleText": "Telugu" } }, { "languageCode": "th", "languageName": { "simpleText": "Thai" } }, { "languageCode": "ti", "languageName": { "simpleText": "Tigrinya" } }, { "languageCode": "ts", "languageName": { "simpleText": "Tsonga" } }, { "languageCode": "tr", "languageName": { "simpleText": "Turkish" } }, { "languageCode": "tk", "languageName": { "simpleText": "Turkmen" } }, { "languageCode": "uk", "languageName": { "simpleText": "Ukrainian" } }, { "languageCode": "ur", "languageName": { "simpleText": "Urdu" } }, { "languageCode": "ug", "languageName": { "simpleText": "Uyghur" } }, { "languageCode": "uz", "languageName": { "simpleText": "Uzbek" } }, { "languageCode": "vi", "languageName": { "simpleText": "Vietnamese" } }, { "languageCode": "cy", "languageName": { "simpleText": "Welsh" } }, { "languageCode": "fy", "languageName": { "simpleText": "Western Frisian" } }, { "languageCode": "xh", "languageName": { "simpleText": "Xhosa" } }, { "languageCode": "yi", "languageName": { "simpleText": "Yiddish" } }, { "languageCode": "yo", "languageName": { "simpleText": "Yoruba" } }, { "languageCode": "zu", "languageName": { "simpleText": "Zulu" } }], "defaultAudioTrackIndex": 0 }},"videoDetails": { "videoId": "GJLlxj_dtq8", "# 155 | } 156 | } 157 | #[test] 158 | fn test_caption() { 159 | let c = Config::default(); 160 | let caption = HTML.caption(c.parser.from, c.parser.to, "en").unwrap(); 161 | assert_eq!(caption.base_url, "https://www.youtube.com/api/timedtext?v=GJLlxj_dtq8&caps=asr&xoaf=5&hl=en-GB&ip=0.0.0.0&ipbits=0&expire=1681082354&sparams=ip,ipbits,expire,v,caps,xoaf&signature=13D068D838F3B1262B96D29751914C9E75100C4C.A99B64907A100E2E5F74ACE0BA586FB82F865CE3&key=yt8&lang=en"); 162 | } 163 | 164 | #[test] 165 | fn test_transcript_parse() { 166 | let doc = Document::parse(TRANSCRIPT).unwrap(); 167 | let parsed = TranscriptParser::parse(&doc).unwrap(); 168 | assert_eq!(parsed.transcripts.len(), 74) 169 | } 170 | #[test] 171 | fn test_transcript_parse_time() { 172 | let doc = Document::parse(TRANSCRIPT).unwrap(); 173 | let parsed = TranscriptParser::parse(&doc).unwrap(); 174 | let elem = parsed.into_iter().next().unwrap(); 175 | assert_eq!(elem.start, Duration::from_millis(0)); 176 | assert_eq!(elem.duration, Duration::from_secs_f32(1.54)) 177 | } 178 | } 179 | -------------------------------------------------------------------------------- /youtube-transcript/src/utils.rs: -------------------------------------------------------------------------------- 1 | use std::time::Duration; 2 | 3 | pub(crate) fn to_human_readable(duration: &Duration) -> String { 4 | let dur = duration.as_nanos(); 5 | match dur { 6 | 0u128..=999 => format!("{}ns", dur), 7 | 1000u128..=999_999 => format!("{}μs", dur / 1000), 8 | 1_000_000u128..=999_999_999 => format!("{}ms", dur / 1_000_000), 9 | 1_000_000_000u128..=59_999_999_999 => format!("{}s", dur / 1_000_000_000), 10 | 60_000_000_000u128..=3_599_999_999_999 => { 11 | format!( 12 | "{}m {}s", 13 | dur as f64 / 60_000_000_000.0, 14 | dur % 60_000_000_000 15 | ) 16 | } 17 | _ => format!( 18 | "{}h {}m", 19 | dur / 3_600_000_000_000, 20 | (dur % 3_600_000_000_000) / 60_000_000_000, 21 | ), 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /youtube-transcript/src/youtube.rs: -------------------------------------------------------------------------------- 1 | use crate::config::Config; 2 | use crate::parser::{HTMLParser, Transcript, TranscriptParser}; 3 | use crate::LangCode; 4 | use reqwest::Client; 5 | use roxmltree::Document; 6 | use std::error::Error; 7 | 8 | /// Youtube container that holds the [`Config`]. 9 | pub struct Youtube { 10 | config: Config, 11 | } 12 | 13 | impl Youtube { 14 | /// extracts [`Transcript`] from the video link provided. 15 | pub async fn transcript<'a>(&self, url: &'a str) -> Result> { 16 | let client = Client::default(); 17 | let response = client.get(url).send().await?; 18 | let text = response.text().await?; 19 | self.transcript_from_text(&text).await 20 | } 21 | /// extracts [`Transcript`] from the youtube raw html text provided. 22 | pub async fn transcript_from_text(&self, text: &str) -> Result> { 23 | let client = Client::default(); 24 | let c = text.caption( 25 | self.config.parser.from, 26 | self.config.parser.to, 27 | self.config.lang_code.into(), 28 | )?; 29 | let response = client.get(c.base_url).send().await?; 30 | let trans_resp = response.text().await?; 31 | let doc = Document::parse(&trans_resp)?; 32 | let t = TranscriptParser::parse(&doc)?; 33 | Ok(t) 34 | } 35 | } 36 | 37 | /// Builder struct for building [`Youtube`] 38 | pub struct YoutubeBuilder { 39 | config: Config, 40 | } 41 | 42 | impl YoutubeBuilder { 43 | /// creates [`YoutubeBuilder`] with default [`Config`] values. 44 | pub fn default() -> Self { 45 | Self { 46 | config: Config::default(), 47 | } 48 | } 49 | 50 | /// set language code 51 | pub fn lang_code(mut self, lang_code: LangCode) -> Self { 52 | self.config.lang_code = lang_code; 53 | self 54 | } 55 | 56 | /// Builds [`Youtube`] 57 | pub fn build(self) -> Youtube { 58 | Youtube { 59 | config: self.config, 60 | } 61 | } 62 | } 63 | 64 | #[cfg(test)] 65 | mod test { 66 | use super::*; 67 | #[tokio::test] 68 | #[ignore = "Requires mocking youtube response"] 69 | async fn test_u_trans() { 70 | let transcript = YoutubeBuilder::default() 71 | .build() 72 | .transcript("https://www.youtube.com/watch?v=GJLlxj_dtq8") 73 | .await 74 | .unwrap(); 75 | assert_eq!( 76 | >::into(transcript), 77 | "Hey, how's it going Dave 2d here?This is a Microsoft Surface go and when they first announced it I was interested in itIt seemed like a pretty compelling device having used it for a little whileI really think this is seriously the best product that Microsoft has put out in a very long time this thing starts at $400I don't think that base configuration is where you want to spend the money though. They have a mid tier one550 quite a bit more but you're getting double the RAM double the storage but significantly faster storageThat is the model that I think most people should pick up if you can afford that price bumpso this unit here, is that mid tier model the$550 unit and IReally like it. Ok, let's go around. This thing build quality is great. It's a surface productIt has a magnesium enclosure fit and finish on this is really well donetheTop surface has these new rounded edges and it actually makes the device a lot more comfortable to holdNot that the original surface products are like uncomfortableBut this small detail just makes it that much more ergonomic and that much more inviting to useIt's a nice touch and I think Microsoft should put this kind ofRounded edge on all of their products because it does make a difference. The screen is a 10 inch screenI thought I'd be a little bit small for what I doBut it actually isn't it is noticeably smaller compared to like a 12 or 13 inch screen, but it doesn't feel particularly crampedIt's still a very usable surface area the bezels around that screen though are thick now visuallyIt's not attractive right having thick bezels. Like this doesn't look goodBut when you're actually using it, you won't notice it you'll be focused on your contentit's just that when this devices off or it's just sitting there and you're kind ofexamining it visually the bezels are thick the panel itself is nice its sharp great colors and brightness andHitting a price point like this with this kind of screenIt could not have been easy. Like we see four or five hundred dollar devices out there that have terrible screensThis thing looks really good. There is pen support as usual and feels relatively lag free to meI'm not an artistBut the surface area feels reasonably sized for people that want to use it for any kind of digital creative workNow on the side are two speakers and they sound really good for this kind of device sizenice body to the soundExcellent stereo separation just from the positioning and you just get really clean audio that gets to a decent volumeYou also get a killer killer webcam $400 gets your webcam of this qualityit's actually one of the best kans I've seen on any laptop period but when you compareThis webcam to something like a 12-inch MacBook. It just blows my mindI mean if you can stick a webcam like this into a $400 device, there's no excuse for other peopleThey should be using really good webcams and no one else is doing it, but surface does so good for themthis device though is not complete without the keyboard and the keyboard is aHundred bucks, which is crazy expensive you think about it. That's like at the base model. That's 20% of the cost, butThat's what we have. Okay, when it's connected up and it connects magnetically. It is an awesome. Awesomeproductivity deviceSo I was concerned that this keyboard would be really small and cramped and just kind of weird feeling because it is a lot smallerThan the regular service devices. It's not cramped. It's excellent. It does take a little bit of time to get used to itBut it is a really comfortable keyboard the trackpad feels good. It's a surface productSo tracking is accurate and gestures work nicely. But the pad is a little small. Maybe it's a visual thingI just wish there's a little bit more surface area to this trackpad. Okay performance on this device isGood, it's not amazing. It's a Pentium Gold chip and most productivity stuffLike emails web browsing or any kind of work-related stuff runs really smoothly on thisSo the drive feeds on the mid-tier model actually really good fast read speed but on the slower drive of the base model the wholeSystem is gonna feel a bit more sluggish and that reason alone makes it worth it to upgrade to the mid-tier modelBattery life is also pretty good getting around seven hours of battery life and to charge itYou can either use the included surface connect adapter or you can use the USB C portI really wish that the included adapter was USB C but its surface connect because well, that's Microsoft's for youOkay gaming performance. I was actually surprised by thisYou're not gonna be able to play some killer triple-a titlesBut light games are pretty good on this thing if you want to pick it up for some casual light gamesIt'll do the trick nowThe surface go is still a surface product through and through so if they're issues you had with surface products in the pastYou may have those same issues with this one. Like if you need more ports, there's still only one portIt's use BC this yearBut it's still only one port if you don't like the kickstand on your lapLike it's not an ideal situation for lap use, but it does work reasonablyWell plus infinite positions up to a certain degree, but I don't knowThis makes it fairly usable for most people I think but if you've had issues in the past same issues nowOverall though really good product. I think for studentsThis is such a good option you get so much versatility on this thingYou get a great keyboard for taking notesYou can pull up course material and stuff in classGood option great for me to consumption for like a secondary device if you want it for thatI think you can't go wrong with this it is however notCheap once you add everything up together like the keyboard and like the mid tier unitIt's not the $400 device that they're kind of marketingSo you kind of have to take that into consideration, but overall I like this thing. Ok. Hope you guys enjoyed this video thumbsWe liked it subs we loved it. See you guys next time" 78 | ); 79 | } 80 | } 81 | --------------------------------------------------------------------------------