├── .github ├── CODEOWNERS └── workflows │ └── deploy.yml ├── reset_db.sh ├── src ├── tests │ ├── mod.rs │ ├── huutonet │ │ ├── mod.rs │ │ ├── api_url.rs │ │ └── parse.rs │ └── tori │ │ ├── mod.rs │ │ ├── api_url.rs │ │ └── parse.rs ├── tori │ ├── mod.rs │ ├── seller.rs │ ├── parse.rs │ ├── vahti.rs │ ├── api.rs │ └── models.rs ├── huutonet │ ├── mod.rs │ ├── seller.rs │ ├── parse.rs │ ├── api.rs │ ├── models.rs │ └── vahti.rs ├── command │ ├── telegram │ │ ├── help.rs │ │ ├── start.rs │ │ ├── vahti.rs │ │ ├── poistavahti.rs │ │ └── mod.rs │ ├── discord │ │ ├── extensions.rs │ │ ├── vahti.rs │ │ ├── poistaesto.rs │ │ ├── mod.rs │ │ ├── poistavahti.rs │ │ └── interaction.rs │ └── mod.rs ├── schema.rs ├── error.rs ├── models.rs ├── delivery │ ├── mod.rs │ ├── telegram.rs │ └── discord.rs ├── itemhistory.rs ├── main.rs ├── vahti.rs └── database.rs ├── .gitignore ├── migrations ├── 20211112185056_initial_migration │ ├── down.sql │ └── up.sql ├── 20211215123733_vahti_blacklist │ ├── down.sql │ └── up.sql ├── 2023-03-02-121436_add_delivery_method │ ├── up.sql │ └── down.sql ├── 2021-12-28-153855_add_site_id │ ├── down.sql │ └── up.sql └── 2021-12-17-203524_vahti_and_blacklist_ids │ ├── down.sql │ └── up.sql ├── .dockerignore ├── media ├── demo.png └── no_image.jpg ├── entrypoint.sh ├── rustfmt.toml ├── diesel.toml ├── .env.example ├── docker-compose.yml ├── sqlx_migrate.sh ├── utils └── vahti_generator.hs ├── LICENSE ├── testdata ├── huutonet │ ├── basic_parse.json │ ├── parse_after.json │ └── parse_multiple.json └── tori │ ├── basic_parse.json │ └── parse_after.json ├── Cargo.toml ├── Dockerfile ├── README.md └── docs └── tori-apispec.md /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @lajp @DrVilepis 2 | -------------------------------------------------------------------------------- /reset_db.sh: -------------------------------------------------------------------------------- 1 | rm -rf database.sqlite* 2 | diesel setup 3 | -------------------------------------------------------------------------------- /src/tests/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod huutonet; 2 | pub mod tori; 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /.env 3 | /vendor 4 | /result 5 | database.sqlite* 6 | -------------------------------------------------------------------------------- /migrations/20211112185056_initial_migration/down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE Vahdit; 2 | -------------------------------------------------------------------------------- /migrations/20211215123733_vahti_blacklist/down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE Blacklist; 2 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !src 3 | !Cargo* 4 | !diesel* 5 | !migrations 6 | !entrypoint.sh -------------------------------------------------------------------------------- /media/demo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Testausserveri/torimies-rs/HEAD/media/demo.png -------------------------------------------------------------------------------- /media/no_image.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Testausserveri/torimies-rs/HEAD/media/no_image.jpg -------------------------------------------------------------------------------- /entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | while [ 1 ]; 3 | do 4 | /app/diesel database setup && break; 5 | done 6 | /app/torimies-rs 7 | -------------------------------------------------------------------------------- /src/tests/huutonet/mod.rs: -------------------------------------------------------------------------------- 1 | mod api_url; 2 | mod parse; 3 | 4 | const API_BASE: &str = "https://api.huuto.net/1.1/items?"; 5 | -------------------------------------------------------------------------------- /src/tests/tori/mod.rs: -------------------------------------------------------------------------------- 1 | mod api_url; 2 | mod parse; 3 | 4 | const API_BASE: &str = "https://api.tori.fi/api/v1.2/public/ads?"; 5 | -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | edition = "2021" 2 | imports_granularity = "Module" 3 | group_imports = "StdExternalCrate" 4 | unstable_features = true 5 | -------------------------------------------------------------------------------- /migrations/2023-03-02-121436_add_delivery_method/up.sql: -------------------------------------------------------------------------------- 1 | -- Your SQL goes here 2 | ALTER TABLE Vahdit 3 | ADD delivery_method Integer NOT NULL DEFAULT 1; 4 | -------------------------------------------------------------------------------- /migrations/2023-03-02-121436_add_delivery_method/down.sql: -------------------------------------------------------------------------------- 1 | -- This file should undo anything in `up.sql` 2 | ALTER TABLE Vahdit 3 | DROP COLUMN delivery_method; 4 | -------------------------------------------------------------------------------- /diesel.toml: -------------------------------------------------------------------------------- 1 | # For documentation on how to configure this file, 2 | # see diesel.rs/guides/configuring-diesel-cli 3 | 4 | [print_schema] 5 | file = "src/schema.rs" 6 | -------------------------------------------------------------------------------- /src/tori/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod api; 2 | mod models; 3 | pub mod parse; 4 | pub mod seller; 5 | pub mod vahti; 6 | 7 | pub const ID: i32 = 1; 8 | pub const NAME: &str = "tori"; 9 | -------------------------------------------------------------------------------- /src/huutonet/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod api; 2 | mod models; 3 | pub mod parse; 4 | pub mod seller; 5 | pub mod vahti; 6 | 7 | pub const ID: i32 = 2; 8 | pub const NAME: &str = "huutonet"; 9 | -------------------------------------------------------------------------------- /migrations/20211215123733_vahti_blacklist/up.sql: -------------------------------------------------------------------------------- 1 | -- Add migration script here 2 | CREATE TABLE Blacklist( 3 | user_id INTEGER NOT NULL, 4 | seller_id INTEGER NOT NULL 5 | ); 6 | -------------------------------------------------------------------------------- /migrations/2021-12-28-153855_add_site_id/down.sql: -------------------------------------------------------------------------------- 1 | -- This file should undo anything in `up.sql` 2 | ALTER TABLE Vahdit 3 | DROP COLUMN site_id; 4 | 5 | ALTER TABLE Blacklists 6 | DROP COLUMN site_id; 7 | -------------------------------------------------------------------------------- /.env.example: -------------------------------------------------------------------------------- 1 | # Required variable depend on chosen feature-set 2 | DATABASE_URL=database.sqlite 3 | DISCORD_TOKEN= 4 | APPLICATION_ID= 5 | UPDATE_INTERVAL=60 6 | TELOXIDE_TOKEN= 7 | FUTURES_MAX_BUFFER_SIZE=10 8 | -------------------------------------------------------------------------------- /migrations/2021-12-28-153855_add_site_id/up.sql: -------------------------------------------------------------------------------- 1 | -- Your SQL goes here 2 | ALTER TABLE Vahdit 3 | ADD site_id Integer NOT NULL DEFAULT 1; 4 | 5 | ALTER TABLE Blacklists 6 | ADD site_id Integer NOT NULL DEFAULT 1; 7 | -------------------------------------------------------------------------------- /migrations/20211112185056_initial_migration/up.sql: -------------------------------------------------------------------------------- 1 | -- Add migration script here 2 | CREATE TABLE Vahdit( 3 | url TEXT NOT NULL, 4 | user_id INTEGER NOT NULL, 5 | last_updated BIGINT NOT NULL 6 | ); 7 | 8 | -------------------------------------------------------------------------------- /src/command/telegram/help.rs: -------------------------------------------------------------------------------- 1 | use teloxide::prelude::*; 2 | use teloxide::utils::command::BotCommands; 3 | 4 | pub async fn run() -> ResponseResult { 5 | Ok(super::TelegramCommand::descriptions().to_string()) 6 | } 7 | -------------------------------------------------------------------------------- /src/command/telegram/start.rs: -------------------------------------------------------------------------------- 1 | use teloxide::prelude::*; 2 | 3 | pub async fn run() -> ResponseResult { 4 | Ok(String::from( 5 | "Get started by adding a Vahti. Use /help for a list of commands", 6 | )) 7 | } 8 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | torimies-rs: 5 | image: ghcr.io/testausserveri/torimies-rs:main 6 | restart: unless-stopped 7 | volumes: 8 | - .env:/app/.env 9 | - ./database.sqlite:/app/database.sqlite 10 | -------------------------------------------------------------------------------- /src/huutonet/seller.rs: -------------------------------------------------------------------------------- 1 | use serde_json::Value; 2 | 3 | use crate::error::Error; 4 | 5 | pub async fn get_seller_name_from_id(sellerid: i32) -> Result { 6 | let url = format!("https://api.huuto.net/1.1/users/{}", sellerid); 7 | let response = reqwest::get(&url).await?.text().await?; 8 | let response_json: Value = serde_json::from_str(&response)?; 9 | Ok(response_json["username"].to_string()) 10 | } 11 | -------------------------------------------------------------------------------- /sqlx_migrate.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # This script is meant to help legacy-users to migrate their 4 | # databases so that diesel is able to use them 5 | # Yes, I know it's a hacky solution, but then again 6 | # it's (hopefully) only ran once on a small user(and data)base :)) 7 | 8 | # NOTE: Please backup your database before running this 9 | 10 | rm -rf migrations/20211112185056_initial_migration 11 | diesel migration run 12 | git restore migrations/20211112185056_initial_migration 13 | -------------------------------------------------------------------------------- /src/command/telegram/vahti.rs: -------------------------------------------------------------------------------- 1 | use teloxide::prelude::*; 2 | 3 | use crate::database::Database; 4 | use crate::vahti::new_vahti; 5 | 6 | pub async fn run(msg: Message, vahti: String, db: Database) -> ResponseResult { 7 | if vahti.is_empty() { 8 | return Ok(String::from("No url provided")); 9 | } 10 | 11 | Ok(new_vahti( 12 | db, 13 | &vahti, 14 | msg.chat.id.0 as u64, 15 | crate::delivery::telegram::ID, 16 | ) 17 | .await 18 | .unwrap_or_else(|e| e.to_string())) 19 | } 20 | -------------------------------------------------------------------------------- /src/schema.rs: -------------------------------------------------------------------------------- 1 | #![allow(non_snake_case)] 2 | // @generated automatically by Diesel CLI. 3 | 4 | diesel::table! { 5 | Blacklists (id) { 6 | id -> Integer, 7 | user_id -> BigInt, 8 | seller_id -> Integer, 9 | site_id -> Integer, 10 | } 11 | } 12 | 13 | diesel::table! { 14 | Vahdit (id) { 15 | id -> Integer, 16 | url -> Text, 17 | user_id -> BigInt, 18 | last_updated -> BigInt, 19 | site_id -> Integer, 20 | delivery_method -> Integer, 21 | } 22 | } 23 | 24 | diesel::allow_tables_to_appear_in_same_query!(Blacklists, Vahdit,); 25 | -------------------------------------------------------------------------------- /migrations/2021-12-17-203524_vahti_and_blacklist_ids/down.sql: -------------------------------------------------------------------------------- 1 | -- This file should undo anything in `up.sql` 2 | CREATE TABLE TempVahdit( 3 | url TEXT NOT NULL, 4 | user_id INTEGER NOT NULL, 5 | last_updated BIGINT NOT NULL 6 | ); 7 | INSERT INTO TempVahdit (url, user_id, last_updated) 8 | SELECT url, user_id, last_updated FROM Vahdit; 9 | DROP TABLE Vahdit; 10 | ALTER TABLE TempVahdit RENAME TO Vahdit; 11 | 12 | CREATE TABLE TempBlacklist( 13 | user_id INTEGER NOT NULL, 14 | seller_id INTEGER NOT NULL 15 | ); 16 | INSERT INTO TempBlacklist (user_id, seller_id) 17 | SELECT user_id, seller_id FROM Blacklists; 18 | DROP TABLE Blacklists; 19 | ALTER TABLE TempBlacklist RENAME TO Blacklist; 20 | -------------------------------------------------------------------------------- /migrations/2021-12-17-203524_vahti_and_blacklist_ids/up.sql: -------------------------------------------------------------------------------- 1 | -- Your SQL goes here 2 | CREATE TABLE TempVahdit( 3 | id INTEGER PRIMARY KEY NOT NULL, 4 | url TEXT NOT NULL, 5 | user_id BIGINT NOT NULL, 6 | last_updated BIGINT NOT NULL 7 | ); 8 | 9 | INSERT INTO TempVahdit (url, user_id, last_updated) 10 | SELECT url, user_id, last_updated FROM Vahdit; 11 | DROP TABLE Vahdit; 12 | ALTER TABLE TempVahdit RENAME TO Vahdit; 13 | 14 | CREATE TABLE TempBlacklist( 15 | id INTEGER PRIMARY KEY NOT NULL, 16 | user_id BIGINT NOT NULL, 17 | seller_id INTEGER NOT NULL 18 | ); 19 | INSERT INTO TempBlacklist (user_id, seller_id) 20 | SELECT user_id, seller_id FROM Blacklist; 21 | DROP TABLE Blacklist; 22 | ALTER TABLE TempBlacklist RENAME TO Blacklists; 23 | -------------------------------------------------------------------------------- /src/error.rs: -------------------------------------------------------------------------------- 1 | use thiserror::Error; 2 | 3 | #[derive(Debug, Error)] 4 | pub enum Error { 5 | #[error("Tori has done some stupiding")] 6 | Tori, 7 | #[error("Discord error {0}")] 8 | Discord(#[from] serenity::Error), 9 | #[error("Database error {0}")] 10 | Database(#[from] diesel::result::Error), 11 | #[error("Database Pool error {0}")] 12 | DbPool(#[from] r2d2::Error), 13 | #[error("Unknown url passed: {0}")] 14 | UnknownUrl(String), 15 | #[error("Json Error {0}")] 16 | Serde(#[from] serde_json::Error), 17 | #[error("Reqwest error: {0}")] 18 | Reqwest(#[from] reqwest::Error), 19 | #[error("The specified Vahti already exists")] 20 | VahtiExists, 21 | #[error("Invalid Item passed")] 22 | InvalidItem, 23 | } 24 | -------------------------------------------------------------------------------- /src/huutonet/parse.rs: -------------------------------------------------------------------------------- 1 | use serde_json::Value; 2 | 3 | use super::models::FullHuutonetItem; 4 | use crate::error::Error; 5 | use crate::vahti::VahtiItem; 6 | 7 | pub fn api_parse_after(search: &str, after: i64) -> Result, Error> { 8 | let response_json: Value = serde_json::from_str(search)?; 9 | let mut items = vec![]; 10 | if let Some(ads) = response_json["items"].as_array() { 11 | for ad in ads { 12 | let fullitem: FullHuutonetItem = serde_json::from_value(ad.to_owned()).unwrap(); 13 | let item = VahtiItem::from(fullitem); 14 | if item.published <= after { 15 | break; 16 | } 17 | items.push(item); 18 | } 19 | } 20 | debug!("Parsed {} items", items.len()); 21 | Ok(items) 22 | } 23 | -------------------------------------------------------------------------------- /src/tori/seller.rs: -------------------------------------------------------------------------------- 1 | use serde_json::Value; 2 | 3 | use crate::error::Error; 4 | 5 | pub async fn get_seller_name_from_id(sellerid: i32) -> Result { 6 | let url = format!( 7 | "https://api.tori.fi/api/v1.2/public/ads?account={}&lim=1", 8 | sellerid 9 | ); 10 | let response = reqwest::get(&url).await?.text().await?; 11 | let response_json: Value = serde_json::from_str(&response)?; 12 | if let Some(ads) = response_json["list_ads"].as_array() { 13 | if ads.is_empty() { 14 | return Ok(String::from("Unknown Seller")); 15 | } 16 | return Ok(ads[0].as_object().unwrap()["ad"]["user"]["account"]["name"] 17 | .as_str() 18 | .unwrap_or("Unknown Seller") 19 | .to_string()); 20 | } 21 | Ok(String::from("Unknown Seller")) 22 | } 23 | -------------------------------------------------------------------------------- /src/models.rs: -------------------------------------------------------------------------------- 1 | #[derive(Queryable, Clone, Debug)] 2 | pub struct DbVahti { 3 | pub id: i32, 4 | pub url: String, 5 | pub user_id: i64, 6 | pub last_updated: i64, 7 | pub site_id: i32, 8 | pub delivery_method: i32, 9 | } 10 | 11 | use crate::schema::Vahdit; 12 | 13 | #[derive(Insertable)] 14 | #[table_name = "Vahdit"] 15 | pub struct NewVahti { 16 | pub url: String, 17 | pub user_id: i64, 18 | pub last_updated: i64, 19 | pub site_id: i32, 20 | pub delivery_method: i32, 21 | } 22 | 23 | #[derive(Queryable, Clone, Debug)] 24 | pub struct Blacklist { 25 | pub id: i64, 26 | pub user_id: i64, 27 | pub seller_id: i32, 28 | pub site_id: i32, 29 | } 30 | 31 | use crate::schema::Blacklists; 32 | 33 | #[derive(Insertable)] 34 | #[table_name = "Blacklists"] 35 | pub struct NewBlacklist { 36 | pub user_id: i64, 37 | pub seller_id: i32, 38 | pub site_id: i32, 39 | } 40 | -------------------------------------------------------------------------------- /src/command/discord/extensions.rs: -------------------------------------------------------------------------------- 1 | use serenity::{async_trait, client}; 2 | 3 | use crate::error::Error; 4 | use crate::Database; 5 | 6 | #[async_trait] 7 | pub trait ClientContextExt { 8 | async fn get_db(&self) -> Result; 9 | } 10 | 11 | #[async_trait] 12 | impl ClientContextExt for client::Context { 13 | async fn get_db(&self) -> Result { 14 | let data = self.data.read().await; 15 | let db = data 16 | .get::() 17 | .expect("No Database in client storage"); 18 | Ok(db.to_owned()) 19 | } 20 | } 21 | 22 | #[async_trait] 23 | impl ClientContextExt for client::Client { 24 | async fn get_db(&self) -> Result { 25 | let data = self.data.read().await; 26 | let db = data 27 | .get::() 28 | .expect("No Database in client storage"); 29 | Ok(db.to_owned()) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/command/mod.rs: -------------------------------------------------------------------------------- 1 | #[cfg(feature = "discord-command")] 2 | pub mod discord; 3 | 4 | #[cfg(feature = "telegram-command")] 5 | pub mod telegram; 6 | 7 | use async_trait::async_trait; 8 | 9 | use crate::error::Error; 10 | 11 | /// This is the Command handler trait. It should be implemented for structs that 12 | /// provide methods for the user to interact with Torimies 13 | /// 14 | /// The command trait should implement the start function 15 | /// and the manager function, which returns a Manager struct for the commander 16 | /// It should be able to handle the events, calling the appropriate functions on it's own. 17 | #[async_trait] 18 | pub trait Command 19 | where 20 | Self: Send + Sync, 21 | { 22 | async fn start(&mut self) -> Result<(), Error>; 23 | fn manager(&self) -> Box; 24 | } 25 | 26 | /// The Manager trait is used for shutting down the corresponding Commander. 27 | #[async_trait] 28 | pub trait Manager 29 | where 30 | Self: Send + Sync, 31 | { 32 | async fn shutdown(&self); 33 | } 34 | -------------------------------------------------------------------------------- /src/command/telegram/poistavahti.rs: -------------------------------------------------------------------------------- 1 | use teloxide::prelude::*; 2 | 3 | use crate::database::Database; 4 | use crate::vahti::remove_vahti; 5 | 6 | pub async fn run(msg: Message, vahti: String, db: Database) -> ResponseResult { 7 | if vahti.is_empty() { 8 | let vahdit = db 9 | .fetch_vahti_entries_by_user_id(msg.chat.id.0) 10 | .await 11 | .unwrap_or(Vec::new()) 12 | .iter() 13 | .map(|v| v.url.clone()) 14 | .collect::>(); 15 | 16 | if vahdit.is_empty() { 17 | return Ok(String::from("You have no registered Vahtis")); 18 | } 19 | 20 | return Ok( 21 | "Please provide a Vahti url, here are your registered Vahtis\n".to_owned() 22 | + &vahdit.join("\n"), 23 | ); 24 | } 25 | 26 | Ok(remove_vahti( 27 | db, 28 | &vahti, 29 | msg.chat.id.0 as u64, 30 | crate::delivery::telegram::ID, 31 | ) 32 | .await 33 | .unwrap_or_else(|e| e.to_string())) 34 | } 35 | -------------------------------------------------------------------------------- /src/tori/parse.rs: -------------------------------------------------------------------------------- 1 | use serde_json::Value; 2 | 3 | use super::models::FullToriItem; 4 | use crate::error::Error; 5 | use crate::vahti::VahtiItem; 6 | 7 | pub fn api_parse_after(json: &str, after: i64) -> Result, Error> { 8 | let response_json: Value = serde_json::from_str(json)?; 9 | let mut items = vec![]; 10 | let mut past_weirdness = false; 11 | 12 | if let Some(ads) = response_json["list_ads"].as_array() { 13 | for ad in ads { 14 | let ad_object = &ad.as_object().ok_or(Error::Tori)?["ad"]; 15 | let fullitem: FullToriItem = serde_json::from_value(ad_object.to_owned())?; 16 | let item = VahtiItem::from(fullitem); 17 | 18 | if item.published <= after { 19 | if past_weirdness { 20 | break; 21 | } 22 | continue; 23 | } else { 24 | past_weirdness = true; 25 | } 26 | 27 | items.push(item); 28 | } 29 | } 30 | debug!("Parsed {} items", items.len()); 31 | Ok(items) 32 | } 33 | -------------------------------------------------------------------------------- /utils/vahti_generator.hs: -------------------------------------------------------------------------------- 1 | -- stack script --resolver lts-20 --package sqlite-simple --package text 2 | 3 | {-# LANGUAGE OverloadedStrings #-} 4 | 5 | import qualified Data.Text as T 6 | import Database.SQLite.Simple 7 | import Control.Monad 8 | import Data.List 9 | 10 | vahtiQuery :: String -> [(Int, String)] 11 | vahtiQuery q = [(1, "https://www.tori.fi/koko_suomi?q=" ++ q), (2, "https://www.huuto.net/haku?words=" ++ q)] 12 | 13 | wordList = ["thinkpad", "lenovo", "xeon", "server", "hp", "elitebook", "i3", "i5", "i7"] 14 | 15 | generate :: [[(Int, String)]] 16 | generate = do 17 | ws <- filter (not . null) $ filterM (const [True, False]) wordList 18 | return . vahtiQuery $ intercalate "+" ws 19 | 20 | main :: IO () 21 | main = do 22 | putStrLn "Generating Vahtis..." 23 | let vahtis = concat generate 24 | 25 | putStrLn $ "Inserting " ++ show (length vahtis) ++ " Vahtis.." 26 | conn <- open "test.sqlite" 27 | mapM_ (execute conn "INSERT INTO Vahdit (site_id, user_id, last_updated, url, delivery_method) VALUES (?, 328625071327412267, 0, ?, 1)") vahtis 28 | -------------------------------------------------------------------------------- /src/huutonet/api.rs: -------------------------------------------------------------------------------- 1 | use serde_json::Value; 2 | 3 | pub fn vahti_to_api(vahti: &str) -> String { 4 | let mut url = String::from("https://api.huuto.net/1.1/items?"); 5 | if vahti.contains('?') { 6 | // Easy parse 7 | url += &vahti[vahti.find('?').unwrap() + 1..]; 8 | } else { 9 | // Difficult parse 10 | let mut args: Vec<&str> = vahti.split('/').collect(); 11 | let args: Vec<&str> = args.drain(4..).collect(); 12 | 13 | let url_end: String = args 14 | .chunks_exact(2) 15 | .map(|arg| format!("&{}={}", arg[0], arg[1])) 16 | .collect(); 17 | 18 | if !url_end.is_empty() { 19 | url += &url_end[1..]; 20 | } 21 | } 22 | url += "&sort=newest"; // You can never be too sure 23 | url 24 | } 25 | 26 | pub async fn is_valid_url(url: &str) -> bool { 27 | let url = vahti_to_api(url); 28 | let response = reqwest::get(&url) 29 | .await 30 | .unwrap() 31 | .json::() 32 | .await 33 | .unwrap(); 34 | response["totalCount"].as_i64().unwrap() > 0 35 | } 36 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Luukas Pörtfors 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/command/discord/vahti.rs: -------------------------------------------------------------------------------- 1 | use serenity::builder::{CreateCommand, CreateCommandOption}; 2 | use serenity::client::Context; 3 | use serenity::model::application::{CommandInteraction, CommandOptionType}; 4 | 5 | use super::extensions::ClientContextExt; 6 | use crate::vahti::new_vahti; 7 | 8 | pub fn register() -> CreateCommand { 9 | CreateCommand::new("vahti") 10 | .description("Luo uusi vahti") 11 | .add_option( 12 | CreateCommandOption::new(CommandOptionType::String, "url", "Hakusivun linkki") 13 | .required(true), 14 | ) 15 | } 16 | 17 | pub async fn run(ctx: &Context, command: &CommandInteraction) -> String { 18 | let mut url = String::new(); 19 | for a in &command.data.options { 20 | match a.name.as_str() { 21 | "url" => url = String::from(a.value.as_str().unwrap()), 22 | _ => unreachable!(), 23 | } 24 | } 25 | 26 | info!("New vahti {}", &url); 27 | 28 | let db = ctx.get_db().await.unwrap(); 29 | 30 | new_vahti( 31 | db, 32 | &url, 33 | u64::from(command.user.id), 34 | crate::delivery::discord::ID, 35 | ) 36 | .await 37 | .unwrap_or_else(|e| e.to_string()) 38 | } 39 | -------------------------------------------------------------------------------- /src/delivery/mod.rs: -------------------------------------------------------------------------------- 1 | #[cfg(feature = "discord-delivery")] 2 | pub mod discord; 3 | 4 | #[cfg(feature = "telegram-delivery")] 5 | pub mod telegram; 6 | 7 | use std::sync::Arc; 8 | 9 | use async_trait::async_trait; 10 | use dashmap::DashMap; 11 | 12 | use crate::error::Error; 13 | use crate::vahti::VahtiItem; 14 | 15 | /// This is the Delivery trait. It should be implemented for 16 | /// structs that provide a method for Torimies to deliver the 17 | /// items gathered from Vahti::update(). 18 | /// 19 | /// The deliver method should take in a Vec of VahtiItems, all of which 20 | /// have the same delivery_method and deliver_to fields 21 | #[async_trait] 22 | pub trait Delivery 23 | where 24 | Self: Send + Sync, 25 | { 26 | async fn deliver(&self, vs: Vec) -> Result<(), Error>; 27 | } 28 | 29 | pub async fn perform_delivery( 30 | delivery: Arc>>, 31 | vs: Vec, 32 | ) -> Result<(), Error> { 33 | if let Some(v) = vs.first() { 34 | assert!(vs.iter().all(|vc| vc.delivery_method == v.delivery_method)); 35 | 36 | return delivery 37 | .get(&v.delivery_method.expect("bug: impossible")) 38 | .expect("Missing delivery-method feature") 39 | .deliver(vs) 40 | .await; 41 | } 42 | 43 | Ok(()) 44 | } 45 | -------------------------------------------------------------------------------- /src/itemhistory.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | use std::sync::{Arc, Mutex}; 3 | 4 | use dashmap::DashMap; 5 | 6 | #[derive(Debug, Clone)] 7 | pub struct ItemHistory { 8 | // (item_id, site_id), timestamp 9 | items: HashMap<(i64, i32), i64>, 10 | } 11 | 12 | // (user_id, delivery_method) => ItemHistory 13 | pub type ItemHistoryStorage = Arc>>>; 14 | 15 | impl ItemHistory { 16 | pub fn new() -> ItemHistory { 17 | Self { 18 | items: HashMap::new(), 19 | } 20 | } 21 | 22 | pub fn add_item(&mut self, id: i64, site_id: i32, timestamp: i64) { 23 | if !self.contains(id, site_id) { 24 | debug!("Adding id: {},{}, timestamp: {}", id, site_id, timestamp); 25 | self.items.insert((id, site_id), timestamp); 26 | } 27 | } 28 | 29 | pub fn contains(&self, id: i64, site_id: i32) -> bool { 30 | self.items.contains_key(&(id, site_id)) 31 | } 32 | 33 | pub fn purge_old(&mut self) { 34 | self.items 35 | .retain(|(_, _), timestamp| timestamp > &mut (chrono::Local::now().timestamp() - 1000)); 36 | } 37 | 38 | pub fn extend(&mut self, other: &Self) { 39 | self.items.extend(other.items.iter()); 40 | self.purge_old() 41 | } 42 | } 43 | 44 | impl Default for ItemHistory { 45 | fn default() -> Self { 46 | Self::new() 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /testdata/huutonet/basic_parse.json: -------------------------------------------------------------------------------- 1 | {"totalCount":1,"updated":"2023-03-09T11:06:48+0200","links":{"self":"https:\/\/api.huuto.net\/1.1\/items?words=thinkpad&page=1&sort=newest&category=1","first":"https:\/\/api.huuto.net\/1.1\/items?words=thinkpad&page=1&sort=newest&category=1","last":"https:\/\/api.huuto.net\/1.1\/items?words=thinkpad&page=1&sort=newest&category=1","previous":null,"next":null,"gallery":"https:\/\/api.huuto.net\/1.1\/galleries\/items?words=thinkpad&sort=newest&category=1","hits":"https:\/\/api.huuto.net\/1.1\/hits?words=thinkpad&sort=newest&category=1"},"items":[{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/575647318","category":"https:\/\/api.huuto.net\/1.1\/categories\/22","alternative":"https:\/\/www.huuto.net\/kohteet\/tekniikan-maailma-20_1993\/575647318","images":"https:\/\/api.huuto.net\/1.1\/items\/575647318\/images"},"id":575647318,"title":"Tekniikan Maailma 20\/1993","category":"Ajoneuvokirjat ja -lehdet","seller":"kodin","sellerId":241366,"currentPrice":4,"buyNowPrice":4,"saleMethod":"buy-now","listTime":"2023-01-18T07:54:48+0200","postalCode":"04920","location":"SAARENTAUS","closingTime":"2023-05-18T07:51:00+0300","bidderCount":0,"offerCount":0,"hasReservePrice":false,"hasReservePriceExceeded":false,"upgrades":[],"images":[{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/575647318\/images\/505225227","thumbnail":"https:\/\/kuvat.huuto.net\/v1\/a777\/9ca312c77fbf51f301afec055e4\/505225227-s.jpg","medium":"https:\/\/kuvat.huuto.net\/v1\/a777\/9ca312c77fbf51f301afec055e4\/505225227-m.jpg","original":null}}]}]} -------------------------------------------------------------------------------- /.github/workflows/deploy.yml: -------------------------------------------------------------------------------- 1 | name: Docker 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | workflow_dispatch: 7 | 8 | env: 9 | REGISTRY: ghcr.io 10 | IMAGE_NAME: ${{ github.repository }} 11 | 12 | 13 | jobs: 14 | build-and-push-image: 15 | name: Build and publish 16 | runs-on: ubuntu-latest 17 | permissions: 18 | contents: read 19 | packages: write 20 | 21 | steps: 22 | - name: Checkout repository 23 | uses: actions/checkout@v2 24 | 25 | - name: Set up QEMU 26 | uses: docker/setup-qemu-action@v1 27 | 28 | - name: Set up Docker Buildx 29 | uses: docker/setup-buildx-action@v1 30 | 31 | - name: Log in to the Container registry 32 | uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 33 | with: 34 | registry: ${{ env.REGISTRY }} 35 | username: ${{ github.actor }} 36 | password: ${{ secrets.GITHUB_TOKEN }} 37 | 38 | - name: Extract metadata (tags, labels) for Docker 39 | id: meta 40 | uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 41 | with: 42 | images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} 43 | 44 | - name: Build and push Docker image 45 | uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc 46 | with: 47 | context: . 48 | platforms: linux/amd64,linux/arm64 49 | push: true 50 | tags: ${{ steps.meta.outputs.tags }} 51 | labels: ${{ steps.meta.outputs.labels }} 52 | cache-from: type=gha 53 | cache-to: type=gha,mode=max 54 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "torimies-rs" 3 | version = "0.2.0" 4 | edition = "2021" 5 | authors = ["Luukas Pörtfors "] 6 | 7 | [features] 8 | default = ["discord", "telegram", "tori", "huutonet"] 9 | discord = ["discord-delivery", "discord-command"] 10 | telegram = ["telegram-delivery", "telegram-command"] 11 | discord-delivery = [] 12 | discord-command = [] 13 | telegram-delivery = [] 14 | telegram-command = [] 15 | tori = [] 16 | huutonet = [] 17 | 18 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 19 | [dependencies.serenity] 20 | version = "0.12" 21 | default-features = false 22 | features = ["client", "gateway", "rustls_backend", "model"] 23 | 24 | [dependencies.tokio] 25 | version = "1.13" 26 | features = ["macros", "rt-multi-thread", "signal"] 27 | 28 | [dependencies.diesel] 29 | version = "1.4.8" 30 | features = ["sqlite", "r2d2"] 31 | 32 | [dependencies.openssl] 33 | version = "0.10" 34 | features = ["vendored"] 35 | 36 | [dependencies.libsqlite3-sys] 37 | version = "0.22.2" 38 | features = ["bundled"] 39 | 40 | [dependencies.teloxide] 41 | version = "0.12.2" 42 | features = ["rustls", "throttle", "macros"] 43 | 44 | [dependencies] 45 | tracing = "0.1" 46 | tracing-subscriber = { version = "0.3", features = ["env-filter"] } 47 | 48 | reqwest = "0.11" 49 | regex = "1.5" 50 | dotenv = "0.15" 51 | futures = "0.3" 52 | chrono = "0.4" 53 | clokwerk = "0.3" 54 | serde_json = "1.0" 55 | serde = "1.0.166" 56 | lazy_static = "1.4" 57 | async-trait = "0.1" 58 | thiserror = "1" 59 | dashmap = "5.4.0" 60 | itertools = "0.10.5" 61 | r2d2 = "0.8.10" 62 | url = "2.4.0" 63 | encoding = "0.2.33" 64 | hex = "0.4.3" 65 | -------------------------------------------------------------------------------- /src/command/discord/poistaesto.rs: -------------------------------------------------------------------------------- 1 | use serenity::builder::{CreateCommand, EditInteractionResponse}; 2 | use serenity::client::Context; 3 | use serenity::model::application::CommandInteraction; 4 | 5 | use super::extensions::ClientContextExt; 6 | use super::interaction::menu_from_options; 7 | 8 | pub fn register() -> CreateCommand { 9 | CreateCommand::new("poistaesto").description("Salli aiemmin estetty myyjä") 10 | } 11 | 12 | pub async fn run(ctx: &Context, command: &CommandInteraction) -> String { 13 | let db = ctx.get_db().await.unwrap(); 14 | let blacklist = db 15 | .fetch_user_blacklist(u64::from(command.user.id) as i64) 16 | .await 17 | .unwrap(); 18 | 19 | let mut blacklist_names = vec![]; 20 | for entry in &blacklist { 21 | blacklist_names.push(match entry.1 { 22 | #[cfg(feature = "tori")] 23 | crate::tori::ID => crate::tori::seller::get_seller_name_from_id(entry.0) 24 | .await 25 | .unwrap_or(String::from("Unknown Seller")), 26 | #[cfg(feature = "huutonet")] 27 | crate::huutonet::ID => crate::huutonet::seller::get_seller_name_from_id(entry.0) 28 | .await 29 | .unwrap_or(String::from("Unknown Seller")), 30 | _ => String::from("Unknown Seller"), 31 | }); 32 | } 33 | 34 | let options = blacklist_names 35 | .iter() 36 | .zip(blacklist.iter().map(|ids| format!("{},{}", ids.0, ids.1))) 37 | .collect::>(); 38 | 39 | let mut edit = EditInteractionResponse::new().content("Valitse poistettava(t) esto/estot"); 40 | if blacklist.is_empty() { 41 | edit = edit.content("Ei estettyjä myyjiä!"); 42 | } else { 43 | edit = edit.components(menu_from_options("unblock_seller", options)); 44 | } 45 | command.edit_response(&ctx.http, edit).await.unwrap(); 46 | 47 | String::new() 48 | } 49 | -------------------------------------------------------------------------------- /src/tests/huutonet/api_url.rs: -------------------------------------------------------------------------------- 1 | use super::API_BASE; 2 | use crate::huutonet::api::vahti_to_api; 3 | 4 | #[test] 5 | fn no_keyword() { 6 | let url = "https://www.huuto.net/haku?words=&area="; 7 | let expected = API_BASE.to_owned() + "words=&area=&sort=newest"; 8 | assert_eq!(vahti_to_api(url), expected); 9 | } 10 | 11 | #[test] 12 | fn basic_query() { 13 | let url = "https://www.huuto.net/haku?words=thinkpad&area="; 14 | let expected = API_BASE.to_owned() + "words=thinkpad&area=&sort=newest"; 15 | assert_eq!(vahti_to_api(url), expected); 16 | } 17 | 18 | #[test] 19 | fn slash_query() { 20 | let url = "https://www.huuto.net/haku/words/thinkpad"; 21 | let expected = API_BASE.to_owned() + "words=thinkpad&sort=newest"; 22 | assert_eq!(vahti_to_api(url), expected); 23 | } 24 | 25 | #[test] 26 | fn query_with_non_ascii() { 27 | let url = "https://www.huuto.net/haku?words=th%C3%B6nkp%C3%A4d"; 28 | let slash_url = "https://www.huuto.net/haku/words/th%C3%B6nkp%C3%A4d"; 29 | let expected = API_BASE.to_owned() + "words=th%C3%B6nkp%C3%A4d&sort=newest"; 30 | 31 | assert_eq!(vahti_to_api(url), expected); 32 | assert_eq!(vahti_to_api(slash_url), expected); 33 | } 34 | 35 | #[test] 36 | fn multiquery1() { 37 | let url = "https://www.huuto.net/haku?words=thinkpad&classification=new&area=uusimaa"; 38 | let slash_url = "https://www.huuto.net/haku/words/thinkpad/classification/new/area/uusimaa"; 39 | let expected = 40 | API_BASE.to_owned() + "words=thinkpad&classification=new&area=uusimaa&sort=newest"; 41 | 42 | assert_eq!(vahti_to_api(url), expected); 43 | assert_eq!(vahti_to_api(slash_url), expected); 44 | } 45 | 46 | #[test] 47 | fn multiquery2() { 48 | let url = "https://www.huuto.net/haku?sort=lowprice&category=502"; 49 | let slash_url = "https://www.huuto.net/haku/sort/lowprice/category/502"; 50 | let expected = API_BASE.to_owned() + "sort=lowprice&category=502&sort=newest"; 51 | 52 | assert_eq!(vahti_to_api(url), expected); 53 | assert_eq!(vahti_to_api(slash_url), expected); 54 | } 55 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM --platform=$BUILDPLATFORM rustlang/rust:nightly-bullseye-slim AS build 2 | 3 | ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER="aarch64-linux-gnu-gcc" 4 | 5 | RUN apt update \ 6 | && apt upgrade -y \ 7 | && apt install -y pkg-config libssl-dev perl make libsqlite3-dev 8 | 9 | ARG TARGETPLATFORM 10 | RUN case "$TARGETPLATFORM" in \ 11 | "linux/amd64") echo "x86_64-unknown-linux-gnu" > /target.txt ;; \ 12 | "linux/arm64") echo "aarch64-unknown-linux-gnu" > /target.txt ;; \ 13 | *) exit 1 ;; \ 14 | esac 15 | 16 | RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \ 17 | dpkg --add-architecture arm64 \ 18 | && apt update \ 19 | && apt install gcc-aarch64-linux-gnu libc6-dev-arm64-cross libsqlite3-dev:arm64 -y; \ 20 | fi 21 | 22 | RUN rustup target add $(cat /target.txt) 23 | 24 | RUN cargo install --target $(cat /target.txt) diesel_cli --no-default-features --features "sqlite" \ 25 | && mkdir /out \ 26 | && cp /usr/local/cargo/bin/diesel /out 27 | 28 | RUN cargo new --bin torimies-rs 29 | 30 | WORKDIR /torimies-rs 31 | 32 | COPY Cargo.toml Cargo.lock ./ 33 | 34 | RUN cargo build --target $(cat /target.txt) --release && rm -rf .git src/ target/$(cat /target.txt)/release/deps/torimies* 35 | 36 | COPY src/ src/ 37 | 38 | RUN cargo build --target $(cat /target.txt) --release && mv target/$(cat /target.txt)/release/torimies-rs /out 39 | 40 | 41 | 42 | FROM --platform=$TARGETPLATFORM debian:bullseye-slim AS runner 43 | 44 | RUN apt update \ 45 | && apt upgrade -y \ 46 | && apt install --no-install-recommends ca-certificates sqlite3 -y \ 47 | && rm -rf /var/lib/apt/lists/* 48 | 49 | RUN adduser \ 50 | --disabled-password \ 51 | --gecos "" \ 52 | --home "/none" \ 53 | --shell "/sbin/nologin" \ 54 | --no-create-home \ 55 | torimies 56 | 57 | WORKDIR /app 58 | 59 | COPY --from=build /out/diesel ./ 60 | COPY --from=build /out/torimies-rs ./ 61 | COPY migrations /app/migrations 62 | COPY entrypoint.sh ./ 63 | 64 | RUN chown -R torimies:torimies /app 65 | 66 | USER torimies 67 | 68 | CMD ["sh", "entrypoint.sh"] 69 | -------------------------------------------------------------------------------- /src/huutonet/models.rs: -------------------------------------------------------------------------------- 1 | #![allow(dead_code)] 2 | use serde::Deserialize; 3 | 4 | use crate::vahti::VahtiItem; 5 | 6 | #[derive(Deserialize, Debug, Default)] 7 | struct HuutonetLinks { 8 | #[serde(rename = "self")] 9 | self_: String, 10 | category: String, 11 | alternative: String, 12 | images: String, 13 | } 14 | 15 | #[derive(Deserialize, Debug, Default)] 16 | struct HuutonetImageLinks { 17 | #[serde(rename = "self")] 18 | self_: String, 19 | thumbnail: String, 20 | medium: String, 21 | original: Option, // Always `null` ? 22 | } 23 | 24 | #[derive(Deserialize, Debug, Default)] 25 | struct HuutonetImage { 26 | links: HuutonetImageLinks, 27 | } 28 | 29 | #[derive(Deserialize, Debug, Default)] 30 | #[serde(rename_all = "camelCase")] 31 | pub struct FullHuutonetItem { 32 | links: HuutonetLinks, 33 | id: i64, 34 | title: String, 35 | category: String, 36 | seller: String, 37 | seller_id: i32, 38 | current_price: f64, 39 | buy_now_price: Option, 40 | sale_method: String, 41 | list_time: String, 42 | postal_code: Option, 43 | location: String, 44 | closing_time: String, 45 | bidder_count: i64, 46 | offer_count: i64, 47 | has_reserve_price: bool, 48 | has_reserve_price_exceeded: bool, 49 | // upgrades: Seems to be an empy vec 50 | images: Vec, 51 | } 52 | 53 | impl From for VahtiItem { 54 | fn from(h: FullHuutonetItem) -> VahtiItem { 55 | let published = chrono::DateTime::parse_from_str(&h.list_time, "%FT%T%:z") 56 | .unwrap() 57 | .timestamp(); 58 | let mut img_url = String::new(); 59 | if !h.images.is_empty() { 60 | img_url = h.images[0].links.medium.clone(); 61 | } 62 | VahtiItem { 63 | delivery_method: None, 64 | vahti_url: None, 65 | deliver_to: None, 66 | site_id: 2, 67 | title: h.title, 68 | url: h.links.alternative, 69 | img_url, 70 | published, 71 | price: h.current_price.round() as i64, 72 | seller_name: h.seller, 73 | seller_id: h.seller_id, 74 | location: h.location, 75 | ad_type: h.sale_method, 76 | ad_id: h.id, 77 | } 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /testdata/huutonet/parse_after.json: -------------------------------------------------------------------------------- 1 | {"totalCount":2,"updated":"2023-03-09T12:10:22+0200","links":{"self":"https:\/\/api.huuto.net\/1.1\/items?words=thinkpad&page=1&sort=newest&category=324","first":"https:\/\/api.huuto.net\/1.1\/items?words=thinkpad&page=1&sort=newest&category=324","last":"https:\/\/api.huuto.net\/1.1\/items?words=thinkpad&page=1&sort=newest&category=324","previous":null,"next":null,"gallery":"https:\/\/api.huuto.net\/1.1\/galleries\/items?words=thinkpad&sort=newest&category=324","hits":"https:\/\/api.huuto.net\/1.1\/hits?words=thinkpad&sort=newest&category=324"},"items":[{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/577660812","category":"https:\/\/api.huuto.net\/1.1\/categories\/333","alternative":"https:\/\/www.huuto.net\/kohteet\/thinkpad-ultradock-telakka\/577660812","images":"https:\/\/api.huuto.net\/1.1\/items\/577660812\/images"},"id":577660812,"title":"Thinkpad ultradock Telakka","category":"Muu kodin elektroniikka","seller":"MTM333","sellerId":3000010,"currentPrice":20,"buyNowPrice":20,"saleMethod":"buy-now","listTime":"2023-02-26T22:20:50+0200","postalCode":"30100","location":"FORSSA","closingTime":"2023-03-12T22:20:00+0200","bidderCount":0,"offerCount":0,"hasReservePrice":false,"hasReservePriceExceeded":false,"upgrades":[],"images":[{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/577660812\/images\/507633260","thumbnail":"https:\/\/kuvat.huuto.net\/v1\/fd97\/58c84442023e8246c7fdad1c93b\/507633260-s.jpg","medium":"https:\/\/kuvat.huuto.net\/v1\/fd97\/58c84442023e8246c7fdad1c93b\/507633260-m.jpg","original":null}}]},{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/573190735","category":"https:\/\/api.huuto.net\/1.1\/categories\/333","alternative":"https:\/\/www.huuto.net\/kohteet\/virtalahteita-hp-ja-nokia-virransaadinlattiamalli\/573190735","images":"https:\/\/api.huuto.net\/1.1\/items\/573190735\/images"},"id":573190735,"title":"VIRTAL\u00c4HTEIT\u00c4 HP ja NOKIA VIRRANS\u00c4\u00c4DIN(lattiamalli)","category":"Muu kodin elektroniikka","seller":"vaasan09","sellerId":609794,"currentPrice":45,"buyNowPrice":45,"saleMethod":"buy-now","listTime":"2022-12-02T14:10:35+0200","postalCode":"00500","location":"HELSINKI","closingTime":"2023-04-01T14:09:25+0300","bidderCount":0,"offerCount":0,"hasReservePrice":false,"hasReservePriceExceeded":false,"upgrades":[],"images":[{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/573190735\/images\/478501343","thumbnail":"https:\/\/kuvat.huuto.net\/v1\/369c\/d14b5cfaef59dfaae96be7b7509\/478501343-s.jpg","medium":"https:\/\/kuvat.huuto.net\/v1\/369c\/d14b5cfaef59dfaae96be7b7509\/478501343-m.jpg","original":null}}]}]} -------------------------------------------------------------------------------- /testdata/tori/basic_parse.json: -------------------------------------------------------------------------------- 1 | {"config_etag":"W/\"90469a3ad869011377b03c2e290c23ce8ea3d6cd\"","counter_map":{"all":1},"list_ads":[{"ad":{"account":{"code":"188169","label":"188169"},"account_ads":{"code":"82","label":"82"},"ad_id":"/private/accounts/188169/ads/79217488","body":"Maalaismaisemin koristeltu peltirasia kakenmoiseen säilytykseen. Mukana pieni elefantti, kameli ja seepra. Siistit ja hyväkuntoiset, rasian läpimitta 20 cm ja korkeus 9 cm. Nouto ja posti ok.","category":{"code":"3105","label":"Säilytysastiat ja rasiat","name":"","path_en":"","parent":""},"company_ad":false,"ad_details":{"delivery_options":{"multiple":[{"code":"delivery_send","label":"Lähetys"}]},"general_condition":{"single":{"code":"good","label":"Hyvä"}}},"full_details":true,"images":[{"base_url":"https://img.tori.fi/image","media_id":"/public/media/ad/9039260397","path":"90/9039260397.jpg","width":1980,"height":1080},{"base_url":"https://img.tori.fi/image","media_id":"/public/media/ad/4864739306","path":"48/4864739306.jpg","width":1980,"height":1080},{"base_url":"https://img.tori.fi/image","media_id":"/public/media/ad/4828470346","path":"48/4828470346.jpg","width":1980,"height":1080}],"list_id":"/public/ads/81076530","list_id_code":"81076530","list_price":{"currency":"EUR","price_value":7,"label":"7 €"},"locations":[{"code":"18","key":"region","label":"Uusimaa","locations":[{"code":"313","key":"area","label":"Helsinki","locations":[{"code":"00630","key":"zipcode","label":"Maunula-Suursuo"}]}]}],"mc_settings":{"use_form":false},"phone_hidden":true,"prices":[{"currency":"EUR","price_value":7,"label":"7 €"}],"status":"active","subject":"Maalaisromanttinen peltipurkki ja eläimiä","thumbnail":{"base_url":"https://img.tori.fi/image","media_id":"/public/media/ad/9039260397","path":"90/9039260397.jpg","width":1980,"height":1080},"type":{"code":"s","label":"Myydään"},"user":{"account":{"name":"H.S.M","created":"tammikuusta 2014"},"uuid":"b981f262-7a29-4b31-93c6-b9a930070e84"},"share_link":"https://www.tori.fi/vi/81076530.htm","pivo":{"enabled":false},"list_time":{"label":"4 maaliskuuta 22:47","value":1614890870}},"labelmap":{"category":"Osasto","delivery_options":"Toimitustapa","general_condition":"Kunto","type":"Ilmoitustyyppi"},"spt_metadata":{"category":"Home and personal \u003e Kitchen accessories and dishes \u003e Containers and cases","contentid":"urn:apps.tori.fi:ClassifiedAd:81076530","details":{"currency":"EUR","locality":"Helsinki","postalCode":"00630","price":"7","region":"Uusimaa"}}}],"proximity_slices":[],"sorting":"date","spt_metadata":{"contentid":"urn:apps.tori.fi:Listing:3100","category":"Home and personal \u003e Kitchen accessories and dishes","filter":{"currency":"EUR","numResults":1}}} 2 | -------------------------------------------------------------------------------- /src/command/discord/mod.rs: -------------------------------------------------------------------------------- 1 | mod extensions; 2 | mod interaction; 3 | mod poistaesto; 4 | mod poistavahti; 5 | mod vahti; 6 | 7 | use std::sync::Arc; 8 | 9 | use async_trait::async_trait; 10 | use serenity::gateway::ShardManager; 11 | use serenity::model::application::Interaction; 12 | use serenity::model::gateway::GatewayIntents; 13 | use serenity::model::prelude::*; 14 | use serenity::prelude::*; 15 | 16 | use crate::command::Command; 17 | use crate::database::Database; 18 | use crate::error::Error; 19 | 20 | pub const NAME: &str = "discord"; 21 | 22 | struct Handler; 23 | 24 | #[async_trait] 25 | impl EventHandler for Handler { 26 | async fn interaction_create(&self, ctx: Context, interaction: Interaction) { 27 | interaction::handle_interaction(ctx, interaction).await; 28 | } 29 | 30 | async fn ready(&self, ctx: Context, ready: Ready) { 31 | info!("Connected as {}", ready.user.name); 32 | let _ = serenity::model::application::Command::set_global_commands( 33 | &ctx.http, 34 | vec![ 35 | vahti::register(), 36 | poistavahti::register(), 37 | poistaesto::register(), 38 | ], 39 | ) 40 | .await; 41 | } 42 | async fn resume(&self, _: Context, _: ResumedEvent) { 43 | info!("Resumed"); 44 | } 45 | } 46 | 47 | pub struct Discord { 48 | pub client: Client, 49 | } 50 | 51 | pub struct Manager { 52 | shard_manager: Arc, 53 | } 54 | 55 | impl Discord { 56 | pub async fn init(db: &Database) -> Result { 57 | let token = 58 | std::env::var("DISCORD_TOKEN").expect("Expected DISCORD_TOKEN in the environment"); 59 | 60 | let application_id: u64 = std::env::var("APPLICATION_ID") 61 | .expect("Expected APPLICATION_ID in the environment") 62 | .parse() 63 | .expect("Invalid APPLICATION_ID"); 64 | 65 | let client = Client::builder(&token, GatewayIntents::non_privileged()) 66 | .application_id(application_id.into()) 67 | .event_handler(Handler) 68 | .await?; 69 | 70 | let mut data = client.data.write().await; 71 | data.insert::(db.to_owned()); 72 | drop(data); 73 | 74 | Ok(Self { client }) 75 | } 76 | } 77 | 78 | #[async_trait] 79 | impl super::Manager for Manager { 80 | async fn shutdown(&self) { 81 | info!("Discord destroy"); 82 | self.shard_manager.shutdown_all().await; 83 | info!("Discord destroy done"); 84 | } 85 | } 86 | 87 | #[async_trait] 88 | impl Command for Discord { 89 | async fn start(&mut self) -> Result<(), Error> { 90 | Ok(self.client.start().await?) 91 | } 92 | 93 | fn manager(&self) -> Box { 94 | Box::new(Manager { 95 | shard_manager: self.client.shard_manager.clone(), 96 | }) 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /src/command/telegram/mod.rs: -------------------------------------------------------------------------------- 1 | mod help; 2 | mod poistavahti; 3 | mod start; 4 | mod vahti; 5 | 6 | use async_trait::async_trait; 7 | use teloxide::adaptors::throttle::Limits; 8 | use teloxide::dispatching::{DefaultKey, ShutdownToken}; 9 | use teloxide::prelude::*; 10 | use teloxide::utils::command::BotCommands; 11 | use teloxide::RequestError; 12 | 13 | use crate::command::Command; 14 | use crate::database::Database; 15 | use crate::error::Error; 16 | 17 | pub const NAME: &str = "telegram"; 18 | 19 | pub struct Telegram { 20 | pub dispatcher: Dispatcher, 21 | pub db: Database, 22 | } 23 | 24 | pub struct Manager { 25 | shutdown_token: ShutdownToken, 26 | } 27 | 28 | impl Telegram { 29 | pub async fn init(db: &Database) -> Result { 30 | let token = 31 | std::env::var("TELOXIDE_TOKEN").expect("Expected TELOXIDE_TOKEN in the environment"); 32 | 33 | let bot = Bot::new(token); 34 | 35 | let _ = bot.set_my_commands(TelegramCommand::bot_commands()).await; 36 | 37 | let handler = Update::filter_message().branch( 38 | dptree::entry() 39 | .filter_command::() 40 | .endpoint(handle), 41 | ); 42 | 43 | let dispatcher = Dispatcher::builder(bot.clone(), handler) 44 | .dependencies(dptree::deps![db.clone()]) 45 | .build(); 46 | 47 | Ok(Self { 48 | dispatcher, 49 | db: db.clone(), 50 | }) 51 | } 52 | } 53 | 54 | #[derive(BotCommands, Clone)] 55 | #[command(rename_rule = "lowercase", description = "Supported commands")] 56 | enum TelegramCommand { 57 | #[command(description = "Display start message")] 58 | Start, 59 | #[command(description = "Display help message")] 60 | Help, 61 | #[command(description = "Add new vahti with `/vahti [url]`")] 62 | Vahti(String), 63 | #[command(description = "Remove a vahti with `/poistavahti [url]`")] 64 | PoistaVahti(String), 65 | } 66 | 67 | async fn handle(bot: Bot, msg: Message, cmd: TelegramCommand, db: Database) -> ResponseResult<()> { 68 | let response = match cmd { 69 | TelegramCommand::Vahti(v) => vahti::run(msg.clone(), v, db).await, 70 | TelegramCommand::PoistaVahti(v) => poistavahti::run(msg.clone(), v, db).await, 71 | TelegramCommand::Help => help::run().await, 72 | TelegramCommand::Start => start::run().await, 73 | } 74 | .unwrap_or(String::from( 75 | "Ran into an unhandled error while processing the command", 76 | )); 77 | 78 | bot.throttle(Limits::default()) 79 | .send_message(msg.chat.id, response) 80 | .disable_web_page_preview(true) 81 | .await?; 82 | Ok(()) 83 | } 84 | 85 | #[async_trait] 86 | impl super::Manager for Manager { 87 | async fn shutdown(&self) { 88 | info!("Telegram destroy"); 89 | self.shutdown_token.shutdown().unwrap().await; 90 | info!("Telegram destroy done"); 91 | } 92 | } 93 | 94 | #[async_trait] 95 | impl Command for Telegram { 96 | async fn start(&mut self) -> Result<(), Error> { 97 | self.dispatcher.dispatch().await; 98 | Ok(()) 99 | } 100 | 101 | fn manager(&self) -> Box { 102 | Box::new(Manager { 103 | shutdown_token: self.dispatcher.shutdown_token(), 104 | }) 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /src/tori/vahti.rs: -------------------------------------------------------------------------------- 1 | use std::sync::LazyLock; 2 | 3 | use async_trait::async_trait; 4 | use regex::Regex; 5 | 6 | use crate::database::Database; 7 | use crate::error::Error; 8 | use crate::itemhistory::ItemHistoryStorage; 9 | use crate::models::DbVahti; 10 | use crate::tori::api::*; 11 | use crate::tori::parse::*; 12 | 13 | pub static TORI_REGEX: LazyLock = 14 | LazyLock::new(|| Regex::new(r"^https://(m\.|www\.)?tori\.fi/.*\?.*$").unwrap()); 15 | 16 | use crate::vahti::{Vahti, VahtiItem}; 17 | 18 | #[derive(Debug, Clone)] 19 | pub struct ToriVahti { 20 | pub id: i32, 21 | pub delivery_method: i32, 22 | pub url: String, 23 | pub user_id: u64, 24 | pub last_updated: i64, 25 | pub site_id: i32, 26 | } 27 | 28 | #[async_trait] 29 | impl Vahti for ToriVahti { 30 | async fn update( 31 | &mut self, 32 | db: &Database, 33 | ihs: ItemHistoryStorage, 34 | ) -> Result, Error> { 35 | debug!("Updating {}", self.url); 36 | let ihref = ihs 37 | .get(&(self.user_id, self.delivery_method)) 38 | .expect("bug: impossible"); 39 | 40 | let res = reqwest::get(vahti_to_api(&self.url)) 41 | .await? 42 | .text() 43 | .await? 44 | .to_string(); 45 | 46 | let mut ih = ihref.lock().unwrap().clone(); 47 | let ret = api_parse_after(&res, self.last_updated)? 48 | .iter() 49 | .filter_map(|i| { 50 | if !ih.contains(i.ad_id, i.site_id) { 51 | ih.add_item(i.ad_id, i.site_id, chrono::Local::now().timestamp()); 52 | 53 | // FIXME: Somewhat sketchy 54 | let mut newi = i.clone(); 55 | newi.vahti_url = Some(self.url.clone()); 56 | newi.deliver_to = Some(self.user_id); 57 | newi.delivery_method = Some(self.delivery_method); 58 | 59 | Some(newi) 60 | } else { 61 | None 62 | } 63 | }) 64 | .collect::>(); 65 | 66 | { 67 | let mut locked = ihref.lock().unwrap(); 68 | ih.extend(&locked); 69 | *locked = ih; 70 | } 71 | 72 | if ret.is_empty() { 73 | return Ok(vec![]); 74 | } 75 | 76 | db.vahti_updated(self.to_db(), ret.iter().map(|i| i.published).max()) 77 | .await?; 78 | 79 | Ok(ret) 80 | } 81 | 82 | fn is_valid_url(&self, url: &str) -> bool { 83 | TORI_REGEX.is_match(url) 84 | } 85 | 86 | async fn validate_url(&self) -> Result { 87 | Ok(is_valid_url(&self.url).await) 88 | } 89 | 90 | fn from_db(v: DbVahti) -> Result { 91 | assert_eq!(v.site_id, super::ID); 92 | 93 | Ok(Self { 94 | id: v.id, 95 | url: v.url, 96 | user_id: v.user_id as u64, 97 | last_updated: v.last_updated, 98 | site_id: super::ID, 99 | delivery_method: v.delivery_method, 100 | }) 101 | } 102 | 103 | fn to_db(&self) -> DbVahti { 104 | DbVahti { 105 | delivery_method: self.delivery_method, 106 | id: self.id, 107 | url: self.url.clone(), 108 | user_id: self.user_id as i64, 109 | last_updated: self.last_updated, 110 | site_id: self.site_id, 111 | } 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /src/huutonet/vahti.rs: -------------------------------------------------------------------------------- 1 | use std::sync::LazyLock; 2 | 3 | use async_trait::async_trait; 4 | use regex::Regex; 5 | 6 | pub static HUUTONET_REGEX: LazyLock = 7 | LazyLock::new(|| Regex::new(r"^https://(www\.)?huuto\.net/haku?.*$").unwrap()); 8 | 9 | use super::api::{is_valid_url, vahti_to_api}; 10 | use super::parse::api_parse_after; 11 | use crate::error::Error; 12 | use crate::itemhistory::ItemHistoryStorage; 13 | use crate::models::DbVahti; 14 | use crate::vahti::{Vahti, VahtiItem}; 15 | use crate::Database; 16 | 17 | #[derive(Debug, Clone)] 18 | pub struct HuutonetVahti { 19 | pub id: i32, 20 | pub url: String, 21 | pub user_id: u64, 22 | pub last_updated: i64, 23 | pub site_id: i32, 24 | pub delivery_method: i32, 25 | } 26 | 27 | #[async_trait] 28 | impl Vahti for HuutonetVahti { 29 | async fn update( 30 | &mut self, 31 | db: &Database, 32 | ihs: ItemHistoryStorage, 33 | ) -> Result, Error> { 34 | debug!("Updating {}", self.url); 35 | let ihref = ihs 36 | .get(&(self.user_id, self.delivery_method)) 37 | .expect("bug: impossible"); 38 | 39 | let res = reqwest::get(vahti_to_api(&self.url)) 40 | .await? 41 | .text() 42 | .await? 43 | .to_string(); 44 | 45 | let mut ih = ihref.lock().unwrap().clone(); 46 | let ret = api_parse_after(&res, self.last_updated)? 47 | .into_iter() 48 | .filter_map(|i| { 49 | if !ih.contains(i.ad_id, i.site_id) { 50 | ih.add_item(i.ad_id, i.site_id, chrono::Local::now().timestamp()); 51 | let mut newi = i.clone(); 52 | newi.vahti_url = Some(self.url.clone()); 53 | newi.deliver_to = Some(self.user_id); 54 | newi.delivery_method = Some(self.delivery_method); 55 | 56 | Some(newi) 57 | } else { 58 | None 59 | } 60 | }) 61 | .map(|mut i| { 62 | i.vahti_url = Some(self.url.clone()); 63 | i.deliver_to = Some(self.user_id); 64 | i.delivery_method = Some(self.delivery_method); 65 | i 66 | }) 67 | .collect::>(); 68 | 69 | { 70 | let mut locked = ihref.lock().unwrap(); 71 | ih.extend(&locked); 72 | *locked = ih; 73 | } 74 | 75 | if ret.is_empty() { 76 | return Ok(vec![]); 77 | } 78 | 79 | db.vahti_updated(self.to_db(), ret.iter().map(|i| i.published).max()) 80 | .await?; 81 | 82 | Ok(ret) 83 | } 84 | 85 | fn is_valid_url(&self, url: &str) -> bool { 86 | HUUTONET_REGEX.is_match(url) 87 | } 88 | 89 | async fn validate_url(&self) -> Result { 90 | Ok(is_valid_url(&self.url).await) 91 | } 92 | 93 | fn from_db(v: DbVahti) -> Result { 94 | assert_eq!(v.site_id, super::ID); 95 | 96 | Ok(Self { 97 | id: v.id, 98 | url: v.url, 99 | user_id: v.user_id as u64, 100 | last_updated: v.last_updated, 101 | site_id: super::ID, 102 | delivery_method: v.delivery_method, 103 | }) 104 | } 105 | 106 | fn to_db(&self) -> DbVahti { 107 | DbVahti { 108 | id: self.id, 109 | url: self.url.clone(), 110 | user_id: self.user_id as i64, 111 | last_updated: self.last_updated, 112 | site_id: self.site_id, 113 | delivery_method: self.delivery_method, 114 | } 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /src/command/discord/poistavahti.rs: -------------------------------------------------------------------------------- 1 | use serenity::all::ReactionType; 2 | use serenity::builder::{ 3 | CreateActionRow, CreateButton, CreateCommand, CreateCommandOption, 4 | CreateInteractionResponseMessage, CreateSelectMenu, CreateSelectMenuKind, 5 | CreateSelectMenuOption, EditInteractionResponse, 6 | }; 7 | use serenity::client::Context; 8 | use serenity::model::application::{CommandInteraction, CommandOptionType}; 9 | 10 | use super::extensions::ClientContextExt; 11 | use crate::vahti::remove_vahti; 12 | 13 | pub fn register() -> CreateCommand { 14 | CreateCommand::new("poistavahti") 15 | .description("Poista olemassaoleva vahti") 16 | .add_option(CreateCommandOption::new( 17 | CommandOptionType::String, 18 | "url", 19 | "Hakusivun linkki", 20 | )) 21 | } 22 | 23 | fn show_select_menu_page(urls: Vec, page: usize) -> Vec { 24 | let options = urls 25 | .iter() 26 | .skip(page * 25) 27 | .take(25) 28 | .map(|url| CreateSelectMenuOption::new(url, url)) 29 | .collect::>(); 30 | let menu = CreateSelectMenu::new( 31 | "remove_vahti_menu", 32 | CreateSelectMenuKind::String { options }, 33 | ); 34 | let buttons = vec![ 35 | CreateButton::new(format!( 36 | "remove_vahti_menu_page_{}", 37 | if page > 0 { page - 1 } else { 0 } 38 | )) 39 | .emoji(ReactionType::Unicode("◀️".to_string())) 40 | .disabled(page == 0), 41 | CreateButton::new(format!("remove_vahti_menu_page_{}", page + 1)) 42 | .emoji(ReactionType::Unicode("▶️".to_string())) 43 | .disabled(page >= urls.len() / 25), 44 | ]; 45 | 46 | vec![ 47 | CreateActionRow::SelectMenu(menu), 48 | CreateActionRow::Buttons(buttons), 49 | ] 50 | } 51 | 52 | pub async fn update_message( 53 | ctx: &Context, 54 | page: usize, 55 | user_id: u64, 56 | ) -> serenity::builder::CreateInteractionResponseMessage { 57 | let db = ctx.get_db().await.unwrap(); 58 | 59 | let vahtilist = db 60 | .fetch_vahti_entries_by_user_id(user_id as i64) 61 | .await 62 | .unwrap(); 63 | 64 | let mut urls = vahtilist.iter().cloned().map(|v| v.url).collect::>(); 65 | urls.sort(); 66 | 67 | if vahtilist.is_empty() { 68 | CreateInteractionResponseMessage::new() 69 | .content("Ei vahteja! Aseta vahti komennolla `/vahti`") 70 | } else { 71 | CreateInteractionResponseMessage::new().components(show_select_menu_page(urls, page)) 72 | } 73 | } 74 | 75 | pub async fn run(ctx: &Context, command: &CommandInteraction) -> String { 76 | let mut url = String::new(); 77 | for a in &command.data.options { 78 | match a.name.as_str() { 79 | "url" => url = String::from(a.value.as_str().unwrap()), 80 | _ => unreachable!(), 81 | } 82 | } 83 | 84 | let db = ctx.get_db().await.unwrap(); 85 | 86 | if !url.is_empty() { 87 | remove_vahti( 88 | db, 89 | &url, 90 | u64::from(command.user.id), 91 | crate::delivery::discord::ID, 92 | ) 93 | .await 94 | .unwrap() 95 | } else { 96 | let db = ctx.get_db().await.unwrap(); 97 | let vahtilist = db 98 | .fetch_vahti_entries_by_user_id(u64::from(command.user.id) as i64) 99 | .await 100 | .unwrap(); 101 | 102 | let urls = vahtilist.iter().cloned().map(|v| v.url).collect::>(); 103 | 104 | let message = if vahtilist.is_empty() { 105 | EditInteractionResponse::new().content("Ei vahteja! Aseta vahti komennolla `/vahti`") 106 | } else { 107 | EditInteractionResponse::new().components(show_select_menu_page(urls, 0)) 108 | }; 109 | 110 | command.edit_response(&ctx.http, message).await.unwrap(); 111 | String::new() 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /src/delivery/telegram.rs: -------------------------------------------------------------------------------- 1 | use async_trait::async_trait; 2 | use chrono::{Local, TimeZone}; 3 | use futures::stream::{self, StreamExt}; 4 | use teloxide::adaptors::throttle::Limits; 5 | use teloxide::prelude::*; 6 | use teloxide::types::{ChatId, InputFile, ParseMode}; 7 | 8 | use crate::delivery::Delivery; 9 | use crate::error::Error; 10 | use crate::vahti::VahtiItem; 11 | 12 | pub struct Telegram { 13 | pub bot: Bot, 14 | } 15 | 16 | pub const ID: i32 = 2; 17 | pub const NAME: &str = "telegram"; 18 | 19 | /// This is the telegram delivery client 20 | /// There will be a separate client for handling commands 21 | impl Telegram { 22 | pub async fn init() -> Result { 23 | let token = 24 | std::env::var("TELOXIDE_TOKEN").expect("Expected TELOXIDE_TOKEN in the environment"); 25 | let bot = Bot::new(token); 26 | Ok(Self { bot }) 27 | } 28 | 29 | pub async fn destroy(self) {} 30 | } 31 | 32 | impl VahtiItem { 33 | fn format_telegram(self) -> String { 34 | let sellerurl = match self.site_id { 35 | #[cfg(feature = "tori")] 36 | crate::tori::ID => { 37 | format!("https://www.tori.fi/li?&aid={}", self.seller_id) 38 | } 39 | #[cfg(feature = "huutonet")] 40 | crate::huutonet::ID => { 41 | format!("https://www.huuto.net/kayttaja/{}", self.seller_id) 42 | } 43 | i => panic!("Unsupported site_id {}", i), 44 | }; 45 | 46 | let mut msg = format!(r#"{}"#, self.url, self.title) + "\n"; 47 | msg.push_str((format!(r#"Hinta: {}€"#, self.price) + "\n").as_str()); 48 | msg.push_str( 49 | (format!( 50 | r#"Myyjä: {}"#, 51 | sellerurl, self.seller_name 52 | ) + "\n") 53 | .as_str(), 54 | ); 55 | msg.push_str((format!(r#"Sijainti: {}"#, self.location) + "\n").as_str()); 56 | msg.push_str( 57 | (format!( 58 | r#"Ilmoitus jätetty: {}"#, 59 | Local 60 | .timestamp_opt(self.published, 0) 61 | .unwrap() 62 | .format("%d/%m/%Y %R") 63 | ) + "\n") 64 | .as_str(), 65 | ); 66 | msg.push_str((format!(r#"Ilmoitustyyppi: {}"#, self.ad_type) + "\n").as_str()); 67 | msg.push_str(&format!( 68 | r#"Avaa Hakusivu"#, 69 | self.vahti_url.unwrap() 70 | )); 71 | 72 | msg 73 | } 74 | } 75 | 76 | #[async_trait] 77 | impl Delivery for Telegram { 78 | async fn deliver(&self, items: Vec) -> Result<(), Error> { 79 | let Some(fst) = items.first() else { 80 | return Ok(()); 81 | }; 82 | 83 | assert!(items.iter().all(|i| i.deliver_to == fst.deliver_to)); 84 | 85 | info!( 86 | "Delivering {} items to {}", 87 | items.len(), 88 | fst.deliver_to.unwrap() 89 | ); 90 | 91 | let recipient = ChatId(fst.deliver_to.unwrap() as i64); 92 | 93 | stream::iter(items.iter().cloned()) 94 | .map(async move |i| { 95 | let file = if i.img_url.is_empty() { 96 | InputFile::file("./media/no_image.jpg") 97 | } else { 98 | InputFile::url(url::Url::parse(&i.img_url).unwrap()) 99 | }; 100 | 101 | self.bot 102 | .clone() 103 | .throttle(Limits::default()) 104 | .send_photo(recipient, file) 105 | .caption(i.clone().format_telegram()) 106 | .parse_mode(ParseMode::Html) 107 | .await 108 | // FIXME: Perhaps don't ignore an error here 109 | .ok() 110 | }) 111 | .buffer_unordered(*crate::FUTURES_MAX_BUFFER_SIZE) 112 | .collect::>() 113 | .await; 114 | 115 | Ok(()) 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /testdata/tori/parse_after.json: -------------------------------------------------------------------------------- 1 | {"config_etag":"W/\"90469a3ad869011377b03c2e290c23ce8ea3d6cd\"","counter_map":{"all":2},"list_ads":[{"ad":{"account":{"code":"174417","label":"174417"},"account_ads":{"code":"4","label":"4"},"ad_id":"/private/accounts/174417/ads/106171673","body":"Elikkä otsikon mukainen laite olis kaupan, maksanut uutena +400 Sopii hyvin esim toimistoon mikäli haluat useamman näytön liittää\n\nLöytyy, 135W Power, 2xHdmi, 2x displayport, Gigabit(1000mb) Ethernet, Kuuloke/Mikrofoni, 4x USB 3.0, 1x USB-C 3.2 gen 1+USB-C kone liitäntä.\n\nhttps://www.multitronic.fi/fi/products/3088231/lenovo-thinkpad-universal-usb-c-smart-dock-telakointiasema-usb-c-hdmi-2-x-dp-gige-135-watt-eurooppa-malleihin-thinkpad-x1-carbon-gen-9-20xw-20xx","category":{"code":"5036","label":"Oheislaitteet","name":"","path_en":"","parent":""},"company_ad":false,"ad_details":{"delivery_options":{"multiple":[{"code":"delivery_send","label":"Lähetys"}]},"general_condition":{"single":{"code":"excellent","label":"Erinomainen"}},"peripheral":{"single":{"code":"other","label":"Muut"}}},"full_details":true,"images":[{"base_url":"https://img.tori.fi/image","media_id":"/public/media/ad/3907440735","path":"39/3907440735.jpg","width":1980,"height":1080}],"list_id":"/public/ads/106717196","list_id_code":"106717196","list_price":{"currency":"EUR","price_value":130,"label":"130 €"},"locations":[{"code":"5","key":"region","label":"Pohjanmaa","locations":[{"code":"88","key":"area","label":"Vaasa","locations":[{"code":"65100","key":"zipcode","label":"Vaasa Keskus"}]}]}],"mc_settings":{"use_form":false},"phone_hidden":true,"prices":[{"currency":"EUR","price_value":130,"label":"130 €"}],"status":"active","subject":"Lenovo ThinkPad Universal USB-C Smart Dock","thumbnail":{"base_url":"https://img.tori.fi/image","media_id":"/public/media/ad/3907440735","path":"39/3907440735.jpg","width":1980,"height":1080},"type":{"code":"s","label":"Myydään"},"user":{"account":{"name":"A.O","created":"joulukuusta 2013"},"uuid":"2c082953-77d2-4f89-bf79-7e628110fd13"},"share_link":"https://www.tori.fi/vi/106717196.htm","pivo":{"enabled":false},"list_time":{"label":"13 joulukuuta 07:17","value":1670908635}},"labelmap":{"category":"Osasto","delivery_options":"Toimitustapa","general_condition":"Kunto","peripheral":"Oheislaite","type":"Ilmoitustyyppi"},"spt_metadata":{"category":"Electronics \u003e Computers and accessories \u003e Computer accessories","contentid":"urn:apps.tori.fi:ClassifiedAd:106717196","details":{"currency":"EUR","locality":"Vaasa","postalCode":"65100","price":"130","region":"Pohjanmaa"}}},{"ad":{"account":{"code":"1747627","label":"1747627"},"account_ads":{"code":"7","label":"7"},"ad_id":"/private/accounts/1747627/ads/96458965","body":"Myydään uusi ja käyttämätön Lenovo 15\" musta ThinkPad Essential Backpack-reppu.","category":{"code":"3063","label":"Laukut ja hatut","name":"","path_en":"","parent":""},"company_ad":false,"ad_details":{"delivery_options":{"multiple":[{"code":"delivery_send","label":"Lähetys"}]},"general_condition":{"single":{"code":"new","label":"Uusi"}}},"images":[{"base_url":"https://img.tori.fi/image","media_id":"/public/media/ad/1378488724","path":"13/1378488724.jpg","width":1980,"height":1080}],"list_id":"/public/ads/97479479","list_id_code":"97479479","list_price":{"currency":"EUR","price_value":40,"label":"40 €"},"locations":[{"code":"5","key":"region","label":"Pohjanmaa","locations":[{"code":"88","key":"area","label":"Vaasa","locations":[{"code":"66510","key":"zipcode","label":"Merikaarto"}]}]}],"mc_settings":{"use_form":false},"phone_hidden":true,"prices":[{"currency":"EUR","price_value":40,"label":"40 €"}],"status":"active","subject":"Lenovo ThinkPad Essential Backpack-reppu","thumbnail":{"base_url":"https://img.tori.fi/image","media_id":"/public/media/ad/1378488724","path":"13/1378488724.jpg","width":1980,"height":1080},"type":{"code":"s","label":"Myydään"},"user":{"account":{"name":"Artem","created":"marraskuusta 2018"},"uuid":"2437f5c8-1660-4d34-a666-58bac8997edb"},"share_link":"https://www.tori.fi/vi/97479479.htm","pivo":{"enabled":false},"list_time":{"label":"1 toukokuuta 17:45","value":1651416320}},"labelmap":{"category":"Osasto","delivery_options":"Toimitustapa","general_condition":"Kunto","type":"Ilmoitustyyppi"},"spt_metadata":{"category":"Home and personal \u003e Accessories and watches \u003e Bags and hats","contentid":"urn:apps.tori.fi:ClassifiedAd:97479479","details":{"currency":"EUR","locality":"Vaasa","postalCode":"66510","price":"40","region":"Pohjanmaa"}}}],"proximity_slices":[],"sorting":"date","spt_metadata":{"contentid":"urn:apps.tori.fi:Listing:0000","filter":{"currency":"EUR","numResults":2}}} 2 | -------------------------------------------------------------------------------- /src/tori/api.rs: -------------------------------------------------------------------------------- 1 | use encoding::all::ISO_8859_2; 2 | use encoding::{DecoderTrap, Encoding}; 3 | use serde_json::Value; 4 | use url::Url; 5 | 6 | const TORI_PRICES: [&str; 9] = ["0", "25", "50", "75", "100", "250", "500", "1000", "2000"]; 7 | 8 | // NOTE: Couldn't find a good crate to do this 9 | fn url_decode(url: &str) -> String { 10 | let mut result = String::new(); 11 | let mut chars = url.chars().peekable(); 12 | 13 | while let Some(c) = chars.next() { 14 | if c == '%' { 15 | if chars.peek() == Some(&'%') { 16 | result.push('%'); 17 | let _ = chars.next(); 18 | } else { 19 | let hex_str = (&mut chars).take(2).collect::(); 20 | let bytes = hex::decode(&hex_str).unwrap(); 21 | result.push_str(&ISO_8859_2.decode(&bytes, DecoderTrap::Ignore).unwrap()) 22 | } 23 | } else { 24 | result.push(c) 25 | } 26 | } 27 | 28 | result 29 | } 30 | 31 | // TODO: Error handling 32 | pub fn vahti_to_api(vahti: &str) -> String { 33 | let url = Url::parse(&url_decode(vahti)).unwrap(); 34 | let orig_params = url 35 | .query_pairs() 36 | .map(|(k, v)| (k.into_owned(), v.into_owned())) 37 | .collect::>(); 38 | 39 | let mut range_start = None; 40 | let mut range_end = None; 41 | 42 | let mut params = orig_params 43 | .clone() 44 | .into_iter() 45 | .filter_map(|(k, v)| match k.as_str() { 46 | "q" => Some((k, v.replace(' ', "+"))), 47 | "cg" => { 48 | if orig_params.iter().any(|(k, _)| k == "c") || v == "0" { 49 | None 50 | } else { 51 | Some((String::from("category"), v)) 52 | } 53 | } 54 | "c" => Some((String::from("category"), v)), 55 | "ps" => { 56 | if let Ok(n) = v.parse::() { 57 | range_start = Some(TORI_PRICES[n]) 58 | } 59 | None 60 | } 61 | "pe" => { 62 | if let Ok(n) = v.parse::() { 63 | range_end = Some(TORI_PRICES[n]) 64 | } 65 | None 66 | } 67 | "ca" => { 68 | match orig_params 69 | .iter() 70 | .find(|(k, _)| k == "w") 71 | .map(|(_, v)| v.parse::().ok().map(|n| n > 100)) 72 | { 73 | Some(Some(true)) => None, 74 | _ => Some((String::from("region"), v)), 75 | } 76 | } 77 | "w" => { 78 | if let Ok(n) = v.parse::() { 79 | if n > 100 { 80 | Some((String::from("region"), (n - 100).to_string())) 81 | } else { 82 | None 83 | } 84 | } else { 85 | None 86 | } 87 | } 88 | "m" => Some((String::from("area"), v)), 89 | "f" => match v.as_str() { 90 | "p" => Some((String::from("company_ad"), String::from("0"))), 91 | "c" => Some((String::from("company_ad"), String::from("1"))), 92 | _ => None, 93 | }, 94 | "st" => Some((String::from("ad_type"), v)), 95 | _ => None, 96 | }) 97 | .collect::>(); 98 | 99 | if range_start.is_some() || range_end.is_some() { 100 | params.push(( 101 | String::from("suborder"), 102 | format!( 103 | "{}-{}", 104 | range_start.unwrap_or_default(), 105 | range_end.unwrap_or_default() 106 | ), 107 | )); 108 | } 109 | 110 | format!( 111 | "https://api.tori.fi/api/v1.2/public/ads?{}", 112 | params 113 | .iter() 114 | .map(|(k, v)| format!("{k}={v}")) 115 | .collect::>() 116 | .join("&") 117 | ) 118 | } 119 | 120 | pub async fn is_valid_url(url: &str) -> bool { 121 | let url = vahti_to_api(url) + "&lim=0"; 122 | let response = reqwest::get(&url) 123 | .await 124 | .unwrap() 125 | .json::() 126 | .await 127 | .unwrap(); 128 | if let Some(counter_map) = response["counter_map"].as_object() { 129 | if let Some(amount) = counter_map["all"].as_i64() { 130 | amount > 0 131 | } else { 132 | false 133 | } 134 | } else { 135 | false 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /src/tori/models.rs: -------------------------------------------------------------------------------- 1 | #![allow(dead_code)] 2 | use serde::Deserialize; 3 | 4 | use crate::vahti::VahtiItem; 5 | 6 | #[derive(Deserialize, Debug, Clone, Default)] 7 | struct ToriAccount { 8 | code: String, 9 | label: String, 10 | } 11 | 12 | #[derive(Deserialize, Debug, Clone, Default)] 13 | struct ToriCategory { 14 | code: String, 15 | label: String, 16 | name: String, 17 | path_en: String, 18 | parent: String, 19 | } 20 | 21 | #[derive(Deserialize, Debug, Clone, Default)] 22 | struct ToriImage { 23 | base_url: String, 24 | media_id: String, 25 | path: String, 26 | width: i64, 27 | height: i64, 28 | } 29 | 30 | #[derive(Deserialize, Debug, Clone, Default)] 31 | struct ToriListPrice { 32 | #[serde(default)] 33 | currency: String, 34 | price_value: i64, 35 | label: String, 36 | } 37 | 38 | #[derive(Deserialize, Debug, Clone, Default)] 39 | struct ToriMcSettings { 40 | use_form: bool, 41 | } 42 | 43 | #[derive(Deserialize, Debug, Clone, Default)] 44 | struct ToriListType { 45 | code: String, 46 | label: String, 47 | } 48 | 49 | #[derive(Deserialize, Debug, Clone, Default)] 50 | struct ToriUserAccount { 51 | name: String, 52 | created: String, 53 | } 54 | 55 | #[derive(Deserialize, Debug, Clone, Default)] 56 | struct ToriUser { 57 | account: ToriUserAccount, 58 | uuid: String, 59 | } 60 | 61 | #[derive(Deserialize, Debug, Clone, Default)] 62 | struct ToriPivo { 63 | enabled: bool, 64 | } 65 | 66 | #[derive(Deserialize, Debug, Clone, Default)] 67 | struct ToriListTime { 68 | label: String, 69 | value: i64, 70 | } 71 | 72 | #[derive(Deserialize, Debug, Clone, Default)] 73 | struct ToriLocation { 74 | code: String, 75 | key: String, 76 | label: String, 77 | #[serde(default)] 78 | locations: Vec, 79 | } 80 | 81 | #[derive(Deserialize, Debug, Clone)] 82 | pub struct FullToriItem { 83 | account: ToriAccount, 84 | #[serde(default)] 85 | account_ads: ToriAccount, 86 | ad_id: String, 87 | #[serde(default)] 88 | body: String, 89 | #[serde(default)] 90 | category: ToriCategory, 91 | #[serde(default)] 92 | company_ad: bool, 93 | //ad_details: Value, // Complicated and not used anyways 94 | #[serde(default)] 95 | full_details: bool, 96 | #[serde(default)] 97 | images: Vec, 98 | #[serde(default)] 99 | list_id: String, 100 | #[serde(default)] 101 | list_id_code: String, 102 | #[serde(default)] 103 | list_price: ToriListPrice, 104 | locations: Vec, 105 | #[serde(default)] 106 | mc_settings: ToriMcSettings, 107 | #[serde(default)] 108 | phone_hidden: bool, 109 | #[serde(default)] 110 | prices: Vec, 111 | #[serde(default)] 112 | status: String, 113 | subject: String, 114 | thumbnail: Option, 115 | r#type: ToriListType, 116 | user: ToriUser, 117 | share_link: String, 118 | #[serde(default)] 119 | pivo: ToriPivo, 120 | list_time: ToriListTime, 121 | } 122 | 123 | impl From for VahtiItem { 124 | fn from(t: FullToriItem) -> VahtiItem { 125 | let img_url = match t.thumbnail { 126 | Some(i) => { 127 | format!( 128 | "https://images.tori.fi/api/v1/imagestori/images{}?rule=medium_660", 129 | &i.path[i.path.find('/').unwrap_or(i.path.len())..] 130 | ) 131 | } 132 | None => String::new(), 133 | }; 134 | 135 | let mut location_vec: Vec = vec![]; 136 | let mut loc = &t.locations[0]; 137 | loop { 138 | location_vec.push(loc.label.clone()); 139 | if loc.locations.is_empty() { 140 | break; 141 | } 142 | loc = &loc.locations[0]; 143 | } 144 | 145 | let mut prevloc = String::new(); 146 | let mut location = String::new(); 147 | for loc_string in location_vec.iter().rev() { 148 | if *loc_string == prevloc { 149 | break; 150 | } 151 | prevloc = loc_string.to_string(); 152 | if location.is_empty() { 153 | location += loc_string; 154 | } else { 155 | location += &format!(", {}", loc_string); 156 | } 157 | } 158 | 159 | VahtiItem { 160 | vahti_url: None, 161 | site_id: super::ID, 162 | deliver_to: None, 163 | delivery_method: None, 164 | title: t.subject, 165 | url: t.share_link, 166 | img_url, 167 | published: t.list_time.value, 168 | price: t.list_price.price_value, 169 | seller_name: t.user.account.name, 170 | seller_id: t.account.code.parse().unwrap(), 171 | location, 172 | ad_type: t.r#type.label, 173 | ad_id: t.ad_id[t.ad_id.rfind('/').unwrap() + 1..].parse().unwrap(), 174 | } 175 | } 176 | } 177 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # torimies-rs 2 | 3 | ## How the bot works 4 | 5 | Users of the bot can create and remove vahti-entries that they have made. Vahti-entries are stored in the sqlite-database of the bot. 6 | The vahtis in the database are periodically checked for new matches on the tori.fi site using an undocumented api endpoint, and new matching listings are then sent to the vahti's creator. 7 | 8 | ![](./media/demo.png) 9 | 10 | ## Features 11 | Available features-flags are 12 | * "tori" 13 | * "huutonet" 14 | * "discord" (both "discord-command" and "discord-delivery") 15 | * "discord-delivery" 16 | * "discord-command" 17 | * "telegram" (both "telegram-command" and "telegram-delivery") 18 | * "telegram-delivery" 19 | * "telegram-command" 20 | 21 | Default features include all the features. 22 | Configure your instance according to your needs with `cargo build --release --no-default-features --features LIST,OF,FEATURES` 23 | Please not that the program does not compile if no site-support is configures (atleast one of features "tori" and "huutonet") 24 | 25 | ## Hosting the bot 26 | ### Discord: 27 | If you do not have a discord application ready create one [here](https://discord.com/developers/applications). Create a bot user for the application if one doesn't already exist. 28 | 29 | When you have your discord application ready, visit the following link to generate an invite link: `https://discord.com/developers/applications/YourAppID/oauth2/url-generator`. 30 | Replace "YourAppID" with the application id of your application. 31 | 32 | The discord application invite link used should have the following scopes: 33 | - `bot` - required for the invite link to be a bot-invite link 34 | - `applications.commands` - required for the bot commands to be usable 35 | 36 | Make sure to create the `.env` file if it does not exist and ensure that it contains all the necessary variables: 37 | * `DATABASE_URL=database.sqlite` (or another location) 38 | * `DISCORD_TOKEN=YourToken` (the token for your discord bot) 39 | * `APPLICATION_ID=YourAppID` (the discord application id) 40 | 41 | Optional variables: 42 | * `UPDATE_INTERVAL=time_in_seconds` (the interval at which the bot updates vahtis, defaults to 60) 43 | * `FUTURES_MAX_BUFFER_SIZE=integer` (the argument given to [buffer\_unordered](https://docs.rs/futures/0.3.28/futures/prelude/stream/trait.StreamExt.html#method.buffer_unordered) defining the amount of concurrent futures. Recommended amount is ~6\*`$(nproc)` and a larger amount may cause problems, defaults to 10) 44 | 45 | ### Telegram: 46 | Create a bot with [@BotFather](https://t.me/botfather) 47 | set `TELOXIDE_TOKEN` value to the API-token. 48 | 49 | Make sure to create the `.env` file if it does not exist and ensure that it contains all the necessary variables: 50 | * `DATABASE_URL=database.sqlite` (or another location) 51 | * `TELOXIDE_TOKEN=YourToken` (the token for your telegram bot) 52 | 53 | Optional variables: 54 | * `UPDATE_INTERVAL=time_in_seconds` (the interval at which the bot updates vahtis, defaults to 60) 55 | * `FUTURES_MAX_BUFFER_SIZE=integer` (the argument given to [buffer\_unordered](https://docs.rs/futures/0.3.28/futures/prelude/stream/trait.StreamExt.html#method.buffer_unordered) defining the amount of concurrent futures. Recommended amount is `50`, raising it above that will most likely bring diminishing returns. Default value is 50) 56 | 57 | ### With Docker 58 | 59 | Bot can be started by running command `docker-compose up -d`. 60 | 61 | Log viewing happens with command `docker-compose logs`. 62 | 63 | `docker-compose up -d --build` rebuilds the Docker image and restarts the container with the new image. 64 | 65 | Bot can be shut down with command `docker-compose down`. 66 | 67 | ### Without Docker 68 | 69 | Before starting the bot you must setup the sqlite-database. This can be done with the `diesel` tool, which is used in these instructions. 70 | 71 | `diesel` can be installed using `cargo install diesel_cli`. 72 | 73 | After installing the `diesel` tool the `reset_db.sh` script can be run 74 | to automatically set up the database, deleting any existing database. 75 | 76 | The binary builds include a pre-initialized database. 77 | 78 | ### Autodeploy 79 | 80 | Use watchtower to pull automatically the latest image. 81 | 82 | #### For databases setup before diesel-migration 83 | 84 | If you have a database with pre-existing data, the `diesel` tool wont be able to apply the migrations. 85 | 86 | In order to run the migrations I've written a simple script that temporarily gets rid of the `initial_migration` 87 | and then runs the migrations. 88 | 89 | **Please remember to change the `DATABASE_URL` to `database.sqlite` instead of `sqlite:database.sqlite` :)** 90 | 91 | Then see `sqlx_migrate.sh` 92 | 93 | ### Running torimies-rs 94 | 95 | **If you are building from source** run `cargo run --release` in the root of the repository. 96 | 97 | **If you are are using a binary build** run `./torimies-rs`. 98 | 99 | ## Using the bot 100 | 101 | The bot has two main commands implemented as application commands (slash-commands) 102 | and those are: 103 | * `/vahti url` Adds a new vahti with the specified url 104 | * `/poistavahti url` Removes the vahti with the specified url 105 | * `/poistaesto` Prompts you with a drop-down menu to select which seller you wish to unblock 106 | 107 | 108 | One additional owner-restricted commmand is also included (this is not a slash-command): 109 | * `!update_all_vahtis` immediately updates all vahtis 110 | 111 | 112 | Please keep in mind that the bot is still considered to be WIP. 113 | We will gladly accept any feedback/feature requests :), just file an [issue](https://github.com/Testausserveri/torimies-rs/issues) and we'll look into it. 114 | -------------------------------------------------------------------------------- /src/tests/tori/api_url.rs: -------------------------------------------------------------------------------- 1 | use super::API_BASE; 2 | use crate::tori::api::vahti_to_api; 3 | 4 | #[test] 5 | fn no_keyword() { 6 | let url = "https://www.tori.fi/koko_suomi?"; 7 | let expected = API_BASE.to_owned(); 8 | assert_eq!(expected, vahti_to_api(url)); 9 | } 10 | 11 | #[test] 12 | fn basic_query() { 13 | let url = "https://www.tori.fi/koko_suomi?q=thinkpad"; 14 | let expected = API_BASE.to_owned() + "q=thinkpad"; 15 | assert_eq!(expected, vahti_to_api(url)); 16 | } 17 | 18 | #[test] 19 | fn query_with_non_ascii() { 20 | let url = "https://www.tori.fi/koko_suomi?q=th%F6nkpad"; 21 | let expected = API_BASE.to_owned() + "q=thönkpad"; 22 | assert_eq!(expected, vahti_to_api(url)); 23 | } 24 | 25 | #[test] 26 | fn query_with_category() { 27 | let url = "https://www.tori.fi/koko_suomi?q=&cg=2030"; 28 | let expected = API_BASE.to_owned() + "q=&category=2030"; 29 | assert_eq!(expected, vahti_to_api(url)); 30 | } 31 | 32 | #[test] 33 | fn query_with_0_category() { 34 | let url = "https://www.tori.fi/koko_suomi?q=&cg=0"; 35 | let expected = API_BASE.to_owned() + "q="; 36 | assert_eq!(expected, vahti_to_api(url)); 37 | } 38 | 39 | #[test] 40 | fn query_with_price_range() { 41 | let url = "https://www.tori.fi/koko_suomi?q=thinkpad&ps=2&pe=4"; 42 | let expected = API_BASE.to_owned() + "q=thinkpad&suborder=50-100"; 43 | assert_eq!(expected, vahti_to_api(url)); 44 | } 45 | 46 | #[test] 47 | fn price_range_no_start() { 48 | let url = "https://www.tori.fi/koko_suomi?q=thinkpad&pe=5"; 49 | let expected = API_BASE.to_owned() + "q=thinkpad&suborder=-250"; 50 | assert_eq!(expected, vahti_to_api(url)); 51 | } 52 | 53 | #[test] 54 | fn price_range_no_end() { 55 | let url = "https://www.tori.fi/koko_suomi?q=thinkpad&ps=6"; 56 | let expected = API_BASE.to_owned() + "q=thinkpad&suborder=500-"; 57 | assert_eq!(expected, vahti_to_api(url)); 58 | } 59 | 60 | #[test] 61 | fn query_with_ad_type() { 62 | let url = "https://www.tori.fi/koko_suomi?q=thinkpad&cg=0&st=s&st=g"; 63 | let expected = API_BASE.to_owned() + "q=thinkpad&ad_type=s&ad_type=g"; 64 | assert_eq!(expected, vahti_to_api(url)); 65 | } 66 | 67 | #[test] 68 | fn query_with_w() { 69 | let url = "https://www.tori.fi/koko_suomi?q=thinkpad&w=3"; 70 | let expected = API_BASE.to_owned() + "q=thinkpad"; 71 | assert_eq!(expected, vahti_to_api(url)); 72 | } 73 | 74 | #[test] 75 | fn query_with_w_region() { 76 | let url = "https://www.tori.fi/koko_suomi?q=thinkpad&w=104"; 77 | let expected = API_BASE.to_owned() + "q=thinkpad®ion=4"; 78 | assert_eq!(expected, vahti_to_api(url)); 79 | } 80 | 81 | #[test] 82 | fn query_with_area() { 83 | let url = "https://www.tori.fi/koko_suomi?q=thinkpad&m=7"; 84 | let expected = API_BASE.to_owned() + "q=thinkpad&area=7"; 85 | assert_eq!(expected, vahti_to_api(url)); 86 | } 87 | 88 | #[test] 89 | fn query_with_ca() { 90 | let url = "https://www.tori.fi/koko_suomi?q=thinkpad&ca=10"; 91 | let expected = API_BASE.to_owned() + "q=thinkpad®ion=10"; 92 | assert_eq!(expected, vahti_to_api(url)); 93 | } 94 | 95 | #[test] 96 | fn query_with_ca_and_w() { 97 | let url = "https://www.tori.fi/koko_suomi?q=thinkpad&w=104&ca=10"; 98 | let expected = API_BASE.to_owned() + "q=thinkpad®ion=4"; 99 | assert_eq!(expected, vahti_to_api(url)); 100 | } 101 | 102 | #[test] 103 | fn query_with_no_argument_name() { 104 | let url = "https://www.tori.fi/koko_suomi?q=thinkpad&=69"; 105 | let expected = API_BASE.to_owned() + "q=thinkpad"; 106 | assert_eq!(expected, vahti_to_api(url)); 107 | } 108 | 109 | #[test] 110 | fn query_with_different_base() { 111 | let url = "https://www.tori.fi/lappi?q=thinkpad"; 112 | let expected = API_BASE.to_owned() + "q=thinkpad"; 113 | assert_eq!(expected, vahti_to_api(url)); 114 | } 115 | 116 | #[test] 117 | fn multiquery1() { 118 | let url = 119 | "https://www.tori.fi/pohjanmaa?q=yoga-matto&cg=0&w=1&st=s&st=k&st=u&st=h&st=g&l=0&md=th"; 120 | let expected = 121 | API_BASE.to_owned() + "q=yoga-matto&ad_type=s&ad_type=k&ad_type=u&ad_type=h&ad_type=g"; 122 | assert_eq!(expected, vahti_to_api(url)); 123 | } 124 | 125 | #[test] 126 | fn multiquery2() { 127 | let url = "https://www.tori.fi/uusimaa?q=vinkulelu+koiralle&cg=0&w=1&st=s&st=k&st=u&st=h&st=g&l=0&md=th"; 128 | let expected = API_BASE.to_owned() 129 | + "q=vinkulelu+koiralle&ad_type=s&ad_type=k&ad_type=u&ad_type=h&ad_type=g"; 130 | assert_eq!(expected, vahti_to_api(url)); 131 | } 132 | 133 | #[test] 134 | fn query_gets_decoded() { 135 | let url = "https://www.tori.fi/koko_suomi?q=th%E4nkpad"; 136 | let expected = API_BASE.to_owned() + "q=thänkpad"; 137 | assert_eq!(expected, vahti_to_api(url)); 138 | } 139 | 140 | #[test] 141 | fn category_from_cg() { 142 | let url = "https://www.tori.fi/koko_suomi?cg=5000"; 143 | let expected = API_BASE.to_owned() + "category=5000"; 144 | assert_eq!(expected, vahti_to_api(url)); 145 | } 146 | 147 | #[test] 148 | fn zero_category_is_ignored() { 149 | let url = "https://www.tori.fi/koko_suomi?cg=0"; 150 | let expected = API_BASE.to_owned(); 151 | assert_eq!(expected, vahti_to_api(url)); 152 | } 153 | 154 | #[test] 155 | fn c_is_prioritized_over_cg() { 156 | let url1 = "https://www.tori.fi/koko_suomi?cg=5010&c=5012"; 157 | let url2 = "https://www.tori.fi/koko_suomi?c=5012&cg=5010"; 158 | let expected = API_BASE.to_owned() + "category=5012"; 159 | 160 | assert_eq!(expected, vahti_to_api(url1)); 161 | assert_eq!(expected, vahti_to_api(url2)); 162 | } 163 | 164 | #[test] 165 | fn ca_region() { 166 | let url = "https://www.tori.fi/li?ca=1"; 167 | let expected = API_BASE.to_owned() + "region=1"; 168 | assert_eq!(expected, vahti_to_api(url)); 169 | } 170 | 171 | #[test] 172 | fn w_region() { 173 | let url = "https://www.tori.fi/li?w=101"; 174 | let expected = API_BASE.to_owned() + "region=1"; 175 | assert_eq!(expected, vahti_to_api(url)); 176 | } 177 | 178 | #[test] 179 | fn w_is_prioritized_over_ca() { 180 | let url1 = "https://www.tori.fi/koko_suomi?w=105&ca=1"; 181 | let url2 = "https://www.tori.fi/koko_suomi?ca=1&w=105"; 182 | let expected = API_BASE.to_owned() + "region=5"; 183 | 184 | assert_eq!(expected, vahti_to_api(url1)); 185 | assert_eq!(expected, vahti_to_api(url2)); 186 | } 187 | 188 | #[test] 189 | fn company_ad() { 190 | let url = "https://www.tori.fi/li?f=c"; 191 | let expected = API_BASE.to_owned() + "company_ad=1"; 192 | assert_eq!(expected, vahti_to_api(url)); 193 | } 194 | 195 | #[test] 196 | fn private_ad() { 197 | let url = "https://www.tori.fi/li?f=p"; 198 | let expected = API_BASE.to_owned() + "company_ad=0"; 199 | assert_eq!(expected, vahti_to_api(url)); 200 | } 201 | 202 | #[test] 203 | fn both_company_and_private_ads() { 204 | let url = "https://www.tori.fi/li?f=a"; 205 | let expected = API_BASE.to_owned(); 206 | assert_eq!(expected, vahti_to_api(url)); 207 | } 208 | -------------------------------------------------------------------------------- /src/delivery/discord.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use async_trait::async_trait; 4 | use chrono::{Local, TimeZone}; 5 | use futures::stream::{self, StreamExt}; 6 | use serenity::builder::{ 7 | CreateActionRow, CreateButton, CreateEmbed, CreateEmbedFooter, CreateMessage, 8 | }; 9 | use serenity::http::Http; 10 | use serenity::model::application::ButtonStyle; 11 | use serenity::model::colour::Color; 12 | 13 | use crate::delivery::Delivery; 14 | use crate::error::Error; 15 | use crate::vahti::VahtiItem; 16 | 17 | pub const ID: i32 = 1; 18 | pub const NAME: &str = "discord"; 19 | 20 | pub struct Discord { 21 | pub http: Arc, 22 | } 23 | 24 | /// This is the discord delivery client 25 | /// There will be a separate client for handling commands 26 | impl Discord { 27 | pub async fn init() -> Result { 28 | let token = 29 | std::env::var("DISCORD_TOKEN").expect("Expected DISCORD_TOKEN in the environment"); 30 | 31 | // NOTE: We don't need a serenity::Client because we don't have to listen to events. 32 | let http = Arc::new(Http::new(&token)); 33 | Ok(Self { http }) 34 | } 35 | 36 | pub async fn destroy(self) {} 37 | } 38 | 39 | impl VahtiItem { 40 | fn embed(self) -> CreateEmbed { 41 | match self.site_id { 42 | #[cfg(feature = "tori")] 43 | crate::tori::ID => { 44 | let color = match self.ad_type.as_str() { 45 | "Myydään" => Color::DARK_GREEN, 46 | "Annetaan" => Color::BLITZ_BLUE, 47 | _ => Color::FADED_PURPLE, 48 | }; 49 | 50 | let e = CreateEmbed::new() 51 | .color(color) 52 | .description(format!("[{}]({})", self.title, self.url)) 53 | .field("Hinta", format!("{} €", self.price), true) 54 | .field( 55 | "Myyjä", 56 | format!( 57 | "[{}](https://www.tori.fi/li?&aid={})", 58 | self.seller_name, self.seller_id 59 | ), 60 | true, 61 | ) 62 | .field("Sijainti", &self.location, true) 63 | .field( 64 | "Ilmoitus Jätetty", 65 | Local 66 | .timestamp_opt(self.published, 0) 67 | .unwrap() 68 | .format("%d/%m/%Y %R") 69 | .to_string(), 70 | true, 71 | ) 72 | .field("Ilmoitustyyppi", self.ad_type.to_string(), true) 73 | .footer(CreateEmbedFooter::new( 74 | self.vahti_url.expect("bug: impossible"), 75 | )); 76 | if !self.img_url.is_empty() { 77 | e.image(&self.img_url) 78 | } else { 79 | e 80 | } 81 | } 82 | #[cfg(feature = "huutonet")] 83 | crate::huutonet::ID => { 84 | let e = CreateEmbed::new() 85 | .color(Color::BLUE) 86 | .description(format!("[{}]({})", self.title, self.url)) 87 | .field("Hinta", format!("{} €", self.price), true) 88 | .field( 89 | "Myyjä", 90 | format!( 91 | "[{}](https://www.huuto.net/kayttaja/{})", 92 | &self.seller_name, self.seller_id 93 | ), 94 | true, 95 | ) 96 | .field("Sijainti", &self.location, true) 97 | .field( 98 | "Ilmoitus Jätetty", 99 | Local 100 | .timestamp_opt(self.published, 0) 101 | .unwrap() 102 | .format("%d/%m/%Y %R") 103 | .to_string(), 104 | true, 105 | ) 106 | .field("Ilmoitustyyppi", self.ad_type.to_string(), true) 107 | .footer(CreateEmbedFooter::new( 108 | self.vahti_url.expect("bug: impossible"), 109 | )); 110 | if !self.img_url.is_empty() { 111 | e.image(&self.img_url) 112 | } else { 113 | e 114 | } 115 | } 116 | i => panic!("Unsupported site_id {}", i), 117 | } 118 | } 119 | } 120 | 121 | #[async_trait] 122 | impl Delivery for Discord { 123 | async fn deliver(&self, items: Vec) -> Result<(), Error> { 124 | let Some(fst) = items.first() else { 125 | return Ok(()); 126 | }; 127 | 128 | assert!(items.iter().all(|i| i.deliver_to == fst.deliver_to)); 129 | 130 | info!( 131 | "Delivering {} items to {}", 132 | items.len(), 133 | fst.deliver_to.unwrap() 134 | ); 135 | 136 | // NOTE: Let's try 5 embeds per message, discord has a limit afaik, but idk 137 | // if the text/character limit will become an issue before the embed limit does 138 | let chunks: Vec> = items.chunks(5).map(|c| c.to_vec()).collect(); 139 | 140 | let http = self.http.clone(); 141 | let recipient = http 142 | .get_user(fst.deliver_to.expect("bug: impossible").into()) 143 | .await?; 144 | 145 | stream::iter(chunks.iter().cloned()) 146 | .map(|is| (is, http.clone(), recipient.clone())) 147 | .map(async move |(items, http, rec)| { 148 | let mut message = CreateMessage::new(); 149 | for item in items { 150 | message = message.add_embed(item.clone().embed()); 151 | } 152 | let buttons = vec![ 153 | CreateButton::new("block_seller") 154 | .label("Estä myyjä") 155 | .style(ButtonStyle::Danger), 156 | CreateButton::new("remove_vahti") 157 | .label("Poista vahti") 158 | .style(ButtonStyle::Danger), 159 | ]; 160 | let row = CreateActionRow::Buttons(buttons); 161 | if cfg!(feature = "discord-command") { 162 | message = message.components(vec![row]); 163 | } 164 | rec.dm(&http, message) 165 | .await 166 | // FIXME: Perhaps don't ignore an error here 167 | .ok() 168 | }) 169 | .buffer_unordered(*crate::FUTURES_MAX_BUFFER_SIZE) 170 | .collect::>() 171 | .await; 172 | 173 | Ok(()) 174 | } 175 | } 176 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | #![feature(lazy_cell, async_closure, iter_array_chunks)] 2 | #![allow(dead_code)] 3 | 4 | #[cfg(test)] 5 | mod tests; 6 | 7 | mod itemhistory; 8 | #[cfg(feature = "tori")] 9 | mod tori; 10 | 11 | #[cfg(feature = "huutonet")] 12 | mod huutonet; 13 | 14 | mod error; 15 | pub mod models; 16 | pub mod schema; 17 | 18 | pub mod command; 19 | pub mod database; 20 | pub mod delivery; 21 | mod vahti; 22 | 23 | #[macro_use] 24 | extern crate tracing; 25 | #[macro_use] 26 | extern crate diesel; 27 | 28 | use std::sync::{Arc, LazyLock, RwLock}; 29 | 30 | use command::{Command, Manager}; 31 | use dashmap::DashMap; 32 | use database::Database; 33 | use delivery::Delivery; 34 | use futures::future::join_all; 35 | use futures::stream::{self, StreamExt}; 36 | 37 | static UPDATE_INTERVAL: LazyLock = LazyLock::new(|| { 38 | std::env::var("UPDATE_INTERVAL") 39 | .unwrap_or(String::from("120")) 40 | .parse() 41 | .expect("Invalid UPDATED_INTERVAL") 42 | }); 43 | 44 | static FUTURES_MAX_BUFFER_SIZE: LazyLock = LazyLock::new(|| { 45 | std::env::var("FUTURES_MAX_BUFFER_SIZE") 46 | .unwrap_or(String::from("50")) 47 | .parse() 48 | .expect("Invalid FUTURES_MAX_BUFFER_SIZE") 49 | }); 50 | 51 | #[derive(PartialEq, Clone)] 52 | enum State { 53 | Running, 54 | Shutdown, 55 | } 56 | 57 | #[derive(Clone)] 58 | struct Torimies { 59 | pub delivery: Arc>>, 60 | pub command: Arc>>, 61 | pub command_manager: Arc>>, 62 | pub database: Database, 63 | pub itemhistorystorage: crate::itemhistory::ItemHistoryStorage, 64 | pub state: Arc>, 65 | } 66 | 67 | // False positive 68 | #[allow(clippy::needless_pass_by_ref_mut)] 69 | async fn update_loop(man: &mut Torimies) { 70 | let mut interval = tokio::time::interval(std::time::Duration::from_secs(*UPDATE_INTERVAL)); 71 | loop { 72 | // Exiting after recieved signal depends on 73 | // 1) the ongoing update 74 | // 2) the following UPDATE_INTERVAL-tick 75 | interval.tick().await; 76 | let mut failcount = 0; 77 | 78 | let state = if let Ok(state) = man.state.read() { 79 | state.clone() 80 | } else { 81 | error!("Failed to read Torimies.state"); 82 | failcount += 1; 83 | if failcount > 2 { 84 | error!("Assuming State::Shutdown"); 85 | State::Shutdown 86 | } else { 87 | State::Running 88 | } 89 | }; 90 | 91 | if state == State::Shutdown { 92 | break; 93 | } 94 | 95 | if let Err(e) = man.update_all_vahtis().await { 96 | error!("Error while updating: {}", e); 97 | } 98 | } 99 | 100 | info!("Update loop exited") 101 | } 102 | 103 | async fn command_loop(man: &Torimies) { 104 | let mut balls = man.command.iter_mut().collect::>(); 105 | let fs = stream::iter(balls.iter_mut()) 106 | .map(async move |c| { 107 | let mut failcount = 0; 108 | while let Err(e) = c.start().await { 109 | error!("Failed to start {} commander {}", c.key(), e); 110 | failcount += 1; 111 | if failcount > 2 { 112 | error!("Giving up with starting {} commander", c.key()); 113 | break; 114 | } 115 | } 116 | }) 117 | .collect::>() 118 | .await; 119 | 120 | join_all(fs).await; 121 | info!("Command loop exited") 122 | } 123 | 124 | async fn ctrl_c_handler(man: &Torimies) { 125 | tokio::signal::ctrl_c() 126 | .await 127 | .expect("Failed to register ctrl+c handler"); 128 | info!("Recieved ctrl+c"); 129 | info!("Setting State to State::Shutdown"); 130 | *man.state.write().unwrap() = State::Shutdown; 131 | 132 | let balls = man.command_manager.iter().collect::>(); 133 | let fs = stream::iter(balls.iter()) 134 | .map(async move |c| c.shutdown().await) 135 | .collect::>() 136 | .await; 137 | join_all(fs).await; 138 | info!("Ctrl+c handler exited"); 139 | } 140 | 141 | impl Torimies { 142 | pub fn new(db: Database) -> Self { 143 | Self { 144 | delivery: Arc::new(DashMap::new()), 145 | command: Arc::new(DashMap::new()), 146 | command_manager: Arc::new(DashMap::new()), 147 | database: db, 148 | itemhistorystorage: Arc::new(DashMap::new()), 149 | state: Arc::new(RwLock::new(State::Running)), 150 | } 151 | } 152 | 153 | fn register_deliverer(&mut self, id: i32, deliverer: T) { 154 | self.delivery.insert(id, Box::new(deliverer)); 155 | } 156 | 157 | fn register_commander( 158 | &mut self, 159 | name: impl ToString, 160 | commander: T, 161 | ) { 162 | self.command_manager 163 | .insert(name.to_string(), commander.manager()); 164 | self.command.insert(name.to_string(), Box::new(commander)); 165 | } 166 | } 167 | 168 | #[tokio::main] 169 | async fn main() { 170 | dotenv::dotenv().expect("Failed to load .env file"); 171 | 172 | tracing_subscriber::fmt::init(); 173 | 174 | let database = Database::new().await; 175 | 176 | let mut the_man = Torimies::new(database); 177 | 178 | #[cfg(feature = "discord-delivery")] 179 | { 180 | let dc = crate::delivery::discord::Discord::init() 181 | .await 182 | .expect("Discord delivery initialization failed"); 183 | 184 | the_man.register_deliverer(crate::delivery::discord::ID, dc); 185 | } 186 | 187 | #[cfg(feature = "discord-command")] 188 | { 189 | let dc = crate::command::discord::Discord::init(&the_man.database.clone()) 190 | .await 191 | .expect("Discord commmand initialization failed"); 192 | 193 | the_man.register_commander(crate::command::discord::NAME, dc); 194 | } 195 | 196 | #[cfg(feature = "telegram-delivery")] 197 | { 198 | let tg = crate::delivery::telegram::Telegram::init() 199 | .await 200 | .expect("Telegram delivery initialization failed"); 201 | 202 | the_man.register_deliverer(crate::delivery::telegram::ID, tg) 203 | } 204 | 205 | #[cfg(feature = "telegram-command")] 206 | { 207 | let tg = crate::command::telegram::Telegram::init(&the_man.database.clone()) 208 | .await 209 | .expect("Telegram commmand initialization failed"); 210 | 211 | the_man.register_commander(crate::command::telegram::NAME, tg); 212 | } 213 | 214 | let the_man2 = the_man.clone(); 215 | let the_man3 = the_man.clone(); 216 | 217 | let update = tokio::task::spawn(async move { update_loop(&mut the_man).await }); 218 | let command = tokio::task::spawn(async move { command_loop(&the_man2).await }); 219 | let ctrl_c = tokio::task::spawn(async move { ctrl_c_handler(&the_man3).await }); 220 | 221 | let _ = futures::join!(update, command, ctrl_c); 222 | } 223 | -------------------------------------------------------------------------------- /src/vahti.rs: -------------------------------------------------------------------------------- 1 | use std::sync::{Arc, LazyLock, Mutex}; 2 | 3 | use async_trait::async_trait; 4 | use futures::stream::{self, StreamExt}; 5 | use itertools::Itertools; 6 | use regex::Regex; 7 | 8 | use crate::database::Database; 9 | use crate::delivery::perform_delivery; 10 | use crate::error::Error; 11 | #[cfg(feature = "huutonet")] 12 | use crate::huutonet::vahti::HuutonetVahti; 13 | use crate::itemhistory::{ItemHistory, ItemHistoryStorage}; 14 | use crate::models::DbVahti; 15 | #[cfg(feature = "tori")] 16 | use crate::tori::vahti::ToriVahti; 17 | use crate::Torimies; 18 | 19 | static SITES: LazyLock, i32)>> = LazyLock::new(|| { 20 | vec![ 21 | #[cfg(feature = "tori")] 22 | (&crate::tori::vahti::TORI_REGEX, crate::tori::ID), 23 | #[cfg(feature = "huutonet")] 24 | (&crate::huutonet::vahti::HUUTONET_REGEX, crate::huutonet::ID), 25 | ] 26 | }); 27 | 28 | // This is the Vahti trait, implementing it (and a couple of other things) 29 | // provides support for a new site 30 | #[async_trait] 31 | pub trait Vahti 32 | where 33 | Self: Sized + Send + Sync, 34 | { 35 | async fn update( 36 | &mut self, 37 | db: &Database, 38 | ihs: ItemHistoryStorage, 39 | ) -> Result, Error>; 40 | async fn validate_url(&self) -> Result; 41 | fn is_valid_url(&self, url: &str) -> bool; 42 | fn from_db(v: DbVahti) -> Result; 43 | fn to_db(&self) -> DbVahti; 44 | } 45 | 46 | #[derive(Clone, Debug, PartialEq)] 47 | pub struct VahtiItem { 48 | pub deliver_to: Option, 49 | pub delivery_method: Option, 50 | pub site_id: i32, 51 | pub title: String, 52 | pub vahti_url: Option, 53 | pub url: String, 54 | pub img_url: String, 55 | pub published: i64, 56 | pub price: i64, 57 | pub seller_name: String, 58 | pub seller_id: i32, 59 | pub location: String, 60 | pub ad_type: String, 61 | pub ad_id: i64, 62 | } 63 | 64 | pub async fn new_vahti( 65 | db: Database, 66 | url: &str, 67 | userid: u64, 68 | delivery_method: i32, 69 | ) -> Result { 70 | let Some(site_id) = SITES 71 | .iter() 72 | .find(|(r, _)| r.is_match(url)) 73 | .map(|(_, sid)| *sid) 74 | else { 75 | return Err(Error::UnknownUrl(url.to_string())); 76 | }; 77 | 78 | if db.fetch_vahti(url, userid as i64).await.is_ok() { 79 | info!("Not adding a pre-defined Vahti {} for user {}", url, userid); 80 | return Err(Error::VahtiExists); 81 | } 82 | 83 | match db 84 | .add_vahti_entry(url, userid as i64, site_id, delivery_method) 85 | .await 86 | { 87 | Ok(_) => Ok(String::from("Vahti added succesfully")), 88 | Err(e) => Err(e), 89 | } 90 | } 91 | 92 | pub async fn remove_vahti( 93 | db: Database, 94 | url: &str, 95 | userid: u64, 96 | delivery_method: i32, 97 | ) -> Result { 98 | if db.fetch_vahti(url, userid as i64).await.is_err() { 99 | info!("Not removing a nonexistant vahti!"); 100 | return Ok( 101 | "A Vahti is not defined with that url. Make sure the url is correct".to_string(), 102 | ); 103 | } 104 | match db 105 | .remove_vahti_entry(url, userid as i64, delivery_method) 106 | .await 107 | { 108 | Ok(_) => Ok("Vahti removed!".to_string()), 109 | Err(e) => Err(e), 110 | } 111 | } 112 | 113 | impl Torimies { 114 | pub async fn update_all_vahtis(&mut self) -> Result<(), Error> { 115 | let vahtis = self.database.fetch_all_vahtis().await?; 116 | self.update_vahtis(vahtis).await?; 117 | Ok(()) 118 | } 119 | 120 | pub async fn update_vahtis(&mut self, vahtis: Vec) -> Result<(), Error> { 121 | info!("Updating {} vahtis", vahtis.len()); 122 | let start = std::time::Instant::now(); 123 | 124 | let ihs = self.itemhistorystorage.clone(); 125 | 126 | // NOTE: pre-populate ItemHistoryStorage to prevent deadlocks on inserts 127 | // this must not be done concurrently and must be done while there are 128 | // no references (mutable or unmutable) into the ihs dashmap 129 | vahtis.iter().for_each(|v| { 130 | if !ihs.contains_key(&(v.user_id as u64, v.delivery_method)) { 131 | ihs.insert( 132 | (v.user_id as u64, v.delivery_method), 133 | Arc::new(Mutex::new(ItemHistory::new())), 134 | ); 135 | } 136 | }); 137 | 138 | let db = self.database.clone(); 139 | let dm = self.delivery.clone(); 140 | 141 | let items = stream::iter(vahtis.iter().cloned()) 142 | .map(|v| (v, ihs.clone(), db.clone())) 143 | .map(async move |(v, ihs, db)| match v.site_id { 144 | #[cfg(feature = "tori")] 145 | crate::tori::ID => { 146 | let Ok(mut tv) = ToriVahti::from_db(v) else { 147 | return vec![]; 148 | }; 149 | 150 | tv.update(&db, ihs.clone()).await.unwrap_or_default() 151 | } 152 | #[cfg(feature = "huutonet")] 153 | crate::huutonet::ID => { 154 | if let Ok(mut hv) = HuutonetVahti::from_db(v) { 155 | hv.update(&db, ihs.clone()).await.unwrap_or_default() 156 | } else { 157 | vec![] 158 | } 159 | } 160 | i => panic!("Unsupported site_id {}", i), 161 | }) 162 | .buffer_unordered(*crate::FUTURES_MAX_BUFFER_SIZE) 163 | .collect::>() 164 | .await; 165 | 166 | info!("Recieving items took {}ms", start.elapsed().as_millis()); 167 | 168 | let groups: Vec> = items 169 | .iter() 170 | .flatten() 171 | .group_by(|v| { 172 | ( 173 | v.deliver_to.expect("bug: impossible"), 174 | v.delivery_method.expect("bug: impossible"), 175 | ) 176 | }) 177 | .into_iter() 178 | .map(|(_, g)| g.cloned().unique_by(|v| v.ad_id).collect()) 179 | .collect(); 180 | 181 | // False positive, because we actually want to .await the future elsewhere 182 | #[allow(clippy::async_yields_async)] 183 | stream::iter( 184 | groups 185 | .iter() 186 | .map(|v| (v, db.clone())) 187 | .map(async move |(v, db)| { 188 | let mut v = v.clone(); 189 | 190 | if let Some(fst) = v.first() { 191 | // NOTE: If db fails, blacklisted sellers are not filtered out 192 | if let Ok(bl) = db 193 | .fetch_user_blacklist(fst.deliver_to.expect("bug: impossible") as i64) 194 | .await 195 | { 196 | v.retain(|i| !bl.contains(&(i.seller_id, i.site_id))); 197 | } 198 | } 199 | v 200 | }) 201 | .map(|v| (v, dm.clone())), 202 | ) 203 | .then(|(v, dm)| async move { perform_delivery(dm.clone(), v.await) }) 204 | .for_each_concurrent(*crate::FUTURES_MAX_BUFFER_SIZE, |d| async move { 205 | d.await.ok(); 206 | }) 207 | .await; 208 | 209 | info!("Update took {}ms", start.elapsed().as_millis()); 210 | Ok(()) 211 | } 212 | } 213 | -------------------------------------------------------------------------------- /src/database.rs: -------------------------------------------------------------------------------- 1 | use core::time::Duration; 2 | use std::collections::BTreeMap; 3 | use std::env; 4 | 5 | use diesel::connection::SimpleConnection; 6 | use diesel::prelude::*; 7 | use diesel::r2d2::{ConnectionManager, CustomizeConnection, Pool}; 8 | use diesel::sqlite::SqliteConnection; 9 | use serenity::prelude::TypeMapKey; 10 | 11 | use crate::error::Error; 12 | use crate::models::*; 13 | 14 | #[derive(Clone)] 15 | pub struct Database { 16 | database: Pool>, 17 | } 18 | 19 | impl TypeMapKey for Database { 20 | type Value = Database; 21 | } 22 | 23 | #[derive(Debug)] 24 | pub struct ConnectionOptions { 25 | pub enable_wal: bool, 26 | pub enable_foreign_keys: bool, 27 | pub busy_timeout: Option, 28 | } 29 | 30 | impl CustomizeConnection for ConnectionOptions { 31 | fn on_acquire(&self, conn: &mut SqliteConnection) -> Result<(), diesel::r2d2::Error> { 32 | (|| { 33 | if self.enable_wal { 34 | conn.batch_execute("PRAGMA journal_mode = WAL; PRAGMA synchronous = NORMAL;")?; 35 | } 36 | if self.enable_foreign_keys { 37 | conn.batch_execute("PRAGMA foreign_keys = ON;")?; 38 | } 39 | if let Some(d) = self.busy_timeout { 40 | conn.batch_execute(&format!("PRAGMA busy_timeout = {};", d.as_millis()))?; 41 | } 42 | Ok(()) 43 | })() 44 | .map_err(diesel::r2d2::Error::QueryError) 45 | } 46 | } 47 | 48 | impl Database { 49 | pub async fn new() -> Database { 50 | dotenv::dotenv().ok(); 51 | 52 | let database_url = env::var("DATABASE_URL").expect("DATABASE_URL must be set"); 53 | 54 | let manager = ConnectionManager::::new(database_url); 55 | let database = Pool::builder() 56 | .max_size(16) 57 | .connection_customizer(Box::new(ConnectionOptions { 58 | enable_wal: true, 59 | enable_foreign_keys: false, 60 | busy_timeout: Some(Duration::from_secs(30)), 61 | })) 62 | .build(manager) 63 | .expect("Failed to create connection pool"); 64 | 65 | Self { database } 66 | } 67 | 68 | pub async fn add_vahti_entry( 69 | &self, 70 | arg_url: &str, 71 | userid: i64, 72 | site_id: i32, 73 | delivery_method: i32, 74 | ) -> Result { 75 | let time = chrono::Local::now().timestamp(); 76 | info!("Adding Vahti `{}` for the user {}", arg_url, userid); 77 | use crate::schema::Vahdit; 78 | let new_vahti = NewVahti { 79 | last_updated: time, 80 | url: arg_url.to_string(), 81 | user_id: userid, 82 | site_id, 83 | delivery_method, 84 | }; 85 | Ok(diesel::insert_into(Vahdit::table) 86 | .values(&new_vahti) 87 | .execute(&self.database.get()?)?) 88 | } 89 | 90 | pub async fn remove_vahti_entry( 91 | &self, 92 | arg_url: &str, 93 | userid: i64, 94 | delivery: i32, 95 | ) -> Result { 96 | info!("Removing Vahti `{}` from the user {}", arg_url, userid); 97 | use crate::schema::Vahdit::dsl::*; 98 | Ok(diesel::delete( 99 | Vahdit.filter( 100 | url.eq(arg_url) 101 | .and(user_id.eq(userid)) 102 | .and(delivery_method.eq(delivery)), 103 | ), 104 | ) 105 | .execute(&self.database.get()?)?) 106 | } 107 | 108 | pub async fn fetch_vahti_entries_by_url(&self, arg_url: &str) -> Result, Error> { 109 | info!("Fetching Vahtis {}...", arg_url); 110 | use crate::schema::Vahdit::dsl::*; 111 | Ok(Vahdit 112 | .filter(url.eq(arg_url)) 113 | .load::(&self.database.get()?)?) 114 | } 115 | 116 | pub async fn fetch_vahti_entries_by_user_id(&self, userid: i64) -> Result, Error> { 117 | info!("Fetching the Vahtis of user {}...", userid); 118 | use crate::schema::Vahdit::dsl::*; 119 | Ok(Vahdit 120 | .filter(user_id.eq(userid)) 121 | .load::(&self.database.get()?)?) 122 | } 123 | 124 | pub async fn fetch_vahti(&self, arg_url: &str, userid: i64) -> Result { 125 | info!("Fetching the user {}'s Vahti {}...", userid, arg_url); 126 | use crate::schema::Vahdit::dsl::*; 127 | Ok(Vahdit 128 | .filter(user_id.eq(userid).and(url.eq(arg_url))) 129 | .first::(&self.database.get()?)?) 130 | } 131 | 132 | pub async fn fetch_all_vahtis(&self) -> Result, Error> { 133 | info!("Fetching all Vahtis..."); 134 | use crate::schema::Vahdit::dsl::*; 135 | Ok(Vahdit.load::(&self.database.get()?)?) 136 | } 137 | 138 | pub async fn fetch_all_vahtis_group(&self) -> Result>, Error> { 139 | // FIXME: This could be done in sql 140 | info!("Fetching all vahtis grouping them by url"); 141 | let vahdit = self.fetch_all_vahtis().await?; 142 | let ret: BTreeMap> = 143 | vahdit.into_iter().fold(BTreeMap::new(), |mut acc, v| { 144 | acc.entry(v.url.clone()).or_default().push(v); 145 | acc 146 | }); 147 | Ok(ret) 148 | } 149 | 150 | pub async fn vahti_updated( 151 | &self, 152 | vahti: DbVahti, 153 | timestamp: Option, 154 | ) -> Result { 155 | info!( 156 | "Updating Vahti {} for the user {}", 157 | vahti.url, vahti.user_id 158 | ); 159 | use crate::schema::Vahdit::dsl::*; 160 | let time = timestamp.unwrap_or_else(|| chrono::Local::now().timestamp()); 161 | info!( 162 | "Newest item {}s ago", 163 | chrono::Local::now().timestamp() - time 164 | ); 165 | Ok(diesel::update( 166 | Vahdit.filter( 167 | url.eq(vahti.url) 168 | .and(user_id.eq(vahti.user_id).and(last_updated.lt(time))), 169 | ), 170 | ) 171 | .set(last_updated.eq(time)) 172 | .execute(&self.database.get()?)?) 173 | } 174 | 175 | pub async fn fetch_user_blacklist(&self, userid: i64) -> Result, Error> { 176 | debug!("Fetching the blacklist for user {}...", userid); 177 | use crate::schema::Blacklists::dsl::*; 178 | Ok(Blacklists 179 | .filter(user_id.eq(userid)) 180 | .select((seller_id, site_id)) 181 | .load::<(i32, i32)>(&self.database.get()?)?) 182 | } 183 | 184 | pub async fn add_seller_to_blacklist( 185 | &self, 186 | userid: i64, 187 | sellerid: i32, 188 | siteid: i32, 189 | ) -> Result { 190 | info!( 191 | "Adding seller {} to the blacklist of user {}", 192 | sellerid, userid 193 | ); 194 | use crate::schema::Blacklists; 195 | let new_entry = NewBlacklist { 196 | user_id: userid, 197 | seller_id: sellerid, 198 | site_id: siteid, 199 | }; 200 | Ok(diesel::insert_into(Blacklists::table) 201 | .values(new_entry) 202 | .execute(&self.database.get()?)?) 203 | } 204 | 205 | pub async fn remove_seller_from_blacklist( 206 | &self, 207 | userid: i64, 208 | sellerid: i32, 209 | siteid: i32, 210 | ) -> Result { 211 | info!( 212 | "Removing seller {} from the blacklist of user {}", 213 | sellerid, userid 214 | ); 215 | use crate::schema::Blacklists::dsl::*; 216 | Ok(diesel::delete( 217 | Blacklists.filter( 218 | user_id 219 | .eq(userid) 220 | .and(seller_id.eq(sellerid)) 221 | .and(site_id.eq(siteid)), 222 | ), 223 | ) 224 | .execute(&self.database.get()?)?) 225 | } 226 | } 227 | -------------------------------------------------------------------------------- /src/command/discord/interaction.rs: -------------------------------------------------------------------------------- 1 | use itertools::Itertools; 2 | use serenity::all::ComponentInteractionDataKind; 3 | use serenity::builder::{ 4 | CreateActionRow, CreateSelectMenu, CreateSelectMenuOption, EditInteractionResponse, 5 | }; 6 | use serenity::model::application::Interaction; 7 | use serenity::prelude::*; 8 | 9 | use super::extensions::ClientContextExt; 10 | 11 | pub fn menu_from_options( 12 | custom_id: &str, 13 | options: Vec<(impl ToString, impl ToString)>, 14 | ) -> Vec { 15 | let menu_options = options 16 | .iter() 17 | .map(|(l, v)| CreateSelectMenuOption::new(l.to_string(), v.to_string())) 18 | .collect::>(); 19 | let menu = CreateSelectMenu::new( 20 | custom_id, 21 | serenity::builder::CreateSelectMenuKind::String { 22 | options: menu_options, 23 | }, 24 | ); 25 | vec![CreateActionRow::SelectMenu(menu)] 26 | } 27 | 28 | pub async fn handle_interaction(ctx: Context, interaction: Interaction) { 29 | match interaction { 30 | Interaction::Command(command) => { 31 | command.defer_ephemeral(&ctx.http).await.unwrap(); 32 | 33 | let content = match command.data.name.as_str() { 34 | "vahti" => super::vahti::run(&ctx, &command).await, 35 | "poistavahti" => super::poistavahti::run(&ctx, &command).await, 36 | "poistaesto" => super::poistaesto::run(&ctx, &command).await, 37 | _ => unreachable!(), 38 | }; 39 | 40 | if !content.is_empty() { 41 | command 42 | .edit_response(&ctx.http, EditInteractionResponse::new().content(&content)) 43 | .await 44 | .unwrap(); 45 | } 46 | } 47 | Interaction::Component(button) => { 48 | if button.data.custom_id == "remove_vahti" { 49 | button.defer_ephemeral(&ctx.http).await.unwrap(); 50 | let message = button.message.clone(); 51 | let urls: Vec<_> = message 52 | .embeds 53 | .iter() 54 | .filter_map(|e| e.footer.as_ref().map(|f| f.text.clone())) 55 | .unique() 56 | .collect(); 57 | 58 | if !urls.is_empty() { 59 | button 60 | .edit_response( 61 | &ctx.http, 62 | EditInteractionResponse::new().components(menu_from_options( 63 | "remove_vahti_menu", 64 | urls.iter().zip(urls.iter()).collect::>(), 65 | )), 66 | ) 67 | .await 68 | .unwrap(); 69 | } else { 70 | button 71 | .edit_response(&ctx.http, 72 | EditInteractionResponse::new().content("Creating Vahti deletion menu failed, try deleting the Vahti manually with /poistavahti") 73 | ) 74 | .await.unwrap(); 75 | } 76 | } else if button.data.custom_id == "block_seller" { 77 | button.defer_ephemeral(&ctx.http).await.unwrap(); 78 | let message = button.message.clone(); 79 | 80 | let urls: Vec<_> = message 81 | .embeds 82 | .iter() 83 | .filter_map(|e| e.footer.as_ref().map(|f| f.text.clone())) 84 | .collect(); 85 | 86 | assert!(!urls.is_empty(), "Cannot determine search url"); 87 | 88 | let sellers = message 89 | .embeds 90 | .iter() 91 | .map(|e| e.fields.iter().find(|f| f.name == "Myyjä")) 92 | .filter_map(|f| f.map(|ff| ff.value.clone())) 93 | .filter_map(|s| match s { 94 | #[cfg(feature = "tori")] 95 | _ if s.contains("https://www.tori.fi/li?&aid=") => Some(( 96 | s[1..s.find(']').unwrap()].to_string(), 97 | format!( 98 | "{},{}", 99 | &s[s.rfind('=').unwrap() + 1..s.find(')').unwrap()], 100 | crate::tori::ID 101 | ), 102 | )), 103 | #[cfg(feature = "huutonet")] 104 | _ if s.contains("https://www.huuto.net/kayttaja/") => Some(( 105 | s[1..s.find(']').unwrap()].to_string(), 106 | format!( 107 | "{},{}", 108 | &s[s.rfind('/').unwrap() + 1..s.find(')').unwrap()], 109 | crate::huutonet::ID 110 | ), 111 | )), 112 | _ => None, 113 | }) 114 | .unique() 115 | .collect::>(); 116 | 117 | button 118 | .edit_response( 119 | &ctx.http, 120 | EditInteractionResponse::new() 121 | .content("Choose the seller to block") 122 | .components(menu_from_options("block_seller_menu", sellers)), 123 | ) 124 | .await 125 | .unwrap(); 126 | } else if button.data.custom_id == "unblock_seller" { 127 | button.defer_ephemeral(&ctx.http).await.unwrap(); 128 | let db = ctx.get_db().await.unwrap(); 129 | let userid = u64::from(button.user.id); 130 | let ids: Vec = match button.data.kind.clone() { 131 | ComponentInteractionDataKind::StringSelect { values } => { 132 | values[0].split(',').map(|s| s.to_string()).collect() 133 | } 134 | _ => unreachable!(), 135 | }; 136 | let sellerid = ids[0].parse::().unwrap(); 137 | let siteid = ids[1].parse::().unwrap(); 138 | 139 | db.remove_seller_from_blacklist(userid.try_into().unwrap(), sellerid, siteid) 140 | .await 141 | .unwrap(); 142 | button 143 | .edit_response( 144 | &ctx.http, 145 | EditInteractionResponse::new().content("Esto poistettu!"), 146 | ) 147 | .await 148 | .unwrap(); 149 | } else if button.data.custom_id == "remove_vahti_menu" { 150 | button.defer_ephemeral(&ctx.http).await.unwrap(); 151 | let userid = u64::from(button.user.id); 152 | let url = match button.data.kind.clone() { 153 | ComponentInteractionDataKind::StringSelect { values } => values[0].to_string(), 154 | _ => unreachable!(), 155 | }; 156 | let db = ctx.get_db().await.unwrap(); 157 | 158 | crate::vahti::remove_vahti(db, &url, userid, crate::delivery::discord::ID) 159 | .await 160 | .unwrap(); 161 | button 162 | .edit_response( 163 | &ctx.http, 164 | EditInteractionResponse::new() 165 | .content(format!("Poistettu vahti: `{}`", url)), 166 | ) 167 | .await 168 | .unwrap(); 169 | } else if button.data.custom_id.starts_with("remove_vahti_menu_page_") { 170 | let page_number: usize = button 171 | .data 172 | .custom_id 173 | .strip_prefix("remove_vahti_menu_page_") 174 | .unwrap() 175 | .parse() 176 | .unwrap(); 177 | 178 | button 179 | .create_response( 180 | &ctx.http, 181 | serenity::builder::CreateInteractionResponse::UpdateMessage( 182 | super::poistavahti::update_message( 183 | &ctx, 184 | page_number, 185 | u64::from(button.user.id), 186 | ) 187 | .await, 188 | ), 189 | ) 190 | .await 191 | .unwrap(); 192 | return; 193 | } else if button.data.custom_id == "block_seller_menu" { 194 | button.defer_ephemeral(&ctx.http).await.unwrap(); 195 | let db = ctx.get_db().await.unwrap(); 196 | let userid = u64::from(button.user.id); 197 | let ids: Vec = match button.data.kind.clone() { 198 | ComponentInteractionDataKind::StringSelect { values } => { 199 | values[0].split(',').map(|s| s.to_string()).collect() 200 | } 201 | _ => unreachable!(), 202 | }; 203 | let sellerid = ids[0].parse::().unwrap(); 204 | let siteid = ids[1].parse::().unwrap(); 205 | 206 | db.add_seller_to_blacklist(userid as i64, sellerid, siteid) 207 | .await 208 | .unwrap(); 209 | button 210 | .edit_response( 211 | &ctx.http, 212 | EditInteractionResponse::new().content("Myyjä estetty!"), 213 | ) 214 | .await 215 | .unwrap(); 216 | } 217 | } 218 | _ => {} 219 | } 220 | } 221 | -------------------------------------------------------------------------------- /src/tests/tori/parse.rs: -------------------------------------------------------------------------------- 1 | use std::fs::File; 2 | use std::io::Read; 3 | 4 | use crate::tori::parse::api_parse_after; 5 | use crate::vahti::VahtiItem; 6 | 7 | #[test] 8 | fn basic_parse() { 9 | let mut file = File::open("testdata/tori/basic_parse.json").expect("Test data not found"); 10 | let mut contents = String::new(); 11 | file.read_to_string(&mut contents).unwrap(); 12 | 13 | let expected = VahtiItem { 14 | deliver_to: None, 15 | delivery_method: None, 16 | site_id: crate::tori::ID, 17 | title: "Maalaisromanttinen peltipurkki ja eläimiä".to_string(), 18 | vahti_url: None, 19 | url: "https://www.tori.fi/vi/81076530.htm".to_string(), 20 | img_url: "https://images.tori.fi/api/v1/imagestori/images/9039260397.jpg?rule=medium_660" 21 | .to_string(), 22 | published: 1614890870, 23 | price: 7, 24 | seller_name: "H.S.M".to_string(), 25 | seller_id: 188169, 26 | location: "Maunula-Suursuo, Helsinki, Uusimaa".to_string(), 27 | ad_type: "Myydään".to_string(), 28 | ad_id: 79217488, 29 | }; 30 | 31 | assert_eq!( 32 | *api_parse_after(&contents, 0).unwrap().first().unwrap(), 33 | expected 34 | ); 35 | } 36 | 37 | #[test] 38 | fn parse_after() { 39 | let mut file = File::open("testdata/tori/parse_after.json").expect("Test data not found"); 40 | let mut contents = String::new(); 41 | file.read_to_string(&mut contents).unwrap(); 42 | 43 | assert_eq!(api_parse_after(&contents, 1651416320).unwrap().len(), 1); 44 | assert_eq!(api_parse_after(&contents, 1651416319).unwrap().len(), 2); 45 | } 46 | 47 | #[test] 48 | fn parse_multiple() { 49 | let mut file = File::open("testdata/tori/parse_multiple.json").expect("Test data not found"); 50 | let mut contents = String::new(); 51 | file.read_to_string(&mut contents).unwrap(); 52 | 53 | let mut expected = vec![ 54 | VahtiItem { 55 | deliver_to: None, 56 | delivery_method: None, 57 | site_id: 1, 58 | title: "Naamiaisasu ".to_string(), 59 | vahti_url: None, 60 | url: "https://www.tori.fi/vi/107951227.htm".to_string(), 61 | img_url: 62 | "https://images.tori.fi/api/v1/imagestori/images/7574231064.jpg?rule=medium_660" 63 | .to_string(), 64 | published: 1674035937, 65 | price: 25, 66 | seller_name: "Erja Latva".to_string(), 67 | seller_id: 289139, 68 | location: "Suvilahti, Vaasa, Pohjanmaa".to_string(), 69 | ad_type: "Myydään".to_string(), 70 | ad_id: 107463388, 71 | }, 72 | VahtiItem { 73 | deliver_to: None, 74 | delivery_method: None, 75 | site_id: 1, 76 | title: "Ninebot by Segway KickScooter sähköpotkulauta F25E".to_string(), 77 | vahti_url: None, 78 | url: "https://www.tori.fi/vi/103805389.htm".to_string(), 79 | img_url: 80 | "https://images.tori.fi/api/v1/imagestori/images/100146113672.jpg?rule=medium_660" 81 | .to_string(), 82 | published: 1673531834, 83 | price: 339, 84 | seller_name: "Gigantti outlet Vaasa".to_string(), 85 | seller_id: 3237298, 86 | location: "Asevelikylä, Vaasa, Pohjanmaa".to_string(), 87 | ad_type: "Myydään".to_string(), 88 | ad_id: 103120642, 89 | }, 90 | VahtiItem { 91 | deliver_to: None, 92 | delivery_method: None, 93 | site_id: 1, 94 | title: "Meta Quest 2 Elite hihna + akku".to_string(), 95 | vahti_url: None, 96 | url: "https://www.tori.fi/vi/108452916.htm".to_string(), 97 | img_url: 98 | "https://images.tori.fi/api/v1/imagestori/images/100181065412.jpg?rule=medium_660" 99 | .to_string(), 100 | published: 1675057180, 101 | price: 143, 102 | seller_name: "Gigantti outlet Vaasa".to_string(), 103 | seller_id: 3237298, 104 | location: "Asevelikylä, Vaasa, Pohjanmaa".to_string(), 105 | ad_type: "Myydään".to_string(), 106 | ad_id: 107987389, 107 | }, 108 | VahtiItem { 109 | deliver_to: None, 110 | delivery_method: None, 111 | site_id: 1, 112 | title: "Sea-doo rxt-x 300 rs".to_string(), 113 | vahti_url: None, 114 | url: "https://www.tori.fi/vi/106281850.htm".to_string(), 115 | img_url: 116 | "https://images.tori.fi/api/v1/imagestori/images/100170569995.jpg?rule=medium_660" 117 | .to_string(), 118 | published: 1674365101, 119 | price: 16990, 120 | seller_name: "Rinta-Joupin Autoliike, Tervajoki".to_string(), 121 | seller_id: 2349504, 122 | location: "Tervajoki, Laihia, Pohjanmaa".to_string(), 123 | ad_type: "Myydään".to_string(), 124 | ad_id: 105715838, 125 | }, 126 | VahtiItem { 127 | deliver_to: None, 128 | delivery_method: None, 129 | site_id: 1, 130 | title: "ASUS PRIME Z790-P D4 ATX emolevy".to_string(), 131 | vahti_url: None, 132 | url: "https://www.tori.fi/vi/107247726.htm".to_string(), 133 | img_url: 134 | "https://images.tori.fi/api/v1/imagestori/images/100171951188.jpg?rule=medium_660" 135 | .to_string(), 136 | published: 1675853738, 137 | price: 268, 138 | seller_name: "Gigantti outlet Vaasa".to_string(), 139 | seller_id: 3237298, 140 | location: "Asevelikylä, Vaasa, Pohjanmaa".to_string(), 141 | ad_type: "Myydään".to_string(), 142 | ad_id: 106730945, 143 | }, 144 | VahtiItem { 145 | deliver_to: None, 146 | delivery_method: None, 147 | site_id: 1, 148 | title: "Bosch Ladattava pölynimuri BBH3ZOO28 (tornadon)".to_string(), 149 | vahti_url: None, 150 | url: "https://www.tori.fi/vi/106947918.htm".to_string(), 151 | img_url: 152 | "https://images.tori.fi/api/v1/imagestori/images/100168416209.jpg?rule=medium_660" 153 | .to_string(), 154 | published: 1675842778, 155 | price: 174, 156 | seller_name: "Gigantti outlet Vaasa".to_string(), 157 | seller_id: 3237298, 158 | location: "Asevelikylä, Vaasa, Pohjanmaa".to_string(), 159 | ad_type: "Myydään".to_string(), 160 | ad_id: 106414054, 161 | }, 162 | VahtiItem { 163 | deliver_to: None, 164 | delivery_method: None, 165 | site_id: 1, 166 | title: "Miele hood 90cm black".to_string(), 167 | vahti_url: None, 168 | url: "https://www.tori.fi/vi/106692075.htm".to_string(), 169 | img_url: "".to_string(), 170 | published: 1675869730, 171 | price: 3329, 172 | seller_name: "Gigantti outlet Vaasa".to_string(), 173 | seller_id: 3237298, 174 | location: "Asevelikylä, Vaasa, Pohjanmaa".to_string(), 175 | ad_type: "Myydään".to_string(), 176 | ad_id: 106144962, 177 | }, 178 | VahtiItem { 179 | deliver_to: None, 180 | delivery_method: None, 181 | site_id: 1, 182 | title: "Ninebot by Segway KickScooter sähköpotkulauta E25D".to_string(), 183 | vahti_url: None, 184 | url: "https://www.tori.fi/vi/101906085.htm".to_string(), 185 | img_url: 186 | "https://images.tori.fi/api/v1/imagestori/images/100134901177.jpg?rule=medium_660" 187 | .to_string(), 188 | published: 1675853818, 189 | price: 402, 190 | seller_name: "Gigantti outlet Vaasa".to_string(), 191 | seller_id: 3237298, 192 | location: "Asevelikylä, Vaasa, Pohjanmaa".to_string(), 193 | ad_type: "Myydään".to_string(), 194 | ad_id: 101130082, 195 | }, 196 | VahtiItem { 197 | deliver_to: None, 198 | delivery_method: None, 199 | site_id: 1, 200 | title: "Mercury F20EPS".to_string(), 201 | vahti_url: None, 202 | url: "https://www.tori.fi/vi/109023706.htm".to_string(), 203 | img_url: 204 | "https://images.tori.fi/api/v1/imagestori/images/100185031992.jpg?rule=medium_660" 205 | .to_string(), 206 | published: 1676283023, 207 | price: 3700, 208 | seller_name: "Rinta-Joupin Autoliike, Tervajoki".to_string(), 209 | seller_id: 2349504, 210 | location: "Tervajoki, Laihia, Pohjanmaa".to_string(), 211 | ad_type: "Myydään".to_string(), 212 | ad_id: 108584455, 213 | }, 214 | VahtiItem { 215 | deliver_to: None, 216 | delivery_method: None, 217 | site_id: 1, 218 | title: "Savo hood a".to_string(), 219 | vahti_url: None, 220 | url: "https://www.tori.fi/vi/101984681.htm".to_string(), 221 | img_url: "".to_string(), 222 | published: 1675873122, 223 | price: 299, 224 | seller_name: "Gigantti outlet Vaasa".to_string(), 225 | seller_id: 3237298, 226 | location: "Asevelikylä, Vaasa, Pohjanmaa".to_string(), 227 | ad_type: "Myydään".to_string(), 228 | ad_id: 101212772, 229 | }, 230 | VahtiItem { 231 | deliver_to: None, 232 | delivery_method: None, 233 | site_id: 1, 234 | title: "E.t.m sports& casuals 52".to_string(), 235 | vahti_url: None, 236 | url: "https://www.tori.fi/vi/109489503.htm".to_string(), 237 | img_url: 238 | "https://images.tori.fi/api/v1/imagestori/images/100188042265.jpg?rule=medium_660" 239 | .to_string(), 240 | published: 1677350539, 241 | price: 20, 242 | seller_name: "moternimies".to_string(), 243 | seller_id: 2695759, 244 | location: "Vanha Vaasa, Vaasa, Pohjanmaa".to_string(), 245 | ad_type: "Myydään".to_string(), 246 | ad_id: 109060376, 247 | }, 248 | VahtiItem { 249 | deliver_to: None, 250 | delivery_method: None, 251 | site_id: 1, 252 | title: "SoFlow sähköpotkulauta SOFLOW01".to_string(), 253 | vahti_url: None, 254 | url: "https://www.tori.fi/vi/99732961.htm".to_string(), 255 | img_url: 256 | "https://images.tori.fi/api/v1/imagestori/images/100124634411.jpg?rule=medium_660" 257 | .to_string(), 258 | published: 1676294880, 259 | price: 297, 260 | seller_name: "Gigantti outlet Vaasa".to_string(), 261 | seller_id: 3237298, 262 | location: "Asevelikylä, Vaasa, Pohjanmaa".to_string(), 263 | ad_type: "Myydään".to_string(), 264 | ad_id: 98836530, 265 | }, 266 | VahtiItem { 267 | deliver_to: None, 268 | delivery_method: None, 269 | site_id: 1, 270 | title: "Audio Pro G10 älykäs monihuonekaiutin (vaaleanharm".to_string(), 271 | vahti_url: None, 272 | url: "https://www.tori.fi/vi/91855652.htm".to_string(), 273 | img_url: 274 | "https://images.tori.fi/api/v1/imagestori/images/100104289690.jpg?rule=medium_660" 275 | .to_string(), 276 | published: 1677846660, 277 | price: 167, 278 | seller_name: "Gigantti outlet Vaasa".to_string(), 279 | seller_id: 3237298, 280 | location: "Asevelikylä, Vaasa, Pohjanmaa".to_string(), 281 | ad_type: "Myydään".to_string(), 282 | ad_id: 90554189, 283 | }, 284 | ]; 285 | 286 | let mut got = api_parse_after(&contents, 0).unwrap(); 287 | 288 | expected.sort_by_key(|v| v.ad_id); 289 | got.sort_by_key(|v| v.ad_id); 290 | 291 | let _ = expected 292 | .iter() 293 | .zip(got.iter()) 294 | .map(|(a, b)| assert_eq!(a, b)) 295 | .collect::>(); 296 | } 297 | -------------------------------------------------------------------------------- /src/tests/huutonet/parse.rs: -------------------------------------------------------------------------------- 1 | use std::fs::File; 2 | use std::io::Read; 3 | 4 | use crate::huutonet::parse::api_parse_after; 5 | use crate::vahti::VahtiItem; 6 | 7 | #[test] 8 | fn basic_parse() { 9 | let mut file = File::open("testdata/huutonet/basic_parse.json").expect("Test data not found"); 10 | let mut contents = String::new(); 11 | file.read_to_string(&mut contents).unwrap(); 12 | 13 | let expected = VahtiItem { 14 | deliver_to: None, 15 | delivery_method: None, 16 | site_id: crate::huutonet::ID, 17 | title: "Tekniikan Maailma 20/1993".to_string(), 18 | vahti_url: None, 19 | url: "https://www.huuto.net/kohteet/tekniikan-maailma-20_1993/575647318".to_string(), 20 | img_url: "https://kuvat.huuto.net/v1/a777/9ca312c77fbf51f301afec055e4/505225227-m.jpg" 21 | .to_string(), 22 | published: 1674021288, 23 | price: 4, 24 | seller_name: "kodin".to_string(), 25 | seller_id: 241366, 26 | location: "SAARENTAUS".to_string(), 27 | ad_type: "buy-now".to_string(), 28 | ad_id: 575647318, 29 | }; 30 | 31 | assert_eq!( 32 | *api_parse_after(&contents, 0).unwrap().first().unwrap(), 33 | expected 34 | ); 35 | } 36 | 37 | #[test] 38 | fn parse_after() { 39 | let mut file = File::open("testdata/huutonet/parse_after.json").expect("Test data not found"); 40 | let mut contents = String::new(); 41 | file.read_to_string(&mut contents).unwrap(); 42 | 43 | assert_eq!(api_parse_after(&contents, 1669983035).unwrap().len(), 1); 44 | assert_eq!(api_parse_after(&contents, 1669983034).unwrap().len(), 2); 45 | } 46 | 47 | #[test] 48 | fn parse_multiple() { 49 | let mut file = 50 | File::open("testdata/huutonet/parse_multiple.json").expect("Test data not found"); 51 | let mut contents = String::new(); 52 | file.read_to_string(&mut contents).unwrap(); 53 | 54 | let mut expected = vec![ 55 | VahtiItem { 56 | deliver_to: None, 57 | delivery_method: None, 58 | site_id: 2, 59 | title: "Lenovo ThinkPad Workstation Dock telakointiasema".to_string(), 60 | vahti_url: None, 61 | url: "https://www.huuto.net/kohteet/lenovo-thinkpad-workstation-dock-telakointiasema/578236742".to_string(), 62 | img_url: "https://kuvat.huuto.net/v1/6082/695e42d788604ea88de676285aa/508366924-m.jpg".to_string(), 63 | published: 1678355586, 64 | price: 13, 65 | seller_name: "ITJari".to_string(), 66 | seller_id: 2732468, 67 | location: "HELSINKI".to_string(), 68 | ad_type: "auction".to_string(), 69 | ad_id: 578236742, 70 | }, 71 | VahtiItem { 72 | deliver_to: None, 73 | delivery_method: None, 74 | site_id: 2, 75 | title: "Sierra Wireless AirPrime 4G LTE".to_string(), 76 | vahti_url: None, 77 | url: "https://www.huuto.net/kohteet/sierra-wireless-airprime-4g-lte/578174408".to_string(), 78 | img_url: "https://kuvat.huuto.net/v1/728e/66af711226504dfd72ba7f58e98/508288586-m.jpg".to_string(), 79 | published: 1678255722, 80 | price: 10, 81 | seller_name: "nick00".to_string(), 82 | seller_id: 2914998, 83 | location: "TAMPERE".to_string(), 84 | ad_type: "buy-now".to_string(), 85 | ad_id: 578174408, 86 | }, 87 | VahtiItem { 88 | deliver_to: None, 89 | delivery_method: None, 90 | site_id: 2, 91 | title: "Lenovo ThinkPad 65W slim -virtalähde".to_string(), 92 | vahti_url: None, 93 | url: "https://www.huuto.net/kohteet/lenovo-thinkpad-65w-slim--virtalahde/578086441".to_string(), 94 | img_url: "https://kuvat.huuto.net/v1/e2d6/c6fef7201d482325a798711b175/508181197-m.jpg".to_string(), 95 | published: 1678114915, 96 | price: 15, 97 | seller_name: "prossu1".to_string(), 98 | seller_id: 2366051, 99 | location: "OULU".to_string(), 100 | ad_type: "auction".to_string(), 101 | ad_id: 578086441, 102 | }, 103 | VahtiItem { 104 | deliver_to: None, 105 | delivery_method: None, 106 | site_id: 2, 107 | title: "Lenovo ThinkPad Quectel SDX24 EM120R-GL WWAN 4G modeemi".to_string(), 108 | vahti_url: None, 109 | url: "https://www.huuto.net/kohteet/lenovo-thinkpad-quectel-sdx24-em120r-gl-wwan-4g-modeemi/578085280".to_string(), 110 | img_url: "https://kuvat.huuto.net/v1/a1d5/c41fa515e150d20db90c614f981/507236793-m.jpg".to_string(), 111 | published: 1678113543, 112 | price: 82, 113 | seller_name: "tarsiger".to_string(), 114 | seller_id: 808553, 115 | location: "VANTAA".to_string(), 116 | ad_type: "buy-now".to_string(), 117 | ad_id: 578085280, 118 | }, 119 | VahtiItem { 120 | deliver_to: None, 121 | delivery_method: None, 122 | site_id: 2, 123 | title: "Thinkpad W541 / P50 170W virtalähde".to_string(), 124 | vahti_url: None, 125 | url: "https://www.huuto.net/kohteet/thinkpad-w541-_-p50--170w-virtalahde/578082963".to_string(), 126 | img_url: "https://kuvat.huuto.net/v1/827b/80cf9ca765dd302da0e5df02144/507234984-m.jpg".to_string(), 127 | published: 1678111203, 128 | price: 42, 129 | seller_name: "tarsiger".to_string(), 130 | seller_id: 808553, 131 | location: "VANTAA".to_string(), 132 | ad_type: "buy-now".to_string(), 133 | ad_id: 578082963, 134 | }, 135 | VahtiItem { 136 | deliver_to: None, 137 | delivery_method: None, 138 | site_id: 2, 139 | title: "Thinkpad näppäimistö UK".to_string(), 140 | vahti_url: None, 141 | url: "https://www.huuto.net/kohteet/thinkpad-nappaimisto-uk/578028877".to_string(), 142 | img_url: "https://kuvat.huuto.net/v1/3569/8614db847be72cec6cf7085879d/508110054-m.jpg".to_string(), 143 | published: 1678024148, 144 | price: 3, 145 | seller_name: "hnetti".to_string(), 146 | seller_id: 1456413, 147 | location: "HELSINKI".to_string(), 148 | ad_type: "auction".to_string(), 149 | ad_id: 578028877, 150 | }, 151 | VahtiItem { 152 | deliver_to: None, 153 | delivery_method: None, 154 | site_id: 2, 155 | title: "Lenovo Thinkpad T15 Gen 2 (20W400HGMX)".to_string(), 156 | vahti_url: None, 157 | url: "https://www.huuto.net/kohteet/lenovo-thinkpad-t15-gen-2-20w400hgmx/578026882".to_string(), 158 | img_url: "https://kuvat.huuto.net/v1/04e4/304693717970e81fa61d0e0b988/508107118-m.jpg".to_string(), 159 | published: 1678022037, 160 | price: 790, 161 | seller_name: "attekorte".to_string(), 162 | seller_id: 24060, 163 | location: "JUUKA".to_string(), 164 | ad_type: "auction".to_string(), 165 | ad_id: 578026882, 166 | }, 167 | VahtiItem { 168 | deliver_to: None, 169 | delivery_method: None, 170 | site_id: 2, 171 | title: "ThinkPad Thunderbolt 3 Workstation Dock Gen 1 + 230W sekä 65".to_string(), 172 | vahti_url: None, 173 | url: "https://www.huuto.net/kohteet/thinkpad-thunderbolt-3-workstation-dock-gen-1--230w-seka-65/578013764".to_string(), 174 | img_url: "https://kuvat.huuto.net/v1/b1ac/d2b7cf0db9ee812a1d4d5a747a6/505840652-m.jpg".to_string(), 175 | published: 1678006781, 176 | price: 125, 177 | seller_name: "tarsiger".to_string(), 178 | seller_id: 808553, 179 | location: "VANTAA".to_string(), 180 | ad_type: "buy-now".to_string(), 181 | ad_id: 578013764, 182 | }, 183 | VahtiItem { 184 | deliver_to: None, 185 | delivery_method: None, 186 | site_id: 2, 187 | title: "Lenovo Thinkpad kosketuslevy E440 L440 T440 W540".to_string(), 188 | vahti_url: None, 189 | url: "https://www.huuto.net/kohteet/lenovo-thinkpad-kosketuslevy-e440-l440-t440-w540/577859186".to_string(), 190 | img_url: "https://kuvat.huuto.net/v1/e985/ac742e77b9ba14af63999f684a7/507888772-m.jpg".to_string(), 191 | published: 1677736062, 192 | price: 10, 193 | seller_name: "nick00".to_string(), 194 | seller_id: 2914998, 195 | location: "TAMPERE".to_string(), 196 | ad_type: "buy-now".to_string(), 197 | ad_id: 577859186, 198 | }, 199 | VahtiItem { 200 | deliver_to: None, 201 | delivery_method: None, 202 | site_id: 2, 203 | title: "16 GB DDR4 2666V SO-DIMM muistia".to_string(), 204 | vahti_url: None, 205 | url: "https://www.huuto.net/kohteet/16-gb-ddr4-2666v-so-dimm-muistia/577756539".to_string(), 206 | img_url: "https://kuvat.huuto.net/v1/947d/a3662a375d2cfc001cc3807f0a8/507751477-m.jpg".to_string(), 207 | published: 1677589947, 208 | price: 50, 209 | seller_name: "nick00".to_string(), 210 | seller_id: 2914998, 211 | location: "TAMPERE".to_string(), 212 | ad_type: "buy-now".to_string(), 213 | ad_id: 577756539, 214 | }, 215 | VahtiItem { 216 | deliver_to: None, 217 | delivery_method: None, 218 | site_id: 2, 219 | title: "Lenovo ThinkPad laturi 135W".to_string(), 220 | vahti_url: None, 221 | url: "https://www.huuto.net/kohteet/lenovo-thinkpad-laturi-135w/577619805".to_string(), 222 | img_url: "https://kuvat.huuto.net/v1/6b2d/0a6af834366a26eaeffb9a53379/507582568-m.jpg".to_string(), 223 | published: 1677406913, 224 | price: 20, 225 | seller_name: "nick00".to_string(), 226 | seller_id: 2914998, 227 | location: "TAMPERE".to_string(), 228 | ad_type: "buy-now".to_string(), 229 | ad_id: 577619805, 230 | }, 231 | VahtiItem { 232 | deliver_to: None, 233 | delivery_method: None, 234 | site_id: 2, 235 | title: "Lenovo Thinkpad X270 M.2 levykelkka + NVMe SSD levy".to_string(), 236 | vahti_url: None, 237 | url: "https://www.huuto.net/kohteet/lenovo-thinkpad-x270-m2-levykelkka--nvme-ssd-levy/577619781".to_string(), 238 | img_url: "https://kuvat.huuto.net/v1/7e21/fa76d89f482de7d00a425a30fc5/507582527-m.jpg".to_string(), 239 | published: 1677406880, 240 | price: 50, 241 | seller_name: "nick00".to_string(), 242 | seller_id: 2914998, 243 | location: "TAMPERE".to_string(), 244 | ad_type: "buy-now".to_string(), 245 | ad_id: 577619781, 246 | }, 247 | VahtiItem { 248 | deliver_to: None, 249 | delivery_method: None, 250 | site_id: 2, 251 | title: "Lenovo Thinkpad T470 M.2 levykelkka + NVMe SSD levy".to_string(), 252 | vahti_url: None, 253 | url: "https://www.huuto.net/kohteet/lenovo-thinkpad-t470-m2-levykelkka--nvme-ssd-levy/577619757".to_string(), 254 | img_url: "https://kuvat.huuto.net/v1/623c/e888f05644235d3d21167448bd2/507582487-m.jpg".to_string(), 255 | published: 1677406857, 256 | price: 50, 257 | seller_name: "nick00".to_string(), 258 | seller_id: 2914998, 259 | location: "TAMPERE".to_string(), 260 | ad_type: "buy-now".to_string(), 261 | ad_id: 577619757, 262 | }, 263 | VahtiItem { 264 | deliver_to: None, 265 | delivery_method: None, 266 | site_id: 2, 267 | title: "Lenovo ThinkPad läppärilaukku 15.6\"".to_string(), 268 | vahti_url: None, 269 | url: "https://www.huuto.net/kohteet/lenovo-thinkpad-lapparilaukku-156/577600532".to_string(), 270 | img_url: "https://kuvat.huuto.net/v1/214e/cf42a5ab8ee533cd6c22c88c2c7/507558554-m.jpg".to_string(), 271 | published: 1677369494, 272 | price: 15, 273 | seller_name: "Melviini".to_string(), 274 | seller_id: 2245306, 275 | location: "HELSINKI".to_string(), 276 | ad_type: "buy-now".to_string(), 277 | ad_id: 577600532, 278 | }, 279 | VahtiItem { 280 | deliver_to: None, 281 | delivery_method: None, 282 | site_id: 2, 283 | title: "Lenovo ThinkPad advanced minidock telakka ja laturi".to_string(), 284 | vahti_url: None, 285 | url: "https://www.huuto.net/kohteet/lenovo-thinkpad-advanced-minidock-telakka-ja-laturi/577519756".to_string(), 286 | img_url: "https://kuvat.huuto.net/v1/2204/5bd90ca1f70cc00a5135ec2ea8a/507461267-m.jpg".to_string(), 287 | published: 1677231594, 288 | price: 30, 289 | seller_name: "Joulubuggi".to_string(), 290 | seller_id: 2942, 291 | location: "TAMPERE".to_string(), 292 | ad_type: "buy-now".to_string(), 293 | ad_id: 577519756, 294 | }, 295 | VahtiItem { 296 | deliver_to: None, 297 | delivery_method: None, 298 | site_id: 2, 299 | title: "Lenovo ThinkPad T540p, 15.5\" 3K (2880 x 1620), IPS".to_string(), 300 | vahti_url: None, 301 | url: "https://www.huuto.net/kohteet/lenovo-thinkpad-t540p-155-3k-2880-x-1620-ips/576827868".to_string(), 302 | img_url: "https://kuvat.huuto.net/v1/a71e/987ba35692c59217306a00185f6/506669279-m.jpg".to_string(), 303 | published: 1676145945, 304 | price: 600, 305 | seller_name: "hammermann".to_string(), 306 | seller_id: 1398678, 307 | location: "TAMPERE".to_string(), 308 | ad_type: "buy-now".to_string(), 309 | ad_id: 576827868, 310 | }, 311 | VahtiItem { 312 | deliver_to: None, 313 | delivery_method: None, 314 | site_id: 2, 315 | title: "IBM Thinkpad X20 + 2 kpl telakka".to_string(), 316 | vahti_url: None, 317 | url: "https://www.huuto.net/kohteet/ibm-thinkpad-x20--2-kpl-telakka/576564494".to_string(), 318 | img_url: "https://kuvat.huuto.net/v1/4c37/1913a47d3811d624df9df4d93ce/506353523-m.jpg".to_string(), 319 | published: 1675671542, 320 | price: 400, 321 | seller_name: "hammermann".to_string(), 322 | seller_id: 1398678, 323 | location: "TAMPERE".to_string(), 324 | ad_type: "buy-now".to_string(), 325 | ad_id: 576564494, 326 | }, 327 | VahtiItem { 328 | deliver_to: None, 329 | delivery_method: None, 330 | site_id: 2, 331 | title: "IBM Thinkpad T43 + laturi + telakka + Win XP Pro".to_string(), 332 | vahti_url: None, 333 | url: "https://www.huuto.net/kohteet/ibm-thinkpad-t43--laturi--telakka---win-xp-pro/576378051".to_string(), 334 | img_url: "https://kuvat.huuto.net/v1/d904/7a54900391694410d3dab1343f6/506108127-m.jpg".to_string(), 335 | published: 1675324587, 336 | price: 250, 337 | seller_name: "hammermann".to_string(), 338 | seller_id: 1398678, 339 | location: "TAMPERE".to_string(), 340 | ad_type: "buy-now".to_string(), 341 | ad_id: 576378051, 342 | }, 343 | VahtiItem { 344 | deliver_to: None, 345 | delivery_method: None, 346 | site_id: 2, 347 | title: "512MB PC100 SODIMM".to_string(), 348 | vahti_url: None, 349 | url: "https://www.huuto.net/kohteet/512mb-pc100-sodimm/573442000".to_string(), 350 | img_url: "https://kuvat.huuto.net/v1/ab36/4c199038f4e8703cfc67aae5654/502037093-m.jpg".to_string(), 351 | published: 1670393684, 352 | price: 9, 353 | seller_name: "countryguy".to_string(), 354 | seller_id: 1304585, 355 | location: "IMATRA".to_string(), 356 | ad_type: "buy-now".to_string(), 357 | ad_id: 573442000, 358 | }, 359 | ]; 360 | 361 | let mut got = api_parse_after(&contents, 0).unwrap(); 362 | 363 | expected.sort_by_key(|v| v.ad_id); 364 | got.sort_by_key(|v| v.ad_id); 365 | 366 | let _ = expected 367 | .iter() 368 | .zip(got.iter()) 369 | .map(|(a, b)| assert_eq!(a, b)) 370 | .collect::>(); 371 | } 372 | -------------------------------------------------------------------------------- /docs/tori-apispec.md: -------------------------------------------------------------------------------- 1 | # Tori.fi api specification 2 | 3 | This document describes our current understanding of the Tori.fi API. 4 | Please note that all of the information here is gathered through reverse-engineering the API 5 | and none of it is confirmed in any way. 6 | 7 | Torimies relies on the Tori.fi API rather than scraping the website. This allows Torimies to 8 | be faster and more accurate than any of its scraper alternatives. 9 | 10 | Using the API also comes with some problems. 11 | Tori.fi has not released any public documentation for the API which means that all features 12 | used by Torimies were discovered through different methods of reverse-engineering the 13 | Tori.fi API. 14 | 15 | ## API Overview 16 | 17 | The Tori-API `BASE\_URL` used by Torimies is `https://api.tori.fi/api/v1.2` 18 | 19 | The Tori.fi API consists of multiple routes and endpoints of which Torimies 20 | only utilizes the ads endpoint found at `$BASE_URL/public/ads` which allows retrieving 21 | a filtered list of Tori.fi-ads using query-parameters for the filtering. 22 | 23 | In a nutshell the Tori.fi API is an excellent example on **how to not design an API** and 24 | it is highly recommended to not proceed further in this document if one has the intention 25 | to preserve their sanity. 26 | 27 | ## `/public/ads` 28 | 29 | The endpoint for fetching a filtered list of ads. 30 | 31 | The filters can be applied with query-parameters. 32 | 33 | The output has the following type: 34 | ```yaml 35 | { 36 | config_etag: string, 37 | counter_map: { 38 | all: int 39 | }, 40 | list_ads: [ 41 | { 42 | ad: { 43 | account: { 44 | code: string, 45 | label: string, 46 | }, 47 | account_ads: { 48 | code: string, 49 | label: string, 50 | }, 51 | ad_id: string, 52 | body: string, 53 | category: { 54 | code: string, 55 | label: string, 56 | name: string, 57 | path_en: string, 58 | parent: string, 59 | aurora_vertical: string, 60 | }, 61 | company_ad: bool, 62 | full_details: bool, 63 | images: [ 64 | { 65 | base_url: string, 66 | media_id: string, 67 | path: string, 68 | width: int, 69 | height: int, 70 | 71 | }, 72 | ... 73 | ], 74 | list_id: string, 75 | list_id_code: string, 76 | list_price: { 77 | currency: string, 78 | price_value: int, 79 | label: string, 80 | }, 81 | locations: [ 82 | { 83 | code: string, 84 | key: string, 85 | label: string 86 | locations: [ 87 | ... 88 | ] 89 | }, 90 | ... 91 | ], 92 | mc_settings: { 93 | use_form: boolean, 94 | }, 95 | phone_hidden: boolean, 96 | prices: [ 97 | { 98 | currency: string, 99 | price_value: int, 100 | label: string, 101 | old_price: { 102 | price_value: int, 103 | label: string, 104 | } 105 | }, 106 | ... 107 | ], 108 | polepos_ad: int, 109 | status: string, 110 | store_details: { 111 | id: string, 112 | name: string, 113 | plan: string, 114 | slogan: string, 115 | address: string, 116 | city: string, 117 | zipcode: string, 118 | category: string, 119 | link: string, 120 | }, 121 | subject: string, 122 | thumbnail: { 123 | base_url: string, 124 | media_id: string, 125 | path: string, 126 | width: int, 127 | height: int, 128 | }, 129 | type: { 130 | code: string, 131 | label: string, 132 | }, 133 | user: { 134 | account: { 135 | name: string, 136 | created: string, 137 | }, 138 | uuid: string, 139 | }, 140 | share_link: string, 141 | link: { 142 | label: string, 143 | url: string, 144 | }, 145 | highlight_price: boolean, 146 | external_integration: { 147 | url: string, 148 | label: string, 149 | type: string, 150 | }, 151 | pivo: { 152 | enable: bool 153 | }, 154 | list_time: { 155 | label: string, 156 | value: int, 157 | } 158 | }, 159 | labelmap: { 160 | category: string, 161 | type: string, 162 | }, 163 | spt_metadata: { 164 | category: string, 165 | contentid: string, 166 | details: { 167 | currency: string, 168 | locality: string, 169 | postalCode: string, 170 | price: string, 171 | region: string, 172 | }, 173 | } 174 | }, 175 | ... 176 | ], 177 | next_page: string, 178 | proximity_slices: [], 179 | sorting: string, 180 | spt_metadata: [ 181 | contentid: string, 182 | filter: { 183 | currency: string, 184 | numResults: int, 185 | } 186 | ] 187 | } 188 | ``` 189 | 190 | ### Fields and descriptions (as far as we know them) 191 | 192 | Below is a table containing some of the fields in the above example response paired with brief descriptions. 193 | Some fields we deem irrelevant are intentionally left out. 194 | 195 | #### DISCLAIMER 196 | All the descriptions provided below are educated guesses and not facts. One is recommended 197 | to do their own research before blindly utilizing any information provided here. 198 | 199 | | Field | Type | Description | 200 | |-------|------|-------------| 201 | | counter\_map | Object | Contains a single field `all` which contains the total count of ads in the response | 202 | | counter\_map.all | int | The total count of ads in the response | 203 | | list\_ads | Array | A list of Objects each of which the fields `ad`, `labelmap` and `spt_metadata`, The Object in the `ad` is described below in great detail | 204 | | sorting | string | The sorting method or whatever value is specified with `?sort=value`, defaults to `date`. The only other value that does something seems to be `price` | 205 | | spt\_metadata | Object | Some metadata...? | 206 | | spt\_metadata.filter | Object | Contains two fields `currency` and `numResults` | 207 | | spt\_metadata.filter.currency | string | Value is `EUR` we are not aware of any way to change this | 208 | | spt\_metadata.filter.numResults | int | Same value as `counter_map.all`, **NOTE: for some reason the name is camelCase :D** | 209 | 210 | ### Ad-Object 211 | 212 | For Torimies' purposes the most important part of the response is the object describing an individual ad. 213 | Therefore it is also the most in-detail part of this document. 214 | 215 | Below is a brief summary of each field and their description (if known). 216 | 217 | #### NOTE 218 | Some of the fields are not always present. This makes for a great deal of fun and deserializer-errors :] 219 | 220 | ##### TODO: 221 | Quite a lot of `ad_details` are missing.. `clothing_*` and `car_*`, `cx_*`, `regdate` to give a few examples. 222 | These will be more important once we start implementing support for Tori Autot. 223 | 224 | 225 | | Field | Type | Description | 226 | |-------|------|-------------| 227 | | account | Object | The publisher of the ad | 228 | | account.code | string | The account id (a number) of the publisher account **as a string** | 229 | | account.label | string | Exactly the same as `account.code` | 230 | | account\_ads | Object | Total ad count of the publisher account | 231 | | ad\_id | string | Not an id, actually a path of format `/private/accounts/{account_id}/ads/{ad_id}` where the *filename* is the actual ad id which is an integer | 232 | | body | string | The ad body, the description written by the publisher | 233 | | category | Object | Describes the category the ad belongs in | 234 | | category.code | string | The category id (a number) **as a string** | 235 | | category.label | string | The category label, for example `Leikkurit ja koneet` | 236 | | category.name | string | An empty string | 237 | | category.path\_en | string | An empty string | 238 | | category.parent | string | An empty string | 239 | | category.aurora\_vertical | string | No idea what this describes, values include `mobility` and `recommerce` | 240 | | company\_ad | bool | Whether the publisher is a company or not | 241 | | ad\_details | Object | Contains some ad-specific details. These often depend on the type of item being sold and therefore are not always present | 242 | | ad\_details.delivery\_options | Object | The available delivery options | 243 | | ad\_details.delivery\_options.multiple | Array | List of available delivery options, `multiple` seems to be always present on `delivery_options` regardless of the actual amount of delivery options| 244 | | ad\_details.delivery\_options.multiple[index] | Object | A delivery option | 245 | | ad\_details.delivery\_options.multiple[index].code | String | The code for a delivery option, for example `delivery_send` | 246 | | ad\_details.delivery\_options.multiple[index].label | String | The label for a delivery option, for example `Lähetys` | 247 | | ad\_details.general\_condition | Object | The general condition of the item | 248 | | ad\_details.general\_condition.single | Object | The general condition of the item | 249 | | ad\_details.general\_condition.single.code | String | The code for the condition, for example `excellent` | 250 | | ad\_details.general\_condition.single.label | String | The label for the condition, for example `Erinomainen` | 251 | | images | Array | List of images associated with the ad | 252 | | images[index] | Object | An individual image object | 253 | | images[index].base\_url | string | The base url for images, the value seems to always be `https://img.tori.fi/image` | 254 | | images[index].media\_id | string | Not an id, actually the base path for the ad images of format `/public/media/ad/{ad_id}` | 255 | | images[index].path | string | A path of format `10/{image_id}` where the `image_id` is a number and actually the only thing we care about in this Object. One can fetch the image from the following url `"https://images.tori.fi/api/v1/imagestori/images/{image_id}?rule=medium_660"` | 256 | | images[index].width | int | The width of the image | 257 | | images[index].height | int | The height of the image | 258 | | list\_id | string | A path of the format `/public/ads/{ad_id}` | 259 | | list\_id\_code | string | The actual `ad_id` (a number) **as a string** | 260 | | list\_price | Object | Contains information about the current price for the ad | 261 | | list\_price.currency | string | The currency for the price, seems to always be `EUR` | 262 | | list\_price.price\_value | int | The value for the price | 263 | | list\_price.label | string | The value paired with a currency symbol, for example `995 €` | 264 | | locations | Array | A list of locations that always contains one item, a location-Object which is a recursive data-type | 265 | | locations[0].code | string | The code for the location (a number) **as a string** | 266 | | locations[0].key | string | The key for the location for example `region`, `area` or `zipcode` | 267 | | locations[0].label | string | The label for the location for example `Kymenlaakso`, `Kouvola` or `Korjala-Kaunisnurmi` | 268 | | locations[0].locations | Array | The sub-locations list of the same type as the `locations`-array (might not be present) | 269 | | mc\_settings | Object | No idea | 270 | | mc\_settings.use\_form | bool | No idea | 271 | | phone\_hidden | bool | Whether or not to hide the phone number??! | 272 | | prices | Array | A List of price-Objects, always of length 1 | 273 | | prices[0] | Object | An object depicting a price (current or historical) for the ad, a recursive data-type | 274 | | prices[0].currency | string | The currency for the price, present on the current price | 275 | | prices[0].price\_value | int | The value for the price | 276 | | prices[0].label | string | The value paired with a currency symbol | 277 | | prices[0].old\_price | Object | Of same type as `prices[0]`, describes an older price. The field is not present if there is no older price | 278 | | status | string | The ad status, seems to always be `active` | 279 | | subject | string | The subject for the ad | 280 | | thumbnail | Object | The ad thumbnail, same type as `images[index]` | 281 | | type | Object | The ad type | 282 | | type.code | string | The code for the ad type, for example `s` | 283 | | type.label | string | The label for the ad type, for example `Myydään` | 284 | | user | Object | The publisher user | 285 | | user.account | Object | The publisher user-account (not the same as `account`) | 286 | | user.account.name | string | The username for the publisher account | 287 | | user.account.created | string | A string depicting the account age, for example `maaliskuusta 2023` | 288 | | user.uuid | string | The uuid for the publisher | 289 | | share\_link | string | The share link for the ad | 290 | | pivo | Object | No idea | 291 | | pivo.enabled | bool | No idea | 292 | | list\_time | Object | The time the listing was published **NOTE: please see further information below** | 293 | | list\_time.label | string | The label for the time, for example `tänään 17:50` | 294 | | list\_time.value | int | A unix EPOCH timestamp for the release time (either UTC or Finnish local time) | 295 | 296 | ### The thing about timestamps 297 | 298 | Yeah.. so there's a thing... 299 | 300 | One could think that the timestamp in `list_time` is assigned to a new ad on creation and 301 | doesn't change after that. That would seem more than reasonable. But as it turn out 302 | that is in fact not the case. 303 | 304 | There seems to be a time period of approximately 10 minutes after a new ad is published to Tori.fi 305 | during which the `list_time` of that specific ad is updated to the time when it is requested. 306 | One can even experience this phenomena live by opening up a new Tori.fi ad, waiting for a minute 307 | and then refreshing the page just to see the creation time update before their own eyes. 308 | 309 | Torimies has to get around this somehow and that is why we implement the `ItemHistory`. 310 | But yeah... this is the stuff that makes one go insane. 311 | 312 | ## Query parameters 313 | 314 | This is a brief overview of the API query options. This only covers the options relevant for Torimies but there 315 | are many more of them. 316 | 317 | [This endpoint](https://api.tori.fi/api/v1.7/public/filters) seems to list out some of the options. 318 | 319 | ### Account 320 | 321 | Torimies uses the ads endpoint with the `account={account_id}` argument to fetch information about a seller account. 322 | This is a bit sub-optimal due to the fact that no account information is available for an account with no active ads. 323 | 324 | ### Limit 325 | 326 | The number of ads returned can be limited with the argument `lim={number}` 327 | 328 | ### Location 329 | Available values listed [here](https://api.tori.fi/api/v1.2/public/regions). 330 | 331 | The location is specified with three parameters. Only the first one of which is supported by the Tori.fi search. 332 | Those parameters are `region`, `area` and `zipcode`. 333 | 334 | #### Examples: 335 | | Location | Arguments | 336 | |----------|-----------| 337 | | Ahvenanmaa | `region=15` | 338 | | Ahvenanmaa, Brändö | `region=15&area=256` | 339 | 340 | Multiple regions and can be selected by just chaining multiple `region` arguments. 341 | 342 | ### Category 343 | Available values listed [here](https://api.tori.fi/api/v1.2/public/categories/filter). 344 | 345 | #### Examples 346 | | Category | Arguments | 347 | |----------|-----------| 348 | | Elektroniikka | `category=5000` | 349 | | Elektroniikka, Puhelimet | `category=5012` | 350 | 351 | ### Ad type 352 | 353 | Specified with the `ad_type` parameter. Valid values include 354 | | Value | Meaning | 355 | |-------|---------| 356 | | s | Myydään | 357 | | k | Ostetaan | 358 | | g | Annetaan | 359 | | u | Vuokrataan | 360 | | h | Halutaan vuokrata | 361 | 362 | ## Tori.fi query parameters 363 | 364 | The most difficult task for Torimies is the conversion between Tori.fi-search urls 365 | and tori api query urls. This is by no means an easy task since the argument conversion rules range from slightly inconvenient 366 | to completely nonsensical. 367 | 368 | Here is the set of conversion rules used by Torimies 369 | 370 | | Search url argument | API-query argument | Description | 371 | |---------------------|--------------------|---------| 372 | | `q` | `q` | The search keywords (uses the ISO-8859-2 encoding) | 373 | | `cg` | `category` | The category, the value of `0` is ignored | 374 | | `c` | `category` | Also the category, used over the `cg` value if present (this contains sub-category information) | 375 | | `ps` and `pe` | `suborder` | Suborder specifies a price range. See the conversion table below | 376 | | `ca` | `region` | The region | 377 | | `m` | `area` | The area | 378 | | `w` | `region` | The region, the conversion is done by subtracting 100 from the w-value, if the w value is below 100 it should be ignored otherwise the value should be used over any `ca` argument | 379 | | `f` | `company_ad` | Whether the ad is from a company. The conversion is the following: `a=>ignore`,`p=>0`,`c=>1` | 380 | | `st` | `ad_type` | The ad type described above | 381 | | Any other argument | ignored/unsupported | Anything else is ignored so that the query isn't messed up | 382 | 383 | ### Price range conversion table 384 | 385 | The price ranges in the tori.fi search url are converted to the suborder argument 386 | according to the following table 387 | 388 | | `ps`/`pe` value | `suborder` value | 389 | |-----------------|------------------| 390 | | 0 | 0 | 391 | | 1 | 25 | 392 | | 2 | 50 | 393 | | 3 | 75 | 394 | | 4 | 100 | 395 | | 5 | 250 | 396 | | 6 | 500 | 397 | | 7 | 1000 | 398 | | 8 | 2000 | 399 | -------------------------------------------------------------------------------- /testdata/huutonet/parse_multiple.json: -------------------------------------------------------------------------------- 1 | {"totalCount":19,"updated":"2023-03-09T12:23:45+0200","links":{"self":"https:\/\/api.huuto.net\/1.1\/items?words=thinkpad&classification=like-new&page=1&sort=newest","first":"https:\/\/api.huuto.net\/1.1\/items?words=thinkpad&classification=like-new&page=1&sort=newest","last":"https:\/\/api.huuto.net\/1.1\/items?words=thinkpad&classification=like-new&page=1&sort=newest","previous":null,"next":null,"gallery":"https:\/\/api.huuto.net\/1.1\/galleries\/items?words=thinkpad&classification=like-new&sort=newest","hits":"https:\/\/api.huuto.net\/1.1\/hits?words=thinkpad&classification=like-new&sort=newest"},"items":[{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/578236742","category":"https:\/\/api.huuto.net\/1.1\/categories\/897","alternative":"https:\/\/www.huuto.net\/kohteet\/lenovo-thinkpad-workstation-dock-telakointiasema\/578236742","images":"https:\/\/api.huuto.net\/1.1\/items\/578236742\/images"},"id":578236742,"title":"Lenovo ThinkPad Workstation Dock telakointiasema","category":"Kannettavien tietokoneiden akut ja tarvikkeet","seller":"ITJari","sellerId":2732468,"currentPrice":13,"buyNowPrice":null,"saleMethod":"auction","listTime":"2023-03-09T11:53:06+0200","postalCode":"00170","location":"HELSINKI","closingTime":"2023-03-10T15:55:00+0200","bidderCount":0,"offerCount":0,"hasReservePrice":false,"hasReservePriceExceeded":false,"upgrades":[],"images":[{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/578236742\/images\/508366924","thumbnail":"https:\/\/kuvat.huuto.net\/v1\/6082\/695e42d788604ea88de676285aa\/508366924-s.jpg","medium":"https:\/\/kuvat.huuto.net\/v1\/6082\/695e42d788604ea88de676285aa\/508366924-m.jpg","original":null}}]},{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/578174408","category":"https:\/\/api.huuto.net\/1.1\/categories\/897","alternative":"https:\/\/www.huuto.net\/kohteet\/sierra-wireless-airprime-4g-lte\/578174408","images":"https:\/\/api.huuto.net\/1.1\/items\/578174408\/images"},"id":578174408,"title":"Sierra Wireless AirPrime 4G LTE","category":"Kannettavien tietokoneiden akut ja tarvikkeet","seller":"nick00","sellerId":2914998,"currentPrice":10,"buyNowPrice":10,"saleMethod":"buy-now","listTime":"2023-03-08T08:08:42+0200","postalCode":"33720","location":"TAMPERE","closingTime":"2023-03-22T08:00:00+0200","bidderCount":0,"offerCount":0,"hasReservePrice":false,"hasReservePriceExceeded":false,"upgrades":[],"images":[{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/578174408\/images\/508288586","thumbnail":"https:\/\/kuvat.huuto.net\/v1\/728e\/66af711226504dfd72ba7f58e98\/508288586-s.jpg","medium":"https:\/\/kuvat.huuto.net\/v1\/728e\/66af711226504dfd72ba7f58e98\/508288586-m.jpg","original":null}}]},{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/578086441","category":"https:\/\/api.huuto.net\/1.1\/categories\/897","alternative":"https:\/\/www.huuto.net\/kohteet\/lenovo-thinkpad-65w-slim--virtalahde\/578086441","images":"https:\/\/api.huuto.net\/1.1\/items\/578086441\/images"},"id":578086441,"title":"Lenovo ThinkPad 65W slim -virtal\u00e4hde","category":"Kannettavien tietokoneiden akut ja tarvikkeet","seller":"prossu1","sellerId":2366051,"currentPrice":15,"buyNowPrice":null,"saleMethod":"auction","listTime":"2023-03-06T17:01:55+0200","postalCode":"90120","location":"OULU","closingTime":"2023-03-19T12:00:00+0200","bidderCount":0,"offerCount":0,"hasReservePrice":false,"hasReservePriceExceeded":false,"upgrades":[],"images":[{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/578086441\/images\/508181197","thumbnail":"https:\/\/kuvat.huuto.net\/v1\/e2d6\/c6fef7201d482325a798711b175\/508181197-s.jpg","medium":"https:\/\/kuvat.huuto.net\/v1\/e2d6\/c6fef7201d482325a798711b175\/508181197-m.jpg","original":null}}]},{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/578085280","category":"https:\/\/api.huuto.net\/1.1\/categories\/897","alternative":"https:\/\/www.huuto.net\/kohteet\/lenovo-thinkpad-quectel-sdx24-em120r-gl-wwan-4g-modeemi\/578085280","images":"https:\/\/api.huuto.net\/1.1\/items\/578085280\/images"},"id":578085280,"title":"Lenovo ThinkPad Quectel SDX24 EM120R-GL WWAN 4G modeemi","category":"Kannettavien tietokoneiden akut ja tarvikkeet","seller":"tarsiger","sellerId":808553,"currentPrice":82,"buyNowPrice":82,"saleMethod":"buy-now","listTime":"2023-03-06T16:39:03+0200","postalCode":"01640","location":"VANTAA","closingTime":"2023-03-20T16:39:01+0200","bidderCount":0,"offerCount":0,"hasReservePrice":false,"hasReservePriceExceeded":false,"upgrades":[],"images":[{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/578085280\/images\/507236793","thumbnail":"https:\/\/kuvat.huuto.net\/v1\/a1d5\/c41fa515e150d20db90c614f981\/507236793-s.jpg","medium":"https:\/\/kuvat.huuto.net\/v1\/a1d5\/c41fa515e150d20db90c614f981\/507236793-m.jpg","original":null}}]},{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/578082963","category":"https:\/\/api.huuto.net\/1.1\/categories\/897","alternative":"https:\/\/www.huuto.net\/kohteet\/thinkpad-w541-_-p50--170w-virtalahde\/578082963","images":"https:\/\/api.huuto.net\/1.1\/items\/578082963\/images"},"id":578082963,"title":"Thinkpad W541 \/ P50 170W virtal\u00e4hde","category":"Kannettavien tietokoneiden akut ja tarvikkeet","seller":"tarsiger","sellerId":808553,"currentPrice":42,"buyNowPrice":42,"saleMethod":"buy-now","listTime":"2023-03-06T16:00:03+0200","postalCode":"01640","location":"VANTAA","closingTime":"2023-03-20T16:00:01+0200","bidderCount":0,"offerCount":0,"hasReservePrice":false,"hasReservePriceExceeded":false,"upgrades":[],"images":[{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/578082963\/images\/507234984","thumbnail":"https:\/\/kuvat.huuto.net\/v1\/827b\/80cf9ca765dd302da0e5df02144\/507234984-s.jpg","medium":"https:\/\/kuvat.huuto.net\/v1\/827b\/80cf9ca765dd302da0e5df02144\/507234984-m.jpg","original":null}}]},{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/578028877","category":"https:\/\/api.huuto.net\/1.1\/categories\/63","alternative":"https:\/\/www.huuto.net\/kohteet\/thinkpad-nappaimisto-uk\/578028877","images":"https:\/\/api.huuto.net\/1.1\/items\/578028877\/images"},"id":578028877,"title":"Thinkpad n\u00e4pp\u00e4imist\u00f6 UK","category":"N\u00e4pp\u00e4imist\u00f6t ja hiiret","seller":"hnetti","sellerId":1456413,"currentPrice":3,"buyNowPrice":null,"saleMethod":"auction","listTime":"2023-03-05T15:49:08+0200","postalCode":"00160","location":"HELSINKI","closingTime":"2023-03-19T15:49:05+0200","bidderCount":0,"offerCount":0,"hasReservePrice":false,"hasReservePriceExceeded":false,"upgrades":[],"images":[{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/578028877\/images\/508110054","thumbnail":"https:\/\/kuvat.huuto.net\/v1\/3569\/8614db847be72cec6cf7085879d\/508110054-s.jpg","medium":"https:\/\/kuvat.huuto.net\/v1\/3569\/8614db847be72cec6cf7085879d\/508110054-m.jpg","original":null}}]},{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/578026882","category":"https:\/\/api.huuto.net\/1.1\/categories\/84","alternative":"https:\/\/www.huuto.net\/kohteet\/lenovo-thinkpad-t15-gen-2-20w400hgmx\/578026882","images":"https:\/\/api.huuto.net\/1.1\/items\/578026882\/images"},"id":578026882,"title":"Lenovo Thinkpad T15 Gen 2 (20W400HGMX)","category":"Kannettavat tietokoneet","seller":"attekorte","sellerId":24060,"currentPrice":790,"buyNowPrice":null,"saleMethod":"auction","listTime":"2023-03-05T15:13:57+0200","postalCode":"83900","location":"JUUKA","closingTime":"2023-03-10T15:13:00+0200","bidderCount":0,"offerCount":0,"hasReservePrice":false,"hasReservePriceExceeded":false,"upgrades":[],"images":[{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/578026882\/images\/508107118","thumbnail":"https:\/\/kuvat.huuto.net\/v1\/04e4\/304693717970e81fa61d0e0b988\/508107118-s.jpg","medium":"https:\/\/kuvat.huuto.net\/v1\/04e4\/304693717970e81fa61d0e0b988\/508107118-m.jpg","original":null}}]},{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/578013764","category":"https:\/\/api.huuto.net\/1.1\/categories\/897","alternative":"https:\/\/www.huuto.net\/kohteet\/thinkpad-thunderbolt-3-workstation-dock-gen-1--230w-seka-65\/578013764","images":"https:\/\/api.huuto.net\/1.1\/items\/578013764\/images"},"id":578013764,"title":"ThinkPad Thunderbolt 3 Workstation Dock Gen 1 + 230W sek\u00e4 65","category":"Kannettavien tietokoneiden akut ja tarvikkeet","seller":"tarsiger","sellerId":808553,"currentPrice":125,"buyNowPrice":125,"saleMethod":"buy-now","listTime":"2023-03-05T10:59:41+0200","postalCode":"01640","location":"VANTAA","closingTime":"2023-03-12T10:59:39+0200","bidderCount":0,"offerCount":0,"hasReservePrice":false,"hasReservePriceExceeded":false,"upgrades":[],"images":[{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/578013764\/images\/505840652","thumbnail":"https:\/\/kuvat.huuto.net\/v1\/b1ac\/d2b7cf0db9ee812a1d4d5a747a6\/505840652-s.jpg","medium":"https:\/\/kuvat.huuto.net\/v1\/b1ac\/d2b7cf0db9ee812a1d4d5a747a6\/505840652-m.jpg","original":null}}]},{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/577859186","category":"https:\/\/api.huuto.net\/1.1\/categories\/897","alternative":"https:\/\/www.huuto.net\/kohteet\/lenovo-thinkpad-kosketuslevy-e440-l440-t440-w540\/577859186","images":"https:\/\/api.huuto.net\/1.1\/items\/577859186\/images"},"id":577859186,"title":"Lenovo Thinkpad kosketuslevy E440 L440 T440 W540","category":"Kannettavien tietokoneiden akut ja tarvikkeet","seller":"nick00","sellerId":2914998,"currentPrice":10,"buyNowPrice":10,"saleMethod":"buy-now","listTime":"2023-03-02T07:47:42+0200","postalCode":"33720","location":"TAMPERE","closingTime":"2023-03-16T07:31:00+0200","bidderCount":0,"offerCount":0,"hasReservePrice":false,"hasReservePriceExceeded":false,"upgrades":[],"images":[{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/577859186\/images\/507888772","thumbnail":"https:\/\/kuvat.huuto.net\/v1\/e985\/ac742e77b9ba14af63999f684a7\/507888772-s.jpg","medium":"https:\/\/kuvat.huuto.net\/v1\/e985\/ac742e77b9ba14af63999f684a7\/507888772-m.jpg","original":null}}]},{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/577756539","category":"https:\/\/api.huuto.net\/1.1\/categories\/61","alternative":"https:\/\/www.huuto.net\/kohteet\/16-gb-ddr4-2666v-so-dimm-muistia\/577756539","images":"https:\/\/api.huuto.net\/1.1\/items\/577756539\/images"},"id":577756539,"title":"16 GB DDR4 2666V SO-DIMM muistia","category":"Muistipiirit","seller":"nick00","sellerId":2914998,"currentPrice":50,"buyNowPrice":50,"saleMethod":"buy-now","listTime":"2023-02-28T15:12:27+0200","postalCode":"33720","location":"TAMPERE","closingTime":"2023-03-14T08:00:00+0200","bidderCount":0,"offerCount":0,"hasReservePrice":false,"hasReservePriceExceeded":false,"upgrades":[],"images":[{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/577756539\/images\/507751477","thumbnail":"https:\/\/kuvat.huuto.net\/v1\/947d\/a3662a375d2cfc001cc3807f0a8\/507751477-s.jpg","medium":"https:\/\/kuvat.huuto.net\/v1\/947d\/a3662a375d2cfc001cc3807f0a8\/507751477-m.jpg","original":null}}]},{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/577619805","category":"https:\/\/api.huuto.net\/1.1\/categories\/897","alternative":"https:\/\/www.huuto.net\/kohteet\/lenovo-thinkpad-laturi-135w\/577619805","images":"https:\/\/api.huuto.net\/1.1\/items\/577619805\/images"},"id":577619805,"title":"Lenovo ThinkPad laturi 135W","category":"Kannettavien tietokoneiden akut ja tarvikkeet","seller":"nick00","sellerId":2914998,"currentPrice":20,"buyNowPrice":20,"saleMethod":"buy-now","listTime":"2023-02-26T12:21:53+0200","postalCode":"33720","location":"TAMPERE","closingTime":"2023-03-12T08:00:00+0200","bidderCount":0,"offerCount":0,"hasReservePrice":false,"hasReservePriceExceeded":false,"upgrades":[],"images":[{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/577619805\/images\/507582568","thumbnail":"https:\/\/kuvat.huuto.net\/v1\/6b2d\/0a6af834366a26eaeffb9a53379\/507582568-s.jpg","medium":"https:\/\/kuvat.huuto.net\/v1\/6b2d\/0a6af834366a26eaeffb9a53379\/507582568-m.jpg","original":null}}]},{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/577619781","category":"https:\/\/api.huuto.net\/1.1\/categories\/58","alternative":"https:\/\/www.huuto.net\/kohteet\/lenovo-thinkpad-x270-m2-levykelkka--nvme-ssd-levy\/577619781","images":"https:\/\/api.huuto.net\/1.1\/items\/577619781\/images"},"id":577619781,"title":"Lenovo Thinkpad X270 M.2 levykelkka + NVMe SSD levy","category":"Kovalevyt","seller":"nick00","sellerId":2914998,"currentPrice":50,"buyNowPrice":50,"saleMethod":"buy-now","listTime":"2023-02-26T12:21:20+0200","postalCode":"33720","location":"TAMPERE","closingTime":"2023-03-12T08:00:00+0200","bidderCount":0,"offerCount":0,"hasReservePrice":false,"hasReservePriceExceeded":false,"upgrades":[],"images":[{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/577619781\/images\/507582527","thumbnail":"https:\/\/kuvat.huuto.net\/v1\/7e21\/fa76d89f482de7d00a425a30fc5\/507582527-s.jpg","medium":"https:\/\/kuvat.huuto.net\/v1\/7e21\/fa76d89f482de7d00a425a30fc5\/507582527-m.jpg","original":null}}]},{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/577619757","category":"https:\/\/api.huuto.net\/1.1\/categories\/58","alternative":"https:\/\/www.huuto.net\/kohteet\/lenovo-thinkpad-t470-m2-levykelkka--nvme-ssd-levy\/577619757","images":"https:\/\/api.huuto.net\/1.1\/items\/577619757\/images"},"id":577619757,"title":"Lenovo Thinkpad T470 M.2 levykelkka + NVMe SSD levy","category":"Kovalevyt","seller":"nick00","sellerId":2914998,"currentPrice":50,"buyNowPrice":50,"saleMethod":"buy-now","listTime":"2023-02-26T12:20:57+0200","postalCode":"33720","location":"TAMPERE","closingTime":"2023-03-12T08:00:00+0200","bidderCount":0,"offerCount":0,"hasReservePrice":false,"hasReservePriceExceeded":false,"upgrades":[],"images":[{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/577619757\/images\/507582487","thumbnail":"https:\/\/kuvat.huuto.net\/v1\/623c\/e888f05644235d3d21167448bd2\/507582487-s.jpg","medium":"https:\/\/kuvat.huuto.net\/v1\/623c\/e888f05644235d3d21167448bd2\/507582487-m.jpg","original":null}}]},{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/577600532","category":"https:\/\/api.huuto.net\/1.1\/categories\/897","alternative":"https:\/\/www.huuto.net\/kohteet\/lenovo-thinkpad-lapparilaukku-156\/577600532","images":"https:\/\/api.huuto.net\/1.1\/items\/577600532\/images"},"id":577600532,"title":"Lenovo ThinkPad l\u00e4pp\u00e4rilaukku 15.6\"","category":"Kannettavien tietokoneiden akut ja tarvikkeet","seller":"Melviini","sellerId":2245306,"currentPrice":15,"buyNowPrice":15,"saleMethod":"buy-now","listTime":"2023-02-26T01:58:14+0200","postalCode":"00600","location":"HELSINKI","closingTime":"2023-03-12T01:58:00+0200","bidderCount":0,"offerCount":0,"hasReservePrice":false,"hasReservePriceExceeded":false,"upgrades":[],"images":[{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/577600532\/images\/507558554","thumbnail":"https:\/\/kuvat.huuto.net\/v1\/214e\/cf42a5ab8ee533cd6c22c88c2c7\/507558554-s.jpg","medium":"https:\/\/kuvat.huuto.net\/v1\/214e\/cf42a5ab8ee533cd6c22c88c2c7\/507558554-m.jpg","original":null}}]},{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/577519756","category":"https:\/\/api.huuto.net\/1.1\/categories\/897","alternative":"https:\/\/www.huuto.net\/kohteet\/lenovo-thinkpad-advanced-minidock-telakka-ja-laturi\/577519756","images":"https:\/\/api.huuto.net\/1.1\/items\/577519756\/images"},"id":577519756,"title":"Lenovo ThinkPad advanced minidock telakka ja laturi","category":"Kannettavien tietokoneiden akut ja tarvikkeet","seller":"Joulubuggi","sellerId":2942,"currentPrice":30,"buyNowPrice":30,"saleMethod":"buy-now","listTime":"2023-02-24T11:39:54+0200","postalCode":"33100","location":"TAMPERE","closingTime":"2023-03-10T11:39:00+0200","bidderCount":0,"offerCount":0,"hasReservePrice":false,"hasReservePriceExceeded":false,"upgrades":[],"images":[{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/577519756\/images\/507461267","thumbnail":"https:\/\/kuvat.huuto.net\/v1\/2204\/5bd90ca1f70cc00a5135ec2ea8a\/507461267-s.jpg","medium":"https:\/\/kuvat.huuto.net\/v1\/2204\/5bd90ca1f70cc00a5135ec2ea8a\/507461267-m.jpg","original":null}}]},{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/576827868","category":"https:\/\/api.huuto.net\/1.1\/categories\/84","alternative":"https:\/\/www.huuto.net\/kohteet\/lenovo-thinkpad-t540p-155-3k-2880-x-1620-ips\/576827868","images":"https:\/\/api.huuto.net\/1.1\/items\/576827868\/images"},"id":576827868,"title":"Lenovo ThinkPad T540p, 15.5\" 3K (2880 x 1620), IPS","category":"Kannettavat tietokoneet","seller":"hammermann","sellerId":1398678,"currentPrice":600,"buyNowPrice":600,"saleMethod":"buy-now","listTime":"2023-02-11T22:05:45+0200","postalCode":"33100","location":"TAMPERE","closingTime":"2023-03-31T09:58:00+0300","bidderCount":0,"offerCount":0,"hasReservePrice":false,"hasReservePriceExceeded":false,"upgrades":[],"images":[{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/576827868\/images\/506669279","thumbnail":"https:\/\/kuvat.huuto.net\/v1\/a71e\/987ba35692c59217306a00185f6\/506669279-s.jpg","medium":"https:\/\/kuvat.huuto.net\/v1\/a71e\/987ba35692c59217306a00185f6\/506669279-m.jpg","original":null}}]},{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/576564494","category":"https:\/\/api.huuto.net\/1.1\/categories\/84","alternative":"https:\/\/www.huuto.net\/kohteet\/ibm-thinkpad-x20--2-kpl-telakka\/576564494","images":"https:\/\/api.huuto.net\/1.1\/items\/576564494\/images"},"id":576564494,"title":"IBM Thinkpad X20 + 2 kpl telakka","category":"Kannettavat tietokoneet","seller":"hammermann","sellerId":1398678,"currentPrice":400,"buyNowPrice":400,"saleMethod":"buy-now","listTime":"2023-02-06T10:19:02+0200","postalCode":"33100","location":"TAMPERE","closingTime":"2023-03-22T06:42:00+0200","bidderCount":0,"offerCount":0,"hasReservePrice":false,"hasReservePriceExceeded":false,"upgrades":[],"images":[{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/576564494\/images\/506353523","thumbnail":"https:\/\/kuvat.huuto.net\/v1\/4c37\/1913a47d3811d624df9df4d93ce\/506353523-s.jpg","medium":"https:\/\/kuvat.huuto.net\/v1\/4c37\/1913a47d3811d624df9df4d93ce\/506353523-m.jpg","original":null}}]},{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/576378051","category":"https:\/\/api.huuto.net\/1.1\/categories\/84","alternative":"https:\/\/www.huuto.net\/kohteet\/ibm-thinkpad-t43--laturi--telakka---win-xp-pro\/576378051","images":"https:\/\/api.huuto.net\/1.1\/items\/576378051\/images"},"id":576378051,"title":"IBM Thinkpad T43 + laturi + telakka + Win XP Pro","category":"Kannettavat tietokoneet","seller":"hammermann","sellerId":1398678,"currentPrice":250,"buyNowPrice":250,"saleMethod":"buy-now","listTime":"2023-02-02T09:56:27+0200","postalCode":"33100","location":"TAMPERE","closingTime":"2023-05-25T12:42:00+0300","bidderCount":0,"offerCount":0,"hasReservePrice":false,"hasReservePriceExceeded":false,"upgrades":[],"images":[{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/576378051\/images\/506108127","thumbnail":"https:\/\/kuvat.huuto.net\/v1\/d904\/7a54900391694410d3dab1343f6\/506108127-s.jpg","medium":"https:\/\/kuvat.huuto.net\/v1\/d904\/7a54900391694410d3dab1343f6\/506108127-m.jpg","original":null}}]},{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/573442000","category":"https:\/\/api.huuto.net\/1.1\/categories\/1112","alternative":"https:\/\/www.huuto.net\/kohteet\/512mb-pc100-sodimm\/573442000","images":"https:\/\/api.huuto.net\/1.1\/items\/573442000\/images"},"id":573442000,"title":"512MB PC100 SODIMM","category":"Retrokoneet ja oheislaitteet","seller":"countryguy","sellerId":1304585,"currentPrice":8.5,"buyNowPrice":8.5,"saleMethod":"buy-now","listTime":"2022-12-07T08:14:44+0200","postalCode":"55120","location":"IMATRA","closingTime":"2023-03-17T00:54:00+0200","bidderCount":0,"offerCount":0,"hasReservePrice":false,"hasReservePriceExceeded":false,"upgrades":[],"images":[{"links":{"self":"https:\/\/api.huuto.net\/1.1\/items\/573442000\/images\/502037093","thumbnail":"https:\/\/kuvat.huuto.net\/v1\/ab36\/4c199038f4e8703cfc67aae5654\/502037093-s.jpg","medium":"https:\/\/kuvat.huuto.net\/v1\/ab36\/4c199038f4e8703cfc67aae5654\/502037093-m.jpg","original":null}}]}]} --------------------------------------------------------------------------------