├── .env.example ├── .github └── workflows │ └── ci.yml ├── .gitignore ├── .tokeignore ├── .vscode └── settings.json ├── Cargo.lock ├── Cargo.toml ├── LICENSE ├── README.md ├── entities ├── Cargo.toml ├── README.md └── src │ ├── check_errors.rs │ ├── health_check.rs │ ├── host.rs │ ├── host_overrides.rs │ ├── instance_stats.rs │ ├── log.rs │ ├── mod.rs │ ├── prelude.rs │ └── state │ ├── error_cache.rs │ ├── mod.rs │ └── scanner.rs ├── migration ├── Cargo.toml ├── README.md └── src │ ├── lib.rs │ ├── m20220101_000001_create_table.rs │ ├── m20230729_010231_datetime_rowid.rs │ ├── m20230729_230909_datetime_int_host.rs │ ├── m20230803_154714_version_url.rs │ ├── m20230829_201916_country.rs │ ├── m20230914_231514_connectivity.rs │ ├── m20231011_231223_errors.rs │ ├── m20231112_142206_stats.rs │ ├── m20250216_185501_overrides.rs │ └── main.rs ├── scanner ├── Cargo.toml ├── src │ ├── about_parser.rs │ ├── cache_update.rs │ ├── cleanup.rs │ ├── instance_check.rs │ ├── instance_parser.rs │ ├── lib.rs │ ├── list_update.rs │ ├── profile_parser.rs │ ├── update_stats.rs │ └── version_check.rs └── test_data │ ├── about.html │ ├── instancelist.html │ ├── instancelist_expected.csv │ └── profile.html ├── screenshot.png ├── server ├── Cargo.toml ├── src │ ├── admin │ │ ├── errors.rs │ │ ├── locks.rs │ │ ├── logs.rs │ │ ├── mod.rs │ │ └── settings.rs │ ├── api.rs │ ├── lib.rs │ └── website.rs ├── static │ ├── admin.js │ ├── bootstrap.min.css │ ├── bootstrap.min.js │ ├── chart.min_4.4.0.js │ ├── chartjs-adapter-moment_1.0.1.js │ ├── dygraph.min.css │ ├── dygraph.min.js │ ├── helpers.min.js │ ├── history.2.js │ ├── instance.js │ ├── login_key.js │ ├── moment.min_2.29.4.js │ ├── robots.txt │ ├── sortable.min.js │ ├── sorting.css │ └── table_toggle.js └── templates │ ├── about.html.j2 │ ├── admin.html.j2 │ ├── admin_logs.html.j2 │ ├── history_admin.html.j2 │ ├── instance_errors.html.j2 │ ├── instance_locks.html.j2 │ ├── instance_settings.html.j2 │ ├── instances.html.j2 │ ├── login.html.j2 │ └── rip.html.j2 └── src └── main.rs /.env.example: -------------------------------------------------------------------------------- 1 | # database connection URI 2 | DATABASE_URL="sqlite:./sqlite.db?mode=rwc" 3 | # listen port 4 | PORT=3645 5 | # URL for nitter instances 6 | NITTER_INSTANCELIST="https://github.com/zedeus/nitter/wiki/Instances" 7 | # relevant for CORS 8 | SITE_URL="http://localhost" 9 | # seconds between instance ping checks 10 | INSTANCE_PING_INTERVAL_S=900 11 | # interval for fetching the instances from the wiki 12 | INSTANCE_LIST_INTERVAL_S=900 13 | # path used for checking account availability 14 | PROFILE_PATH="/jack/with_replies" 15 | # path used for checking RSS availability 16 | RSS_PATH="/jack/rss" 17 | # about page 18 | ABOUT_PATH="/about" 19 | # profile name to find during a health check 20 | PROFILE_NAME='@jack' 21 | # minimum amount of posts to find during a health check 22 | PROFILE_POSTS_MIN=5 23 | # regex content to search for to verify RSS availability 24 | RSS_CONTENT=', 15 | pub http_status: Option, 16 | } 17 | 18 | #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] 19 | pub enum Relation { 20 | #[sea_orm( 21 | belongs_to = "super::host::Entity", 22 | from = "Column::Host", 23 | to = "super::host::Column::Id", 24 | on_update = "Cascade", 25 | on_delete = "Cascade" 26 | )] 27 | Host, 28 | } 29 | 30 | impl Related for Entity { 31 | fn to() -> RelationDef { 32 | Relation::Host.def() 33 | } 34 | } 35 | 36 | impl ActiveModelBehavior for ActiveModel {} 37 | -------------------------------------------------------------------------------- /entities/src/health_check.rs: -------------------------------------------------------------------------------- 1 | //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3 2 | 3 | use sea_orm::{entity::prelude::*, FromQueryResult}; 4 | use sea_query::{Alias, Order, Query, SimpleExpr}; 5 | use serde::Serialize; 6 | 7 | use crate::check_errors; 8 | 9 | #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize)] 10 | #[sea_orm(table_name = "health_check")] 11 | pub struct Model { 12 | #[sea_orm(primary_key, auto_increment = false)] 13 | pub time: i64, 14 | #[sea_orm(primary_key, auto_increment = false)] 15 | pub host: i32, 16 | pub resp_time: Option, 17 | pub healthy: bool, 18 | pub response_code: Option, 19 | } 20 | 21 | #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] 22 | pub enum Relation { 23 | #[sea_orm( 24 | belongs_to = "super::host::Entity", 25 | from = "Column::Host", 26 | to = "super::host::Column::Id", 27 | on_update = "Cascade", 28 | on_delete = "Cascade" 29 | )] 30 | Host, 31 | } 32 | 33 | impl Related for Entity { 34 | fn to() -> RelationDef { 35 | Relation::Host.def() 36 | } 37 | } 38 | 39 | impl ActiveModelBehavior for ActiveModel {} 40 | 41 | #[derive(Debug, FromQueryResult, Serialize)] 42 | pub struct HealthyAmount { 43 | pub time: i64, 44 | pub alive: i64, 45 | pub dead: i64, 46 | } 47 | 48 | impl HealthyAmount { 49 | /// Fetch health check graph data for all or selected hosts in the selected time range. 50 | pub async fn fetch( 51 | db: &DatabaseConnection, 52 | from: Option, 53 | to: Option, 54 | hosts: Option<&[i32]>, 55 | ) -> Result, DbErr> { 56 | let builder = db.get_database_backend(); 57 | let mut stmt: sea_query::SelectStatement = Query::select(); 58 | stmt.column(self::Column::Time) 59 | .expr_as( 60 | SimpleExpr::Custom("SUM(healthy)".to_string()), 61 | Alias::new("alive"), 62 | ) 63 | .expr_as( 64 | SimpleExpr::Custom("SUM(1-healthy)".to_string()), 65 | Alias::new("dead"), 66 | ) 67 | .group_by_col(self::Column::Time) 68 | .from(self::Entity); 69 | if let (Some(from), Some(to)) = (from, to) { 70 | stmt.and_where(self::Column::Time.between(from.timestamp(), to.timestamp())); 71 | } 72 | if let Some(hosts) = hosts { 73 | stmt.and_where(self::Column::Host.is_in(hosts.iter().map(|v| *v))); 74 | } 75 | stmt.group_by_col(self::Column::Time) 76 | .order_by(self::Column::Time, Order::Asc); 77 | Self::find_by_statement(builder.build(&stmt)).all(db).await 78 | } 79 | } 80 | 81 | #[cfg(test)] 82 | mod test { 83 | 84 | use super::*; 85 | use chrono::Utc; 86 | use migration::MigratorTrait; 87 | use sea_orm::{ConnectOptions, Database}; 88 | 89 | pub(crate) async fn db_init() -> DatabaseConnection { 90 | let db = Database::connect(ConnectOptions::new( 91 | "sqlite:./test_db.db?mode=rwc".to_owned(), 92 | )) 93 | .await 94 | .unwrap(); 95 | migration::Migrator::up(&db, None).await.unwrap(); 96 | db 97 | } 98 | #[tokio::test(flavor = "multi_thread", worker_threads = 1)] 99 | #[ignore] 100 | async fn test_fetch_instance_list() { 101 | let db = db_init().await; 102 | let time_now = Utc::now(); 103 | let time_3h = time_now 104 | .checked_sub_signed(chrono::Duration::days(14)) 105 | .unwrap(); 106 | let hosts = vec![1, 2, 3]; 107 | HealthyAmount::fetch(&db, Some(time_3h), Some(time_now), Some(&hosts)) 108 | .await 109 | .unwrap(); 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /entities/src/host.rs: -------------------------------------------------------------------------------- 1 | //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3 2 | 3 | use sea_orm::entity::prelude::*; 4 | use serde::Serialize; 5 | 6 | #[derive(Copy, Clone, Default, Debug, DeriveEntity)] 7 | pub struct Entity; 8 | 9 | impl EntityName for Entity { 10 | fn table_name(&self) -> &str { 11 | "host" 12 | } 13 | } 14 | 15 | #[derive(Clone, Debug, PartialEq, DeriveModel, DeriveActiveModel, Eq, Serialize)] 16 | #[sea_orm(table_name = "host")] 17 | pub struct Model { 18 | #[sea_orm(primary_key)] 19 | pub id: i32, 20 | pub domain: String, 21 | pub url: String, 22 | pub enabled: bool, 23 | pub rss: bool, 24 | pub version: Option, 25 | pub country: String, 26 | pub version_url: Option, 27 | pub connectivity: Option, 28 | /// Last time the url and enabled were updated, *not* the rss 29 | pub updated: i64, 30 | } 31 | 32 | #[derive(Copy, Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize)] 33 | #[sea_orm(rs_type = "i32", db_type = "Integer")] 34 | pub enum Connectivity { 35 | #[sea_orm(num_value = 0)] 36 | All = 0, 37 | #[sea_orm(num_value = 1)] 38 | IPv4 = 1, 39 | #[sea_orm(num_value = 2)] 40 | IPv6 = 2, 41 | } 42 | 43 | #[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)] 44 | pub enum Column { 45 | Id, 46 | Domain, 47 | Url, 48 | Version, 49 | Country, 50 | VersionUrl, 51 | Enabled, 52 | Connectivity, 53 | Rss, 54 | Updated, 55 | } 56 | 57 | #[derive(Copy, Clone, Debug, EnumIter, DerivePrimaryKey)] 58 | pub enum PrimaryKey { 59 | Id, 60 | } 61 | 62 | impl PrimaryKeyTrait for PrimaryKey { 63 | type ValueType = i32; 64 | fn auto_increment() -> bool { 65 | true 66 | } 67 | } 68 | 69 | #[derive(Copy, Clone, Debug, EnumIter)] 70 | pub enum Relation { 71 | UpdateCheck, 72 | CheckErrors, 73 | HostOverrides, 74 | InstanceStats, 75 | } 76 | 77 | impl ColumnTrait for Column { 78 | type EntityName = Entity; 79 | fn def(&self) -> ColumnDef { 80 | match self { 81 | // required for updated -> integer 82 | Self::Id => ColumnType::Integer.def(), 83 | Self::Domain => ColumnType::String(None).def(), 84 | Self::Url => ColumnType::String(None).def(), 85 | Self::Version => ColumnType::String(None).def().null(), 86 | Self::Country => ColumnType::String(None).def(), 87 | Self::VersionUrl => ColumnType::String(None).def().null(), 88 | Self::Enabled => ColumnType::Integer.def(), 89 | Self::Rss => ColumnType::Integer.def(), 90 | Self::Updated => ColumnType::Integer.def(), 91 | Self::Connectivity => ColumnType::Integer.def().null(), 92 | } 93 | } 94 | 95 | // fn select_as(&self, expr: Expr) -> SimpleExpr { 96 | // Column::CaseInsensitiveText => expr.cast_as(Alias::new("text")), 97 | // _ => self.select_enum_as(expr), 98 | // } 99 | 100 | // /// Cast value of a column into the correct type for database storage. 101 | // fn save_as(&self, val: Expr) -> SimpleExpr { 102 | // Column::CaseInsensitiveText => val.cast_as(Alias::new("citext")), 103 | // _ => self.save_enum_as(val), 104 | // } 105 | } 106 | 107 | impl RelationTrait for Relation { 108 | fn def(&self) -> RelationDef { 109 | match self { 110 | Self::UpdateCheck => Entity::has_many(super::health_check::Entity).into(), 111 | Self::CheckErrors => Entity::has_many(super::check_errors::Entity).into(), 112 | Self::HostOverrides => Entity::has_many(super::host_overrides::Entity).into(), 113 | Self::InstanceStats => Entity::has_many(super::instance_stats::Entity).into(), 114 | } 115 | } 116 | } 117 | 118 | impl Related for Entity { 119 | fn to() -> RelationDef { 120 | Relation::UpdateCheck.def() 121 | } 122 | } 123 | 124 | impl Related for Entity { 125 | fn to() -> RelationDef { 126 | Relation::CheckErrors.def() 127 | } 128 | } 129 | 130 | impl Related for Entity { 131 | fn to() -> RelationDef { 132 | Relation::HostOverrides.def() 133 | } 134 | } 135 | 136 | impl Related for Entity { 137 | fn to() -> RelationDef { 138 | Relation::InstanceStats.def() 139 | } 140 | } 141 | 142 | impl ActiveModelBehavior for ActiveModel {} 143 | -------------------------------------------------------------------------------- /entities/src/host_overrides.rs: -------------------------------------------------------------------------------- 1 | //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3 2 | 3 | use sea_orm::entity::prelude::*; 4 | use serde::Serialize; 5 | 6 | use crate::host; 7 | 8 | /// Module of override keys 9 | pub mod keys { 10 | use std::{collections::HashMap, sync::LazyLock}; 11 | 12 | use sea_orm::{ 13 | ActiveModelTrait, ActiveValue, ColumnTrait, DatabaseConnection, DbErr, EntityTrait, 14 | QueryFilter, 15 | }; 16 | use sea_query::OnConflict; 17 | use serde::Serialize; 18 | use tracing::warn; 19 | 20 | use super::{Column, Entity}; 21 | use crate::host; 22 | pub type Result = std::result::Result; 23 | 24 | /// Override key to mark host as blocking health checks 25 | pub const KEY_BAD_HOST: &str = "BAD_HOST"; 26 | /// Expected value for a true value 27 | pub const VAL_BOOL_TRUE: &str = "true"; 28 | /// Override key for host health path to check 29 | pub const KEY_HOST_HEALTH_PATH: &str = "HEALTH_PATH"; 30 | /// Override key for host health query to check 31 | pub const KEY_HOST_HEALTH_QUERY: &str = "HEALTH_QUERY"; 32 | /// Override key for host bearer token 33 | pub const KEY_HOST_BEARER: &str = "BEARER_TOKEN"; 34 | pub const LOCKED_TRUE: i32 = 1; 35 | pub const LOCKED_FALSE: i32 = 0; 36 | /// Overrides that aren't locked by default 37 | pub static NOT_LOCKED_DEFAULTS: &'static [&'static str] = 38 | &[KEY_HOST_HEALTH_PATH, KEY_HOST_BEARER, KEY_HOST_HEALTH_QUERY]; 39 | 40 | pub static ALL_OVERRIDES: &'static [&'static str] = 41 | &[KEY_BAD_HOST, KEY_HOST_BEARER, KEY_HOST_HEALTH_PATH, KEY_HOST_HEALTH_QUERY]; 42 | 43 | #[derive(Serialize, Clone, Copy)] 44 | pub enum ValueType { 45 | String, 46 | Bool, 47 | } 48 | 49 | static KEY_TYPES: LazyLock> = LazyLock::new(|| { 50 | HashMap::from([ 51 | (KEY_BAD_HOST, ValueType::Bool), 52 | (KEY_HOST_HEALTH_PATH, ValueType::String), 53 | (KEY_HOST_HEALTH_QUERY, ValueType::String), 54 | (KEY_HOST_BEARER, ValueType::String), 55 | ]) 56 | }); 57 | 58 | #[derive(Serialize)] 59 | pub struct Override { 60 | pub value: Option, 61 | pub locked: bool, 62 | pub value_type: ValueType, 63 | } 64 | 65 | impl Override { 66 | fn new(value_type: ValueType, key: &&str) -> Self { 67 | Self { 68 | value: Default::default(), 69 | locked: !NOT_LOCKED_DEFAULTS.contains(key), 70 | value_type, 71 | } 72 | } 73 | } 74 | /// Host overrides for a single host 75 | pub struct HostOverrides { 76 | host: i32, 77 | /// Key -> Override 78 | entries: HashMap, 79 | } 80 | impl HostOverrides { 81 | /// Load HostOverrides for a given domain - will be empty if not found 82 | pub async fn load_by_domain(domain: &str, db: &DatabaseConnection) -> Result { 83 | let host = host::Entity::find() 84 | .filter(host::Column::Domain.eq(domain)) 85 | .one(db) 86 | .await?; 87 | match host { 88 | None => Ok(Self { 89 | host: -1, 90 | entries: HashMap::new(), 91 | }), 92 | Some(host) => Self::load(&host, db).await, 93 | } 94 | } 95 | 96 | pub async fn load(host: &host::Model, db: &DatabaseConnection) -> Result { 97 | let mut overrides: HashMap = KEY_TYPES 98 | .iter() 99 | .map(|(key, value_type)| (key.to_string(), Override::new(*value_type, key))) 100 | .collect(); 101 | 102 | let entries = Entity::find() 103 | .filter(Column::Host.eq(host.id)) 104 | .all(db) 105 | .await?; 106 | 107 | entries.into_iter().for_each(|fetched_entry| { 108 | if let Some(entry) = overrides.get_mut(&fetched_entry.key) { 109 | entry.value = fetched_entry.value; 110 | entry.locked = fetched_entry.locked == LOCKED_TRUE; 111 | } else { 112 | warn!( 113 | host = host.id, 114 | key = fetched_entry.key, 115 | "Found unknown override key!" 116 | ); 117 | } 118 | }); 119 | Ok(Self { 120 | host: host.id, 121 | entries: overrides, 122 | }) 123 | } 124 | 125 | /// Retrieve [KEY_HOST_BEARER] 126 | pub fn bearer(&self) -> Option<&str> { 127 | self.get(KEY_HOST_BEARER) 128 | } 129 | 130 | /// Retrieve [KEY_HOST_HEALTH_PATH] 131 | pub fn health_path(&self) -> Option<&str> { 132 | self.get(KEY_HOST_HEALTH_PATH) 133 | } 134 | 135 | /// Retrieve [KEY_HOST_HEALTH_QUERY] 136 | pub fn health_query(&self) -> Option<&str> { 137 | self.get(KEY_HOST_HEALTH_QUERY) 138 | } 139 | 140 | /// Retrieve override value for key 141 | pub fn get(&self, key: &str) -> Option<&str> { 142 | self.entries 143 | .get(key) 144 | .and_then(|v| v.value.as_deref()) 145 | } 146 | 147 | pub fn entries(&self) -> &HashMap { 148 | &self.entries 149 | } 150 | 151 | pub fn host(&self) -> i32 { 152 | self.host 153 | } 154 | 155 | /// Returns whether an entry is locked, will be true if not found 156 | pub async fn is_locked(&self, key: &str) -> bool { 157 | self.entries.get(key).map(|v| v.locked).unwrap_or(true) 158 | } 159 | 160 | pub async fn update_value( 161 | &mut self, 162 | key: String, 163 | value: Option, 164 | db: &DatabaseConnection, 165 | ) -> Result<()> { 166 | match self.entries.get_mut(&key) { 167 | Some(entry) => { 168 | if entry.value == value { 169 | return Ok(()); 170 | } 171 | entry.value = value; 172 | let locked_val = match entry.locked { 173 | true => LOCKED_TRUE, 174 | false => LOCKED_FALSE, 175 | }; 176 | super::Entity::insert(super::ActiveModel { 177 | host: ActiveValue::Set(self.host), 178 | key: ActiveValue::Set(key), 179 | locked: ActiveValue::Set(locked_val), 180 | value: ActiveValue::Set(entry.value.clone()), 181 | }) 182 | .on_conflict( 183 | dbg!(OnConflict::columns([Column::Key, Column::Host])) 184 | .update_columns([Column::Locked, Column::Value]) 185 | .to_owned(), 186 | ) 187 | .exec(db) 188 | .await?; 189 | Ok(()) 190 | } 191 | None => todo!(), 192 | } 193 | } 194 | } 195 | } 196 | 197 | #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize)] 198 | #[sea_orm(table_name = "host_overrides")] 199 | pub struct Model { 200 | #[sea_orm(primary_key, auto_increment = false)] 201 | pub host: i32, 202 | #[sea_orm(primary_key)] 203 | pub key: String, 204 | pub locked: i32, 205 | pub value: Option, 206 | } 207 | 208 | #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] 209 | pub enum Relation { 210 | #[sea_orm( 211 | belongs_to = "super::host::Entity", 212 | from = "Column::Host", 213 | to = "super::host::Column::Id", 214 | on_update = "Cascade", 215 | on_delete = "Cascade" 216 | )] 217 | Host, 218 | } 219 | 220 | impl Related for Entity { 221 | fn to() -> RelationDef { 222 | Relation::Host.def() 223 | } 224 | } 225 | 226 | impl ActiveModelBehavior for ActiveModel {} 227 | -------------------------------------------------------------------------------- /entities/src/instance_stats.rs: -------------------------------------------------------------------------------- 1 | //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3 2 | 3 | use sea_orm::{entity::prelude::*, FromQueryResult}; 4 | use sea_query::{Alias, Order, Query, SimpleExpr}; 5 | use serde::Serialize; 6 | 7 | #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize)] 8 | #[sea_orm(table_name = "instance_stats")] 9 | pub struct Model { 10 | #[sea_orm(primary_key, auto_increment = false)] 11 | pub time: i64, 12 | #[sea_orm(primary_key, auto_increment = false)] 13 | pub host: i32, 14 | pub limited_accs: i32, 15 | pub total_accs: i32, 16 | pub total_requests: i64, 17 | pub req_photo_rail: i32, 18 | pub req_user_screen_name: i32, 19 | pub req_search: i32, 20 | pub req_list_tweets: i32, 21 | pub req_user_media: i32, 22 | pub req_tweet_detail: i32, 23 | pub req_list: i32, 24 | pub req_user_tweets: i32, 25 | pub req_user_tweets_and_replies: i32, 26 | } 27 | 28 | #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] 29 | pub enum Relation { 30 | #[sea_orm( 31 | belongs_to = "super::host::Entity", 32 | from = "Column::Host", 33 | to = "super::host::Column::Id", 34 | on_update = "Cascade", 35 | on_delete = "Cascade" 36 | )] 37 | Host, 38 | } 39 | 40 | impl Related for Entity { 41 | fn to() -> RelationDef { 42 | Relation::Host.def() 43 | } 44 | } 45 | 46 | impl ActiveModelBehavior for ActiveModel {} 47 | 48 | #[derive(Debug, FromQueryResult, Serialize)] 49 | pub struct StatsAmount { 50 | pub time: i64, 51 | pub limited_accs_max: i32, 52 | pub limited_accs_avg: i32, 53 | pub total_accs_max: i32, 54 | pub total_accs_avg: i32, 55 | pub total_requests_max: i64, 56 | pub total_requests_avg: i64, 57 | pub req_photo_rail_max: i32, 58 | pub req_photo_rail_avg: i32, 59 | pub req_user_screen_name_max: i32, 60 | pub req_user_screen_name_avg: i32, 61 | pub req_search_max: i32, 62 | pub req_search_avg: i32, 63 | pub req_list_tweets_max: i32, 64 | pub req_list_tweets_avg: i32, 65 | pub req_user_media_max: i32, 66 | pub req_user_media_avg: i32, 67 | pub req_tweet_detail_max: i32, 68 | pub req_tweet_detail_avg: i32, 69 | pub req_list_max: i32, 70 | pub req_list_avg: i32, 71 | pub req_user_tweets_max: i32, 72 | pub req_user_tweets_avg: i32, 73 | pub req_user_tweets_and_replies_max: i32, 74 | pub req_user_tweets_and_replies_avg: i32, 75 | } 76 | 77 | impl StatsAmount { 78 | /// Fetch health check graph data for all or selected hosts in the selected time range. 79 | pub async fn fetch( 80 | db: &DatabaseConnection, 81 | from: DateTimeUtc, 82 | to: DateTimeUtc, 83 | hosts: Option<&[i32]>, 84 | ) -> Result, DbErr> { 85 | let builder = db.get_database_backend(); 86 | let columns = [ 87 | "limited_accs", 88 | "total_accs", 89 | "total_requests", 90 | "req_photo_rail", 91 | "req_user_screen_name", 92 | "req_search", 93 | "req_list_tweets", 94 | "req_user_media", 95 | "req_tweet_detail", 96 | "req_list", 97 | "req_user_tweets", 98 | "req_user_tweets_and_replies", 99 | ]; 100 | let mut stmt: sea_query::SelectStatement = Query::select(); 101 | let col_stmt = stmt.column(self::Column::Time); 102 | for col in columns { 103 | col_stmt 104 | .expr_as( 105 | SimpleExpr::Custom(format!("MAX({col})")), 106 | Alias::new(format!("{col}_max")), 107 | ) 108 | .expr_as( 109 | SimpleExpr::Custom(format!("CAST(ifnull(AVG({col}),0) as int)")), 110 | Alias::new(format!("{col}_avg")), 111 | ); 112 | } 113 | col_stmt 114 | .group_by_col(self::Column::Time) 115 | .from(self::Entity) 116 | .and_where(self::Column::Time.between(from.timestamp(), to.timestamp())); 117 | if let Some(hosts) = hosts { 118 | stmt.and_where(self::Column::Host.is_in(hosts.iter().map(|v| *v))); 119 | } 120 | stmt.group_by_col(self::Column::Time) 121 | .order_by(self::Column::Time, Order::Asc); 122 | StatsAmount::find_by_statement(builder.build(&stmt)) 123 | .all(db) 124 | .await 125 | } 126 | } 127 | 128 | #[derive(Debug, FromQueryResult, Serialize)] 129 | pub struct StatsCSVEntry { 130 | pub time: i64, 131 | pub limited_accs_avg: i32, 132 | pub total_accs_avg: i32, 133 | pub total_requests_avg: i64, 134 | } 135 | 136 | impl StatsCSVEntry { 137 | /// Fetch health check graph data for all or selected hosts in the selected time range. 138 | pub async fn fetch(db: &DatabaseConnection) -> Result, DbErr> { 139 | let builder = db.get_database_backend(); 140 | let columns = ["limited_accs", "total_accs", "total_requests"]; 141 | let mut stmt: sea_query::SelectStatement = Query::select(); 142 | stmt.column(self::Column::Time); 143 | for col in columns { 144 | stmt.expr_as( 145 | SimpleExpr::Custom(format!("CAST (AVG({col}) as int)")), 146 | Alias::new(format!("{col}_avg")), 147 | ); 148 | } 149 | stmt.group_by_col(self::Column::Time).from(self::Entity); 150 | stmt.group_by_col(self::Column::Time) 151 | .order_by(self::Column::Time, Order::Asc); 152 | Self::find_by_statement(builder.build(&stmt)).all(db).await 153 | } 154 | } 155 | -------------------------------------------------------------------------------- /entities/src/log.rs: -------------------------------------------------------------------------------- 1 | //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3 2 | 3 | use sea_orm::entity::prelude::*; 4 | use serde::Serialize; 5 | 6 | #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize)] 7 | #[sea_orm(table_name = "log")] 8 | pub struct Model { 9 | #[sea_orm(primary_key, auto_increment = false)] 10 | pub user_host: i32, 11 | pub host_affected: Option, 12 | pub key: String, 13 | #[sea_orm(primary_key, auto_increment = false)] 14 | pub time: i64, 15 | pub new_value: Option, 16 | } 17 | 18 | #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] 19 | pub enum Relation { 20 | #[sea_orm( 21 | belongs_to = "super::host::Entity", 22 | from = "Column::UserHost", 23 | to = "super::host::Column::Id", 24 | on_update = "Cascade", 25 | on_delete = "Restrict" 26 | )] 27 | Host2, 28 | #[sea_orm( 29 | belongs_to = "super::host::Entity", 30 | from = "Column::HostAffected", 31 | to = "super::host::Column::Id", 32 | on_update = "Cascade", 33 | on_delete = "Cascade" 34 | )] 35 | Host1, 36 | } 37 | 38 | impl ActiveModelBehavior for ActiveModel {} 39 | -------------------------------------------------------------------------------- /entities/src/mod.rs: -------------------------------------------------------------------------------- 1 | //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3 2 | 3 | pub mod prelude; 4 | 5 | pub mod check_errors; 6 | pub mod health_check; 7 | pub mod host; 8 | pub mod host_overrides; 9 | pub mod instance_stats; 10 | pub mod log; 11 | 12 | // has to be re-added on entity regeneration 13 | pub mod state; -------------------------------------------------------------------------------- /entities/src/prelude.rs: -------------------------------------------------------------------------------- 1 | //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3 2 | 3 | pub use super::check_errors::Entity as CheckErrors; 4 | pub use super::health_check::Entity as HealthCheck; 5 | pub use super::host::Entity as Host; 6 | pub use super::host_overrides::Entity as HostOverrides; 7 | pub use super::instance_stats::Entity as InstanceStats; 8 | pub use super::log::Entity as Log; 9 | -------------------------------------------------------------------------------- /entities/src/state/error_cache.rs: -------------------------------------------------------------------------------- 1 | use chrono::Utc; 2 | use sea_orm::prelude::DateTimeUtc; 3 | use serde::Serialize; 4 | 5 | #[derive(Debug, Serialize, Clone, Default)] 6 | pub struct HostError { 7 | pub time: DateTimeUtc, 8 | pub message: String, 9 | pub http_body: Option, 10 | pub http_status: Option, 11 | } 12 | 13 | impl HostError { 14 | pub fn new(message: String, http_body: String, http_status: u16) -> Self { 15 | Self { 16 | time: Utc::now(), 17 | message, 18 | http_body: Some(http_body), 19 | http_status: Some(http_status as _), 20 | } 21 | } 22 | 23 | /// HostError from only a message 24 | pub fn new_message(message: String) -> Self { 25 | Self { 26 | time: Utc::now(), 27 | message, 28 | http_body: None, 29 | http_status: None, 30 | } 31 | } 32 | 33 | /// HostError without body 34 | pub fn new_without_body(message: String, http_status: u16) -> Self { 35 | Self { 36 | time: Utc::now(), 37 | message, 38 | http_body: None, 39 | http_status: Some(http_status as _), 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /entities/src/state/mod.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: AGPL-3.0-only 2 | //! Global state and structures. 3 | //! For build process decoupling 4 | 5 | use std::sync::{Arc, RwLock}; 6 | 7 | use chrono::Utc; 8 | use sea_orm::prelude::DateTimeUtc; 9 | use serde::Serialize; 10 | 11 | use crate::host::Connectivity; 12 | 13 | /// Log for recent host errors 14 | pub mod error_cache; 15 | pub mod scanner; 16 | 17 | pub type Cache = RwLock; 18 | 19 | pub type AppState = Arc; 20 | 21 | pub struct InnerState { 22 | pub cache: RwLock, 23 | } 24 | 25 | pub fn new() -> AppState { 26 | Arc::new(InnerState { 27 | cache: RwLock::new(CacheData { 28 | hosts: vec![], 29 | last_update: Utc::now(), 30 | latest_commit: String::new(), 31 | }), 32 | }) 33 | } 34 | 35 | #[derive(Debug, Serialize)] 36 | pub struct CacheData { 37 | pub hosts: Vec, 38 | pub last_update: DateTimeUtc, 39 | pub latest_commit: String, 40 | } 41 | 42 | #[derive(Debug, Serialize)] 43 | pub struct CacheHost { 44 | pub url: String, 45 | pub domain: String, 46 | pub points: i32, 47 | pub rss: bool, 48 | pub recent_pings: Vec>, 49 | pub ping_max: Option, 50 | pub ping_min: Option, 51 | pub ping_avg: Option, 52 | pub version: Option, 53 | pub version_url: Option, 54 | pub healthy: bool, 55 | pub last_healthy: Option, 56 | /// Whether the source is from the normal upstream repo 57 | pub is_upstream: bool, 58 | /// Whether the source is from the latest upstream commit 59 | pub is_latest_version: bool, 60 | /// Whether this host is known to be bad (ip blocking) 61 | pub is_bad_host: bool, 62 | /// Country from the wiki 63 | pub country: String, 64 | /// Last health checks time formatted, healthy 65 | pub recent_checks: Vec<(String, bool)>, 66 | /// Percentage of healthy checks since first seen 67 | pub healthy_percentage_overall: u8, 68 | pub connectivity: Option, 69 | /// Internal: show last-seen information 70 | pub __show_last_seen: bool, 71 | } 72 | -------------------------------------------------------------------------------- /entities/src/state/scanner.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: AGPL-3.0-only 2 | use std::{sync::Arc, time::Duration}; 3 | pub type ScannerConfig = Arc; 4 | #[derive(Debug)] 5 | pub struct Config { 6 | /// time until next instance list fetch 7 | pub list_fetch_interval: Duration, 8 | /// time until next instance ping check 9 | pub instance_check_interval: Duration, 10 | /// time until next instance statistics check 11 | pub instance_stats_interval: Duration, 12 | /// instances list URL 13 | pub instance_list_url: String, 14 | /// profile path for health check 15 | pub profile_path: String, 16 | /// rss path for health check 17 | pub rss_path: String, 18 | /// about path for version check 19 | pub about_path: String, 20 | /// Expected profile name for a valid profile health check 21 | pub profile_name: String, 22 | /// Expected minimum of timeline posts for a valid profile health check 23 | pub profile_posts_min: usize, 24 | /// Expected string for a valid RSS health check 25 | pub rss_content: String, 26 | /// List of additional hosts to include during health checks 27 | pub additional_hosts: Vec, 28 | /// Country to use for additional hosts 29 | pub additional_host_country: String, 30 | /// Website URL of this service 31 | pub website_url: String, 32 | /// Duration to average the ping/response times over 33 | pub ping_range: chrono::Duration, 34 | /// don't emit errors for hosts which are already listed as down 35 | pub auto_mute: bool, 36 | /// Git URL for source fetching 37 | pub source_git_url: String, 38 | /// Git branch to fetch the current commit from 39 | pub source_git_branch: String, 40 | /// Interval to run cleanup operations in, to remove old data 41 | pub cleanup_interval: Duration, 42 | /// Amount of latest errors to keep per instance/host 43 | pub error_retention_per_host: usize, 44 | /// Path for connectivity checks 45 | pub connectivity_path: String, 46 | } 47 | 48 | impl Config { 49 | pub fn test_defaults() -> ScannerConfig { 50 | Arc::new(Config { 51 | instance_stats_interval: Duration::from_secs(15 * 60), 52 | list_fetch_interval: Duration::from_secs(15 * 60), 53 | instance_check_interval: Duration::from_secs(15 * 60), 54 | instance_list_url: String::from("https://github.com/zedeus/nitter/wiki/Instances"), 55 | profile_path: String::from("/jack"), 56 | rss_path: String::from("/jack/rss"), 57 | about_path: String::from("/about"), 58 | profile_name: String::from("@jack"), 59 | profile_posts_min: 5, 60 | rss_content: String::from(r#"` 8 | 9 | Run the following command to re-generate the entities. 10 | `sea-orm-cli generate entity -o entities/src --with-serde serialize` 11 | 12 | Note that you have to check for unwanted changes such as `DateTime` & `Date` types. These are likely replaced with String by the sea-orm CLI. Same with additional modules like `web`. 13 | 14 | May require setting the ENV `$Env:DATABASE_URL='sqlite:./sqlite.db?mode=rwc'` 15 | 16 | - Generate a new migration file 17 | ```sh 18 | cargo run -- migrate generate MIGRATION_NAME 19 | ``` 20 | - Apply all pending migrations 21 | ```sh 22 | cargo run 23 | ``` 24 | ```sh 25 | cargo run -- up 26 | ``` 27 | - Apply first 10 pending migrations 28 | ```sh 29 | cargo run -- up -n 10 30 | ``` 31 | - Rollback last applied migrations 32 | ```sh 33 | cargo run -- down 34 | ``` 35 | - Rollback last 10 applied migrations 36 | ```sh 37 | cargo run -- down -n 10 38 | ``` 39 | - Drop all tables from the database, then reapply all migrations 40 | ```sh 41 | cargo run -- fresh 42 | ``` 43 | - Rollback all applied migrations, then reapply all migrations 44 | ```sh 45 | cargo run -- refresh 46 | ``` 47 | - Rollback all applied migrations 48 | ```sh 49 | cargo run -- reset 50 | ``` 51 | - Check the status of all migrations 52 | ```sh 53 | cargo run -- status 54 | ``` 55 | -------------------------------------------------------------------------------- /migration/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub use sea_orm_migration::prelude::*; 2 | 3 | mod m20220101_000001_create_table; 4 | mod m20230729_010231_datetime_rowid; 5 | mod m20230729_230909_datetime_int_host; 6 | mod m20230803_154714_version_url; 7 | mod m20230829_201916_country; 8 | mod m20230914_231514_connectivity; 9 | mod m20231011_231223_errors; 10 | mod m20231112_142206_stats; 11 | mod m20250216_185501_overrides; 12 | 13 | pub struct Migrator; 14 | 15 | #[async_trait::async_trait] 16 | impl MigratorTrait for Migrator { 17 | fn migrations() -> Vec> { 18 | vec![ 19 | Box::new(m20220101_000001_create_table::Migration), 20 | Box::new(m20230729_010231_datetime_rowid::Migration), 21 | Box::new(m20230729_230909_datetime_int_host::Migration), 22 | Box::new(m20230803_154714_version_url::Migration), 23 | Box::new(m20230829_201916_country::Migration), 24 | Box::new(m20230914_231514_connectivity::Migration), 25 | Box::new(m20231011_231223_errors::Migration), 26 | Box::new(m20231112_142206_stats::Migration), 27 | Box::new(m20250216_185501_overrides::Migration), 28 | ] 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /migration/src/m20220101_000001_create_table.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: AGPL-3.0-only 2 | use sea_orm_migration::prelude::*; 3 | 4 | #[derive(DeriveMigrationName)] 5 | pub struct Migration; 6 | 7 | #[async_trait::async_trait] 8 | impl MigrationTrait for Migration { 9 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 10 | manager 11 | .create_table( 12 | Table::create() 13 | .table(Host::Table) 14 | .if_not_exists() 15 | .col( 16 | ColumnDef::new(Host::Id) 17 | .integer() 18 | .not_null() 19 | .auto_increment() 20 | .primary_key(), 21 | ) 22 | .col( 23 | ColumnDef::new(Host::Domain) 24 | .string() 25 | .not_null() 26 | .unique_key(), 27 | ) 28 | .col(ColumnDef::new(Host::URL).string().not_null()) 29 | .col(ColumnDef::new(Host::Version).string()) 30 | .col(ColumnDef::new(Host::Enabled).boolean().not_null()) 31 | .col(ColumnDef::new(Host::RSS).boolean().not_null()) 32 | .col(ColumnDef::new(Host::Updated).date_time().not_null()) 33 | .to_owned(), 34 | ) 35 | .await 36 | .unwrap(); 37 | manager 38 | .create_table( 39 | Table::create() 40 | .table(UpdateCheck::Table) 41 | .if_not_exists() 42 | .col(ColumnDef::new(UpdateCheck::Time).date_time().not_null()) 43 | .col(ColumnDef::new(UpdateCheck::Host).integer().not_null()) 44 | .col(ColumnDef::new(UpdateCheck::RespTime).integer()) 45 | .col(ColumnDef::new(UpdateCheck::Healthy).boolean().not_null()) 46 | .col(ColumnDef::new(UpdateCheck::ResponseCode).integer()) 47 | .foreign_key( 48 | ForeignKey::create() 49 | .name("FK_updatecheck_host") 50 | .from(UpdateCheck::Table, UpdateCheck::Host) 51 | .to(Host::Table, Host::Id) 52 | .on_delete(ForeignKeyAction::Cascade) 53 | .on_update(ForeignKeyAction::Cascade), 54 | ) 55 | .primary_key( 56 | Index::create() 57 | .col(UpdateCheck::Host) 58 | .col(UpdateCheck::Time) 59 | .name("pk_updatecheck"), 60 | ) 61 | .to_owned(), 62 | ) 63 | .await 64 | .unwrap(); 65 | Ok(()) 66 | } 67 | 68 | async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { 69 | manager 70 | .drop_table( 71 | Table::drop() 72 | .table(UpdateCheck::Table) 73 | .if_exists() 74 | .to_owned(), 75 | ) 76 | .await 77 | .unwrap(); 78 | manager 79 | .drop_table(Table::drop().table(Host::Table).if_exists().to_owned()) 80 | .await 81 | } 82 | } 83 | 84 | /// Learn more at https://docs.rs/sea-query#iden 85 | #[derive(Iden)] 86 | enum Host { 87 | Table, 88 | Id, 89 | Domain, 90 | URL, 91 | RSS, 92 | Version, 93 | Enabled, 94 | Updated, 95 | } 96 | 97 | #[derive(Iden)] 98 | enum UpdateCheck { 99 | Table, 100 | Host, 101 | Time, 102 | Healthy, 103 | RespTime, 104 | ResponseCode, 105 | } 106 | -------------------------------------------------------------------------------- /migration/src/m20230729_010231_datetime_rowid.rs: -------------------------------------------------------------------------------- 1 | use std::borrow::Cow; 2 | 3 | use sea_orm_migration::{ 4 | prelude::*, 5 | sea_orm::{prelude::DateTimeUtc, DbBackend, FromQueryResult, RuntimeErr, Statement}, 6 | }; 7 | 8 | #[derive(DeriveMigrationName)] 9 | pub struct Migration; 10 | 11 | #[derive(Debug, FromQueryResult)] 12 | pub struct UpdateCheck { 13 | time: DateTimeUtc, 14 | host: i32, 15 | resp_time: Option, 16 | healthy: bool, 17 | response_code: Option, 18 | } 19 | 20 | #[async_trait::async_trait] 21 | impl MigrationTrait for Migration { 22 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 23 | let cmd = r#"CREATE TABLE "health_check_new" ( 24 | "time" integer NOT NULL, 25 | "host" integer NOT NULL, 26 | "resp_time" integer, 27 | "healthy" integer NOT NULL, 28 | "response_code" integer, 29 | CONSTRAINT "pk_health_check_new" PRIMARY KEY ("host", "time"), 30 | FOREIGN KEY ("host") REFERENCES "host" ("id") ON DELETE CASCADE ON UPDATE CASCADE 31 | ) WITHOUT ROWID, STRICT;"#; 32 | let db = manager.get_connection(); 33 | db.execute_unprepared("BEGIN EXCLUSIVE").await?; 34 | db.execute_unprepared(cmd).await?; 35 | tracing::info!("fetching all healtcheck entries.."); 36 | let data = UpdateCheck::find_by_statement(Statement::from_sql_and_values( 37 | DbBackend::Sqlite, 38 | r#"SELECT * FROM update_check"#, 39 | [], 40 | )) 41 | .all(db) 42 | .await?; 43 | 44 | tracing::info!("migrating {} entries to temp table..", data.len()); 45 | let mut duplicates = 0; 46 | for entry in data.into_iter() { 47 | if let Err(err) = db 48 | .execute(Statement::from_sql_and_values( 49 | DbBackend::Sqlite, 50 | r#"INSERT INTO health_check_new ( 51 | time, 52 | host, 53 | resp_time, 54 | healthy, 55 | response_code) VALUES ($1,$2,$3,$4,$5)"#, 56 | [ 57 | entry.time.timestamp().into(), 58 | entry.host.into(), 59 | entry.resp_time.into(), 60 | entry.healthy.into(), 61 | entry.response_code.into(), 62 | ], 63 | )) 64 | .await 65 | { 66 | if let DbErr::Exec(RuntimeErr::SqlxError(e)) = &err { 67 | if let Some(err) = e.as_database_error() { 68 | if err.code() == Some(Cow::Borrowed("1555")) { 69 | duplicates += 1; 70 | continue; 71 | } 72 | } 73 | } 74 | tracing::info!(entry=?entry); 75 | return Err(err); 76 | } 77 | } 78 | tracing::info!("dropped {} duplicate timestamps..", duplicates); 79 | tracing::info!("dropping old table.."); 80 | db.execute_unprepared("DROP TABLE update_check").await?; 81 | 82 | db.execute_unprepared(&cmd.replace("health_check_new", "health_check")) 83 | .await?; 84 | tracing::info!("inserting back into final table.."); 85 | db.execute_unprepared( 86 | r#"INSERT INTO health_check 87 | SELECT * 88 | FROM health_check_new WHERE true"#, 89 | ) 90 | .await?; 91 | tracing::info!("dropping temp table.."); 92 | db.execute_unprepared("DROP TABLE health_check_new").await?; 93 | tracing::info!("cleaning up db.."); 94 | db.execute_unprepared("COMMIT TRANSACTION").await?; 95 | db.execute_unprepared("VACUUM").await?; 96 | Ok(()) 97 | } 98 | 99 | async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> { 100 | panic!("Can't migrate down"); 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /migration/src/m20230729_230909_datetime_int_host.rs: -------------------------------------------------------------------------------- 1 | use sea_orm_migration::{ 2 | prelude::*, 3 | sea_orm::{prelude::DateTimeUtc, DbBackend, FromQueryResult, Statement}, 4 | }; 5 | 6 | #[derive(DeriveMigrationName)] 7 | pub struct Migration; 8 | 9 | #[derive(Debug, FromQueryResult)] 10 | pub struct Host { 11 | pub id: i32, 12 | pub domain: String, 13 | pub url: String, 14 | pub enabled: bool, 15 | pub rss: bool, 16 | pub version: Option, 17 | pub updated: DateTimeUtc, 18 | } 19 | 20 | #[async_trait::async_trait] 21 | impl MigrationTrait for Migration { 22 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 23 | let cmd = r#"CREATE TABLE "host_new" ( 24 | "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, 25 | "domain" text NOT NULL UNIQUE, 26 | "url" text NOT NULL, 27 | "version" text, 28 | "enabled" integer NOT NULL, 29 | "rss" integer NOT NULL, 30 | "updated" integer NOT NULL 31 | ) STRICT;"#; 32 | let db = manager.get_connection(); 33 | db.execute_unprepared("PRAGMA foreign_keys = 0").await?; 34 | db.execute_unprepared("BEGIN EXCLUSIVE").await?; 35 | db.execute_unprepared(cmd).await?; 36 | tracing::info!("fetching all host entries.."); 37 | let data = Host::find_by_statement(Statement::from_sql_and_values( 38 | DbBackend::Sqlite, 39 | r#"SELECT * FROM host"#, 40 | [], 41 | )) 42 | .all(db) 43 | .await?; 44 | 45 | tracing::info!("migrating {} entries to temp table..", data.len()); 46 | for entry in data.into_iter() { 47 | db.execute(Statement::from_sql_and_values( 48 | DbBackend::Sqlite, 49 | r#"INSERT INTO host_new ( 50 | id, 51 | domain, 52 | url, 53 | version, 54 | enabled, 55 | rss, 56 | updated ) VALUES ($1,$2,$3,$4,$5,$6,$7)"#, 57 | [ 58 | entry.id.into(), 59 | entry.domain.into(), 60 | entry.url.into(), 61 | entry.version.into(), 62 | entry.enabled.into(), 63 | entry.rss.into(), 64 | entry.updated.timestamp().into(), 65 | ], 66 | )) 67 | .await?; 68 | } 69 | tracing::info!("dropping old table.."); 70 | db.execute_unprepared("DROP TABLE host").await?; 71 | 72 | db.execute_unprepared(&cmd.replace("host_new", "host")) 73 | .await?; 74 | tracing::info!("inserting back into final table.."); 75 | db.execute_unprepared( 76 | r#"INSERT INTO host 77 | SELECT * 78 | FROM host_new WHERE true"#, 79 | ) 80 | .await?; 81 | tracing::info!("dropping temp table.."); 82 | db.execute_unprepared("DROP TABLE host_new").await?; 83 | tracing::info!("cleaning up db.."); 84 | db.execute_unprepared("COMMIT TRANSACTION").await?; 85 | db.execute_unprepared("PRAGMA foreign_keys = 1").await?; 86 | db.execute_unprepared("VACUUM").await?; 87 | Ok(()) 88 | } 89 | 90 | async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> { 91 | panic!("Can't migrate down"); 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /migration/src/m20230803_154714_version_url.rs: -------------------------------------------------------------------------------- 1 | use sea_orm_migration::prelude::*; 2 | 3 | #[derive(DeriveMigrationName)] 4 | pub struct Migration; 5 | 6 | #[async_trait::async_trait] 7 | impl MigrationTrait for Migration { 8 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 9 | let cmd = r#"ALTER TABLE "host" ADD COLUMN "version_url" text;"#; 10 | let db = manager.get_connection(); 11 | db.execute_unprepared("BEGIN EXCLUSIVE").await?; 12 | tracing::info!("adding version_url column.."); 13 | db.execute_unprepared(cmd).await?; 14 | db.execute_unprepared("COMMIT TRANSACTION").await?; 15 | db.execute_unprepared("VACUUM").await?; 16 | Ok(()) 17 | } 18 | 19 | async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> { 20 | panic!("Can't migrate down"); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /migration/src/m20230829_201916_country.rs: -------------------------------------------------------------------------------- 1 | use sea_orm_migration::prelude::*; 2 | 3 | #[derive(DeriveMigrationName)] 4 | pub struct Migration; 5 | 6 | #[async_trait::async_trait] 7 | impl MigrationTrait for Migration { 8 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 9 | let cmd = r#"ALTER TABLE "host" ADD COLUMN "country" TEXT NOT NULL DEFAULT '';"#; 10 | let db = manager.get_connection(); 11 | db.execute_unprepared("BEGIN EXCLUSIVE").await?; 12 | tracing::info!("adding version_url column.."); 13 | db.execute_unprepared(cmd).await?; 14 | db.execute_unprepared("COMMIT TRANSACTION").await?; 15 | db.execute_unprepared("VACUUM").await?; 16 | Ok(()) 17 | } 18 | 19 | async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> { 20 | panic!("Can't migrate down"); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /migration/src/m20230914_231514_connectivity.rs: -------------------------------------------------------------------------------- 1 | use sea_orm_migration::prelude::*; 2 | 3 | #[derive(DeriveMigrationName)] 4 | pub struct Migration; 5 | 6 | #[async_trait::async_trait] 7 | impl MigrationTrait for Migration { 8 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 9 | let cmd = r#"ALTER TABLE "host" ADD COLUMN "connectivity" INT;"#; 10 | let db = manager.get_connection(); 11 | db.execute_unprepared("BEGIN EXCLUSIVE").await?; 12 | tracing::info!("adding connectivity column.."); 13 | db.execute_unprepared(cmd).await?; 14 | db.execute_unprepared("COMMIT TRANSACTION").await?; 15 | db.execute_unprepared("VACUUM").await?; 16 | Ok(()) 17 | } 18 | 19 | async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> { 20 | panic!("Can't migrate down"); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /migration/src/m20231011_231223_errors.rs: -------------------------------------------------------------------------------- 1 | use sea_orm_migration::prelude::*; 2 | 3 | #[derive(DeriveMigrationName)] 4 | pub struct Migration; 5 | 6 | #[async_trait::async_trait] 7 | impl MigrationTrait for Migration { 8 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 9 | let cmd = r#"CREATE TABLE "check_errors" ( 10 | "time" integer NOT NULL, 11 | "host" integer NOT NULL, 12 | "message" text NOT NULL, 13 | "http_body" text, 14 | "http_status" integer, 15 | CONSTRAINT "pk_check_errors" PRIMARY KEY ("host", "time"), 16 | FOREIGN KEY ("host") REFERENCES "host" ("id") ON DELETE CASCADE ON UPDATE CASCADE 17 | ) WITHOUT ROWID, STRICT;"#; 18 | let db = manager.get_connection(); 19 | db.execute_unprepared("BEGIN EXCLUSIVE").await?; 20 | tracing::info!("adding check_errors table.."); 21 | db.execute_unprepared(cmd).await?; 22 | db.execute_unprepared("COMMIT TRANSACTION").await?; 23 | db.execute_unprepared("VACUUM").await?; 24 | Ok(()) 25 | } 26 | 27 | async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> { 28 | panic!("Can't migrate down"); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /migration/src/m20231112_142206_stats.rs: -------------------------------------------------------------------------------- 1 | use sea_orm_migration::prelude::*; 2 | 3 | #[derive(DeriveMigrationName)] 4 | pub struct Migration; 5 | 6 | #[async_trait::async_trait] 7 | impl MigrationTrait for Migration { 8 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 9 | let cmd = r#"CREATE TABLE "instance_stats" ( 10 | "time" integer NOT NULL, 11 | "host" integer NOT NULL, 12 | "limited_accs" integer NOT NULL, 13 | "total_accs" integer NOT NULL, 14 | "total_requests" integer NOT NULL, 15 | "req_photo_rail" integer NOT NULL, 16 | "req_user_screen_name" integer NOT NULL, 17 | "req_search" integer NOT NULL, 18 | "req_list_tweets" integer NOT NULL, 19 | "req_user_media" integer NOT NULL, 20 | "req_tweet_detail" integer NOT NULL, 21 | "req_list" integer NOT NULL, 22 | "req_user_tweets" integer NOT NULL, 23 | "req_user_tweets_and_replies" integer NOT NULL, 24 | CONSTRAINT "pk_instance_stats" PRIMARY KEY ("host", "time"), 25 | FOREIGN KEY ("host") REFERENCES "host" ("id") ON DELETE CASCADE ON UPDATE CASCADE 26 | ) WITHOUT ROWID, STRICT;"#; 27 | let db = manager.get_connection(); 28 | db.execute_unprepared("BEGIN EXCLUSIVE").await?; 29 | tracing::info!("adding instance_stats table.."); 30 | db.execute_unprepared(cmd).await?; 31 | db.execute_unprepared("COMMIT TRANSACTION").await?; 32 | db.execute_unprepared("VACUUM").await?; 33 | Ok(()) 34 | } 35 | 36 | async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> { 37 | panic!("Can't migrate down"); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /migration/src/m20250216_185501_overrides.rs: -------------------------------------------------------------------------------- 1 | use sea_orm_migration::prelude::*; 2 | 3 | #[derive(DeriveMigrationName)] 4 | pub struct Migration; 5 | 6 | #[async_trait::async_trait] 7 | impl MigrationTrait for Migration { 8 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 9 | let cmd_host_overrides = r#"CREATE TABLE "host_overrides" ( 10 | "host" integer NOT NULL, 11 | "key" text NOT NULL, 12 | "locked" integer NOT NULL, 13 | "value" text, 14 | CONSTRAINT "pk_override_hostkey" PRIMARY KEY ("host", "key"), 15 | FOREIGN KEY ("host") REFERENCES "host" ("id") ON DELETE CASCADE ON UPDATE CASCADE 16 | ) STRICT;"#; 17 | let cmd_overrides_index = r#"CREATE INDEX "index_overrides_key" ON host_overrides ("key");"#; 18 | let cmd_log = r#"CREATE TABLE "log" ( 19 | "user_host" integer NOT NULL, 20 | "host_affected" integer, 21 | "key" text NOT NULL, 22 | "time" integer NOT NULL, 23 | "new_value" text, 24 | CONSTRAINT "pk_log_usertime" PRIMARY KEY ("user_host", "time"), 25 | FOREIGN KEY ("host_affected") REFERENCES "host" ("id") ON DELETE CASCADE ON UPDATE CASCADE, 26 | FOREIGN KEY ("user_host") REFERENCES "host" ("id") ON DELETE RESTRICT ON UPDATE CASCADE 27 | ) STRICT;"#; 28 | let cmd_log_index = r#"CREATE INDEX "index_log_time" ON log ("time");"#; 29 | let db = manager.get_connection(); 30 | db.execute_unprepared("BEGIN EXCLUSIVE").await?; 31 | tracing::info!("adding host overrides table.."); 32 | db.execute_unprepared(cmd_host_overrides).await?; 33 | tracing::info!("adding host overrides index.."); 34 | db.execute_unprepared(cmd_overrides_index).await?; 35 | tracing::info!("adding log table.."); 36 | db.execute_unprepared(cmd_log).await?; 37 | tracing::info!("adding log index.."); 38 | db.execute_unprepared(cmd_log_index).await?; 39 | db.execute_unprepared("COMMIT TRANSACTION").await?; 40 | db.execute_unprepared("VACUUM").await?; 41 | Ok(()) 42 | } 43 | 44 | async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> { 45 | panic!("Can't migrate down"); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /migration/src/main.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: AGPL-3.0-only 2 | use sea_orm_migration::prelude::*; 3 | 4 | #[async_std::main] 5 | async fn main() { 6 | cli::run_cli(migration::Migrator).await; 7 | } 8 | -------------------------------------------------------------------------------- /scanner/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "scanner" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | reqwest = { workspace = true, features = [ 10 | "deflate", 11 | "gzip", 12 | "brotli", 13 | "cookies", 14 | "rustls-tls", 15 | ] } 16 | sea-orm = { workspace = true, features = [ 17 | "sqlx-sqlite", 18 | "runtime-tokio-native-tls", 19 | "macros", 20 | ] } 21 | sea-query = { workspace = true } 22 | thiserror = { workspace = true } 23 | miette = { workspace = true } 24 | tracing = { workspace = true } 25 | tokio = { workspace = true, features = ["full"] } 26 | scraper = "0.17.1" 27 | chrono = { workspace = true } 28 | regex = { workspace = true } 29 | git2 = "0.19.0" 30 | # testing 31 | serde = { workspace = true, features = ["derive"] } 32 | # health parsing 33 | serde_json = "1" 34 | 35 | [dev-dependencies] 36 | tracing-test = { workspace = true } 37 | csv = "1.2.2" 38 | 39 | [dev-dependencies.migration] 40 | path = "../migration" 41 | 42 | [dependencies.entities] 43 | path = "../entities" 44 | -------------------------------------------------------------------------------- /scanner/src/about_parser.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: AGPL-3.0-only 2 | use regex::{Regex, RegexBuilder}; 3 | use scraper::{Html, Selector}; 4 | use thiserror::Error; 5 | 6 | use crate::instance_parser::EXPECT_CSS_SELCTOR; 7 | 8 | pub type Result = std::result::Result; 9 | 10 | #[derive(Error, Debug)] 11 | pub enum AboutParseError { 12 | #[error("No p containing a version found!")] 13 | NoAboutElement, 14 | #[error("No a element found!")] 15 | NoCommitLinkFound, 16 | #[error("Missing test! Found '{0}'")] 17 | InvalidCommitFormat(String), 18 | #[error("No valid href!")] 19 | NoValidHref, 20 | } 21 | 22 | pub(crate) struct AboutParser { 23 | selector_p: Selector, 24 | selector_a: Selector, 25 | regex: Regex, 26 | } 27 | 28 | pub struct AboutParsed { 29 | pub version_name: String, 30 | pub url: String, 31 | } 32 | 33 | impl AboutParser { 34 | /// Returns the text of the version element of nitters about site 35 | pub fn parse_about_version(&self, html: &str) -> Result { 36 | let fragment = Html::parse_fragment(html); 37 | // get all

elements 38 | let p_elem = fragment 39 | .select(&self.selector_p) 40 | .find(|t| t.text().any(|text| text.contains("Version"))) 41 | .ok_or(AboutParseError::NoAboutElement)?; 42 | 43 | let mut a_elems = p_elem.select(&self.selector_a); 44 | let link = a_elems.next().ok_or(AboutParseError::NoCommitLinkFound)?; 45 | let url = link 46 | .value() 47 | .attr("href") 48 | .map(|v| v.trim().to_owned()) 49 | .ok_or(AboutParseError::NoValidHref)?; 50 | let link_text = link.text().fold(String::new(), |mut acc, text| { 51 | acc.push_str(text); 52 | acc 53 | }); 54 | if !self.regex.is_match(&link_text) { 55 | return Err(AboutParseError::InvalidCommitFormat(link_text)); 56 | } 57 | Ok(AboutParsed { 58 | url, 59 | version_name: link_text, 60 | }) 61 | } 62 | 63 | pub fn new() -> Self { 64 | let mut builder = RegexBuilder::new(r#"^((\d+\.\d+\.\d+)|[a-zA-Z0-9]{7,})"#); 65 | builder.case_insensitive(true); 66 | Self { 67 | selector_p: Selector::parse("p").expect(EXPECT_CSS_SELCTOR), 68 | selector_a: Selector::parse("a").expect(EXPECT_CSS_SELCTOR), 69 | regex: builder.build().expect("failed to generate regex"), 70 | } 71 | } 72 | } 73 | 74 | #[cfg(test)] 75 | mod test { 76 | use super::*; 77 | #[test] 78 | fn parse() { 79 | let html = include_str!("../test_data/about.html"); 80 | let parser = AboutParser::new(); 81 | let res = parser.parse_about_version(html).unwrap(); 82 | assert_eq!(&res.version_name, "2023.07.22-72d8f35"); 83 | assert_eq!( 84 | res.url, 85 | String::from("https://github.com/zedeus/nitter/commit/72d8f35") 86 | ); 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /scanner/src/cleanup.rs: -------------------------------------------------------------------------------- 1 | use entities::check_errors; 2 | use entities::host; 3 | use sea_orm::ColumnTrait; 4 | use sea_orm::EntityTrait; 5 | use sea_orm::Order; 6 | use sea_orm::QueryFilter; 7 | use sea_query::Query; 8 | use tokio::time::interval; 9 | 10 | use crate::Result; 11 | use crate::Scanner; 12 | 13 | impl Scanner { 14 | /// Setup scheduled job for cleaning up old data 15 | pub(crate) fn schedule_cleanup(&self) -> Result<()> { 16 | let c = self.clone(); 17 | tokio::spawn(async move { 18 | let mut interval = interval(c.inner.config.cleanup_interval); 19 | loop { 20 | interval.tick().await; 21 | if let Err(e) = c.cleanup().await { 22 | tracing::error!(error=%e); 23 | } 24 | } 25 | }); 26 | Ok(()) 27 | } 28 | /// Perform cleanup of outdated data 29 | async fn cleanup(&self) -> Result<()> { 30 | self.cleanup_errors().await?; 31 | Ok(()) 32 | } 33 | 34 | /// Remove all but recent host error entries 35 | async fn cleanup_errors(&self) -> Result<()> { 36 | // I wish this was easier without row ids or giant SQL queries 37 | let hosts = host::Entity::find().all(&self.inner.db).await?; 38 | for host in hosts { 39 | let res = if host.enabled { 40 | check_errors::Entity::delete_many() 41 | .filter(check_errors::Column::Host.eq(host.id)) 42 | .filter( 43 | check_errors::Column::Time.not_in_subquery( 44 | Query::select() 45 | .column(check_errors::Column::Time) 46 | .from(check_errors::Entity) 47 | .and_where(check_errors::Column::Host.eq(host.id)) 48 | .order_by(check_errors::Column::Time, Order::Desc) 49 | .limit(self.inner.config.error_retention_per_host as _) 50 | .to_owned(), 51 | ), 52 | ) 53 | .exec(&self.inner.db) 54 | .await? 55 | } else { 56 | check_errors::Entity::delete_many() 57 | .filter(check_errors::Column::Host.eq(host.id)) 58 | .exec(&self.inner.db) 59 | .await? 60 | }; 61 | if res.rows_affected > 0 { 62 | tracing::debug!( 63 | host = host.id, 64 | host.enabled, 65 | deleted_errors = res.rows_affected 66 | ); 67 | } 68 | } 69 | 70 | Ok(()) 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /scanner/src/instance_check.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: AGPL-3.0-only 2 | //! Instance health/uptime checking code 3 | use std::time::Instant; 4 | 5 | use chrono::Utc; 6 | use entities::state::error_cache::HostError; 7 | use entities::{check_errors, health_check, host_overrides}; 8 | use entities::{host, prelude::*}; 9 | use entities::host_overrides::keys::HostOverrides; 10 | use reqwest::Url; 11 | use sea_orm::prelude::DateTimeUtc; 12 | use sea_orm::ColumnTrait; 13 | use sea_orm::EntityTrait; 14 | use sea_orm::QueryFilter; 15 | use sea_orm::{ActiveModelTrait, ActiveValue}; 16 | use tokio::task::JoinSet; 17 | use tracing::instrument; 18 | 19 | use crate::about_parser::AboutParsed; 20 | use crate::Result; 21 | use crate::Scanner; 22 | 23 | impl Scanner { 24 | /// Check uptime for host and create a new uptime entry in the database 25 | pub(crate) async fn check_uptime(&mut self) -> Result<()> { 26 | let start = Instant::now(); 27 | let hosts = Host::find() 28 | .filter(host::Column::Enabled.eq(true)) 29 | .all(&self.inner.db) 30 | .await?; 31 | 32 | let mut join_set = JoinSet::new(); 33 | 34 | let last_check = self.query_latest_check(&self.inner.db).await?; 35 | 36 | for model in hosts.into_iter() { 37 | let scanner = self.clone(); 38 | let muted_host = last_check 39 | .iter() 40 | .find(|v| v.host == model.id) 41 | .map_or(false, |check| !check.healthy); 42 | join_set.spawn(async move { 43 | if let Err(e) = scanner.health_check_host(model, muted_host).await { 44 | 45 | } 46 | }); 47 | } 48 | // wait till all of them are finished, preventing DoS 49 | let tasks = join_set.len(); 50 | while let Some(_) = join_set.join_next().await {} 51 | let end = Instant::now(); 52 | let took_ms = end.saturating_duration_since(start).as_millis(); 53 | *self.inner.last_uptime_check.lock().unwrap() = Utc::now(); 54 | tracing::debug!(hosts = tasks, took_ms = took_ms, "checked uptime"); 55 | Ok(()) 56 | } 57 | 58 | #[instrument] 59 | async fn health_check_host(&self, host: host::Model, muted: bool) -> Result<()> { 60 | let now = Utc::now(); 61 | let mut url = match Url::parse(&host.url) { 62 | Err(e) => { 63 | if !muted { 64 | tracing::error!(error=?e, url=host.url,"failed to parse instance URL"); 65 | } 66 | self.insert_failed_health_check( 67 | host.id, 68 | now, 69 | HostError::new_message(format!("Not a valid URL")), 70 | None, 71 | ) 72 | .await; 73 | return Ok(()); 74 | } 75 | Ok(v) => v, 76 | }; 77 | url.set_path(&self.inner.config.profile_path); 78 | let overrides = HostOverrides::load(&host, &self.inner.db).await?; 79 | let start = Instant::now(); 80 | let fetch_res = self.fetch_url(url.as_str(), overrides.bearer()).await; 81 | let end = Instant::now(); 82 | let took_ms = end.saturating_duration_since(start).as_millis(); 83 | match fetch_res { 84 | Err(e) => { 85 | if !muted { 86 | tracing::info!( 87 | host = host.url, 88 | took = took_ms, 89 | "couldn't ping host: {e}, marking as dead" 90 | ); 91 | } 92 | self.insert_failed_health_check( 93 | host.id, 94 | now, 95 | e.to_host_error(), 96 | Some(took_ms as _), 97 | ) 98 | .await; 99 | } 100 | Ok((http_code, content)) => { 101 | if !muted { 102 | tracing::trace!(host = host.url, took = took_ms); 103 | } 104 | // check for valid profile 105 | match self.inner.profile_parser.parse_profile_content(&content) { 106 | Err(e) => { 107 | if !muted { 108 | tracing::debug!( 109 | error=?e, 110 | content = content, 111 | "host doesn't contain a valid profile" 112 | ); 113 | } 114 | self.insert_failed_health_check( 115 | host.id, 116 | now, 117 | HostError::new(e.to_string(), content, http_code), 118 | Some(took_ms as _), 119 | ) 120 | .await; 121 | } 122 | Ok(profile_content) => { 123 | if self.inner.config.profile_name != profile_content.name 124 | || self.inner.config.profile_posts_min > profile_content.post_count 125 | { 126 | if !muted { 127 | tracing::debug!( 128 | profile_content = ?profile_content, 129 | "host doesn't contain expected profile content" 130 | ); 131 | } 132 | self.insert_failed_health_check( 133 | host.id, 134 | now, 135 | HostError::new( 136 | format!("profile content mismatch"), 137 | content, 138 | http_code, 139 | ), 140 | Some(took_ms as _), 141 | ) 142 | .await; 143 | } else { 144 | // create successful uptime entry 145 | if let Err(e) = (health_check::ActiveModel { 146 | time: ActiveValue::Set(now.timestamp()), 147 | host: ActiveValue::Set(host.id), 148 | resp_time: ActiveValue::Set(Some(took_ms as _)), 149 | response_code: ActiveValue::Set(Some(http_code as _)), 150 | healthy: ActiveValue::Set(true), 151 | } 152 | .insert(&self.inner.db) 153 | .await) 154 | { 155 | tracing::error!(host=host.id, error=?e,"Failed to insert update check"); 156 | } 157 | } 158 | } 159 | } 160 | } 161 | } 162 | Ok(()) 163 | } 164 | 165 | /// Check if rss is available 166 | pub(crate) async fn has_rss(&self, url: &mut Url, api_token: Option<&str>, mute: bool) -> bool { 167 | url.set_path(&self.inner.config.rss_path); 168 | match self.fetch_url(url.as_str(), api_token).await { 169 | Ok((code, content)) => match self.inner.rss_check_regex.is_match(&content) { 170 | true => return true, 171 | false => { 172 | if !mute { 173 | // 404 = disabled 174 | tracing::debug!( 175 | url = url.as_str(), 176 | code = code, 177 | content = content, 178 | "rss content not found" 179 | ); 180 | } 181 | return false; 182 | } 183 | }, 184 | Err(e) => { 185 | if !mute && e.http_status_code() != Some(404) { 186 | tracing::debug!(error=?e,url=url.as_str(),"fetching rss feed failed"); 187 | } 188 | return false; 189 | } 190 | } 191 | } 192 | 193 | /// Check nitter version 194 | pub(crate) async fn nitter_version(&self, url: &mut Url, api_token: Option<&str>, mute: bool) -> Option { 195 | url.set_path(&self.inner.config.about_path); 196 | match self.fetch_url(url.as_str(), api_token).await { 197 | Ok((code, content)) => match self.inner.about_parser.parse_about_version(&content) { 198 | Ok(v) => Some(v), 199 | Err(e) => { 200 | if !mute { 201 | tracing::debug!(url=url.as_str(),code,content,error=?e,"failed parsing version from about page"); 202 | } 203 | None 204 | } 205 | }, 206 | Err(e) => { 207 | if !mute { 208 | tracing::debug!(url=url.as_str(),error=?e,"failed fetching about page"); 209 | } 210 | None 211 | } 212 | } 213 | } 214 | 215 | async fn insert_failed_health_check( 216 | &self, 217 | host: i32, 218 | time: DateTimeUtc, 219 | host_error: HostError, 220 | resp_time: Option, 221 | ) { 222 | if let Err(e) = (health_check::ActiveModel { 223 | time: ActiveValue::Set(time.timestamp()), 224 | host: ActiveValue::Set(host), 225 | resp_time: ActiveValue::Set(resp_time), 226 | healthy: ActiveValue::Set(false), 227 | response_code: ActiveValue::Set(host_error.http_status), 228 | } 229 | .insert(&self.inner.db) 230 | .await) 231 | { 232 | tracing::error!(error=?e,"Failed to insert update check"); 233 | } 234 | if let Err(e) = (check_errors::ActiveModel { 235 | time: ActiveValue::Set(host_error.time.timestamp()), 236 | host: ActiveValue::Set(host), 237 | message: ActiveValue::Set(host_error.message), 238 | http_body: ActiveValue::Set(host_error.http_body), 239 | http_status: ActiveValue::Set(host_error.http_status), 240 | } 241 | .insert(&self.inner.db) 242 | .await) 243 | { 244 | tracing::error!(host=host, error=?e,"Failed to insert error for host"); 245 | } 246 | } 247 | } 248 | -------------------------------------------------------------------------------- /scanner/src/instance_parser.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: AGPL-3.0-only 2 | use std::collections::HashMap; 3 | 4 | use reqwest::Url; 5 | use scraper::{ElementRef, Html, Selector}; 6 | use thiserror::Error; 7 | 8 | pub type Result = std::result::Result; 9 | 10 | pub static EXPECT_CSS_SELCTOR: &'static str = "failed to parse css selector"; 11 | static CHECKBOX: &'static str = "✅"; 12 | 13 | type InstanceMap = HashMap; 14 | 15 | #[derive(Error, Debug)] 16 | pub enum InstanceListError { 17 | #[error("No div #wiki-body found!")] 18 | NoWikiDiv, 19 | #[error("No table found containing instances!")] 20 | NoInstanceTable, 21 | #[error("Abort-on-err on, malformed table row found!")] 22 | MalformedRow, 23 | } 24 | #[derive(Debug, Eq, PartialEq)] 25 | #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] 26 | pub struct InstanceParsed { 27 | /// URL without any login stuff 28 | pub domain: String, 29 | /// connection URL 30 | pub url: String, 31 | /// whether this instance is marked as online 32 | pub online: bool, 33 | /// the SSL provider this instance supposedly has 34 | pub ssl_provider: String, 35 | /// the country for this instance 36 | pub country: String, 37 | } 38 | 39 | /// Instance parser. 40 | pub(crate) struct InstanceParser { 41 | selector_wiki: Selector, 42 | selector_table: Selector, 43 | selector_tr: Selector, 44 | selector_td: Selector, 45 | selector_a: Selector, 46 | } 47 | 48 | impl InstanceParser { 49 | pub fn new() -> Self { 50 | Self { 51 | selector_wiki: Selector::parse(r#"div[id="wiki-body"]"#).expect(EXPECT_CSS_SELCTOR), 52 | selector_table: Selector::parse("table").expect(EXPECT_CSS_SELCTOR), 53 | selector_tr: Selector::parse("tbody > tr").expect(EXPECT_CSS_SELCTOR), 54 | selector_td: Selector::parse("td").expect(EXPECT_CSS_SELCTOR), 55 | selector_a: Selector::parse("a").expect(EXPECT_CSS_SELCTOR), 56 | } 57 | } 58 | 59 | /// Parse a html rendered version of the instance list 60 | /// 61 | /// *abort_on_err* is just for testing and return an error for any malformed table entry 62 | pub fn parse_instancelist( 63 | &self, 64 | html: &str, 65 | additional_instances: &[String], 66 | additional_instances_country: &str, 67 | abort_on_err: bool, 68 | ) -> Result { 69 | let fragment = Html::parse_fragment(html); 70 | // wiki body by div ID 71 | let mut wiki_divs = fragment.select(&self.selector_wiki); 72 | // first result 73 | let first_wiki = wiki_divs.next().ok_or(InstanceListError::NoWikiDiv)?; 74 | // all element 75 | let mut tables = first_wiki.select(&self.selector_table); 76 | // find the one with "Online" text inside 77 | let instance_table = tables 78 | .find(|t| t.text().any(|text| text.contains("Online"))) 79 | .ok_or(InstanceListError::NoInstanceTable)?; 80 | 81 | let mut instances = HashMap::with_capacity(50); 82 | // iterate over all > inside 83 | for row in instance_table.select(&self.selector_tr) { 84 | match self.parse_row(row) { 85 | Ok(instance) => { 86 | if let Some(old) = instances.insert(instance.domain.clone(), instance) { 87 | tracing::warn!(domain = old.domain, "Parsed duplicate instance domain!"); 88 | } 89 | } 90 | Err(e) => { 91 | if abort_on_err { 92 | return Err(e); 93 | } 94 | continue; 95 | } 96 | } 97 | } 98 | 99 | for entry in additional_instances { 100 | match Url::parse(entry.as_ref()) { 101 | Ok(v) => { 102 | if let Some(domain) = v.domain() { 103 | instances.insert( 104 | domain.to_owned(), 105 | InstanceParsed { 106 | domain: domain.to_owned(), 107 | url: entry.clone(), 108 | online: true, 109 | ssl_provider: String::new(), 110 | country: additional_instances_country.to_owned(), 111 | }, 112 | ); 113 | } 114 | } 115 | Err(e) => tracing::warn!(instance=entry,error=?e,"Ignoring additional instance"), 116 | } 117 | } 118 | 119 | Ok(instances) 120 | } 121 | 122 | /// Parse a single instance table row 123 | fn parse_row(&self, row: ElementRef) -> Result { 124 | let mut cols = row.select(&self.selector_td); 125 | // get first URL column 126 | let url: String = match cols.next() { 127 | None => { 128 | tracing::error!(row=?row.html(),"Parsed instance missing URL row, skipping!"); 129 | return Err(InstanceListError::MalformedRow); 130 | } 131 | Some(col) => { 132 | // find first inside and get its "href" attribute 133 | let mut a_elems = col.select(&self.selector_a); 134 | match a_elems.next().and_then(|v| v.value().attr("href")) { 135 | None => { 136 | tracing::error!(row=?row.html(),"Parsed instance missing valid URL element, skipping!"); 137 | return Err(InstanceListError::MalformedRow); 138 | } 139 | Some(url_value) => { 140 | // trim whitespace and strip `/` at the end 141 | let trimmed = url_value.trim(); 142 | trimmed.strip_suffix("/").unwrap_or(trimmed).to_owned() 143 | } 144 | } 145 | } 146 | }; 147 | // parse URL to strip everything apart from the domain 148 | let domain = match Url::parse(&url) { 149 | Ok(parsed_url) => parsed_url.domain().map(|v| v.to_owned()).ok_or_else(|| { 150 | tracing::error!(url = url, "Parsed instance URL has no domain"); 151 | InstanceListError::MalformedRow 152 | })?, 153 | Err(e) => { 154 | tracing::error!(url=url,error=?e,"Parsed instance URL is not valid"); 155 | return Err(InstanceListError::MalformedRow); 156 | } 157 | }; 158 | 159 | // map all remaining cols into Strings 160 | let columns: Vec<_> = cols 161 | .map(|col| { 162 | col.text().fold(String::new(), |mut acc, text| { 163 | acc.push_str(text); 164 | acc 165 | }) 166 | }) 167 | .collect(); 168 | if columns.len() < 4 { 169 | tracing::error!(instance_data=?columns,"Parsed instance missing fields, skipping!"); 170 | return Err(InstanceListError::MalformedRow); 171 | } 172 | let instance = InstanceParsed { 173 | domain, 174 | url, 175 | online: columns[0] == CHECKBOX, 176 | ssl_provider: columns[3].clone(), 177 | country: columns[2].clone(), 178 | }; 179 | Ok(instance) 180 | } 181 | } 182 | 183 | #[cfg(test)] 184 | mod test { 185 | use csv; 186 | use std::collections::HashMap; 187 | use tracing_test::traced_test; 188 | 189 | use super::*; 190 | #[test] 191 | #[traced_test] 192 | fn parse() { 193 | // update test html via test_fetch_instance_list 194 | let html = include_str!("../test_data/instancelist.html"); 195 | let parser = InstanceParser::new(); 196 | let res = parser.parse_instancelist(html, &[], "", true).unwrap(); 197 | 198 | // writeback for new tests 199 | // write_data(res.values()); 200 | 201 | // adjust when updating the test html 202 | let expected: HashMap = expected_data() 203 | .into_iter() 204 | .map(|instance| (instance.domain.clone(), instance)) 205 | .collect(); 206 | assert_eq!(res.len(), expected.len()); 207 | for (_, instance) in res.iter() { 208 | assert_eq!(Some(instance), expected.get(&instance.domain)); 209 | } 210 | } 211 | 212 | fn expected_data() -> Vec { 213 | let file = std::fs::File::open("test_data/instancelist_expected.csv").unwrap(); 214 | let mut rdr = csv::Reader::from_reader(file); 215 | let mut vec = Vec::new(); 216 | for result in rdr.deserialize() { 217 | // Notice that we need to provide a type hint for automatic 218 | // deserialization. 219 | let record: InstanceParsed = result.unwrap(); 220 | vec.push(record); 221 | } 222 | vec 223 | } 224 | } 225 | -------------------------------------------------------------------------------- /scanner/src/list_update.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: AGPL-3.0-only 2 | //! Updates the list of available instances, fetching all required fields 3 | 4 | use std::collections::HashMap; 5 | use std::time::Duration; 6 | use std::time::Instant; 7 | 8 | use chrono::Utc; 9 | use entities::host; 10 | use entities::host_overrides::keys::HostOverrides; 11 | use entities::prelude::Host; 12 | use reqwest::Url; 13 | use sea_orm::{ 14 | ActiveModelTrait, ActiveValue, ColumnTrait, EntityTrait, QueryFilter, TransactionTrait, 15 | }; 16 | use sea_query::OnConflict; 17 | use tokio::task::JoinSet; 18 | use tracing::instrument; 19 | 20 | use crate::Result; 21 | use crate::Scanner; 22 | 23 | impl Scanner { 24 | /// Fetches the list of all instances from the wiki. 25 | /// Updates all fields for host::Model, including connectivity, rss, version and enabled. 26 | #[instrument] 27 | pub(crate) async fn update_instacelist(&mut self) -> Result<()> { 28 | let start = Instant::now(); 29 | let html: String = self.fetch_instance_list().await?; 30 | let parsed_instances = self.inner.instance_parser.parse_instancelist( 31 | &html, 32 | &self.inner.config.additional_hosts, 33 | &self.inner.config.additional_host_country, 34 | false, 35 | )?; 36 | 37 | let transaction = self.inner.db.begin().await?; 38 | 39 | // find all currently enabled instances 40 | let enabled_hosts = Host::find() 41 | .filter(host::Column::Enabled.eq(true)) 42 | .all(&transaction) 43 | .await?; 44 | // make a diff and remove the ones not found while parsing 45 | let time: chrono::DateTime = Utc::now(); 46 | let mut removed = 0; 47 | for host in enabled_hosts.iter() { 48 | if !parsed_instances.contains_key(&host.domain) { 49 | host::ActiveModel { 50 | id: ActiveValue::Set(host.id), 51 | enabled: ActiveValue::Set(false), 52 | updated: ActiveValue::Set(time.timestamp()), 53 | ..Default::default() 54 | } 55 | .update(&transaction) 56 | .await?; 57 | removed += 1; 58 | } 59 | } 60 | 61 | // now update/insert the existing ones 62 | 63 | let found_instances: usize = parsed_instances.len(); 64 | // find last update checks to detect spam 65 | let last_status = self.query_latest_check(&transaction).await?; 66 | // prevent blocking the DB forever during host updates 67 | // worst case we disable hosts that should be disabled and add no new ones 68 | transaction.commit().await?; 69 | 70 | let mut join_set = JoinSet::new(); 71 | for (_, instance) in parsed_instances { 72 | let overrides = HostOverrides::load_by_domain(&instance.domain, &self.inner.db).await?; 73 | 74 | // TODO: parallelize this! 75 | let scanner_c = self.clone(); 76 | // detect already offline host and prevent log spam 77 | let muted_host = match self.inner.config.auto_mute { 78 | false => false, 79 | true => last_status 80 | .iter() 81 | .find(|v| v.domain == instance.domain) 82 | .map_or(false, |check| !check.healthy), 83 | }; 84 | // tracing::trace!(muted_host,instance=?instance,last_status=?last_status); 85 | join_set.spawn(async move { 86 | let (connectivity, rss, version, version_url) = match Url::parse(&instance.url) { 87 | Err(_) => { 88 | if !muted_host { 89 | tracing::info!(url = instance.url, "Instance URL invalid"); 90 | } 91 | (None, false, None, None) 92 | } 93 | Ok(mut url) => { 94 | let connectivity = scanner_c.check_connectivity(&mut url).await; 95 | // prevent DoS 96 | tokio::time::sleep(Duration::from_secs(1)).await; 97 | let rss = scanner_c 98 | .has_rss(&mut url, overrides.bearer(), muted_host) 99 | .await; 100 | tokio::time::sleep(Duration::from_secs(1)).await; 101 | match scanner_c 102 | .nitter_version(&mut url, overrides.bearer(), muted_host) 103 | .await 104 | { 105 | Some(version) => ( 106 | connectivity, 107 | rss, 108 | Some(version.version_name), 109 | Some(version.url), 110 | ), 111 | None => (connectivity, rss, None, None), 112 | } 113 | } 114 | }; 115 | 116 | host::ActiveModel { 117 | id: ActiveValue::NotSet, 118 | domain: ActiveValue::Set(instance.domain), 119 | country: ActiveValue::Set(instance.country), 120 | url: ActiveValue::Set(instance.url), 121 | enabled: ActiveValue::Set(true), 122 | version: ActiveValue::Set(version), 123 | version_url: ActiveValue::Set(version_url), 124 | rss: ActiveValue::Set(rss), 125 | updated: ActiveValue::Set(time.timestamp()), 126 | connectivity: ActiveValue::Set(connectivity), 127 | } 128 | }); 129 | } 130 | 131 | let mut update_models = Vec::with_capacity(join_set.len()); 132 | while let Some(update_model) = join_set.join_next().await.map(|v| v.unwrap()) { 133 | update_models.push(update_model); 134 | } 135 | // insert all at once 136 | let transaction = self.inner.db.begin().await?; 137 | Host::insert_many(update_models) 138 | .on_conflict( 139 | OnConflict::column(host::Column::Domain) 140 | .update_columns([ 141 | host::Column::Enabled, 142 | host::Column::Updated, 143 | host::Column::Url, 144 | host::Column::Rss, 145 | host::Column::Version, 146 | host::Column::VersionUrl, 147 | host::Column::Country, 148 | host::Column::Connectivity, 149 | ]) 150 | .to_owned(), 151 | ) 152 | .exec(&transaction) 153 | .await?; 154 | 155 | transaction.commit().await?; 156 | let end = Instant::now(); 157 | let took_ms = end.saturating_duration_since(start).as_millis(); 158 | { 159 | *self.inner.last_list_fetch.lock().unwrap() = Utc::now(); 160 | } 161 | tracing::debug!( 162 | removed = removed, 163 | found = found_instances, 164 | took_ms = took_ms 165 | ); 166 | Ok(()) 167 | } 168 | 169 | /// Check ipv4/6 connectivity of host 170 | async fn check_connectivity(&self, url: &mut Url) -> Option { 171 | url.set_path(&self.inner.config.connectivity_path); 172 | let ipv4 = self 173 | .inner 174 | .client_ipv4 175 | .get(url.as_str()) 176 | .send() 177 | .await 178 | .map_or(false, |res| res.status().is_success()); 179 | // prevent DoS 180 | tokio::time::sleep(Duration::from_secs(1)).await; 181 | let ipv6 = self 182 | .inner 183 | .client_ipv6 184 | .get(url.as_str()) 185 | .send() 186 | .await 187 | .map_or(false, |res| res.status().is_success()); 188 | 189 | match (ipv4, ipv6) { 190 | (true, true) => Some(host::Connectivity::All), 191 | (false, true) => Some(host::Connectivity::IPv6), 192 | (true, false) => Some(host::Connectivity::IPv4), 193 | (false, false) => None, 194 | } 195 | } 196 | } 197 | 198 | #[cfg(test)] 199 | mod test { 200 | use super::*; 201 | use entities::state::scanner::Config; 202 | use tracing_test::traced_test; 203 | 204 | use crate::{test::db_init, Scanner}; 205 | 206 | #[tokio::test(flavor = "multi_thread", worker_threads = 1)] 207 | #[traced_test] 208 | #[ignore] 209 | async fn connectivity_test() { 210 | let db = db_init().await; 211 | let scanner = Scanner::new(db, Config::test_defaults(), entities::state::new()) 212 | .await 213 | .unwrap(); 214 | assert_eq!( 215 | scanner 216 | .check_connectivity(&mut Url::parse("https://v4.ipv6test.app").unwrap()) 217 | .await, 218 | Some(host::Connectivity::IPv4) 219 | ); 220 | assert_eq!( 221 | scanner 222 | .check_connectivity(&mut Url::parse("https://ipv6test.app").unwrap()) 223 | .await, 224 | Some(host::Connectivity::All) 225 | ); 226 | assert_eq!( 227 | scanner 228 | .check_connectivity(&mut Url::parse("https://v6.ipv6test.app").unwrap()) 229 | .await, 230 | Some(host::Connectivity::IPv6) 231 | ); 232 | } 233 | } 234 | -------------------------------------------------------------------------------- /scanner/src/profile_parser.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: AGPL-3.0-only 2 | //! Parse profile pages for verification 3 | use scraper::{Html, Selector}; 4 | use thiserror::Error; 5 | 6 | use crate::instance_parser::EXPECT_CSS_SELCTOR; 7 | 8 | pub type Result = std::result::Result; 9 | 10 | #[derive(Error, Debug)] 11 | pub enum ProfileParseError { 12 | #[error("No profile-card div found!")] 13 | NoProfileCard, 14 | #[error("No timeline div found!")] 15 | NoTimeline, 16 | } 17 | 18 | pub(crate) struct ProfileParser { 19 | selector_profile_card_name: Selector, 20 | selector_timeline: Selector, 21 | selector_timeline_item: Selector, 22 | } 23 | 24 | #[derive(Debug)] 25 | pub struct ProfileParsed { 26 | pub post_count: usize, 27 | pub name: String, 28 | } 29 | 30 | impl ProfileParser { 31 | /// Returns the health-check relevant part of a nitter account profile 32 | pub fn parse_profile_content(&self, html: &str) -> Result { 33 | let fragment = Html::parse_fragment(html); 34 | // get profile info div 35 | let mut profile_card_name_divs = fragment.select(&self.selector_profile_card_name); 36 | let first_card = profile_card_name_divs 37 | .next() 38 | .ok_or(ProfileParseError::NoProfileCard)?; 39 | let profile_name = first_card.text().fold(String::new(), |mut acc, text| { 40 | acc.push_str(text); 41 | acc 42 | }); 43 | 44 | // find timeline div 45 | let mut timeline_divs = fragment.select(&self.selector_timeline); 46 | let first_timeline_div = timeline_divs.next().ok_or(ProfileParseError::NoTimeline)?; 47 | // select timeline-item divs inside 48 | let timeline_items = first_timeline_div.select(&self.selector_timeline_item); 49 | let timeline_item_count = timeline_items.count(); 50 | 51 | Ok(ProfileParsed { 52 | post_count: timeline_item_count, 53 | name: profile_name, 54 | }) 55 | } 56 | 57 | pub fn new() -> Self { 58 | Self { 59 | selector_profile_card_name: Selector::parse(".profile-card-username") 60 | .expect(EXPECT_CSS_SELCTOR), 61 | selector_timeline: Selector::parse(".timeline").expect(EXPECT_CSS_SELCTOR), 62 | selector_timeline_item: Selector::parse(".timeline-item").expect(EXPECT_CSS_SELCTOR), 63 | } 64 | } 65 | } 66 | 67 | #[cfg(test)] 68 | mod test { 69 | use super::*; 70 | #[test] 71 | fn parse() { 72 | let html = include_str!("../test_data/profile.html"); 73 | let parser = ProfileParser::new(); 74 | let res = parser.parse_profile_content(html).unwrap(); 75 | assert_eq!(&res.name, "@jack"); 76 | assert_eq!(res.post_count, 20); 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /scanner/src/update_stats.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: AGPL-3.0-only 2 | 3 | //! for .health endpoint statistics gathering 4 | 5 | use std::time::Instant; 6 | 7 | use chrono::Utc; 8 | use entities::host_overrides::keys::HostOverrides; 9 | use entities::prelude::Host; 10 | use entities::{host, instance_stats}; 11 | use reqwest::Url; 12 | use sea_orm::prelude::DateTimeUtc; 13 | use sea_orm::{ActiveValue, ColumnTrait, EntityTrait, QueryFilter}; 14 | use serde::Deserialize; 15 | use tokio::task::JoinSet; 16 | 17 | use crate::{Result, Scanner, ScannerError}; 18 | 19 | /// Instance stats reported by .health 20 | #[derive(Debug, Deserialize)] 21 | struct InstanceStats { 22 | accounts: InstanceStatsAccs, 23 | requests: RequestStats, 24 | } 25 | 26 | #[derive(Debug, Deserialize)] 27 | struct InstanceStatsAccs { 28 | total: i32, 29 | limited: i32, 30 | oldest: DateTimeUtc, 31 | newest: DateTimeUtc, 32 | average: DateTimeUtc, 33 | } 34 | 35 | #[derive(Debug, Deserialize)] 36 | struct RequestStats { 37 | total: i64, 38 | apis: APIStats, 39 | } 40 | 41 | /// Instance api stats reported by .health 42 | #[derive(Debug, Deserialize)] 43 | #[allow(non_snake_case)] 44 | struct APIStats { 45 | pub photoRail: i32, 46 | pub userScreenName: i32, 47 | pub search: i32, 48 | pub listTweets: i32, 49 | pub userMedia: i32, 50 | pub tweetDetail: i32, 51 | pub list: i32, 52 | pub userTweets: i32, 53 | pub userTweetsAndReplies: i32, 54 | } 55 | 56 | impl Scanner { 57 | pub(crate) async fn check_health(&self) -> Result<()> { 58 | let hosts = Host::find() 59 | .filter(host::Column::Enabled.eq(true)) 60 | .all(&self.inner.db) 61 | .await?; 62 | let start = Instant::now(); 63 | 64 | let mut join_set = JoinSet::new(); 65 | let time = Utc::now(); 66 | for model in hosts.into_iter() { 67 | let scanner = self.clone(); 68 | join_set.spawn(async move { 69 | let res = scanner.fetch_instance_stats(time, &model).await; 70 | if let Err(e) = &res { 71 | tracing::debug!(host=model.id, error=?e,"Failed to fetch instance stats"); 72 | } 73 | res.ok() 74 | }); 75 | } 76 | 77 | let mut stat_data = Vec::with_capacity(join_set.len()); 78 | while let Some(join_res) = join_set.join_next().await { 79 | if let Some(data) = join_res? { 80 | stat_data.push(data); 81 | } 82 | } 83 | tracing::debug!(db_stats_entries = stat_data.len()); 84 | if !stat_data.is_empty() { 85 | instance_stats::Entity::insert_many(stat_data) 86 | .exec(&self.inner.db) 87 | .await?; 88 | } 89 | 90 | let end = Instant::now(); 91 | let duration = end - start; 92 | { 93 | *self.inner.last_stats_fetch.lock().unwrap() = Utc::now(); 94 | } 95 | tracing::debug!(duration=?duration,"stats check finished"); 96 | Ok(()) 97 | } 98 | 99 | async fn fetch_instance_stats( 100 | &self, 101 | time: DateTimeUtc, 102 | host: &host::Model, 103 | ) -> Result { 104 | let overrides = HostOverrides::load(&host, &self.inner.db).await?; 105 | let mut url = Url::parse(&host.url).map_err(|e| ScannerError::InstanceUrlParse)?; 106 | url.set_path(".health"); 107 | if let Some(url_override) = overrides.health_path() { 108 | url.set_path(url_override); 109 | } 110 | if let Some(path_override) = overrides.health_query() { 111 | url.set_query(Some(path_override)); 112 | } 113 | let (_code, body) = self.fetch_url(url.as_str(), overrides.bearer()).await?; 114 | 115 | let stats_data: InstanceStats = 116 | serde_json::from_str(&body).map_err(|e| ScannerError::StatsParsing(e, body))?; 117 | 118 | let stats_model = instance_stats::ActiveModel { 119 | time: ActiveValue::Set(time.timestamp()), 120 | host: ActiveValue::Set(host.id), 121 | limited_accs: ActiveValue::Set(stats_data.accounts.limited), 122 | total_accs: ActiveValue::Set(stats_data.accounts.total), 123 | total_requests: ActiveValue::Set(stats_data.requests.total), 124 | req_photo_rail: ActiveValue::Set(stats_data.requests.apis.photoRail), 125 | req_user_screen_name: ActiveValue::Set(stats_data.requests.apis.userScreenName), 126 | req_search: ActiveValue::Set(stats_data.requests.apis.search), 127 | req_list_tweets: ActiveValue::Set(stats_data.requests.apis.listTweets), 128 | req_user_media: ActiveValue::Set(stats_data.requests.apis.userMedia), 129 | req_tweet_detail: ActiveValue::Set(stats_data.requests.apis.tweetDetail), 130 | req_list: ActiveValue::Set(stats_data.requests.apis.list), 131 | req_user_tweets: ActiveValue::Set(stats_data.requests.apis.userTweets), 132 | req_user_tweets_and_replies: ActiveValue::Set( 133 | stats_data.requests.apis.userTweetsAndReplies, 134 | ), 135 | }; 136 | 137 | Ok(stats_model) 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /scanner/src/version_check.rs: -------------------------------------------------------------------------------- 1 | use entities::state::scanner::ScannerConfig; 2 | use git2::{Direction, Remote}; 3 | 4 | use crate::{Result, ScannerError}; 5 | 6 | pub(crate) struct CurrentVersion { 7 | pub version: String, 8 | config: ScannerConfig, 9 | } 10 | 11 | const SHORT_COMMIT_LEN: usize = 7; 12 | 13 | impl CurrentVersion { 14 | /// Whether a nitter version URL is from the same repo 15 | pub fn is_same_repo(&self, url: &str) -> bool { 16 | url.starts_with(self.config.source_git_url.trim_end_matches(".git")) 17 | } 18 | /// Whether a nitter version URL is pointing to the same version as this one 19 | pub fn is_same_version(&self, url: &str) -> bool { 20 | // look for last path segment (don't parse) and look if that matches 21 | // from the start 22 | self.is_same_repo(url) 23 | && match url.split('/').last() { 24 | Some(other_version) => { 25 | if other_version.len() == self.version.len() { 26 | // we get the long version 27 | other_version.starts_with(&self.version) 28 | } else if other_version.len() == SHORT_COMMIT_LEN { 29 | other_version.starts_with(&self.version[..SHORT_COMMIT_LEN]) 30 | } else { 31 | // everything else is no short commit id.. 32 | false 33 | } 34 | } 35 | None => false, 36 | } 37 | } 38 | } 39 | 40 | pub(crate) fn fetch_git_state(config: ScannerConfig) -> Result { 41 | let mut remote = Remote::create_detached(config.source_git_url.as_str())?; 42 | 43 | remote.connect(Direction::Fetch)?; 44 | 45 | let reference = format!("refs/heads/{}", config.source_git_branch); 46 | let commit = remote 47 | .list()? 48 | .into_iter() 49 | .find(|v| v.name() == &reference) 50 | .map(|v| v.oid().to_string()); 51 | 52 | remote.disconnect()?; 53 | 54 | Ok(commit 55 | .map(|commit| CurrentVersion { 56 | version: commit, 57 | config, 58 | }) 59 | .ok_or(ScannerError::GitBranchNotFound)?) 60 | } 61 | 62 | #[cfg(test)] 63 | mod test { 64 | use entities::state::scanner::Config; 65 | 66 | use super::fetch_git_state; 67 | 68 | #[test] 69 | fn test_git() { 70 | fetch_git_state(Config::test_defaults()).unwrap(); 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /scanner/test_data/about.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | nitter 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 42 |
43 |

About

44 |

Nitter is a free and open source alternative Twitter front-end focused on 45 | privacy and performance. The source is available on GitHub at 46 | https://github.com/zedeus/nitter

47 |
    48 |
  • No JavaScript or ads
    • 49 |
  • All requests go through the backend, client never talks to Twitter
    • 50 |
  • Prevents Twitter from tracking your IP or JavaScript fingerprint
    • 51 |
  • Uses Twitter's unofficial API (no rate limits or developer account required)
    • 52 |
  • Lightweight (for @nim_lang, 60KB vs 784KB from twitter.com)
    • 53 |
  • RSS feeds
    • 54 |
  • Themes
    • 55 |
  • Mobile support (responsive design)
    • 56 |
  • AGPLv3 licensed, no proprietary instances permitted
    • 57 |
58 |

Nitter's GitHub wiki contains 59 | instances and 60 | browser extensions 61 | maintained by the community.

62 |

Why use Nitter?

63 |

It's impossible to use Twitter without JavaScript enabled. For privacy-minded 64 | folks, preventing JavaScript analytics and IP-based tracking is important, but 65 | apart from using a VPN and uBlock/uMatrix, it's impossible. Despite being behind 66 | a VPN and using heavy-duty adblockers, you can get accurately tracked with your 67 | browser's fingerprint, 68 | no JavaScript required. This all became 69 | particularly important after Twitter removed the 70 | ability 71 | for users to control whether their data gets sent to advertisers.

72 |

Using an instance of Nitter (hosted on a VPS for example), you can browse 73 | Twitter without JavaScript while retaining your privacy. In addition to 74 | respecting your privacy, Nitter is on average around 15 times lighter than 75 | Twitter, and in most cases serves pages faster (eg. timelines load 2-4x faster).

76 |

In the future a simple account system will be added that lets you follow Twitter 77 | users, allowing you to have a clean chronological timeline without needing a 78 | Twitter account.

79 |

Donating

80 |

Liberapay: https://liberapay.com/zedeus
81 | Patreon: https://patreon.com/nitter
82 | BTC: bc1qp7q4qz0fgfvftm5hwz3vy284nue6jedt44kxya
83 | ETH: 0x66d84bc3fd031b62857ad18c62f1ba072b011925
84 | LTC: ltc1qhsz5nxw6jw9rdtw9qssjeq2h8hqk2f85rdgpkr
85 | XMR: 42hKayRoEAw4D6G6t8mQHPJHQcXqofjFuVfavqKeNMNUZfeJLJAcNU19i1bGdDvcdN6romiSscWGWJCczFLe9RFhM3d1zpL

86 |

Contact

87 |

Feel free to join our Matrix channel.

88 | 89 |

Instance info

90 |

Version 2023.07.22-72d8f35

91 |
92 | 93 | -------------------------------------------------------------------------------- /scanner/test_data/instancelist_expected.csv: -------------------------------------------------------------------------------- 1 | domain,url,online,ssl_provider,country 2 | xcancel.com,https://xcancel.com,true,Let's Encrypt,🇺🇸 3 | nitter.privacydev.net,https://nitter.privacydev.net,true,Let's Encrypt,🇫🇷 4 | nitter.poast.org,https://nitter.poast.org,true,Let's Encrypt,🇺🇸 5 | lightbrd.com,https://lightbrd.com,true,Cloudflare,🇹🇷 6 | nitter.lucabased.xyz,https://nitter.lucabased.xyz,true,Cloudflare,🇩🇪 7 | nitter.space,https://nitter.space,true,Cloudflare,🇺🇸 8 | nitter.lunar.icu,https://nitter.lunar.icu,true,Cloudflare,🇩🇪 9 | nitter.kavin.rocks,https://nitter.kavin.rocks,true,Let's Encrypt,🇮🇳 10 | nitter.moomoo.me,https://nitter.moomoo.me,true,Let's Encrypt,🇺🇸 11 | -------------------------------------------------------------------------------- /screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/0xpr03/nitter-status/dbdcb5f7f9cf52027c0cb7adb919f1e8dc327f8a/screenshot.png -------------------------------------------------------------------------------- /server/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "server" 3 | version = "0.1.0" 4 | edition = "2021" 5 | publish = false 6 | 7 | [lib] 8 | 9 | [dependencies] 10 | axum = { version = "0.6.4", features = ["json", "macros", "query", "headers", "multipart"] } 11 | axum-extra = { version = "0.7.5" } 12 | sea-orm = { workspace = true, features = [ "sqlx-sqlite", "runtime-tokio-native-tls", "macros" ] } 13 | hyper = { version = "0.14.20", features = ["full"] } 14 | tower = "0.4" 15 | tower-http = { version = "0.4.2", features = ["full"] } 16 | tokio = { workspace = true, features = ["full"] } 17 | chrono = { workspace = true } 18 | tracing = { workspace = true } 19 | tracing-subscriber = { workspace = true, features = ["env-filter"] } 20 | serde = { workspace = true, features = ["derive"] } 21 | reqwest = { workspace = true, features = ["deflate","gzip","brotli","cookies", "rustls-tls"] } 22 | sha2 = "0.10" 23 | constant_time_eq = "0.3" 24 | base16ct = "0.2" 25 | thiserror = { workspace = true } 26 | trust-dns-resolver = { version = "0.23.0", features = ["dns-over-rustls"] } 27 | 28 | # templating 29 | tera = "1.19.0" 30 | 31 | # login 32 | tower-sessions = {version = "0.3.1", features = [ "sqlite-store","tokio-rt" ]} 33 | time = { workspace = true } 34 | 35 | # login rate limits 36 | tower_governor = "0.1" 37 | 38 | [dependencies.entities] 39 | path = "../entities" -------------------------------------------------------------------------------- /server/src/admin/errors.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use crate::Result; 4 | use axum::{ 5 | extract::{Path, State}, 6 | response::{Html, IntoResponse}, 7 | }; 8 | use entities::{check_errors, state::AppState}; 9 | use sea_orm::ColumnTrait; 10 | use sea_orm::DatabaseConnection; 11 | use sea_orm::EntityTrait; 12 | use sea_orm::QueryFilter; 13 | use sea_orm::QueryOrder; 14 | use sea_orm::QuerySelect; 15 | use tower_sessions::Session; 16 | 17 | use crate::{admin::get_specific_login_host, ServerError}; 18 | 19 | pub async fn errors_view( 20 | State(ref app_state): State, 21 | State(ref template): State>, 22 | State(ref db): State, 23 | Path(instance): Path, 24 | session: Session, 25 | ) -> Result { 26 | tracing::info!(?session); 27 | 28 | let (host, _login) = get_specific_login_host(instance, &session, db).await?; 29 | 30 | let errors = check_errors::Entity::find() 31 | .filter(check_errors::Column::Host.eq(host.id)) 32 | .order_by_desc(check_errors::Column::Time) 33 | .limit(20) 34 | .all(db) 35 | .await?; 36 | 37 | let mut context = tera::Context::new(); 38 | let res = { 39 | let guard = app_state 40 | .cache 41 | .read() 42 | .map_err(|_| ServerError::MutexFailure)?; 43 | let time = guard.last_update.format("%Y.%m.%d %H:%M").to_string(); 44 | context.insert("last_updated", &time); 45 | context.insert("ERRORS", &errors); 46 | context.insert("HOST_DOMAIN", &host.domain); 47 | context.insert("HOST_ID", &instance); 48 | 49 | let res = Html(template.render("instance_errors.html.j2", &context)?).into_response(); 50 | drop(guard); 51 | res 52 | }; 53 | Ok(res) 54 | } 55 | -------------------------------------------------------------------------------- /server/src/admin/locks.rs: -------------------------------------------------------------------------------- 1 | use std::{collections::HashMap, sync::Arc}; 2 | 3 | use axum::{ 4 | extract::{Path, State}, 5 | response::{Html, IntoResponse, Redirect}, 6 | Form, 7 | }; 8 | use entities::host_overrides::{ 9 | self, 10 | keys::{HostOverrides, LOCKED_FALSE, LOCKED_TRUE}, 11 | }; 12 | use sea_orm::{sea_query::OnConflict, ActiveValue, DatabaseConnection, EntityTrait}; 13 | use tower_sessions::Session; 14 | 15 | use crate::{Result, ServerError}; 16 | 17 | use super::get_specific_login_host; 18 | 19 | pub async fn locks_view( 20 | State(ref template): State>, 21 | State(ref db): State, 22 | Path(instance): Path, 23 | session: Session, 24 | ) -> Result { 25 | let (host, login) = get_specific_login_host(instance, &session, db).await?; 26 | 27 | if !login.admin { 28 | return Err(ServerError::MissingPermission); 29 | } 30 | 31 | let overrides = HostOverrides::load(&host, db).await?; 32 | 33 | let mut context = tera::Context::new(); 34 | context.insert("HOST_DOMAIN", &host.domain); 35 | context.insert("HOST_ID", &instance); 36 | context.insert("OVERRIDES", overrides.entries()); 37 | 38 | let res = Html(template.render("instance_locks.html.j2", &context)?).into_response(); 39 | Ok(res) 40 | } 41 | 42 | /// Override Form 43 | // #[derive(Deserialize, Debug)] 44 | pub type LocksFormInput = HashMap; 45 | 46 | pub async fn post_locks( 47 | State(db): State, 48 | Path(instance): Path, 49 | session: Session, 50 | Form(input): Form, 51 | ) -> Result { 52 | let (host, login) = get_specific_login_host(instance, &session, &db).await?; 53 | 54 | if !login.admin { 55 | return Err(ServerError::MissingPermission); 56 | } 57 | 58 | let overrides = HostOverrides::load(&host, &db).await?; 59 | 60 | let mut updated = Vec::with_capacity(input.len()); 61 | 62 | for (key, _value) in overrides.entries() { 63 | let locked = match input.contains_key(key) { 64 | true => LOCKED_TRUE, 65 | false => LOCKED_FALSE, 66 | }; 67 | updated.push(host_overrides::ActiveModel { 68 | host: ActiveValue::Set(host.id), 69 | key: ActiveValue::Set(key.to_string()), 70 | locked: ActiveValue::Set(locked), 71 | value: ActiveValue::Set(None), 72 | }); 73 | } 74 | host_overrides::Entity::insert_many(updated) 75 | .on_conflict( 76 | OnConflict::columns([host_overrides::Column::Host, host_overrides::Column::Key]) 77 | .update_columns([host_overrides::Column::Locked]) 78 | .to_owned(), 79 | ) 80 | .exec(&db) 81 | .await?; 82 | 83 | Ok(Redirect::to(&format!("/admin/instance/locks/{}", instance)).into_response()) 84 | } 85 | -------------------------------------------------------------------------------- /server/src/admin/logs.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | collections::HashMap, 3 | sync::Arc, 4 | }; 5 | 6 | use crate::{Result, ServerError}; 7 | use axum::{ 8 | extract::State, 9 | response::{Html, IntoResponse}, 10 | }; 11 | use chrono::{TimeZone, Utc}; 12 | use entities::{ 13 | host, 14 | log, 15 | }; 16 | use sea_orm::EntityTrait; 17 | use sea_orm::{DatabaseConnection, QueryOrder, QuerySelect}; 18 | use serde::Serialize; 19 | use tower_sessions::Session; 20 | 21 | use super::get_session_login; 22 | 23 | #[derive(Serialize)] 24 | struct LogEntry { 25 | time: String, 26 | by_user_host: String, 27 | key: String, 28 | for_host: Option, 29 | value: Option, 30 | } 31 | #[derive(Serialize)] 32 | struct ForHost { 33 | domain: String, 34 | id: i32, 35 | } 36 | 37 | pub async fn log_view( 38 | State(ref template): State>, 39 | State(ref db): State, 40 | session: Session, 41 | ) -> Result { 42 | let login = get_session_login(&session)?; 43 | if !login.admin { 44 | return Err(ServerError::MissingPermission); 45 | } 46 | 47 | let mut host_cache = HostCache::default(); 48 | 49 | let logs_raw = log::Entity::find() 50 | .order_by_desc(log::Column::Time) 51 | .all(db) 52 | .await?; 53 | let mut logs = Vec::with_capacity(logs_raw.len()); 54 | 55 | for entry in logs_raw { 56 | let by_host = host_cache.get(entry.user_host, db).await?; 57 | let for_host = match entry.host_affected { 58 | None => None, 59 | Some(v) => Some(ForHost { 60 | domain: host_cache.get(v, db).await?, 61 | id: v, 62 | }), 63 | }; 64 | let time = Utc 65 | .timestamp_opt(entry.time, 0) 66 | .unwrap() 67 | .format("%Y-%m-%d %H:%M:%S") 68 | .to_string(); 69 | logs.push(LogEntry { 70 | by_user_host: by_host, 71 | for_host, 72 | key: entry.key, 73 | time, 74 | value: entry.new_value, 75 | }); 76 | } 77 | 78 | let mut context = tera::Context::new(); 79 | context.insert("LOGS", &logs); 80 | 81 | let res = Html(template.render("admin_logs.html.j2", &context)?).into_response(); 82 | Ok(res) 83 | } 84 | 85 | #[derive(Default)] 86 | struct HostCache(HashMap); 87 | 88 | impl HostCache { 89 | async fn get(&mut self, host_id: i32, db: &DatabaseConnection) -> Result { 90 | if let Some(v) = self.0.get(&host_id) { 91 | return Ok(v.clone()); 92 | } 93 | let domain: Option = host::Entity::find_by_id(host_id) 94 | .select_only() 95 | .column(host::Column::Domain) 96 | .into_tuple() 97 | .one(db) 98 | .await?; 99 | let Some(domain) = domain else { 100 | return Err(ServerError::HostNotFound(host_id)); 101 | }; 102 | self.0.insert(host_id, domain.clone()); 103 | Ok(domain) 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /server/src/admin/settings.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | sync::Arc, 3 | time::{SystemTime, UNIX_EPOCH}, 4 | }; 5 | 6 | use crate::{Result, ServerError}; 7 | use axum::{ 8 | extract::{Path, State}, 9 | response::{Html, IntoResponse}, 10 | Form, 11 | }; 12 | use chrono::Utc; 13 | use entities::{ 14 | host_overrides::{self, keys::*}, 15 | log, 16 | }; 17 | use sea_orm::sea_query::OnConflict; 18 | use sea_orm::ActiveModelTrait; 19 | use sea_orm::EntityTrait; 20 | use sea_orm::{ActiveValue, DatabaseConnection}; 21 | use serde::Deserialize; 22 | use tower_sessions::Session; 23 | use tracing::trace; 24 | 25 | use super::get_specific_login_host; 26 | 27 | pub async fn settings_view( 28 | State(ref template): State>, 29 | State(ref db): State, 30 | Path(instance): Path, 31 | session: Session, 32 | ) -> Result { 33 | let (host, login) = get_specific_login_host(instance, &session, db).await?; 34 | 35 | let overrides = HostOverrides::load(&host, db).await?; 36 | 37 | let mut context = tera::Context::new(); 38 | context.insert("HOST_DOMAIN", &host.domain); 39 | context.insert("HOST_ID", &instance); 40 | context.insert("IS_ADMIN", &login.admin); 41 | context.insert("OVERRIDES", overrides.entries()); 42 | 43 | let res = Html(template.render("instance_settings.html.j2", &context)?).into_response(); 44 | Ok(res) 45 | } 46 | 47 | /// Override Form 48 | #[derive(Deserialize, Debug)] 49 | pub struct OverrideFormInput { 50 | /// Some if checked a checkbox, none otherwise 51 | value: Option, 52 | key: String, 53 | } 54 | 55 | pub async fn post_settings( 56 | State(template): State>, 57 | State(db): State, 58 | Path(instance): Path, 59 | session: Session, 60 | Form(input): Form, 61 | ) -> Result { 62 | trace!(form=?input,host=instance,"post_override"); 63 | let (host, login) = get_specific_login_host(instance, &session, &db).await?; 64 | 65 | let overrides = HostOverrides::load(&host, &db).await?; 66 | let Some(entry) = overrides.entries().get(&input.key) else { 67 | trace!("unknown override key"); 68 | return Err(ServerError::InvalidOverrideKey); 69 | }; 70 | 71 | if entry.locked && !login.admin { 72 | trace!(locked = entry.locked, "missing permissions"); 73 | return Err(ServerError::MissingPermission); 74 | } 75 | 76 | let value = match (input.value.as_deref().map(|v| v.trim()), entry.value_type) { 77 | (None, _) => None, 78 | // don't insert empty strings 79 | (Some(""), ValueType::String) => None, 80 | (value, ValueType::String) => value, 81 | (Some(VAL_BOOL_TRUE), ValueType::Bool) => Some("true"), 82 | // don't allow arbitrary data 83 | (Some(_), ValueType::Bool) => Some("false"), 84 | } 85 | .map(|v| v.to_owned()); 86 | 87 | // only for first-time inserts, not updated! 88 | let locked_value = ActiveValue::Set(match entry.locked { 89 | true => LOCKED_TRUE, 90 | false => LOCKED_FALSE, 91 | }); 92 | 93 | let time = Utc::now().timestamp(); 94 | let user_host = login.hosts.iter().next().unwrap(); 95 | log::ActiveModel { 96 | user_host: ActiveValue::Set(*user_host), 97 | host_affected: ActiveValue::Set(Some(host.id)), 98 | key: ActiveValue::Set(input.key.clone()), 99 | time: ActiveValue::Set(time), 100 | new_value: ActiveValue::Set(value.clone()), 101 | } 102 | .insert(&db) 103 | .await?; 104 | 105 | let model = host_overrides::ActiveModel { 106 | host: ActiveValue::Set(host.id), 107 | key: ActiveValue::Set(input.key), 108 | locked: locked_value, 109 | value: ActiveValue::Set(value), 110 | }; 111 | host_overrides::Entity::insert(model) 112 | .on_conflict( 113 | OnConflict::columns([host_overrides::Column::Host, host_overrides::Column::Key]) 114 | .update_columns([host_overrides::Column::Value]) 115 | .to_owned(), 116 | ) 117 | .exec(&db) 118 | .await?; 119 | 120 | settings_view(State(template), State(db), Path(instance), session).await 121 | } 122 | -------------------------------------------------------------------------------- /server/src/api.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: AGPL-3.0-only 2 | use crate::{Result, ServerError}; 3 | use axum::response::IntoResponse; 4 | use axum::{extract::State, Json}; 5 | use chrono::{TimeZone, Utc}; 6 | use entities::state::AppState; 7 | use entities::{health_check, instance_stats}; 8 | use hyper::http::HeaderValue; 9 | use sea_orm::DatabaseConnection; 10 | use serde::Serialize; 11 | use std::fmt::Write; 12 | use std::sync::Arc; 13 | 14 | pub async fn instances( 15 | State(ref app_state): State, 16 | State(ref config): State>, 17 | ) -> Result { 18 | let mut res = { 19 | let guard = app_state 20 | .cache 21 | .read() 22 | .map_err(|_| ServerError::MutexFailure)?; 23 | let res = Json(&*guard).into_response(); 24 | drop(guard); 25 | res 26 | }; 27 | res.headers_mut().insert( 28 | "cache-control", 29 | HeaderValue::from_str(&format!("public, max-age={}", config.max_age)).unwrap(), 30 | ); 31 | res.headers_mut().insert( 32 | "X-Robots-Tag", 33 | HeaderValue::from_static("noindex, nofollow"), 34 | ); 35 | Ok(res) 36 | } 37 | 38 | pub async fn graph_csv_health( 39 | State(ref db): State, 40 | State(ref config): State>, 41 | ) -> Result { 42 | let start = std::time::Instant::now(); 43 | let healthy_data = health_check::HealthyAmount::fetch(db, None, None, None).await?; 44 | let queried = std::time::Instant::now(); 45 | let mut data = String::with_capacity(8 * healthy_data.len()); 46 | 47 | data.push_str("Date,Healthy,Dead\n"); 48 | 49 | for entry in healthy_data { 50 | let time = Utc 51 | .timestamp_opt(entry.time, 0) 52 | .unwrap() 53 | .format("%Y-%m-%dT%H:%M:%SZ"); 54 | writeln!(&mut data, "{time},{},{}", entry.alive, entry.dead) 55 | .map_err(|e| ServerError::CSV(e.to_string()))?; 56 | } 57 | let formatted = std::time::Instant::now(); 58 | let query_time = queried - start; 59 | let format_time = formatted - queried; 60 | tracing::debug!(?query_time, ?format_time); 61 | 62 | let mut res = data.into_response(); 63 | res.headers_mut() 64 | .insert("content-type", HeaderValue::from_str("text/csv").unwrap()); 65 | res.headers_mut().insert( 66 | "cache-control", 67 | HeaderValue::from_str(&format!("public, max-age={}", config.max_age)).unwrap(), 68 | ); 69 | res.headers_mut().insert( 70 | "X-Robots-Tag", 71 | HeaderValue::from_static("noindex, nofollow"), 72 | ); 73 | Ok(res) 74 | } 75 | 76 | pub async fn graph_csv_stats( 77 | State(ref db): State, 78 | State(ref config): State>, 79 | ) -> Result { 80 | let start = std::time::Instant::now(); 81 | let healthy_data = instance_stats::StatsCSVEntry::fetch(db).await?; 82 | let queried = std::time::Instant::now(); 83 | let mut data = String::with_capacity(8 * healthy_data.len()); 84 | 85 | data.push_str("Date,Tokens AVG,Limited Tokens AVG,Requests AVG\n"); 86 | 87 | for entry in healthy_data { 88 | let time = Utc 89 | .timestamp_opt(entry.time, 0) 90 | .unwrap() 91 | .format("%Y-%m-%dT%H:%M:%SZ"); 92 | writeln!( 93 | &mut data, 94 | "{time},{},{},{}", 95 | entry.total_accs_avg, entry.limited_accs_avg, entry.total_requests_avg 96 | ) 97 | .map_err(|e| ServerError::CSV(e.to_string()))?; 98 | } 99 | let formatted = std::time::Instant::now(); 100 | let query_time = queried - start; 101 | let format_time = formatted - queried; 102 | tracing::debug!(?query_time, ?format_time); 103 | 104 | let mut res = data.into_response(); 105 | res.headers_mut() 106 | .insert("content-type", HeaderValue::from_str("text/csv").unwrap()); 107 | res.headers_mut().insert( 108 | "cache-control", 109 | HeaderValue::from_str(&format!("public, max-age={}", config.max_age)).unwrap(), 110 | ); 111 | res.headers_mut().insert( 112 | "X-Robots-Tag", 113 | HeaderValue::from_static("noindex, nofollow"), 114 | ); 115 | Ok(res) 116 | } 117 | -------------------------------------------------------------------------------- /server/src/lib.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: AGPL-3.0-only 2 | use std::{borrow::Cow, collections::HashMap, net::SocketAddr, sync::Arc, time::SystemTimeError}; 3 | 4 | use axum::{ 5 | error_handling::HandleErrorLayer, 6 | extract::DefaultBodyLimit, 7 | http::HeaderValue, 8 | response::{Html, Redirect}, 9 | routing::{get, get_service, post}, 10 | BoxError, Router, 11 | }; 12 | use chrono::TimeZone; 13 | use entities::state::{scanner::ScannerConfig, AppState}; 14 | use hyper::{header, StatusCode}; 15 | use reqwest::Client; 16 | use sea_orm::DatabaseConnection; 17 | use tera::{from_value, to_value, Tera}; 18 | use thiserror::Error; 19 | use tower::ServiceBuilder; 20 | use tower_governor::{governor::GovernorConfigBuilder, GovernorLayer}; 21 | use tower_http::{ 22 | cors::CorsLayer, 23 | limit::RequestBodyLimitLayer, 24 | services::{ServeDir, ServeFile}, 25 | set_header::SetResponseHeaderLayer, 26 | trace::TraceLayer, 27 | }; 28 | use tower_sessions::{cookie::SameSite, SessionManagerLayer, SqliteStore}; 29 | 30 | mod admin; 31 | mod api; 32 | mod website; 33 | 34 | const LOGIN_URL: &'static str = "/admin/login"; 35 | const ADMIN_OVERVIEW_URL: &'static str = "/admin"; 36 | 37 | #[derive(Debug)] 38 | pub struct Config { 39 | pub site_url: String, 40 | pub max_age: usize, 41 | pub session_ttl_seconds: u64, 42 | pub login_token_name: String, 43 | /// Domains for admin hosts 44 | pub admin_domains: Vec, 45 | /// Additional host keys to auth against 46 | pub admin_keys: HashMap, 47 | pub session_db_uri: String, 48 | } 49 | 50 | #[derive(Clone, axum::extract::FromRef)] 51 | struct WebState { 52 | db: DatabaseConnection, 53 | config: Arc, 54 | scanner_config: ScannerConfig, 55 | app_state: AppState, 56 | templates: Arc, 57 | login_client: Client, 58 | } 59 | 60 | /// Start webserver 61 | pub async fn start( 62 | addr: &SocketAddr, 63 | db: DatabaseConnection, 64 | config: Config, 65 | scanner_config: ScannerConfig, 66 | app_state: AppState, 67 | ) -> Result<()> { 68 | #[cfg(debug_assertions)] 69 | let session_secure = false; 70 | #[cfg(not(debug_assertions))] 71 | let session_secure = true; 72 | if !session_secure { 73 | tracing::warn!("debug build, sessions are not secure!"); 74 | } 75 | 76 | let pool = tower_sessions::sqlx::SqlitePool::connect(&config.session_db_uri) 77 | .await 78 | .expect("failed to initialize session store"); 79 | let session_store = SqliteStore::new(pool); 80 | session_store 81 | .migrate() 82 | .await 83 | .expect("failed to migrate session store"); 84 | tokio::task::spawn( 85 | session_store 86 | .clone() 87 | .continuously_delete_expired(tokio::time::Duration::from_secs(60)), 88 | ); 89 | 90 | let session_service = ServiceBuilder::new() 91 | .layer(HandleErrorLayer::new(|e: axum::BoxError| async move { 92 | tracing::debug!(session_error=?e); 93 | StatusCode::BAD_REQUEST 94 | })) 95 | .layer( 96 | SessionManagerLayer::new(session_store) 97 | .with_secure(session_secure) 98 | .with_path("/admin".to_string()) 99 | .with_name("admin_login") 100 | .with_same_site(SameSite::Strict) 101 | .with_max_age(time::Duration::seconds(config.session_ttl_seconds as _)), 102 | ); 103 | 104 | let user_agent = format!("nitter-status (+{}/about)", scanner_config.website_url); 105 | let login_client = Client::builder() 106 | .cookie_store(false) 107 | .brotli(true) 108 | .deflate(true) 109 | .gzip(true) 110 | .use_rustls_tls() 111 | .user_agent(user_agent) 112 | .connect_timeout(std::time::Duration::from_secs(3)) 113 | .timeout(std::time::Duration::from_secs(10)) 114 | .build() 115 | .unwrap(); 116 | 117 | let config = Arc::new(config); 118 | let mut tera = Tera::new("server/templates/*")?; 119 | tera.autoescape_on(vec![".html.j2"]); 120 | tera.register_function("fmt_date", fmt_date); 121 | let state = WebState { 122 | config: config.clone(), 123 | db, 124 | app_state, 125 | scanner_config, 126 | templates: Arc::new(tera), 127 | login_client, 128 | }; 129 | 130 | let per_ip_governor_conf = Box::new( 131 | GovernorConfigBuilder::default() 132 | .per_second(2) 133 | .burst_size(2) 134 | .finish() 135 | .unwrap(), 136 | ); 137 | let rate_limit_layer = ServiceBuilder::new() 138 | .layer(HandleErrorLayer::new(|e: BoxError| async move { 139 | tower_governor::errors::display_error(e) // too many requests 140 | })) 141 | .layer(GovernorLayer { 142 | config: Box::leak(per_ip_governor_conf), 143 | }); 144 | 145 | let router = Router::new() 146 | .nest_service( 147 | "/static", 148 | ServeDir::new("server/static").append_index_html_on_directories(false), 149 | ) 150 | .route("/api/v1/instances", get(api::instances)) 151 | .route("/api/csv/health", get(api::graph_csv_health)) 152 | .route("/api/csv/stats", get(api::graph_csv_stats)) 153 | .nest(ADMIN_OVERVIEW_URL, Router::new() 154 | .route("/", get(admin::overview)) 155 | .route("/instance/errors/:instance", get(admin::errors_view)) 156 | .route("/instance/settings/:instance", get(admin::settings_view).post(admin::post_settings)) 157 | .route("/instance/locks/:instance", get(admin::locks_view).post(admin::post_locks)) 158 | .route("/api/history/:instance", post(admin::history_json_specific)) 159 | .route("/api/history", post(admin::history_json)) 160 | .route("/login", get(admin::login_view).post(admin::login).route_layer(rate_limit_layer)) 161 | .route("/logs", get(admin::log_view)) 162 | .route("/logout", get(admin::logout)) 163 | .layer(ServiceBuilder::new().layer(SetResponseHeaderLayer::overriding(header::CACHE_CONTROL, HeaderValue::from_static("no-cache")))) 164 | .layer(session_service) 165 | ) 166 | .route("/about", get(website::about)) 167 | // keep redirect for old nitter instances redirecting to it 168 | .route("/rip", get(|| async { Redirect::temporary("/") })) 169 | .route( 170 | "/robots.txt", 171 | get_service(ServeFile::new("server/static/robots.txt")), 172 | ) 173 | .route("/instances", get(website::instances)) 174 | .route("/", get(website::instances)) 175 | .layer( 176 | ServiceBuilder::new() 177 | .layer(DefaultBodyLimit::disable()) 178 | .layer(RequestBodyLimitLayer::new(2usize.pow(20) * 2)) 179 | .layer(TraceLayer::new_for_http()) 180 | .layer(cors_policy(&config.site_url)) 181 | .layer(SetResponseHeaderLayer::overriding( 182 | header::CONTENT_SECURITY_POLICY, 183 | "default-src 'self'; child-src 'none'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:;" 184 | .parse::() 185 | .expect("Invalid CSP header value"), 186 | )), 187 | ) 188 | .with_state(state.clone()); 189 | tracing::debug!("Starting server with config {:?}", *config); 190 | tracing::info!("listening on http://{}", addr); 191 | axum::Server::bind(&addr) 192 | .serve(router.into_make_service_with_connect_info::()) 193 | .await 194 | .expect("Failed to start webserver"); 195 | Ok(()) 196 | } 197 | 198 | fn cors_policy(_site_url: &str) -> CorsLayer { 199 | CorsLayer::new() 200 | .allow_origin(tower_http::cors::Any) 201 | .allow_methods(tower_http::cors::Any) 202 | .allow_headers(tower_http::cors::Any) 203 | } 204 | 205 | type Result> = std::result::Result; 206 | 207 | #[derive(Error, Debug)] 208 | pub enum ServerError { 209 | #[error("Failed to access mutex")] 210 | MutexFailure, 211 | #[error("Failed to perform templating: {0:?}")] 212 | Templating(#[from] tera::Error), 213 | #[error("Not logged in")] 214 | NoLogin, 215 | #[error("Internal Error during DB request: {0:?}")] 216 | DBError(#[from] sea_orm::DbErr), 217 | #[error("Internal Error session handling: {0:?}")] 218 | SessionError(#[from] tower_sessions::session::SessionError), 219 | #[error("Host '{0}' can't be found, logging out user!")] 220 | HostNotFound(i32), 221 | #[error("No permission to access this resource")] 222 | MissingPermission, 223 | #[error("Failed to create CSV")] 224 | CSV(String), 225 | #[error("Invalid override key")] 226 | InvalidOverrideKey, 227 | #[error("Failed getting system time: {0}")] 228 | TimeError(#[from] SystemTimeError), 229 | } 230 | 231 | impl axum::response::IntoResponse for ServerError { 232 | fn into_response(self) -> axum::response::Response { 233 | use ServerError::*; 234 | let msg = match &self { 235 | NoLogin => { 236 | // *resp.status_mut() = StatusCode::FOUND; // have to use a 301, [Redirect] 307 won't work for referrer 237 | return Redirect::temporary(LOGIN_URL).into_response(); 238 | } 239 | MissingPermission => ( 240 | StatusCode::FORBIDDEN, 241 | Cow::Borrowed("Missing permission to access this resource"), 242 | ), 243 | InvalidOverrideKey => ( 244 | StatusCode::BAD_REQUEST, 245 | Cow::Borrowed("Invalid override key"), 246 | ), 247 | CSV(_) | MutexFailure | Templating(_) | DBError(_) | SessionError(_) 248 | | HostNotFound(_) | TimeError(_) => ( 249 | StatusCode::INTERNAL_SERVER_ERROR, 250 | Cow::Borrowed("Internal Server Error"), 251 | ), 252 | }; 253 | if msg.0 == StatusCode::INTERNAL_SERVER_ERROR { 254 | tracing::error!(?self); 255 | } 256 | msg.into_response() 257 | } 258 | } 259 | 260 | /// Tera template function to format unix timestamps 261 | fn fmt_date(args: &HashMap) -> tera::Result { 262 | match args.get("value") { 263 | Some(time_val) => match from_value::(time_val.clone()) { 264 | Ok(time_i64) => { 265 | let format = match args.get("fmt") { 266 | None => "%Y.%m.%d %H:%M:%S", 267 | Some(v) => v 268 | .as_str() 269 | .ok_or_else(|| tera::Error::from("fmt has to be a string"))?, 270 | }; 271 | let time = chrono::Utc 272 | .timestamp_opt(time_i64, 0) 273 | .single() 274 | .ok_or_else(|| tera::Error::from("Invalid timestamp"))?; 275 | Ok(to_value(time.format(format).to_string()).unwrap()) 276 | } 277 | Err(_) => Err("timestamp not an i64".into()), 278 | }, 279 | None => Err("no value provided".into()), 280 | } 281 | } 282 | -------------------------------------------------------------------------------- /server/src/website.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: AGPL-3.0-only 2 | use std::sync::Arc; 3 | use std::time::Instant; 4 | 5 | use crate::Result; 6 | use crate::ServerError; 7 | use axum::response::IntoResponse; 8 | use axum::{extract::State, response::Html}; 9 | use entities::state::scanner::ScannerConfig; 10 | use entities::state::AppState; 11 | use hyper::http::HeaderValue; 12 | 13 | pub async fn instances( 14 | State(ref app_state): State, 15 | State(ref template): State>, 16 | State(ref config): State>, 17 | ) -> Result { 18 | let mut context = tera::Context::new(); 19 | let mut res = { 20 | let guard = app_state 21 | .cache 22 | .read() 23 | .map_err(|_| ServerError::MutexFailure)?; 24 | context.insert("instances", &guard.hosts); 25 | let time = guard.last_update.format("%Y.%m.%d %H:%M").to_string(); 26 | context.insert("last_updated", &time); 27 | let start = Instant::now(); 28 | let res = Html(template.render("instances.html.j2", &context)?).into_response(); 29 | let end = Instant::now(); 30 | drop(guard); 31 | let templating_time = end - start; 32 | tracing::trace!(templating_time = templating_time.as_millis()); 33 | res 34 | }; 35 | res.headers_mut().insert( 36 | "cache-control", 37 | HeaderValue::from_str(&format!("public, max-age={}", config.max_age)).unwrap(), 38 | ); 39 | Ok(res) 40 | } 41 | 42 | pub async fn about( 43 | State(ref app_state): State, 44 | State(ref template): State>, 45 | State(ref scanner_config): State, 46 | ) -> Result { 47 | let mut context = tera::Context::new(); 48 | let mut paths = Vec::with_capacity(5); 49 | paths.push(&scanner_config.about_path); 50 | paths.push(&scanner_config.rss_path); 51 | paths.push(&scanner_config.profile_path); 52 | paths.push(&scanner_config.connectivity_path); 53 | context.insert("checked_paths", &paths); 54 | context.insert( 55 | "uptime_interval_s", 56 | &scanner_config.instance_check_interval.as_secs(), 57 | ); 58 | context.insert( 59 | "wiki_interval_s", 60 | &scanner_config.list_fetch_interval.as_secs(), 61 | ); 62 | context.insert( 63 | "ping_avg_interval_h", 64 | &scanner_config.ping_range.num_hours(), 65 | ); 66 | { 67 | let guard = app_state 68 | .cache 69 | .read() 70 | .map_err(|_| ServerError::MutexFailure)?; 71 | context.insert("latest_commit", &guard.latest_commit); 72 | } 73 | 74 | let mut res = Html(template.render("about.html.j2", &context)?).into_response(); 75 | res.headers_mut().insert( 76 | "cache-control", 77 | HeaderValue::from_static("public, max-age=900"), 78 | ); 79 | Ok(res) 80 | } 81 | 82 | pub async fn rip( 83 | State(ref app_state): State, 84 | State(ref template): State>, 85 | State(ref scanner_config): State, 86 | ) -> Result { 87 | let mut context = tera::Context::new(); 88 | { 89 | let guard = app_state 90 | .cache 91 | .read() 92 | .map_err(|_| ServerError::MutexFailure)?; 93 | let time = guard.last_update.format("%Y.%m.%d %H:%M").to_string(); 94 | context.insert("last_updated", &time); 95 | } 96 | 97 | let mut res = Html(template.render("rip.html.j2", &context)?).into_response(); 98 | res.headers_mut().insert( 99 | "cache-control", 100 | HeaderValue::from_static("public, max-age=900"), 101 | ); 102 | Ok(res) 103 | } 104 | -------------------------------------------------------------------------------- /server/static/admin.js: -------------------------------------------------------------------------------- 1 | document.addEventListener("DOMContentLoaded", function() { 2 | const startDateInput = document.getElementById('startDate'); 3 | const endDateInput = document.getElementById('endDate'); 4 | const submitTime = document.getElementById('submitDateRange'); 5 | 6 | // Set initial values in UTC 7 | const initialEndDate = moment.utc(); 8 | const initialStartDate = moment().subtract(30, 'days').utc(); 9 | startDateInput.value = initialStartDate.format('YYYY-MM-DD'); 10 | endDateInput.value = initialEndDate.format('YYYY-MM-DD');; 11 | 12 | submitTime.addEventListener('click', function() { 13 | const startDate = moment(startDateInput.value, 'YYYY-MM-DD'); 14 | const endDate = moment(endDateInput.value, 'YYYY-MM-DD'); 15 | 16 | if (!startDate.isValid() || !endDate.isValid()) { 17 | alert('Invalid date format. Please use the YYYY-MM-DD format.'); 18 | } else if (startDate.isAfter(endDate)) { 19 | alert('Invalid date range. Start date must be before the end date.'); 20 | } else { 21 | fetchDataAndCreateChart(startDate,endDate); 22 | } 23 | }); 24 | fetchDataAndCreateChart(initialStartDate,initialEndDate); 25 | }); 26 | 27 | let chart = undefined; 28 | async function fetchDataAndCreateChart(startDate,endDate) { 29 | let data; 30 | try { 31 | const response = await fetch('/admin/api/history', { 32 | method: "POST", 33 | headers: { 34 | "Content-Type": "application/json", 35 | }, 36 | body: JSON.stringify({"start": startDate.utc(), "end": endDate.add(1, 'days').utc()}) 37 | }); 38 | data = await response.json(); 39 | } catch (error) { 40 | console.error('Failed to fetch data:', error); 41 | } 42 | // Create arrays for y-values for both datasets 43 | const healthyData = data.global.map(entry => { 44 | return {x: moment.unix(entry.time), y: entry.alive}; 45 | }); 46 | 47 | const unhealthyData = data.global.map(entry => { 48 | return {x: moment.unix(entry.time), y: entry.dead}; 49 | }); 50 | 51 | let statsData = []; 52 | let statsKeys = []; 53 | if (data.stats.length > 0) { 54 | 55 | statsKeys = Object.keys(data.stats[0]).filter(key => key !== 'time'); 56 | statsData = statsKeys.map(key => ({"key": key, data: data.stats.map(entry => ({x: moment.unix(entry.time), y: entry[key]}))})); 57 | } 58 | 59 | 60 | const userUnhealthyData = data.user.filter(entry => entry.dead > 0).map(entry => ({"x": entry.time * 1000, "y": entry.dead})); 61 | const paramData = {"healthyData": healthyData, "unhealthyData": unhealthyData, "user": userUnhealthyData, "stats": statsData, "statsKeys": statsKeys}; 62 | 63 | createChartOverview(paramData); 64 | } 65 | 66 | function mapKey(key) { 67 | const keyMappings = { 68 | "limited_accs_max": "Max Lim. Accs", 69 | "limited_accs_avg": "Avg Lim. Accs", 70 | "total_accs_max": "Max Accs", 71 | "total_accs_avg": "Avg Accs", 72 | "total_requests_max": "Max Requests Req.", 73 | "total_requests_avg": "Avg Requests Req.", 74 | "req_photo_rail_max": "Max PhotoRail Req.", 75 | "req_photo_rail_avg": "Avg PhotoRail Req.", 76 | "req_user_screen_name_max": "Max UserScreenName Req.", 77 | "req_user_screen_name_avg": "Avg UserScreenName Req.", 78 | "req_search_max": "Max Search Req.", 79 | "req_search_avg": "Avg Search Req.", 80 | "req_list_tweets_max": "Max ListTweets Req.", 81 | "req_list_tweets_avg": "Avg ListTweets Req.", 82 | "req_user_media_max": "Max UserMedia Req.", 83 | "req_user_media_avg": "Avg UserMedia Req.", 84 | "req_tweet_detail_max": "Max TweetDetail Req.", 85 | "req_tweet_detail_avg": "Avg TweetDetail Req.", 86 | "req_list_max": "Max List Req.", 87 | "req_list_avg": "Avg List Req.", 88 | "req_user_tweets_max": "Max UserTweets Req.", 89 | "req_user_tweets_avg": "Avg UserTweets Req.", 90 | "req_user_tweets_and_replies_max": "Max UserTweetsAndReplies Req.", 91 | "req_user_tweets_and_replies_avg": "Avg UserTweetsAndReplies Req.", 92 | } 93 | return keyMappings[key]; 94 | } 95 | 96 | function nameToColor(name) { 97 | const colors = ["#fafa6e" 98 | ,"#eafb71" 99 | ,"#dafb75" 100 | ,"#cafb7b" 101 | ,"#b9fa81" 102 | ,"#a8f989" 103 | ,"#97f890" 104 | ,"#85f799" 105 | ,"#71f5a1" 106 | ,"#5cf3aa" 107 | ,"#42f1b3" 108 | ,"#15efbc" 109 | ,"#00ecc4" 110 | ,"#00e9cd" 111 | ,"#00e6d5" 112 | ,"#00e3dd" 113 | ,"#00e0e4" 114 | ,"#00dcea" 115 | ,"#00d8f0" 116 | ,"#00d5f5" 117 | ,"#00d0f9" 118 | ,"#00ccfd" 119 | ,"#00c8ff" 120 | ,"#00c3ff" 121 | ,"#00beff" 122 | ,"#00b9ff" 123 | ,"#00b4ff"]; 124 | var hash = hashStr(name); 125 | var index = hash % colors.length; 126 | return colors[index]; 127 | } 128 | 129 | function hashStr(str) { 130 | var hash = 0; 131 | for (var i = 0; i < str.length; i++) { 132 | var charCode = str.charCodeAt(i); 133 | hash += charCode; 134 | } 135 | return hash; 136 | } 137 | 138 | function createChartOverview(data) { 139 | const ctx = document.getElementById('graph-health').getContext('2d'); 140 | let high_data = data.stats.length > 1000; 141 | if (chart) { 142 | chart.destroy(); 143 | } 144 | let datasets = [{ 145 | label: 'Healthy', 146 | data: data.healthyData, 147 | borderColor: 'green', 148 | backgroundColor: 'rgba(0, 255, 0, 0.2)', 149 | fill: true 150 | }, 151 | { 152 | label: 'Unhealthy', 153 | data: data.unhealthyData, 154 | borderColor: 'orange', 155 | backgroundColor: 'rgba(241, 90, 34, 0.2)', 156 | fill: true 157 | }, 158 | { 159 | label: 'Own Instances Unhealthy', 160 | data: data.user, 161 | pointRadius: 5, 162 | pointBackgroundColor: 'rgba(255, 0, 0, 1)', 163 | fill: false, 164 | datasetIndex: "x", 165 | }]; 166 | datasets = datasets.concat(data.stats.map(entry => ({ 167 | yAxisID: 'yStats', 168 | label: mapKey(entry.key), 169 | data: entry.data, 170 | borderColor: nameToColor(entry.key), 171 | fill: false, 172 | hidden: (entry.key !== 'total_requests_avg' && entry.key !== 'limited_accs_avg'), 173 | }))); 174 | console.log(datasets); 175 | chart = new Chart(ctx, { 176 | type: 'line', 177 | data: { 178 | datasets: datasets, 179 | }, 180 | options: { 181 | //responsive: false, 182 | //maintainAspectRatio: false, 183 | plugins: { 184 | colors: { 185 | enabled: true 186 | }, 187 | decimation: { 188 | enabled: high_data, 189 | algorithm: 'min-max', 190 | }, 191 | tooltip: { 192 | // callbacks: { 193 | // label: function(context) { 194 | // console.log(context); 195 | // if (context.dataset) { 196 | // if (context.dataset.yAxisID == 'yStats') { 197 | // let dataset = context.dataset.data[context.dataIndex]; 198 | // if (!dataset) { 199 | // console.warn(context.dataset); 200 | // console.warn(context.dataset.data[context.dataIndex]); 201 | // } 202 | // let mainValue = dataset.y; 203 | // let detailValue = dataset.additional.total_requests_max; 204 | // return [`${context.dataset.label}: ${mainValue}`,`Max Total Requests: ${detailValue}`]; 205 | // } else { 206 | // return `${context.dataset.label}: ${context.formattedValue}`; 207 | // } 208 | // } else { 209 | // console.log(context); 210 | // return null; 211 | // } 212 | // // let label = context.dataset.label || ''; 213 | 214 | // // if (label) { 215 | // // label += ': '; 216 | // // } 217 | // // if (context.parsed.y !== null) { 218 | // // label += new Intl.NumberFormat('en-US', { style: 'currency', currency: 'USD' }).format(context.parsed.y); 219 | // // } 220 | // return 'asdf'; 221 | // } 222 | // } 223 | } 224 | }, 225 | animation: !high_data, 226 | scales: { 227 | x: { 228 | type: 'time', 229 | time: { 230 | unit: 'day' 231 | }, 232 | autoSkip: true, 233 | }, 234 | y: { 235 | beginAtZero: true 236 | }, 237 | yStats: { 238 | beginAtZero: false 239 | } 240 | }, 241 | interaction: { 242 | mode: 'index', 243 | intersect: false 244 | }, 245 | } 246 | }); 247 | } -------------------------------------------------------------------------------- /server/static/chartjs-adapter-moment_1.0.1.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * chartjs-adapter-moment v1.0.1 3 | * https://www.chartjs.org 4 | * (c) 2022 chartjs-adapter-moment Contributors 5 | * Released under the MIT license 6 | */ 7 | !function(e,t){"object"==typeof exports&&"undefined"!=typeof module?t(require("moment"),require("chart.js")):"function"==typeof define&&define.amd?define(["moment","chart.js"],t):t((e="undefined"!=typeof globalThis?globalThis:e||self).moment,e.Chart)}(this,(function(e,t){"use strict";function n(e){return e&&"object"==typeof e&&"default"in e?e:{default:e}}var f=n(e);const a={datetime:"MMM D, YYYY, h:mm:ss a",millisecond:"h:mm:ss.SSS a",second:"h:mm:ss a",minute:"h:mm a",hour:"hA",day:"MMM D",week:"ll",month:"MMM YYYY",quarter:"[Q]Q - YYYY",year:"YYYY"};t._adapters._date.override("function"==typeof f.default?{_id:"moment",formats:function(){return a},parse:function(e,t){return"string"==typeof e&&"string"==typeof t?e=f.default(e,t):e instanceof f.default||(e=f.default(e)),e.isValid()?e.valueOf():null},format:function(e,t){return f.default(e).format(t)},add:function(e,t,n){return f.default(e).add(t,n).valueOf()},diff:function(e,t,n){return f.default(e).diff(f.default(t),n)},startOf:function(e,t,n){return e=f.default(e),"isoWeek"===t?(n=Math.trunc(Math.min(Math.max(0,n),6)),e.isoWeekday(n).startOf("day").valueOf()):e.startOf(t).valueOf()},endOf:function(e,t){return f.default(e).endOf(t).valueOf()}}:{})})); 8 | -------------------------------------------------------------------------------- /server/static/dygraph.min.css: -------------------------------------------------------------------------------- 1 | .dygraph-legend{position:absolute;font-size:14px;z-index:10;width:250px;background:#fff;line-height:normal;text-align:left;overflow:hidden}.dygraph-legend[dir=rtl]{text-align:right}.dygraph-legend-line{display:inline-block;position:relative;bottom:.5ex;padding-left:1em;height:1px;border-bottom-width:2px;border-bottom-style:solid}.dygraph-legend-dash{display:inline-block;position:relative;bottom:.5ex;height:1px;border-bottom-width:2px;border-bottom-style:solid}.dygraph-roller{position:absolute;z-index:10}.dygraph-annotation{position:absolute;z-index:10;overflow:hidden}.dygraph-default-annotation{border:1px solid #000;background-color:#fff;text-align:center}.dygraph-axis-label{z-index:10;line-height:normal;overflow:hidden;color:#000}.dygraph-title{font-weight:700;z-index:10;text-align:center}.dygraph-xlabel{text-align:center}.dygraph-label-rotate-left{text-align:center;transform:rotate(90deg);-webkit-transform:rotate(90deg);-moz-transform:rotate(90deg);-o-transform:rotate(90deg);-ms-transform:rotate(90deg)}.dygraph-label-rotate-right{text-align:center;transform:rotate(-90deg);-webkit-transform:rotate(-90deg);-moz-transform:rotate(-90deg);-o-transform:rotate(-90deg);-ms-transform:rotate(-90deg)} 2 | /*# sourceMappingURL=dygraph.min.css.map */ -------------------------------------------------------------------------------- /server/static/helpers.min.js: -------------------------------------------------------------------------------- 1 | export{H as HALF_PI,b2 as INFINITY,P as PI,b1 as PITAU,b4 as QUARTER_PI,b3 as RAD_PER_DEG,T as TAU,b5 as TWO_THIRDS_PI,R as _addGrace,X as _alignPixel,a2 as _alignStartEnd,p as _angleBetween,b6 as _angleDiff,_ as _arrayUnique,a8 as _attachContext,as as _bezierCurveTo,ap as _bezierInterpolation,ax as _boundSegment,an as _boundSegments,a5 as _capitalize,am as _computeSegments,a9 as _createResolver,aK as _decimalPlaces,aV as _deprecated,aa as _descriptors,ah as _elementsEqual,N as _factorize,aO as _filterBetween,I as _getParentNode,q as _getStartAndCountOfVisiblePoints,W as _int16Range,aj as _isBetween,ai as _isClickEvent,M as _isDomSupported,C as _isPointInArea,S as _limitValue,aN as _longestText,aP as _lookup,B as _lookupByKey,V as _measureText,aT as _merger,aU as _mergerIf,ay as _normalizeAngle,y as _parseObjectDataRadialScale,aq as _pointInLine,ak as _readValueToProps,A as _rlookupByKey,w as _scaleRangesChanged,aG as _setMinAndMaxByKey,aW as _splitKey,ao as _steppedInterpolation,ar as _steppedLineTo,aB as _textX,a1 as _toLeftRightCenter,al as _updateBezierControlPoints,au as addRoundedRectPath,aJ as almostEquals,aI as almostWhole,Q as callback,af as clearCanvas,Y as clipArea,aS as clone,c as color,j as createContext,ad as debounce,h as defined,aE as distanceBetweenPoints,at as drawPoint,aD as drawPointLegend,F as each,e as easingEffects,O as finiteOrDefault,a$ as fontString,o as formatNumber,D as getAngleFromPoint,aR as getHoverColor,G as getMaximumSize,z as getRelativePosition,az as getRtlAdapter,a_ as getStyle,b as isArray,g as isFinite,a7 as isFunction,k as isNullOrUndef,x as isNumber,i as isObject,aQ as isPatternOrGradient,l as listenArrayEvents,aM as log10,a4 as merge,ab as mergeIf,aH as niceNum,aF as noop,aA as overrideTextDirection,J as readUsedSize,Z as renderText,r as requestAnimFrame,a as resolve,f as resolveObjectKey,aC as restoreTextDirection,ae as retinaScale,ag as setsEqual,s as sign,aY as splineCurve,aZ as splineCurveMonotone,K as supportsEventListenerOptions,L as throttled,U as toDegrees,n as toDimension,a0 as toFont,aX as toFontString,b0 as toLineHeight,E as toPadding,m as toPercentage,t as toRadians,av as toTRBL,aw as toTRBLCorners,ac as uid,$ as unclipArea,u as unlistenArrayEvents,v as valueOrDefault}from"./chunks/helpers.segment.js";import"@kurkle/color"; -------------------------------------------------------------------------------- /server/static/history.2.js: -------------------------------------------------------------------------------- 1 | document.addEventListener("DOMContentLoaded", function (event) { 2 | var loadGraphBtn = document.getElementById("loadGraphBtn"); 3 | if (loadGraphBtn) { 4 | loadGraphBtn.addEventListener("click", function () { 5 | var elements = document.querySelectorAll('.graph'); 6 | elements.forEach(function (element) { 7 | element.style.display = 'block'; 8 | }); 9 | loadGraphBtn.style.display = "none"; 10 | loadHealthGraph(); 11 | loadStatsGraph(); 12 | }); 13 | } 14 | }); 15 | 16 | async function loadHealthGraph() { 17 | let graphDiv = document.getElementById('graph-health'); 18 | try { 19 | let g = new Dygraph( 20 | graphDiv, 21 | "/api/csv/health", 22 | { 23 | title: 'Historic Instance Healthiness', 24 | showRangeSelector: true, 25 | width: '100%', 26 | rangeSelectorPlotFillColor: 'MediumSlateBlue', 27 | rangeSelectorPlotFillGradientColor: 'rgba(123, 104, 238, 0)', 28 | colorValue: 0.9, 29 | fillAlpha: 0.4, 30 | colors: ['#008000', '#ffa500'], 31 | ylabel: 'Nitter Instances', 32 | } 33 | ); 34 | g.ready(function () { 35 | g.setAnnotations([ 36 | { 37 | series: "Healthy", 38 | x: "2023-08-15T07:10:17Z", 39 | shortText: "G", 40 | text: "First API Change" 41 | }, 42 | { 43 | series: "Dead", 44 | x: "2023-10-21T14:37:44Z", 45 | shortText: "C", 46 | text: "Wiki Cleanup" 47 | }, 48 | { 49 | series: "Healthy", 50 | x: "2024-01-25T15:24:16Z", 51 | shortText: "R", 52 | text: "API Shutdown" 53 | } 54 | ]); 55 | }); 56 | } catch (error) { 57 | console.error('Failed to fetch data:', error); 58 | graphDiv.textContent = "Failed to load data."; 59 | } 60 | } 61 | async function loadStatsGraph() { 62 | let graphDiv = document.getElementById('graph-stats'); 63 | try { 64 | let g = new Dygraph( 65 | graphDiv, 66 | "/api/csv/stats", 67 | { 68 | title: 'Historic Average Instance Statistics', 69 | showRangeSelector: true, 70 | width: '100%', 71 | rangeSelectorPlotFillColor: 'MediumSlateBlue', 72 | rangeSelectorPlotFillGradientColor: 'rgba(123, 104, 238, 0)', 73 | colorValue: 0.9, 74 | fillAlpha: 0.4, 75 | colors: ['#008000', '#ffa500', '#6495ED'], 76 | series: { 77 | 'Tokens AVG': { 78 | axis: 'y' 79 | }, 80 | 'Limited Tokens AVG': { 81 | axis: 'y' 82 | }, 83 | 'Requests AVG': { 84 | axis: 'y2' 85 | }, 86 | }, 87 | axes: { 88 | y: { 89 | axisLabelWidth: 60, 90 | logscale: "y log scale", 91 | }, 92 | y2: { 93 | // set axis-related properties here 94 | drawGrid: false, 95 | drawAxis: false, 96 | logscale: "y log scale", 97 | }, 98 | }, 99 | ylabel: 'Tokens', 100 | y2label: 'Requests', 101 | } 102 | ); 103 | g.ready(function () { 104 | g.setAnnotations([ 105 | { 106 | series: "Healthy", 107 | x: "2023-08-15T07:10:17Z", 108 | shortText: "G", 109 | text: "First API Change" 110 | }, 111 | { 112 | series: "Dead", 113 | x: "2023-10-21T14:37:44Z", 114 | shortText: "C", 115 | text: "Wiki Cleanup" 116 | }, 117 | { 118 | series: "Healthy", 119 | x: "2024-01-25T15:24:16Z", 120 | shortText: "R", 121 | text: "API Shutdown" 122 | } 123 | ]); 124 | }); 125 | } catch (error) { 126 | console.error('Failed to fetch data:', error); 127 | graphDiv.textContent = "Failed to load data."; 128 | } 129 | } -------------------------------------------------------------------------------- /server/static/instance.js: -------------------------------------------------------------------------------- 1 | let chart = undefined; 2 | document.addEventListener("DOMContentLoaded", function() { 3 | const startDateInput = document.getElementById('startDate'); 4 | const endDateInput = document.getElementById('endDate'); 5 | const submitTime = document.getElementById('submitDateRange'); 6 | 7 | // Set initial values in UTC 8 | const initialEndDate = moment.utc(); 9 | const initialStartDate = moment().subtract(14, 'days').utc(); 10 | startDateInput.value = initialStartDate.format('YYYY-MM-DD'); 11 | endDateInput.value = initialEndDate.format('YYYY-MM-DD');; 12 | 13 | submitTime.addEventListener('click', function() { 14 | const startDate = moment(startDateInput.value, 'YYYY-MM-DD'); 15 | const endDate = moment(endDateInput.value, 'YYYY-MM-DD'); 16 | 17 | if (!startDate.isValid() || !endDate.isValid()) { 18 | alert('Invalid date format. Please use the YYYY-MM-DD format.'); 19 | } else if (startDate.isAfter(endDate)) { 20 | alert('Invalid date range. Start date must be before the end date.'); 21 | } else { 22 | fetchDataAndCreateChart(startDate,endDate); 23 | } 24 | }); 25 | fetchDataAndCreateChart(initialStartDate,initialEndDate); 26 | }); 27 | 28 | function mapKey(key) { 29 | const keyMappings = { 30 | "limited_accs": "Limited Accs", 31 | "total_accs": "Total Accs", 32 | "total_requests": "Total Requests", 33 | "req_photo_rail": "PhotoRail Requests", 34 | "req_user_screen_name": "UserScreeName Req.", 35 | "req_search": "Search Requests", 36 | "req_list_tweets": "ListTeweets Req.", 37 | "req_user_media": "UserMedia Req.", 38 | "req_tweet_detail": "TweetDetail Req.", 39 | "req_list": "List Requests", 40 | "req_user_tweets": "UserTweets Req.", 41 | "req_user_tweets_and_replies": "UserTweetsAndReplies Req.", 42 | } 43 | return keyMappings[key]; 44 | } 45 | 46 | async function fetchDataAndCreateChart(startDate,endDate) { 47 | let jsonData; 48 | try { 49 | const response = await fetch('/admin/api/history/'+document.getElementById('graph').getAttribute('data-host'), { 50 | method: "POST", 51 | headers: { 52 | "Content-Type": "application/json", 53 | }, 54 | body: JSON.stringify({"start": startDate.utc(), "end": endDate.add(1, 'days').utc()}) 55 | }); 56 | jsonData = await response.json(); 57 | 58 | } catch (error) { 59 | console.error('Failed to fetch data:', error); 60 | } 61 | if (jsonData) { 62 | createChartOverview(jsonData); 63 | } 64 | } 65 | 66 | function createChartOverview(jsonData) { 67 | const healthyData = jsonData.health.filter(entry => entry.healthy).map(entry => ({x: moment.unix(entry.time), y: entry.resp_time})); 68 | const unhealthyData = jsonData.health.filter(entry => !entry.healthy).map(entry => ({x: moment.unix(entry.time), y: entry.resp_time})); 69 | const statsData = jsonData.stats.map(entry => ({x: moment.unix(entry.time), y: entry.total_requests})); 70 | 71 | const ctx = document.getElementById('graph').getContext('2d'); 72 | if (chart) { 73 | chart.destroy(); 74 | } 75 | chart = new Chart(ctx, { 76 | type: 'line', 77 | data: { 78 | datasets: [ 79 | { 80 | label: 'Healthy Response Time (ms)', 81 | data: healthyData, 82 | borderColor: 'green', 83 | fill: false, 84 | }, 85 | { 86 | label: 'Unhealthy Response Time (ms)', 87 | data: unhealthyData, 88 | borderColor: 'red', 89 | fill: false, 90 | }, 91 | { 92 | label: 'Total API Requests', 93 | data: statsData, 94 | borderColor: 'blue', 95 | fill: false, 96 | yAxisID: 'yStats', 97 | }, 98 | ] 99 | }, 100 | options: { 101 | interaction: { 102 | mode: 'index', 103 | intersect: false 104 | }, 105 | scales: { 106 | x: { 107 | type: 'time', 108 | // time: { 109 | // // unit: 'hour' 110 | // }, 111 | scaleLabel: { 112 | display: true, 113 | labelString: 'Time' 114 | } 115 | }, 116 | y: { 117 | scaleLabel: { 118 | display: true, 119 | labelString: 'Response Time' 120 | } 121 | }, 122 | yStats: { 123 | scaleLabel: { 124 | display: true, 125 | labelString: 'Requests' 126 | } 127 | } 128 | } 129 | } 130 | }); 131 | 132 | document.getElementById('graph').onclick = function (evt) { 133 | const activePoints = chart.getElementsAtEventForMode(evt, 'index', {intersect: true}); 134 | if (activePoints.length > 0) { 135 | const timestamp = jsonData[activePoints[0]._index].time; 136 | document.getElementById("error_"+timestamp).scrollIntoView(); 137 | } 138 | }; 139 | } -------------------------------------------------------------------------------- /server/static/login_key.js: -------------------------------------------------------------------------------- 1 | async function sha256(message) { 2 | const msgBuffer = new TextEncoder().encode(message); 3 | const hashBuffer = await crypto.subtle.digest('SHA-256', msgBuffer); 4 | const hashArray = Array.from(new Uint8Array(hashBuffer)); 5 | const hashHex = hashArray.map(b => b.toString(16).padStart(2, '0')).join(''); 6 | return hashHex; 7 | } 8 | 9 | async function generate(e) { 10 | if (e) { 11 | e.preventDefault(); 12 | } 13 | let key = self.crypto.randomUUID(); 14 | let hash = await sha256(key); 15 | console.log(key); 16 | console.log(hash); 17 | for (const elem of document.getElementsByClassName("ex-key")) { 18 | elem.innerHTML = key; 19 | } 20 | 21 | for (const elem of document.getElementsByClassName("ex-hash")) { 22 | elem.innerHTML = hash; 23 | } 24 | } 25 | 26 | document.getElementById('generate').addEventListener('click', generate); 27 | 28 | generate(); -------------------------------------------------------------------------------- /server/static/robots.txt: -------------------------------------------------------------------------------- 1 | User-Agent: * 2 | 3 | Disallow: /api/ 4 | Disallow: /admin/ 5 | 6 | Allow: / 7 | 8 | Crawl-delay: 5 9 | 10 | User-agent: AI2Bot 11 | User-agent: Ai2Bot-Dolma 12 | User-agent: Amazonbot 13 | User-agent: anthropic-ai 14 | User-agent: Applebot 15 | User-agent: Applebot-Extended 16 | User-agent: Bytespider 17 | User-agent: CCBot 18 | User-agent: ChatGPT-User 19 | User-agent: Claude-Web 20 | User-agent: ClaudeBot 21 | User-agent: cohere-ai 22 | User-agent: cohere-training-data-crawler 23 | User-agent: Crawlspace 24 | User-agent: Diffbot 25 | User-agent: DuckAssistBot 26 | User-agent: FacebookBot 27 | User-agent: FriendlyCrawler 28 | User-agent: Google-Extended 29 | User-agent: GoogleOther 30 | User-agent: GoogleOther-Image 31 | User-agent: GoogleOther-Video 32 | User-agent: GPTBot 33 | User-agent: iaskspider/2.0 34 | User-agent: ICC-Crawler 35 | User-agent: ImagesiftBot 36 | User-agent: img2dataset 37 | User-agent: ISSCyberRiskCrawler 38 | User-agent: Kangaroo Bot 39 | User-agent: Meta-ExternalAgent 40 | User-agent: Meta-ExternalFetcher 41 | User-agent: OAI-SearchBot 42 | User-agent: omgili 43 | User-agent: omgilibot 44 | User-agent: PanguBot 45 | User-agent: PerplexityBot 46 | User-agent: PetalBot 47 | User-agent: Scrapy 48 | User-agent: SemrushBot-OCOB 49 | User-agent: SemrushBot-SWA 50 | User-agent: Sidetrade indexer bot 51 | User-agent: Timpibot 52 | User-agent: VelenPublicWebCrawler 53 | User-agent: Webzio-Extended 54 | User-agent: YouBot 55 | Disallow: / 56 | -------------------------------------------------------------------------------- /server/static/sortable.min.js: -------------------------------------------------------------------------------- 1 | document.addEventListener("click",function(c){try{function h(a,b){return a.nodeName===b?a:h(a.parentNode,b)}var y=c.shiftKey||c.altKey,d=h(c.target,"TH"),m=d.parentNode,n=m.parentNode,f=n.parentNode;function p(a,b){a.classList.remove("dir-d");a.classList.remove("dir-u");b&&a.classList.add(b)}function q(a){return y&&a.dataset.sortAlt||a.dataset.sort||a.textContent}if("THEAD"===n.nodeName&&f.classList.contains("sortable")&&!d.classList.contains("no-sort")){var r,e=m.cells,t=parseInt(d.dataset.sortTbr); 2 | for(c=0;c { 7 | checkbox.addEventListener('change', function () { 8 | const columnName = checkbox.getAttribute('data-name'); 9 | let checked = checkbox.checked; 10 | toggleColumn(columnName, checked); 11 | saveSettings(); 12 | }); 13 | }); 14 | 15 | function saveSettings() { 16 | let data = {}; 17 | toggleColumnCheckboxes.forEach(checkbox => { 18 | const columnName = checkbox.getAttribute('data-name'); 19 | let checked = checkbox.checked; 20 | data[columnName] = checked; 21 | }); 22 | setCookie(COOKIE_NAME, JSON.stringify(data), 30); 23 | } 24 | 25 | // Function to toggle the visibility of a column based on its name or id 26 | function toggleColumn(name, checked) { 27 | const cellsTd = dataTable.querySelectorAll(`td[data-name="${name}"]`); 28 | 29 | cellsTd.forEach(cell => { 30 | cell.style.display = checked ? '' : 'none'; 31 | }); 32 | const cells = dataTable.querySelectorAll(`th[data-name="${name}"]`); 33 | 34 | cells.forEach(cell => { 35 | cell.style.display = checked ? '' : 'none'; 36 | }); 37 | } 38 | 39 | const COOKIE_NAME = "table_settings"; 40 | document.addEventListener("DOMContentLoaded", function (event) { 41 | let val = getCookie(COOKIE_NAME); 42 | if (val) { 43 | let res = JSON.parse(val); 44 | console.log(res); 45 | for (const [key, value] of Object.entries(res)) { 46 | console.log(key, value); 47 | toggleColumn(key, value); 48 | toggleColumnCheckboxes.forEach(checkbox => { 49 | if (checkbox.getAttribute('data-name') == key) { 50 | checkbox.checked = value; 51 | } 52 | }); 53 | } 54 | } else { 55 | toggleColumnCheckboxes.forEach(checkbox => { 56 | const columnName = checkbox.getAttribute('data-name'); 57 | let checked = checkbox.checked; 58 | toggleColumn(columnName, checked); 59 | }); 60 | } 61 | }); 62 | 63 | function setCookie(name, value, days) { 64 | var expires = ""; 65 | if (days) { 66 | var date = new Date(); 67 | date.setTime(date.getTime() + (days * 24 * 60 * 60 * 1000)); 68 | expires = "; expires=" + date.toUTCString(); 69 | } 70 | document.cookie = name + "=" + (value || "") + expires + "; path=/"; 71 | } 72 | 73 | function getCookie(name) { 74 | var nameEQ = name + "="; 75 | var ca = document.cookie.split(';'); 76 | for (var i = 0; i < ca.length; i++) { 77 | var c = ca[i]; 78 | while (c.charAt(0) == ' ') c = c.substring(1, c.length); 79 | if (c.indexOf(nameEQ) == 0) return c.substring(nameEQ.length, c.length); 80 | } 81 | return null; 82 | } -------------------------------------------------------------------------------- /server/templates/about.html.j2: -------------------------------------------------------------------------------- 1 | {# SPDX-License-Identifier: AGPL-3.0-only #} 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | About Nitter Instance Health 12 | 13 | 14 | 15 |
16 |

Nitter instance uptime & health tracker

17 |

Home

18 |

Uptime check interval: {{uptime_interval_s}}s.

19 |

Instance re-fetching interval, including RSS & version check: {{wiki_interval_s}}s.

20 |

Fetched latest commit: {{latest_commit}}

21 | 22 |

The following paths are checked per instance:

23 |
    24 | {%- for path in checked_paths %} 25 |
  • {{path}}
    • 26 | {%- endfor %} 27 |
28 | 29 |

Table Explanations

30 |
    31 |
  • Country for the host country reported in the instance wiki.
    • 32 |
  • Healthy stands for hosts which are reachable and pass a content check.
    33 | Known bad hosts are marked with a ❓. These instances prevent healthchecks (for example through long caching). 34 |
    • 35 |
  • Average Time is the response time average over the last {{ping_avg_interval_h}} hours. This is 36 | not a network ping.
    • 37 |
  • All Time % for all time percentage of the instance being healthy.
    • 38 |
  • RSS whether the host has RSS feeds enabled.
    • 39 |
  • LSH Last Seen Healthy, for the last time an instance was seen healthy.
    • 40 |
  • Nitter Version which nitter version the host reports.
    • 41 |
  • Connectivity the IP connectivity support. One of All, IPv4, 42 | IPv6 43 |
    • 44 |
  • Points is a weighted instance score based on the availability over the last 3h, 30 and 120 days, 45 | together with the version.
    • 46 |
47 | 48 |

Settings for visible columns are stored locally per client.

49 | 50 |

API

51 |

52 | The same data as visible in the website/table can also be fetched as JSON from /api/v1/instances (link). 54 | Note that the data only changes in the intervals stated above. Thus requesting it 55 | very often will get you rate limited. 56 |

57 |

58 | Please do not use instances listed here for scraping(?) ! 60 | Instead host your own nitter instance, so public instances 61 | can be used by less tech savy people and not get overrun by you. You can get help with that at the listed matrix 62 | channel. 63 |

64 |

The purpose of this API is to serve services like Twiiit, Farside or people looking for indication that their RSS 65 | feed is down - but not scrapers. Scraping will bring the downfall of public instances.

66 | 67 |

Sourcecode

68 |

License: AGPL3

69 |
70 | 71 | 72 | -------------------------------------------------------------------------------- /server/templates/admin.html.j2: -------------------------------------------------------------------------------- 1 | {# SPDX-License-Identifier: AGPL-3.0-only #} 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | Admin Interface 11 | 12 | 13 |
14 |

Admin Interface

15 |

Logout Add more instances

16 | {% if is_admin %}

Settings Logs

{% endif %} 17 | 18 |

Instances

19 | {% if is_admin %}

root mode

{% endif %} 20 |
21 |
22 |
23 |
24 |
25 | 26 | 27 |
28 |
29 | 30 | 31 |
32 |
33 | 34 | 35 |
36 |
37 |
38 | 39 | 40 | 41 | 42 | 43 | 44 | {% for instance in instances -%} 45 | 46 | 47 | {# #} 48 | 49 | 50 | {%- endfor %} 51 | 52 |
Instance
{{instance.domain}}HistoryErrors Settings
53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | -------------------------------------------------------------------------------- /server/templates/admin_logs.html.j2: -------------------------------------------------------------------------------- 1 | {#- SPDX-License-Identifier: AGPL-3.0-only -#} 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | Admin Logs 12 | 13 | 14 | 15 |

16 |

Override Logs

17 |

root mode

18 |

Home

19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | {% for entry in LOGS %} 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | {% endfor %} 39 | 40 |
DateUserHostKeyNew Value
{{entry.time}}{{entry.by_user_host}}{{entry.for_host.domain}} {% if entry.for_host%}link{% endif %}{{entry.key}}{% if entry.value %}{{entry.value}}{% else %}null{% endif %}
41 |
42 | 43 | 44 | -------------------------------------------------------------------------------- /server/templates/history_admin.html.j2: -------------------------------------------------------------------------------- 1 | {# SPDX-License-Identifier: AGPL-3.0-only #} 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | Admin Interface 11 | 12 | 13 |
14 |

Admin Interface

15 |

Overview Logout Add more instances

16 | 17 |

Last errors from {{HOST_DOMAIN}}

18 |
19 | 20 |
21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /server/templates/instance_errors.html.j2: -------------------------------------------------------------------------------- 1 | {# SPDX-License-Identifier: AGPL-3.0-only #} 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | Admin Interface 11 | 12 | 13 |
14 |

Admin Interface

15 |

Overview Logout Add more instances

16 |
17 |
18 |
19 |
20 |
21 | 22 | 23 |
24 |
25 | 26 | 27 |
28 |
29 | 30 |
31 |
32 |

Last errors from {{HOST_DOMAIN}}

33 |
34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | {% for error in ERRORS -%} 45 | 46 | 47 | 48 | 49 | 50 | 51 | {%- endfor %} 52 | 53 |
Time UTCMessageHttp BodyHttp Status
{{fmt_date(value=error.time)}}{{error.message}}{{error.http_body}}{{error.http_status}}
54 |
55 | 56 | 57 | 58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /server/templates/instance_locks.html.j2: -------------------------------------------------------------------------------- 1 | {#- SPDX-License-Identifier: AGPL-3.0-only -#} 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | Locks | {{HOST_DOMAIN}} 12 | 13 | 14 | 15 |
16 |

Override Locks | {{HOST_DOMAIN}}

17 |

root mode

18 |

Home

19 |
20 | {% for key, value in OVERRIDES %} 21 |
22 |
23 | 25 | 28 |
29 |
30 | {% endfor %} 31 |
32 | 33 |
34 |
35 |
36 | 37 | 38 | -------------------------------------------------------------------------------- /server/templates/instance_settings.html.j2: -------------------------------------------------------------------------------- 1 | {#- SPDX-License-Identifier: AGPL-3.0-only -#} 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | Settings | {{HOST_DOMAIN}} 12 | 13 | 14 | 15 |
16 |

Settings for {{HOST_DOMAIN}}

17 |

Home {% if IS_ADMIN %}Lockdown Settings{% endif %}

18 | {% if IS_ADMIN %}

root mode

{% endif %} 19 |
20 | {%- set key = "BAD_HOST" -%} 21 | {%- set disabled = OVERRIDES[key].locked and not IS_ADMIN -%} 22 |
23 |
24 | 26 | 29 |
30 |
31 | 32 |
33 | 34 |
35 |
36 |

Overrides

37 |

Leave settings blank to stop overriding them from their defaults.

38 | 39 |
40 | {%- set key = "BEARER_TOKEN" -%} 41 | {%- set disabled = OVERRIDES[key].locked and not IS_ADMIN -%} 42 |
43 | 44 |
45 | Authorization: Bearer 46 | 48 |
49 |
50 | 51 |
52 | 53 |
54 |
55 |
56 | {%- set key = "HEALTH_PATH" -%} 57 | {%- set disabled = OVERRIDES[key].locked and not IS_ADMIN -%} 58 |
59 | 60 | 62 |
63 | 64 |
65 | 66 |
67 |
68 |
69 | {%- set key = "HEALTH_QUERY" -%} 70 | {%- set disabled = OVERRIDES[key].locked and not IS_ADMIN -%} 71 |
72 | 73 | 75 |
76 | 77 |
78 | 79 |
80 |
81 |
82 | 83 | 84 | -------------------------------------------------------------------------------- /server/templates/instances.html.j2: -------------------------------------------------------------------------------- 1 | {# SPDX-License-Identifier: AGPL-3.0-only #} 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 20 | 21 | Nitter Instance Health 22 | 23 | 24 | 25 |
26 |

Nitter Instance Uptime & Health

27 | 30 |

About

31 |

Please do NOT use these instances for 32 | scraping, host nitter yourself.

33 |

Last Updated {{last_updated}} UTC.

34 |

Customize the visible columns down below.

35 |
36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | {% for host in instances -%} 53 | 54 | 55 | 56 | 66 | 98 | 99 | 100 | 102 | {%- if host.healthy and not host.version -%} 103 | {%- set version = "Dmissing" -%} 104 | {%- elif host.version and not host.is_upstream -%} 105 | {%- set version = "Aforeign" -%} 106 | {%- elif host.is_latest_version -%} 107 | {%- set version = "Blatest" -%} 108 | {%- elif host.version and not host.is_latest_version -%} 109 | {%- set version = "Coutdated" -%} 110 | {%- else -%} 111 | {%- set version = "Eunknown" -%} 112 | {%- endif -%} 113 | 131 | 132 | 133 | 134 | {%- endfor -%} 135 | 136 |
InstanceCountryHealthyHealth HistoryAverage TimeAll Time %RSSNitter VersionConnectivityPoints
{{host.domain}}{{host.country}} 57 | {%- if host.is_bad_host -%} 58 |
❓
60 | {%- elif host.healthy -%} 61 | ✅ 62 | {%- else -%} 63 | ❌ 64 | {%- endif -%} 65 | 		67 | {%- set height = 28 -%} 68 | {%- set width = 110 -%} 69 | {%- set width_bar = 5 -%} 70 | {%- set offset = 5 -%} 71 | {%- if not host.__show_last_seen -%} 72 | 73 | {%- for check in host.recent_checks -%} 74 | {%- if check.1 -%} 75 | {%- set title = "Healthy " ~ check.0 -%} 76 | {%- set color_bar = "#2fcc66" -%} 77 | {%- else -%} 78 | {%- set title = "Unhealthy " ~ check.0 -%} 79 | {%- set color_bar = "#ff6225" -%} 80 | {%- endif -%} 81 | 83 | {{title}} 84 | 85 | {%- endfor -%} 86 | 87 | {%- else -%} 88 | {# #} 89 | {%- if host.last_healthy -%} 90 | LSH: {{host.last_healthy | truncate(length=16, end="") | replace(from="-", to=".") | replace(from="T", 91 | to=" ")}} 92 | UTC 93 | {%- else -%} 94 | Never seen healthy. 95 | {%- endif -%} 96 | {%- endif -%} 97 | {%- if not host.__show_last_seen -%}{{host.ping_avg}}ms{% endif %}{{host.healthy_percentage_overall}}%{% if host.rss -%} ✅ {%- else -%} ❌ {%- endif -%} 114 | {%- if host.version_url -%} 115 | 116 | {{host.version | truncate(length=18) | default(value="missing version")}} 117 | 118 | {%- else -%} 119 | missing version 120 | {%- endif -%} 121 | {%- if version == "Dmissing" -%} 122 | missing 123 | {%- elif version == "Aforeign" -%} 124 | custom 125 | {%- elif version == "Blatest" -%} 126 | latest 127 | {# {%- elif version == "Coutdated" -%} 128 | outdated #} 129 | {%- endif -%} 130 | {{host.connectivity}}{{host.points}}
137 |
138 | 139 |
140 | Visible Columns: 141 |
142 | 143 | 146 |
147 |
148 | 149 | 152 |
153 |
154 | 155 | 158 |
159 |
160 | 161 | 164 |
165 |
166 | 167 | 170 |
171 |
172 |
173 | 174 | 175 | 176 | 177 | 178 | 179 | -------------------------------------------------------------------------------- /server/templates/login.html.j2: -------------------------------------------------------------------------------- 1 | {# SPDX-License-Identifier: AGPL-3.0-only #} 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | Admin Interface 11 | 12 | 13 |
14 |

Login for Instance owners

15 |

Get access to your instance history and errors by loggin in.

16 |

Home

17 | 18 | {% if ERROR %} 19 |
20 |

Failed to login

21 |
{{ERROR}}
22 | {% if QUOTE %}
{{QUOTE}}
{% endif %} 23 |
24 | {% endif %} 25 | 26 |
27 |
28 | 29 | 30 |
Instance domain, without https.
31 |
32 |
33 | 34 | 35 |
Private token to verify against your host.
36 |
37 |
38 | 39 | 42 |
43 |
44 | 45 | 48 |
49 | 50 | 51 |
52 | 53 |

Setting up the login token:

54 |
    55 |

  1. Create a random value, and its corresponding SHA256 hash: 56 | 57 |

    63 | 64 |

    The output is like this: 65 | 66 | 

    00000000-0000-0000-0000-000000000000
67 |         12B9377CBE7E5C94E8A70D9D23929523D14AFA954793130F8A3959C7B849ACA8
    68 | 69 |

    The first line is your random value, and the second line's hex part is the hash. 70 |

    2. 71 | 72 |

  2. For your instance nitter.example.com, either: 73 | 74 |

      75 |
    • 76 |

      create a new TXT record with the name {{VERIFY_TOKEN_NAME}}.nitter and the value 6c9872185d6975f0f51d7d16a6428aadb1df494af0f68166f790ef0d51b0bc8f 77 | 78 |

      Verify that dig -t txt {{VERIFY_TOKEN_NAME}}.nitter.example.com resolves 79 | 80 |

    • OR: create a file https://nitter.example.com/.well-known/{{VERIFY_TOKEN_NAME}} with the content 6c9872185d6975f0f51d7d16a6428aadb1df494af0f68166f790ef0d51b0bc8f 81 |

    82 |
    3. 83 | 84 |

    License: AGPL3

    85 | 86 | 87 | 88 | -------------------------------------------------------------------------------- /server/templates/rip.html.j2: -------------------------------------------------------------------------------- 1 | {# SPDX-License-Identifier: AGPL-3.0-only #} 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 25 | 26 | Nitter Is Dead 27 | 28 | 29 | 30 |
    31 |

    All Nitter public instances shut down.

    32 |

    There are no public instances left for you to use.

    33 | 34 |

    Why ?

    35 |

    Nitter has already been struggling to keep up with the changes Twitter made on their side after 2022.

    36 | 37 |

    In January 2024, the Twitter API used by Nitter to fetch data from Twitter got shut down.

    38 | 39 |

    Without this, there is no way to run instances on the scale needed for the public instances that got listed here. 40 |

    41 | 42 |

    That's unfortunate. But I still want to use Nitter !

    43 |

    If you want to continue running Nitter, you can do so using a normal Twitter account.

    44 | 45 |

    Do note however, that this cannot and will not scale, and is in a legal grey-area. Proceed at your own risk.

    46 | 47 |

    See here and here for setting up 49 | Nitter if you feel brave enough and don't fear the command line.

    50 | 51 |

    But there are some instances ?

    52 |

    Through the thankless work of some there is a tiny amount of instances left. These will be slow and can not 53 | expose 54 | RSS to prevent the huge volume of bot attacks. The latter one also makes captchas and other anti-botting methods a 55 | requirement.

    56 |
    57 | 58 |
    59 |
    60 | 61 |
    62 |
    63 |
    64 |
    65 | 66 |
    67 |
    68 |

    Instances About

    69 |
    70 | 71 | 72 |
    73 | 74 | 75 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: AGPL-3.0-only 2 | use std::{collections::HashMap, env::var, time::Duration}; 3 | 4 | use entities::state::scanner::ScannerConfig; 5 | use miette::{bail, Context, IntoDiagnostic}; 6 | use migration::MigratorTrait; 7 | use sea_orm::{ConnectOptions, ConnectionTrait, Database, DatabaseBackend, DatabaseConnection}; 8 | use std::sync::Arc; 9 | use tracing::info; 10 | 11 | use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; 12 | 13 | fn main() -> miette::Result<()> { 14 | #[cfg(debug_assertions)] 15 | let build_mode = "debug mode"; 16 | #[cfg(not(debug_assertions))] 17 | let build_mode = "release mode"; 18 | println!( 19 | "Starting {} {} licensed under {}, {build_mode}", 20 | env!("CARGO_PKG_NAME"), 21 | env!("CARGO_PKG_VERSION"), 22 | env!("CARGO_PKG_LICENSE") 23 | ); 24 | dotenvy::dotenv() 25 | .into_diagnostic() 26 | .wrap_err_with(|| "Failed to load .env file!")?; 27 | 28 | let rt = tokio::runtime::Builder::new_multi_thread() 29 | .enable_all() 30 | .thread_name(concat!(env!("CARGO_PKG_NAME"), "-wrk")) 31 | .build() 32 | .into_diagnostic() 33 | .wrap_err_with(|| "Failed to initialize async runtime!")?; 34 | 35 | rt.block_on(_main()) 36 | } 37 | 38 | async fn _main() -> miette::Result<()> { 39 | tracing_subscriber::registry() 40 | .with(tracing_subscriber::EnvFilter::new( 41 | var("RUST_LOG").unwrap_or_else(|_| { 42 | #[cfg(debug_assertions)] 43 | return format!( 44 | "warn,tower_http=debug,migration=debug,scanner=trace,server=trace,{}=trace", 45 | env!("CARGO_PKG_NAME") 46 | ) 47 | .into(); 48 | #[cfg(not(debug_assertions))] 49 | return format!( 50 | "warn,tower_http=debug,migration=debug,scanner=info,server=info,{}=debug", 51 | env!("CARGO_PKG_NAME") 52 | ) 53 | .into(); 54 | }), 55 | )) 56 | .with(tracing_subscriber::fmt::layer()) 57 | .init(); 58 | 59 | tracing::debug!("connecting to database"); 60 | let dburl = require_env_str("DATABASE_URL")?; 61 | let mut db_opts = ConnectOptions::new(dburl); 62 | db_opts.connect_timeout(Duration::from_secs(2)); 63 | let pool = Database::connect(db_opts) 64 | .await 65 | .into_diagnostic() 66 | .wrap_err("Failed connecting to database")?; 67 | 68 | let port: u16 = require_env_str("PORT")? 69 | .parse() 70 | .expect("PORT must be a number"); 71 | 72 | let scanner_config = read_scanner_cfg()?; 73 | 74 | let server_config = read_server_config(scanner_config.instance_check_interval.as_secs() as _)?; 75 | 76 | test_init(&pool).await?; 77 | 78 | tracing::info!("migrating db"); 79 | migration::Migrator::up(&pool, None) 80 | .await 81 | .into_diagnostic() 82 | .wrap_err_with(|| "Failed to perform database migration!")?; 83 | 84 | let cache = entities::state::new(); 85 | 86 | let disable_health_checks = require_env_str("DISABLE_HEALTH_CHECKS")? == "true"; 87 | 88 | scanner::run_scanner( 89 | pool.clone(), 90 | scanner_config.clone(), 91 | cache.clone(), 92 | disable_health_checks, 93 | ) 94 | .await 95 | .wrap_err("Crash starting background scanner")?; 96 | 97 | let addr = std::net::SocketAddr::from(([127, 0, 0, 1], port)); 98 | server::start(&addr, pool, server_config, scanner_config, cache) 99 | .await 100 | .into_diagnostic()?; 101 | 102 | tracing::info!("shutting down"); 103 | 104 | Ok(()) 105 | } 106 | 107 | fn read_scanner_cfg() -> miette::Result { 108 | let nitter_instancelist: String = require_env_str("NITTER_INSTANCELIST")?; 109 | let instance_ping_interval: u64 = require_env_str("INSTANCE_PING_INTERVAL_S")? 110 | .parse() 111 | .expect("INSTANCE_PING_INTERVAL_S must be a number"); 112 | let instance_list_interval: u64 = require_env_str("INSTANCE_LIST_INTERVAL_S")? 113 | .parse() 114 | .expect("INSTANCE_LIST_INTERVAL_S must be a number"); 115 | let ping_range: u32 = require_env_str("PING_RANGE_H")? 116 | .parse() 117 | .expect("PING_RANGE_H must be a number"); 118 | 119 | let profile_path = require_env_str("PROFILE_PATH")?; 120 | let rss_path = require_env_str("RSS_PATH")?; 121 | let about_path = require_env_str("ABOUT_PATH")?; 122 | let profile_name = require_env_str("PROFILE_NAME")?; 123 | let profile_posts_min = require_env_str("PROFILE_POSTS_MIN")? 124 | .parse() 125 | .expect("PROFILE_POSTS_MIN must be a positive number"); 126 | let additional_hosts: Vec = require_env_vec_str("ADDITIONAL_HOSTS")?; 127 | let additional_host_country = require_env_str("ADDITIONAL_HOSTS_COUNTRY")?; 128 | let rss_content = require_env_str("RSS_CONTENT")?; 129 | let auto_mute = require_env_str("AUTO_MUTE")? == "true"; 130 | let source_git_branch = require_env_str("ORIGIN_SOURCE_GIT_BRANCH")?; 131 | let source_git_url = require_env_str("ORIGIN_SOURCE_GIT_URL")?; 132 | let cleanup_interval: u64 = require_env_str("CLEANUP_INTERVAL_S")? 133 | .parse() 134 | .expect("CLEANUP_INTERVAL_S must be a number"); 135 | let error_retention_per_host: usize = require_env_str("ERROR_RETENTION_PER_HOST")? 136 | .parse() 137 | .expect("CLEANUP_INTERVAL_S must be a number"); 138 | let instance_stats_interval: u64 = require_env_str("STATS_INTERVAL_S")? 139 | .parse() 140 | .expect("STATS_INTERVAL_S must be a positive number"); 141 | 142 | Ok(Arc::new(entities::state::scanner::Config { 143 | instance_stats_interval: Duration::from_secs(instance_stats_interval), 144 | list_fetch_interval: Duration::from_secs(instance_list_interval), 145 | instance_check_interval: Duration::from_secs(instance_ping_interval), 146 | instance_list_url: nitter_instancelist, 147 | profile_path, 148 | rss_path, 149 | about_path, 150 | profile_name, 151 | profile_posts_min, 152 | rss_content, 153 | additional_hosts, 154 | additional_host_country, 155 | website_url: require_env_str("SITE_URL")?, 156 | ping_range: chrono::Duration::hours(ping_range as _), 157 | auto_mute, 158 | source_git_branch, 159 | source_git_url, 160 | cleanup_interval: Duration::from_secs(cleanup_interval), 161 | error_retention_per_host, 162 | connectivity_path: String::from("/"), 163 | })) 164 | } 165 | 166 | async fn test_init(db: &DatabaseConnection) -> miette::Result<()> { 167 | let res = db 168 | .query_one(sea_orm::Statement::from_string( 169 | DatabaseBackend::Sqlite, 170 | "SELECT sqlite_version() as version;".to_owned(), 171 | )) 172 | .await 173 | .into_diagnostic()?; 174 | let v = res.unwrap(); 175 | let db_version: String = v.try_get("", "version").unwrap(); 176 | tracing::debug!(db_version); 177 | 178 | Ok(()) 179 | } 180 | 181 | fn read_server_config(instance_ping_interval: usize) -> miette::Result { 182 | let site_url = require_env_str("SITE_URL")?; 183 | let session_ttl_seconds = require_env_str("SESSION_TTL_SECONDS")? 184 | .parse() 185 | .expect("SESSION_TTL_SECONDS must be a positive number"); 186 | let login_token_name = require_env_str("LOGIN_TOKEN_NAME")?; 187 | let admin_domains: Vec = require_env_str("ADMIN_DOMAINS")? 188 | .split(",") 189 | .map(|v| v.trim().to_string()) 190 | .collect(); 191 | let admin_keys: HashMap = require_env_str("ADMIN_KEYS")? 192 | .split(",") 193 | .map(|v| { 194 | v.trim() 195 | .split_once('=') 196 | .map(|(a, b)| (a.to_owned(), b.to_owned())) 197 | .expect("ADMIN_KEYS have to be in host=key format!") 198 | }) 199 | .collect(); 200 | let session_db_uri = require_env_str("SESSION_DB_URI")?; 201 | 202 | // sanity check all hosts with an admin key are part of the admin hosts 203 | for host in admin_keys.keys() { 204 | if !admin_domains.contains(host) { 205 | bail!("Found admin key for host {host} that is not in the admin hosts!"); 206 | } 207 | } 208 | 209 | info!( 210 | "Loaded admin domains {:?}, loaded additional admin auth keys for {:?}", 211 | admin_domains, 212 | admin_keys.keys() 213 | ); 214 | 215 | Ok(server::Config { 216 | site_url, 217 | max_age: instance_ping_interval, 218 | session_ttl_seconds, 219 | login_token_name, 220 | admin_domains, 221 | admin_keys, 222 | session_db_uri, 223 | }) 224 | } 225 | 226 | fn require_env_vec_str(name: &str) -> miette::Result> { 227 | Ok(require_env_str(name)? 228 | .trim() 229 | .split(",") 230 | .map(|v| v.trim().to_owned()) 231 | .collect()) 232 | } 233 | 234 | fn require_env_str(name: &str) -> miette::Result { 235 | var(name).map_err(|v| miette::miette!("missing `{}` in environment: {:?}", name, v)) 236 | } 237 | --------------------------------------------------------------------------------