Use {link} or try again later.", cfg) 88 | 89 | extend unsupported, "" 90 | extend preferences, "" 91 | extend resolver, "" 92 | extend rss, "" 93 | extend search, "" 94 | extend timeline, "" 95 | extend list, "" 96 | extend status, "" 97 | extend media, "" 98 | extend embed, "" 99 | extend debug, "" 100 | -------------------------------------------------------------------------------- /src/views/status.nim: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: AGPL-3.0-only 2 | import karax/[karaxdsl, vdom] 3 | 4 | import ".."/[types, formatters] 5 | import tweet, timeline 6 | 7 | proc renderEarlier(thread: Chain): VNode = 8 | buildHtml(tdiv(class="timeline-item more-replies earlier-replies")): 9 | a(class="more-replies-text", href=getLink(thread.content[0])): 10 | text "earlier replies" 11 | 12 | proc renderMoreReplies(thread: Chain): VNode = 13 | let link = getLink(thread.content[^1]) 14 | buildHtml(tdiv(class="timeline-item more-replies")): 15 | if thread.content[^1].available: 16 | a(class="more-replies-text", href=link): 17 | text "more replies" 18 | else: 19 | a(class="more-replies-text"): 20 | text "more replies" 21 | 22 | proc renderReplyThread(thread: Chain; prefs: Prefs; path: string): VNode = 23 | buildHtml(tdiv(class="reply thread thread-line")): 24 | for i, tweet in thread.content: 25 | let last = (i == thread.content.high and not thread.hasMore) 26 | renderTweet(tweet, prefs, path, index=i, last=last) 27 | 28 | if thread.hasMore: 29 | renderMoreReplies(thread) 30 | 31 | proc renderReplies*(replies: Result[Chain]; prefs: Prefs; path: string): VNode = 32 | buildHtml(tdiv(class="replies", id="r")): 33 | for thread in replies.content: 34 | if thread.content.len == 0: continue 35 | renderReplyThread(thread, prefs, path) 36 | 37 | if replies.bottom.len > 0: 38 | renderMore(Query(), replies.bottom, focus="#r") 39 | 40 | proc renderConversation*(conv: Conversation; prefs: Prefs; path: string): VNode = 41 | let hasAfter = conv.after.content.len > 0 42 | let threadId = conv.tweet.threadId 43 | buildHtml(tdiv(class="conversation")): 44 | tdiv(class="main-thread"): 45 | if conv.before.content.len > 0: 46 | tdiv(class="before-tweet thread-line"): 47 | let first = conv.before.content[0] 48 | if threadId != first.id and (first.replyId > 0 or not first.available): 49 | renderEarlier(conv.before) 50 | for i, tweet in conv.before.content: 51 | renderTweet(tweet, prefs, path, index=i) 52 | 53 | tdiv(class="main-tweet", id="m"): 54 | let afterClass = if hasAfter: "thread thread-line" else: "" 55 | renderTweet(conv.tweet, prefs, path, class=afterClass, mainTweet=true) 56 | 57 | if hasAfter: 58 | tdiv(class="after-tweet thread-line"): 59 | let 60 | total = conv.after.content.high 61 | hasMore = conv.after.hasMore 62 | for i, tweet in conv.after.content: 63 | renderTweet(tweet, prefs, path, index=i, 64 | last=(i == total and not hasMore), afterTweet=true) 65 | 66 | if hasMore: 67 | renderMoreReplies(conv.after) 68 | 69 | if not prefs.hideReplies: 70 | if not conv.replies.beginning: 71 | renderNewer(Query(), getLink(conv.tweet), focus="#r") 72 | if conv.replies.content.len > 0 or conv.replies.bottom.len > 0: 73 | renderReplies(conv.replies, prefs, path) 74 | 75 | renderToTop(focus="#m") 76 | -------------------------------------------------------------------------------- /src/experimental/types/unifiedcard.nim: -------------------------------------------------------------------------------- 1 | import options, tables 2 | from ../../types import VideoType, VideoVariant 3 | 4 | type 5 | UnifiedCard* = object 6 | componentObjects*: Table[string, Component] 7 | destinationObjects*: Table[string, Destination] 8 | mediaEntities*: Table[string, MediaEntity] 9 | appStoreData*: Table[string, seq[AppStoreData]] 10 | 11 | ComponentType* = enum 12 | details 13 | media 14 | swipeableMedia 15 | buttonGroup 16 | appStoreDetails 17 | twitterListDetails 18 | communityDetails 19 | mediaWithDetailsHorizontal 20 | unknown 21 | 22 | Component* = object 23 | kind*: ComponentType 24 | data*: ComponentData 25 | 26 | ComponentData* = object 27 | id*: string 28 | appId*: string 29 | mediaId*: string 30 | destination*: string 31 | title*: Text 32 | subtitle*: Text 33 | name*: Text 34 | memberCount*: int 35 | mediaList*: seq[MediaItem] 36 | topicDetail*: tuple[title: Text] 37 | 38 | MediaItem* = object 39 | id*: string 40 | destination*: string 41 | 42 | Destination* = object 43 | kind*: string 44 | data*: tuple[urlData: UrlData] 45 | 46 | UrlData* = object 47 | url*: string 48 | vanity*: string 49 | 50 | MediaType* = enum 51 | photo, video, model3d 52 | 53 | MediaEntity* = object 54 | kind*: MediaType 55 | mediaUrlHttps*: string 56 | videoInfo*: Option[VideoInfo] 57 | 58 | VideoInfo* = object 59 | durationMillis*: int 60 | variants*: seq[VideoVariant] 61 | 62 | AppType* = enum 63 | androidApp, iPhoneApp, iPadApp 64 | 65 | AppStoreData* = object 66 | kind*: AppType 67 | id*: string 68 | title*: Text 69 | category*: Text 70 | 71 | Text = object 72 | content: string 73 | 74 | HasTypeField = Component | Destination | MediaEntity | AppStoreData 75 | 76 | converter fromText*(text: Text): string = text.content 77 | 78 | proc renameHook*(v: var HasTypeField; fieldName: var string) = 79 | if fieldName == "type": 80 | fieldName = "kind" 81 | 82 | proc enumHook*(s: string; v: var ComponentType) = 83 | v = case s 84 | of "details": details 85 | of "media": media 86 | of "swipeable_media": swipeableMedia 87 | of "button_group": buttonGroup 88 | of "app_store_details": appStoreDetails 89 | of "twitter_list_details": twitterListDetails 90 | of "community_details": communityDetails 91 | of "media_with_details_horizontal": mediaWithDetailsHorizontal 92 | else: echo "ERROR: Unknown enum value (ComponentType): ", s; unknown 93 | 94 | proc enumHook*(s: string; v: var AppType) = 95 | v = case s 96 | of "android_app": androidApp 97 | of "iphone_app": iPhoneApp 98 | of "ipad_app": iPadApp 99 | else: echo "ERROR: Unknown enum value (AppType): ", s; androidApp 100 | 101 | proc enumHook*(s: string; v: var MediaType) = 102 | v = case s 103 | of "video": video 104 | of "photo": photo 105 | of "model3d": model3d 106 | else: echo "ERROR: Unknown enum value (MediaType): ", s; photo 107 | -------------------------------------------------------------------------------- /tests/test_profile.py: -------------------------------------------------------------------------------- 1 | from base import BaseTestCase, Profile 2 | from parameterized import parameterized 3 | 4 | profiles = [ 5 | ['mobile_test', 'Test account', 6 | 'Test Account. test test Testing username with @mobile_test_2 and a #hashtag', 7 | 'San Francisco, CA', 'example.com/foobar', 'Joined October 2009', '100'], 8 | ['mobile_test_2', 'mobile test 2', '', '', '', 'Joined January 2011', '13'] 9 | ] 10 | 11 | verified = [['jack'], ['elonmusk']] 12 | 13 | protected = [ 14 | ['mobile_test_7', 'mobile test 7', ''], 15 | ['Poop', 'Randy', 'Social media fanatic.'] 16 | ] 17 | 18 | invalid = [['thisprofiledoesntexist'], ['%']] 19 | 20 | banner_color = [ 21 | ['nim_lang', '22, 25, 32'], 22 | ['rustlang', '35, 31, 32'] 23 | ] 24 | 25 | banner_image = [ 26 | ['mobile_test', 'profile_banners%2F82135242%2F1384108037%2F1500x500'] 27 | ] 28 | 29 | 30 | class ProfileTest(BaseTestCase): 31 | @parameterized.expand(profiles) 32 | def test_data(self, username, fullname, bio, location, website, joinDate, mediaCount): 33 | self.open_nitter(username) 34 | self.assert_exact_text(fullname, Profile.fullname) 35 | self.assert_exact_text(f'@{username}', Profile.username) 36 | 37 | tests = [ 38 | (bio, Profile.bio), 39 | (location, Profile.location), 40 | (website, Profile.website), 41 | (joinDate, Profile.joinDate), 42 | (mediaCount + " Photos and videos", Profile.mediaCount) 43 | ] 44 | 45 | for text, selector in tests: 46 | if len(text) > 0: 47 | self.assert_exact_text(text, selector) 48 | else: 49 | self.assert_element_absent(selector) 50 | 51 | @parameterized.expand(verified) 52 | def test_verified(self, username): 53 | self.open_nitter(username) 54 | self.assert_element_visible(Profile.verified) 55 | 56 | @parameterized.expand(protected) 57 | def test_protected(self, username, fullname, bio): 58 | self.open_nitter(username) 59 | self.assert_element_visible(Profile.protected) 60 | self.assert_exact_text(fullname, Profile.fullname) 61 | self.assert_exact_text(f'@{username}', Profile.username) 62 | 63 | if len(bio) > 0: 64 | self.assert_text(bio, Profile.bio) 65 | else: 66 | self.assert_element_absent(Profile.bio) 67 | 68 | @parameterized.expand(invalid) 69 | def test_invalid_username(self, username): 70 | self.open_nitter(username) 71 | self.assert_text(f'User "{username}" not found') 72 | 73 | def test_suspended(self): 74 | self.open_nitter('user') 75 | self.assert_text('User "user" has been suspended') 76 | 77 | @parameterized.expand(banner_color) 78 | def test_banner_color(self, username, color): 79 | self.open_nitter(username) 80 | banner = self.find_element(Profile.banner + ' a') 81 | self.assertIn(color, banner.value_of_css_property('background-color')) 82 | 83 | @parameterized.expand(banner_image) 84 | def test_banner_image(self, username, url): 85 | self.open_nitter(username) 86 | banner = self.find_element(Profile.banner + ' img') 87 | self.assertIn(url, banner.get_attribute('src')) 88 | -------------------------------------------------------------------------------- /tests/base.py: -------------------------------------------------------------------------------- 1 | from seleniumbase import BaseCase 2 | 3 | 4 | class Card(object): 5 | def __init__(self, tweet=''): 6 | card = tweet + '.card ' 7 | self.link = card + 'a' 8 | self.title = card + '.card-title' 9 | self.description = card + '.card-description' 10 | self.destination = card + '.card-destination' 11 | self.image = card + '.card-image' 12 | 13 | 14 | class Quote(object): 15 | def __init__(self, tweet=''): 16 | quote = tweet + '.quote ' 17 | namerow = quote + '.fullname-and-username ' 18 | self.link = quote + '.quote-link' 19 | self.fullname = namerow + '.fullname' 20 | self.username = namerow + '.username' 21 | self.text = quote + '.quote-text' 22 | self.media = quote + '.quote-media-container' 23 | self.unavailable = quote + '.quote.unavailable' 24 | 25 | 26 | class Tweet(object): 27 | def __init__(self, tweet=''): 28 | namerow = tweet + '.tweet-header ' 29 | self.fullname = namerow + '.fullname' 30 | self.username = namerow + '.username' 31 | self.date = namerow + '.tweet-date' 32 | self.text = tweet + '.tweet-content.media-body' 33 | self.retweet = tweet + '.retweet-header' 34 | self.reply = tweet + '.replying-to' 35 | 36 | 37 | class Profile(object): 38 | fullname = '.profile-card-fullname' 39 | username = '.profile-card-username' 40 | protected = '.icon-lock' 41 | verified = '.verified-icon' 42 | banner = '.profile-banner' 43 | bio = '.profile-bio' 44 | location = '.profile-location' 45 | website = '.profile-website' 46 | joinDate = '.profile-joindate' 47 | mediaCount = '.photo-rail-header' 48 | 49 | 50 | class Timeline(object): 51 | newest = 'div[class="timeline-item show-more"]' 52 | older = 'div[class="show-more"]' 53 | end = '.timeline-end' 54 | none = '.timeline-none' 55 | protected = '.timeline-protected' 56 | photo_rail = '.photo-rail-grid' 57 | 58 | 59 | class Conversation(object): 60 | main = '.main-tweet' 61 | before = '.before-tweet' 62 | after = '.after-tweet' 63 | replies = '.replies' 64 | thread = '.reply' 65 | tweet = '.timeline-item' 66 | tweet_text = '.tweet-content' 67 | 68 | 69 | class Poll(object): 70 | votes = '.poll-info' 71 | choice = '.poll-meter' 72 | value = 'poll-choice-value' 73 | option = 'poll-choice-option' 74 | leader = 'leader' 75 | 76 | 77 | class Media(object): 78 | container = '.attachments' 79 | row = '.gallery-row' 80 | image = '.still-image' 81 | video = '.gallery-video' 82 | gif = '.gallery-gif' 83 | 84 | 85 | class BaseTestCase(BaseCase): 86 | def setUp(self): 87 | super(BaseTestCase, self).setUp() 88 | 89 | def tearDown(self): 90 | super(BaseTestCase, self).tearDown() 91 | 92 | def open_nitter(self, page=''): 93 | self.open(f'http://localhost:8080/{page}') 94 | 95 | def search_username(self, username): 96 | self.open_nitter() 97 | self.update_text('.search-bar input[type=text]', username) 98 | self.submit('.search-bar form') 99 | 100 | 101 | def get_timeline_tweet(num=1): 102 | return Tweet(f'.timeline > div:nth-child({num}) ') 103 | -------------------------------------------------------------------------------- /src/query.nim: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: AGPL-3.0-only 2 | import strutils, strformat, sequtils, tables, uri 3 | 4 | import types 5 | 6 | const 7 | validFilters* = @[ 8 | "media", "images", "twimg", "videos", 9 | "native_video", "consumer_video", "pro_video", 10 | "links", "news", "quote", "mentions", 11 | "replies", "retweets", "nativeretweets", 12 | "verified", "safe" 13 | ] 14 | 15 | emptyQuery* = "include:nativeretweets" 16 | 17 | template `@`(param: string): untyped = 18 | if param in pms: pms[param] 19 | else: "" 20 | 21 | proc initQuery*(pms: Table[string, string]; name=""): Query = 22 | result = Query( 23 | kind: parseEnum[QueryKind](@"f", tweets), 24 | text: @"q", 25 | filters: validFilters.filterIt("f-" & it in pms), 26 | excludes: validFilters.filterIt("e-" & it in pms), 27 | since: @"since", 28 | until: @"until", 29 | near: @"near" 30 | ) 31 | 32 | if name.len > 0: 33 | result.fromUser = name.split(",") 34 | 35 | proc getMediaQuery*(name: string): Query = 36 | Query( 37 | kind: media, 38 | filters: @["twimg", "native_video"], 39 | fromUser: @[name], 40 | sep: "OR" 41 | ) 42 | 43 | 44 | proc getFavoritesQuery*(name: string): Query = 45 | Query( 46 | kind: favorites, 47 | fromUser: @[name] 48 | ) 49 | 50 | proc getReplyQuery*(name: string): Query = 51 | Query( 52 | kind: replies, 53 | fromUser: @[name] 54 | ) 55 | 56 | proc genQueryParam*(query: Query): string = 57 | var 58 | filters: seq[string] 59 | param: string 60 | 61 | if query.kind == users: 62 | return query.text 63 | 64 | for i, user in query.fromUser: 65 | param &= &"from:{user} " 66 | if i < query.fromUser.high: 67 | param &= "OR " 68 | 69 | if query.fromUser.len > 0 and query.kind in {posts, media}: 70 | param &= "filter:self_threads OR-filter:replies " 71 | 72 | if "nativeretweets" notin query.excludes: 73 | param &= "include:nativeretweets " 74 | 75 | for f in query.filters: 76 | filters.add "filter:" & f 77 | for e in query.excludes: 78 | if e == "nativeretweets": continue 79 | filters.add "-filter:" & e 80 | for i in query.includes: 81 | filters.add "include:" & i 82 | 83 | result = strip(param & filters.join(&" {query.sep} ")) 84 | if query.since.len > 0: 85 | result &= " since:" & query.since 86 | if query.until.len > 0: 87 | result &= " until:" & query.until 88 | if query.near.len > 0: 89 | result &= &" near:\"{query.near}\" within:15mi" 90 | if query.text.len > 0: 91 | if result.len > 0: 92 | result &= " " & query.text 93 | else: 94 | result = query.text 95 | 96 | proc genQueryUrl*(query: Query): string = 97 | if query.kind notin {tweets, users}: return 98 | 99 | var params = @[&"f={query.kind}"] 100 | if query.text.len > 0: 101 | params.add "q=" & encodeUrl(query.text) 102 | for f in query.filters: 103 | params.add &"f-{f}=on" 104 | for e in query.excludes: 105 | params.add &"e-{e}=on" 106 | for i in query.includes.filterIt(it != "nativeretweets"): 107 | params.add &"i-{i}=on" 108 | 109 | if query.since.len > 0: 110 | params.add "since=" & query.since 111 | if query.until.len > 0: 112 | params.add "until=" & query.until 113 | if query.near.len > 0: 114 | params.add "near=" & query.near 115 | 116 | if params.len > 0: 117 | result &= params.join("&") 118 | -------------------------------------------------------------------------------- /src/experimental/parser/unifiedcard.nim: -------------------------------------------------------------------------------- 1 | import std/[options, tables, strutils, strformat, sugar] 2 | import jsony 3 | import ../types/unifiedcard 4 | from ../../types import Card, CardKind, Video 5 | from ../../utils import twimg, https 6 | 7 | proc getImageUrl(entity: MediaEntity): string = 8 | entity.mediaUrlHttps.dup(removePrefix(twimg), removePrefix(https)) 9 | 10 | proc parseDestination(id: string; card: UnifiedCard; result: var Card) = 11 | let destination = card.destinationObjects[id].data 12 | result.dest = destination.urlData.vanity 13 | result.url = destination.urlData.url 14 | 15 | proc parseDetails(data: ComponentData; card: UnifiedCard; result: var Card) = 16 | data.destination.parseDestination(card, result) 17 | 18 | result.text = data.title 19 | if result.text.len == 0: 20 | result.text = data.name 21 | 22 | proc parseMediaDetails(data: ComponentData; card: UnifiedCard; result: var Card) = 23 | data.destination.parseDestination(card, result) 24 | 25 | result.kind = summary 26 | result.image = card.mediaEntities[data.mediaId].getImageUrl 27 | result.text = data.topicDetail.title 28 | result.dest = "Topic" 29 | 30 | proc parseAppDetails(data: ComponentData; card: UnifiedCard; result: var Card) = 31 | let app = card.appStoreData[data.appId][0] 32 | 33 | case app.kind 34 | of androidApp: 35 | result.url = "http://play.google.com/store/apps/details?id=" & app.id 36 | of iPhoneApp, iPadApp: 37 | result.url = "https://itunes.apple.com/app/id" & app.id 38 | 39 | result.text = app.title 40 | result.dest = app.category 41 | 42 | proc parseListDetails(data: ComponentData; result: var Card) = 43 | result.dest = &"List · {data.memberCount} Members" 44 | 45 | proc parseCommunityDetails(data: ComponentData; result: var Card) = 46 | result.dest = &"Community · {data.memberCount} Members" 47 | 48 | proc parseMedia(component: Component; card: UnifiedCard; result: var Card) = 49 | let mediaId = 50 | if component.kind == swipeableMedia: 51 | component.data.mediaList[0].id 52 | else: 53 | component.data.id 54 | 55 | let rMedia = card.mediaEntities[mediaId] 56 | case rMedia.kind: 57 | of photo: 58 | result.kind = summaryLarge 59 | result.image = rMedia.getImageUrl 60 | of video: 61 | let videoInfo = rMedia.videoInfo.get 62 | result.kind = promoVideo 63 | result.video = some Video( 64 | available: true, 65 | thumb: rMedia.getImageUrl, 66 | durationMs: videoInfo.durationMillis, 67 | variants: videoInfo.variants 68 | ) 69 | of model3d: 70 | result.title = "Unsupported 3D model ad" 71 | 72 | proc parseUnifiedCard*(json: string): Card = 73 | let card = json.fromJson(UnifiedCard) 74 | 75 | for component in card.componentObjects.values: 76 | case component.kind 77 | of details, communityDetails, twitterListDetails: 78 | component.data.parseDetails(card, result) 79 | of appStoreDetails: 80 | component.data.parseAppDetails(card, result) 81 | of mediaWithDetailsHorizontal: 82 | component.data.parseMediaDetails(card, result) 83 | of media, swipeableMedia: 84 | component.parseMedia(card, result) 85 | of buttonGroup: 86 | discard 87 | of ComponentType.unknown: 88 | echo "ERROR: Unknown component type: ", json 89 | 90 | case component.kind 91 | of twitterListDetails: 92 | component.data.parseListDetails(result) 93 | of communityDetails: 94 | component.data.parseCommunityDetails(result) 95 | else: discard 96 | -------------------------------------------------------------------------------- /src/consts.nim: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: AGPL-3.0-only 2 | import uri, sequtils 3 | 4 | const 5 | auth* = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw" 6 | 7 | api = parseUri("https://api.twitter.com") 8 | activate* = $(api / "1.1/guest/activate.json") 9 | 10 | userShow* = api / "1.1/users/show.json" 11 | photoRail* = api / "1.1/statuses/media_timeline.json" 12 | status* = api / "1.1/statuses/show" 13 | search* = api / "2/search/adaptive.json" 14 | 15 | timelineApi = api / "2/timeline" 16 | timeline* = timelineApi / "profile" 17 | mediaTimeline* = timelineApi / "media" 18 | favorites* = timelineApi / "favorites" 19 | listTimeline* = timelineApi / "list.json" 20 | tweet* = timelineApi / "conversation" 21 | 22 | graphql = api / "graphql" 23 | graphTweet* = graphql / "6lWNh96EXDJCXl05SAtn_g/TweetDetail" 24 | graphUser* = graphql / "7mjxD3-C6BxitPMVQ6w0-Q/UserByScreenName" 25 | graphUserById* = graphql / "I5nvpI91ljifos1Y3Lltyg/UserByRestId" 26 | graphList* = graphql / "JADTh6cjebfgetzvF3tQvQ/List" 27 | graphListBySlug* = graphql / "ErWsz9cObLel1BF-HjuBlA/ListBySlug" 28 | graphListMembers* = graphql / "Ke6urWMeCV2UlKXGRy4sow/ListMembers" 29 | 30 | timelineParams* = { 31 | "include_profile_interstitial_type": "0", 32 | "include_blocking": "0", 33 | "include_blocked_by": "0", 34 | "include_followed_by": "0", 35 | "include_want_retweets": "0", 36 | "include_mute_edge": "0", 37 | "include_can_dm": "0", 38 | "include_can_media_tag": "1", 39 | "skip_status": "1", 40 | "cards_platform": "Web-12", 41 | "include_cards": "1", 42 | "include_composer_source": "false", 43 | "include_reply_count": "1", 44 | "tweet_mode": "extended", 45 | "include_entities": "true", 46 | "include_user_entities": "true", 47 | "include_ext_media_color": "false", 48 | "send_error_codes": "true", 49 | "simple_quoted_tweet": "true", 50 | "include_quote_count": "true" 51 | }.toSeq 52 | 53 | searchParams* = { 54 | "query_source": "typed_query", 55 | "pc": "1", 56 | "spelling_corrections": "1" 57 | }.toSeq 58 | ## top: nothing 59 | ## latest: "tweet_search_mode: live" 60 | ## user: "result_filter: user" 61 | ## photos: "result_filter: photos" 62 | ## videos: "result_filter: videos" 63 | 64 | tweetVariables* = """{ 65 | "focalTweetId": "$1", 66 | $2 67 | "includePromotedContent": false, 68 | "withBirdwatchNotes": false, 69 | "withDownvotePerspective": false, 70 | "withReactionsMetadata": false, 71 | "withReactionsPerspective": false, 72 | "withSuperFollowsTweetFields": false, 73 | "withSuperFollowsUserFields": false, 74 | "withVoice": false, 75 | "withV2Timeline": true 76 | }""" 77 | 78 | tweetFeatures* = """{ 79 | "graphql_is_translatable_rweb_tweet_is_translatable_enabled": false, 80 | "responsive_web_graphql_timeline_navigation_enabled": false, 81 | "standardized_nudges_misinfo": false, 82 | "verified_phone_label_enabled": false, 83 | "responsive_web_twitter_blue_verified_badge_is_enabled": false, 84 | "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false, 85 | "view_counts_everywhere_api_enabled": false, 86 | "responsive_web_edit_tweet_api_enabled": false, 87 | "tweetypie_unmention_optimization_enabled": false, 88 | "vibe_api_enabled": false, 89 | "longform_notetweets_consumption_enabled": true, 90 | "responsive_web_text_conversations_enabled": false, 91 | "responsive_web_enhance_cards_enabled": false, 92 | "interactive_text_enabled": false 93 | }""" 94 | -------------------------------------------------------------------------------- /src/sass/timeline.scss: -------------------------------------------------------------------------------- 1 | @import '_variables'; 2 | 3 | .timeline-container { 4 | @include panel(100%, 600px); 5 | } 6 | 7 | .timeline { 8 | background-color: var(--bg_panel); 9 | 10 | > div:not(:first-child) { 11 | border-top: 1px solid var(--border_grey); 12 | } 13 | } 14 | 15 | .timeline-header { 16 | width: 100%; 17 | background-color: var(--bg_panel); 18 | text-align: center; 19 | padding: 8px; 20 | display: block; 21 | font-weight: bold; 22 | margin-bottom: 5px; 23 | box-sizing: border-box; 24 | 25 | button { 26 | float: unset; 27 | } 28 | } 29 | 30 | .timeline-banner img { 31 | width: 100%; 32 | } 33 | 34 | .timeline-description { 35 | font-weight: normal; 36 | } 37 | 38 | .tab { 39 | align-items: center; 40 | display: flex; 41 | flex-wrap: wrap; 42 | list-style: none; 43 | margin: 0 0 5px 0; 44 | background-color: var(--bg_panel); 45 | padding: 0; 46 | } 47 | 48 | .tab-item { 49 | flex: 1 1 0; 50 | text-align: center; 51 | margin-top: 0; 52 | 53 | a { 54 | border-bottom: .1rem solid transparent; 55 | color: var(--tab); 56 | display: block; 57 | padding: 8px 0; 58 | text-decoration: none; 59 | font-weight: bold; 60 | 61 | &:hover { 62 | text-decoration: none; 63 | } 64 | 65 | &.active { 66 | border-bottom-color: var(--tab_selected); 67 | color: var(--tab_selected); 68 | } 69 | } 70 | 71 | &.active a { 72 | border-bottom-color: var(--tab_selected); 73 | color: var(--tab_selected); 74 | } 75 | 76 | &.wide { 77 | flex-grow: 1.2; 78 | flex-basis: 50px; 79 | } 80 | } 81 | 82 | .timeline-footer { 83 | background-color: var(--bg_panel); 84 | padding: 6px 0; 85 | } 86 | 87 | .timeline-protected { 88 | text-align: center; 89 | 90 | p { 91 | margin: 8px 0; 92 | } 93 | 94 | h2 { 95 | color: var(--accent); 96 | font-size: 20px; 97 | font-weight: 600; 98 | } 99 | } 100 | 101 | .timeline-none { 102 | color: var(--accent); 103 | font-size: 20px; 104 | font-weight: 600; 105 | text-align: center; 106 | } 107 | 108 | .timeline-end { 109 | background-color: var(--bg_panel); 110 | color: var(--accent); 111 | font-size: 16px; 112 | font-weight: 600; 113 | text-align: center; 114 | } 115 | 116 | .show-more { 117 | background-color: var(--bg_panel); 118 | text-align: center; 119 | padding: .75em 0; 120 | display: block !important; 121 | 122 | a { 123 | background-color: var(--darkest_grey); 124 | display: inline-block; 125 | height: 2em; 126 | padding: 0 2em; 127 | line-height: 2em; 128 | 129 | &:hover { 130 | background-color: var(--darker_grey); 131 | } 132 | } 133 | } 134 | 135 | .top-ref { 136 | background-color: var(--bg_color); 137 | border-top: none !important; 138 | 139 | .icon-down { 140 | font-size: 20px; 141 | display: flex; 142 | justify-content: center; 143 | text-decoration: none; 144 | 145 | &:hover { 146 | color: var(--accent_light); 147 | } 148 | 149 | &::before { 150 | transform: rotate(180deg) translateY(-1px); 151 | } 152 | } 153 | } 154 | 155 | .timeline-item { 156 | overflow-wrap: break-word; 157 | border-left-width: 0; 158 | min-width: 0; 159 | padding: .75em; 160 | display: flex; 161 | position: relative; 162 | } 163 | -------------------------------------------------------------------------------- /src/views/renderutils.nim: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: AGPL-3.0-only 2 | import strutils, strformat 3 | import karax/[karaxdsl, vdom, vstyles] 4 | import ".."/[types, utils] 5 | 6 | const smallWebp* = "?name=small&format=webp" 7 | 8 | proc getSmallPic*(url: string): string = 9 | result = url 10 | if "?" notin url and not url.endsWith("placeholder.png"): 11 | result &= smallWebp 12 | result = getPicUrl(result) 13 | 14 | proc icon*(icon: string; text=""; title=""; class=""; href=""): VNode = 15 | var c = "icon-" & icon 16 | if class.len > 0: c = &"{c} {class}" 17 | buildHtml(tdiv(class="icon-container")): 18 | if href.len > 0: 19 | a(class=c, title=title, href=href) 20 | else: 21 | span(class=c, title=title) 22 | 23 | if text.len > 0: 24 | text " " & text 25 | 26 | proc linkUser*(user: User, class=""): VNode = 27 | let 28 | isName = "username" notin class 29 | href = "/" & user.username 30 | nameText = if isName: user.fullname 31 | else: "@" & user.username 32 | 33 | buildHtml(a(href=href, class=class, title=nameText)): 34 | text nameText 35 | if isName and user.verified: 36 | icon "ok", class="verified-icon", title="Verified account" 37 | if isName and user.protected: 38 | text " " 39 | icon "lock", title="Protected account" 40 | 41 | proc linkText*(text: string; class=""): VNode = 42 | let url = if "http" notin text: https & text else: text 43 | buildHtml(): 44 | a(href=url, class=class): text text 45 | 46 | proc hiddenField*(name, value: string): VNode = 47 | buildHtml(): 48 | input(name=name, style={display: "none"}, value=value) 49 | 50 | proc refererField*(path: string): VNode = 51 | hiddenField("referer", path) 52 | 53 | proc buttonReferer*(action, text, path: string; class=""; `method`="post"): VNode = 54 | buildHtml(form(`method`=`method`, action=action, class=class)): 55 | refererField path 56 | button(`type`="submit"): 57 | text text 58 | 59 | proc genCheckbox*(pref, label: string; state: bool): VNode = 60 | buildHtml(label(class="pref-group checkbox-container")): 61 | text label 62 | if state: input(name=pref, `type`="checkbox", checked="") 63 | else: input(name=pref, `type`="checkbox") 64 | span(class="checkbox") 65 | 66 | proc genInput*(pref, label, state, placeholder: string; class=""; autofocus=true): VNode = 67 | let p = placeholder 68 | buildHtml(tdiv(class=("pref-group pref-input " & class))): 69 | if label.len > 0: 70 | label(`for`=pref): text label 71 | if autofocus and state.len == 0: 72 | input(name=pref, `type`="text", placeholder=p, value=state, autofocus="") 73 | else: 74 | input(name=pref, `type`="text", placeholder=p, value=state) 75 | 76 | proc genSelect*(pref, label, state: string; options: seq[string]): VNode = 77 | buildHtml(tdiv(class="pref-group pref-input")): 78 | label(`for`=pref): text label 79 | select(name=pref): 80 | for opt in options: 81 | if opt == state: 82 | option(value=opt, selected=""): text opt 83 | else: 84 | option(value=opt): text opt 85 | 86 | proc genDate*(pref, state: string): VNode = 87 | buildHtml(span(class="date-input")): 88 | input(name=pref, `type`="date", value=state) 89 | icon "calendar" 90 | 91 | proc genImg*(url: string; class=""): VNode = 92 | buildHtml(): 93 | img(src=getPicUrl(url), class=class, alt="") 94 | 95 | proc getTabClass*(query: Query; tab: QueryKind): string = 96 | result = "tab-item" 97 | if query.kind == tab: 98 | result &= " active" 99 | 100 | proc getAvatarClass*(prefs: Prefs): string = 101 | if prefs.squareAvatars: 102 | "avatar" 103 | else: 104 | "avatar round" 105 | -------------------------------------------------------------------------------- /src/sass/index.scss: -------------------------------------------------------------------------------- 1 | @import '_variables'; 2 | 3 | @import 'tweet/_base'; 4 | @import 'profile/_base'; 5 | @import 'general'; 6 | @import 'navbar'; 7 | @import 'inputs'; 8 | @import 'timeline'; 9 | @import 'search'; 10 | 11 | body { 12 | // colors 13 | --bg_color: #{$bg_color}; 14 | --fg_color: #{$fg_color}; 15 | --fg_faded: #{$fg_faded}; 16 | --fg_dark: #{$fg_dark}; 17 | --fg_nav: #{$fg_nav}; 18 | 19 | --bg_panel: #{$bg_panel}; 20 | --bg_elements: #{$bg_elements}; 21 | --bg_overlays: #{$bg_overlays}; 22 | --bg_hover: #{$bg_hover}; 23 | 24 | --grey: #{$grey}; 25 | --dark_grey: #{$dark_grey}; 26 | --darker_grey: #{$darker_grey}; 27 | --darkest_grey: #{$darkest_grey}; 28 | --border_grey: #{$border_grey}; 29 | 30 | --accent: #{$accent}; 31 | --accent_light: #{$accent_light}; 32 | --accent_dark: #{$accent_dark}; 33 | --accent_border: #{$accent_border}; 34 | 35 | --play_button: #{$play_button}; 36 | --play_button_hover: #{$play_button_hover}; 37 | 38 | --more_replies_dots: #{$more_replies_dots}; 39 | --error_red: #{$error_red}; 40 | 41 | --verified_blue: #{$verified_blue}; 42 | --icon_text: #{$icon_text}; 43 | 44 | --tab: #{$fg_color}; 45 | --tab_selected: #{$accent}; 46 | 47 | --profile_stat: #{$fg_color}; 48 | 49 | background-color: var(--bg_color); 50 | color: var(--fg_color); 51 | font-family: $font_0, $font_1, $font_2, $font_3; 52 | font-size: 14px; 53 | line-height: 1.3; 54 | margin: 0; 55 | } 56 | 57 | * { 58 | outline: unset; 59 | margin: 0; 60 | text-decoration: none; 61 | } 62 | 63 | h1 { 64 | display: inline; 65 | } 66 | 67 | h2, h3 { 68 | font-weight: normal; 69 | } 70 | 71 | p { 72 | margin: 14px 0; 73 | } 74 | 75 | a { 76 | color: var(--accent); 77 | 78 | &:hover { 79 | text-decoration: underline; 80 | } 81 | } 82 | 83 | fieldset { 84 | border: 0; 85 | padding: 0; 86 | margin-top: -0.6em; 87 | } 88 | 89 | legend { 90 | width: 100%; 91 | padding: .6em 0 .3em 0; 92 | border: 0; 93 | font-size: 16px; 94 | font-weight: 600; 95 | border-bottom: 1px solid var(--border_grey); 96 | margin-bottom: 8px; 97 | } 98 | 99 | .preferences .note { 100 | border-top: 1px solid var(--border_grey); 101 | border-bottom: 1px solid var(--border_grey); 102 | padding: 6px 0 8px 0; 103 | margin-bottom: 8px; 104 | margin-top: 16px; 105 | } 106 | 107 | ul { 108 | padding-left: 1.3em; 109 | } 110 | 111 | .container { 112 | display: flex; 113 | flex-wrap: wrap; 114 | box-sizing: border-box; 115 | padding-top: 50px; 116 | margin: auto; 117 | min-height: 100vh; 118 | } 119 | 120 | .icon-container { 121 | display: inline; 122 | } 123 | 124 | .overlay-panel { 125 | max-width: 600px; 126 | width: 100%; 127 | margin: 0 auto; 128 | margin-top: 10px; 129 | background-color: var(--bg_overlays); 130 | padding: 10px 15px; 131 | align-self: start; 132 | 133 | ul { 134 | margin-bottom: 14px; 135 | } 136 | 137 | p { 138 | word-break: break-word; 139 | } 140 | } 141 | 142 | .verified-icon { 143 | color: var(--icon_text); 144 | background-color: var(--verified_blue); 145 | border-radius: 50%; 146 | flex-shrink: 0; 147 | margin: 2px 0 3px 3px; 148 | padding-top: 2px; 149 | height: 12px; 150 | width: 14px; 151 | font-size: 8px; 152 | display: inline-block; 153 | text-align: center; 154 | vertical-align: middle; 155 | } 156 | 157 | @media(max-width: 600px) { 158 | .preferences-container { 159 | max-width: 95vw; 160 | } 161 | 162 | .nav-item, .nav-item .icon-container { 163 | font-size: 16px; 164 | } 165 | } 166 | -------------------------------------------------------------------------------- /src/sass/inputs.scss: -------------------------------------------------------------------------------- 1 | @import '_variables'; 2 | @import '_mixins'; 3 | 4 | button { 5 | @include input-colors; 6 | background-color: var(--bg_elements); 7 | color: var(--fg_color); 8 | border: 1px solid var(--accent_border); 9 | padding: 3px 6px; 10 | font-size: 14px; 11 | cursor: pointer; 12 | float: right; 13 | } 14 | 15 | input[type="text"], 16 | input[type="date"], 17 | select { 18 | @include input-colors; 19 | background-color: var(--bg_elements); 20 | padding: 1px 4px; 21 | color: var(--fg_color); 22 | border: 1px solid var(--accent_border); 23 | border-radius: 0; 24 | font-size: 14px; 25 | } 26 | 27 | input[type="text"] { 28 | height: 16px; 29 | } 30 | 31 | select { 32 | height: 20px; 33 | padding: 0 2px; 34 | line-height: 1; 35 | } 36 | 37 | input[type="date"]::-webkit-inner-spin-button { 38 | display: none; 39 | } 40 | 41 | input[type="date"]::-webkit-clear-button { 42 | margin-left: 17px; 43 | filter: grayscale(100%); 44 | filter: hue-rotate(120deg); 45 | } 46 | 47 | input::-webkit-calendar-picker-indicator { 48 | opacity: 0; 49 | } 50 | 51 | input::-webkit-datetime-edit-day-field:focus, 52 | input::-webkit-datetime-edit-month-field:focus, 53 | input::-webkit-datetime-edit-year-field:focus { 54 | background-color: var(--accent); 55 | color: var(--fg_color); 56 | outline: none; 57 | } 58 | 59 | .date-range { 60 | .date-input { 61 | display: inline-block; 62 | position: relative; 63 | } 64 | 65 | .icon-container { 66 | pointer-events: none; 67 | position: absolute; 68 | top: 2px; 69 | right: 5px; 70 | } 71 | 72 | .search-title { 73 | margin: 0 2px; 74 | } 75 | } 76 | 77 | .icon-button button { 78 | color: var(--accent); 79 | text-decoration: none; 80 | background: none; 81 | border: none; 82 | float: none; 83 | padding: unset; 84 | padding-left: 4px; 85 | 86 | &:hover { 87 | color: var(--accent_light); 88 | } 89 | } 90 | 91 | .checkbox { 92 | position: absolute; 93 | top: 1px; 94 | right: 0; 95 | height: 17px; 96 | width: 17px; 97 | background-color: var(--bg_elements); 98 | border: 1px solid var(--accent_border); 99 | 100 | &:after { 101 | content: ""; 102 | position: absolute; 103 | display: none; 104 | } 105 | } 106 | 107 | .checkbox-container { 108 | display: block; 109 | position: relative; 110 | margin-bottom: 5px; 111 | cursor: pointer; 112 | user-select: none; 113 | padding-right: 22px; 114 | 115 | input { 116 | position: absolute; 117 | opacity: 0; 118 | cursor: pointer; 119 | height: 0; 120 | width: 0; 121 | 122 | &:checked ~ .checkbox:after { 123 | display: block; 124 | } 125 | } 126 | 127 | &:hover input ~ .checkbox { 128 | border-color: var(--accent); 129 | } 130 | 131 | &:active input ~ .checkbox { 132 | border-color: var(--accent_light); 133 | } 134 | 135 | .checkbox:after { 136 | left: 2px; 137 | bottom: 0; 138 | font-size: 13px; 139 | font-family: $font_4; 140 | content: '\e803'; 141 | } 142 | } 143 | 144 | .pref-group { 145 | display: inline; 146 | } 147 | 148 | .preferences { 149 | button { 150 | margin: 6px 0 3px 0; 151 | } 152 | 153 | label { 154 | padding-right: 150px; 155 | } 156 | 157 | select { 158 | position: absolute; 159 | top: 0; 160 | right: 0; 161 | display: block; 162 | -moz-appearance: none; 163 | -webkit-appearance: none; 164 | appearance: none; 165 | } 166 | 167 | input[type="text"] { 168 | position: absolute; 169 | right: 0; 170 | max-width: 140px; 171 | } 172 | 173 | .pref-group { 174 | display: block; 175 | } 176 | 177 | .pref-input { 178 | position: relative; 179 | margin-bottom: 6px; 180 | } 181 | 182 | .pref-reset { 183 | float: left; 184 | } 185 | } 186 | -------------------------------------------------------------------------------- /src/routes/media.nim: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: AGPL-3.0-only 2 | import uri, strutils, httpclient, os, hashes, base64, re 3 | import asynchttpserver, asyncstreams, asyncfile, asyncnet 4 | 5 | import jester 6 | 7 | import router_utils 8 | import ".."/[types, formatters, utils] 9 | 10 | export asynchttpserver, asyncstreams, asyncfile, asyncnet 11 | export httpclient, os, strutils, asyncstreams, base64, re 12 | 13 | const 14 | m3u8Mime* = "application/vnd.apple.mpegurl" 15 | maxAge* = "max-age=604800" 16 | 17 | proc safeFetch*(url: string): Future[string] {.async.} = 18 | let client = newAsyncHttpClient() 19 | try: result = await client.getContent(url) 20 | except: discard 21 | finally: client.close() 22 | 23 | template respond*(req: asynchttpserver.Request; headers) = 24 | var msg = "HTTP/1.1 200 OK\c\L" 25 | for k, v in headers: 26 | msg.add(k & ": " & v & "\c\L") 27 | 28 | msg.add "\c\L" 29 | yield req.client.send(msg) 30 | 31 | proc proxyMedia*(req: jester.Request; url: string): Future[HttpCode] {.async.} = 32 | result = Http200 33 | let 34 | request = req.getNativeReq() 35 | client = newAsyncHttpClient() 36 | 37 | try: 38 | let res = await client.get(url) 39 | if res.status != "200 OK": 40 | return Http404 41 | 42 | let hashed = $hash(url) 43 | if request.headers.getOrDefault("If-None-Match") == hashed: 44 | return Http304 45 | 46 | let contentLength = 47 | if res.headers.hasKey("content-length"): 48 | res.headers["content-length", 0] 49 | else: 50 | "" 51 | 52 | let headers = newHttpHeaders({ 53 | "Content-Type": res.headers["content-type", 0], 54 | "Content-Length": contentLength, 55 | "Cache-Control": maxAge, 56 | "ETag": hashed 57 | }) 58 | 59 | respond(request, headers) 60 | 61 | var (hasValue, data) = (true, "") 62 | while hasValue: 63 | (hasValue, data) = await res.bodyStream.read() 64 | if hasValue: 65 | await request.client.send(data) 66 | data.setLen 0 67 | except HttpRequestError, ProtocolError, OSError: 68 | result = Http404 69 | finally: 70 | client.close() 71 | 72 | template check*(code): untyped = 73 | if code != Http200: 74 | resp code 75 | else: 76 | enableRawMode() 77 | break route 78 | 79 | proc decoded*(req: jester.Request; index: int): string = 80 | let 81 | based = req.matches[0].len > 1 82 | encoded = req.matches[index] 83 | if based: decode(encoded) 84 | else: decodeUrl(encoded) 85 | 86 | proc createMediaRouter*(cfg: Config) = 87 | router media: 88 | get "/pic/?": 89 | resp Http404 90 | 91 | get re"^\/pic\/orig\/(enc)?\/?(.+)": 92 | var url = decoded(request, 1) 93 | if "twimg.com" notin url: 94 | url.insert(twimg) 95 | if not url.startsWith(https): 96 | url.insert(https) 97 | url.add("?name=orig") 98 | 99 | let uri = parseUri(url) 100 | cond isTwitterUrl(uri) == true 101 | 102 | let code = await proxyMedia(request, url) 103 | check code 104 | 105 | get re"^\/pic\/(enc)?\/?(.+)": 106 | var url = decoded(request, 1) 107 | if "twimg.com" notin url: 108 | url.insert(twimg) 109 | if not url.startsWith(https): 110 | url.insert(https) 111 | 112 | let uri = parseUri(url) 113 | cond isTwitterUrl(uri) == true 114 | 115 | let code = await proxyMedia(request, url) 116 | check code 117 | 118 | get re"^\/video\/(enc)?\/?(.+)\/(.+)$": 119 | let url = decoded(request, 2) 120 | cond "http" in url 121 | 122 | if getHmac(url) != request.matches[1]: 123 | resp showError("Failed to verify signature", cfg) 124 | 125 | if ".mp4" in url or ".ts" in url or ".m4s" in url: 126 | let code = await proxyMedia(request, url) 127 | check code 128 | 129 | var content: string 130 | if ".vmap" in url: 131 | let m3u8 = getM3u8Url(await safeFetch(url)) 132 | if m3u8.len > 0: 133 | content = await safeFetch(url) 134 | else: 135 | resp Http404 136 | 137 | if ".m3u8" in url: 138 | let vid = await safeFetch(url) 139 | content = proxifyVideo(vid, cookiePref(proxyVideos)) 140 | 141 | resp content, m3u8Mime 142 | -------------------------------------------------------------------------------- /tests/test_tweet_media.py: -------------------------------------------------------------------------------- 1 | from base import BaseTestCase, Poll, Media 2 | from parameterized import parameterized 3 | from selenium.webdriver.common.by import By 4 | 5 | poll = [ 6 | ['nim_lang/status/1064219801499955200', 'Style insensitivity', '91', 1, [ 7 | ('47%', 'Yay'), ('53%', 'Nay') 8 | ]], 9 | 10 | ['polls/status/1031986180622049281', 'What Tree Is Coolest?', '3,322', 1, [ 11 | ('30%', 'Oak'), ('42%', 'Bonsai'), ('5%', 'Hemlock'), ('23%', 'Apple') 12 | ]] 13 | ] 14 | 15 | image = [ 16 | ['mobile_test/status/519364660823207936', 'BzUnaDFCUAAmrjs'], 17 | ['mobile_test_2/status/324619691039543297', 'BIFH45vCUAAQecj'] 18 | ] 19 | 20 | gif = [ 21 | ['elonmusk/status/1141367104702038016', 'D9bzUqoUcAAfUgf'], 22 | ['Proj_Borealis/status/1136595194621677568', 'D8X_PJAXUAAavPT'] 23 | ] 24 | 25 | video_m3u8 = [ 26 | ['d0m96/status/1078373829917974528', '9q1-v9w8-ft3awgD.jpg'], 27 | ['SpaceX/status/1138474014152712192', 'ocJJj2uu4n1kyD2Y.jpg'] 28 | ] 29 | 30 | gallery = [ 31 | ['mobile_test/status/451108446603980803', [ 32 | ['BkKovdrCUAAEz79', 'BkKovdcCEAAfoBO'] 33 | ]], 34 | 35 | ['mobile_test/status/471539824713691137', [ 36 | ['Bos--KNIQAAA7Li', 'Bos--FAIAAAWpah'], 37 | ['Bos--IqIQAAav23'] 38 | ]], 39 | 40 | ['mobile_test/status/469530783384743936', [ 41 | ['BoQbwJAIUAA0QCY', 'BoQbwN1IMAAuTiP'], 42 | ['BoQbwarIAAAlaE-', 'BoQbwh_IEAA27ef'] 43 | ]] 44 | ] 45 | 46 | 47 | class MediaTest(BaseTestCase): 48 | @parameterized.expand(poll) 49 | def test_poll(self, tweet, text, votes, leader, choices): 50 | self.open_nitter(tweet) 51 | self.assert_text(text, '.main-tweet') 52 | self.assert_text(votes, Poll.votes) 53 | 54 | poll_choices = self.find_elements(Poll.choice) 55 | for i, (v, o) in enumerate(choices): 56 | choice = poll_choices[i] 57 | value = choice.find_element(By.CLASS_NAME, Poll.value) 58 | option = choice.find_element(By.CLASS_NAME, Poll.option) 59 | choice_class = choice.get_attribute('class') 60 | 61 | self.assert_equal(v, value.text) 62 | self.assert_equal(o, option.text) 63 | 64 | if i == leader: 65 | self.assertIn(Poll.leader, choice_class) 66 | else: 67 | self.assertNotIn(Poll.leader, choice_class) 68 | 69 | @parameterized.expand(image) 70 | def test_image(self, tweet, url): 71 | self.open_nitter(tweet) 72 | self.assert_element_visible(Media.container) 73 | self.assert_element_visible(Media.image) 74 | 75 | image_url = self.get_image_url(Media.image + ' img') 76 | self.assertIn(url, image_url) 77 | 78 | @parameterized.expand(gif) 79 | def test_gif(self, tweet, gif_id): 80 | self.open_nitter(tweet) 81 | self.assert_element_visible(Media.container) 82 | self.assert_element_visible(Media.gif) 83 | 84 | url = self.get_attribute('source', 'src') 85 | thumb = self.get_attribute('video', 'poster') 86 | self.assertIn(gif_id + '.mp4', url) 87 | self.assertIn(gif_id + '.jpg', thumb) 88 | 89 | @parameterized.expand(video_m3u8) 90 | def test_video_m3u8(self, tweet, thumb): 91 | # no url because video playback isn't supported yet 92 | self.open_nitter(tweet) 93 | self.assert_element_visible(Media.container) 94 | self.assert_element_visible(Media.video) 95 | 96 | video_thumb = self.get_attribute(Media.video + ' img', 'src') 97 | self.assertIn(thumb, video_thumb) 98 | 99 | @parameterized.expand(gallery) 100 | def test_gallery(self, tweet, rows): 101 | self.open_nitter(tweet) 102 | self.assert_element_visible(Media.container) 103 | self.assert_element_visible(Media.row) 104 | self.assert_element_visible(Media.image) 105 | 106 | gallery_rows = self.find_elements(Media.row) 107 | self.assert_equal(len(rows), len(gallery_rows)) 108 | 109 | for i, row in enumerate(gallery_rows): 110 | images = row.find_elements(By.CSS_SELECTOR, 'img') 111 | self.assert_equal(len(rows[i]), len(images)) 112 | for j, image in enumerate(images): 113 | self.assertIn(rows[i][j], image.get_attribute('src')) 114 | -------------------------------------------------------------------------------- /src/views/profile.nim: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: AGPL-3.0-only 2 | import strutils, strformat 3 | import karax/[karaxdsl, vdom, vstyles] 4 | 5 | import renderutils, search 6 | import ".."/[types, utils, formatters] 7 | 8 | proc renderStat(num: int; class: string; text=""): VNode = 9 | let t = if text.len > 0: text else: class 10 | buildHtml(li(class=class)): 11 | span(class="profile-stat-header"): text capitalizeAscii(t) 12 | span(class="profile-stat-num"): 13 | text insertSep($num, ',') 14 | 15 | proc renderUserCard*(user: User; prefs: Prefs): VNode = 16 | buildHtml(tdiv(class="profile-card")): 17 | tdiv(class="profile-card-info"): 18 | let 19 | url = getPicUrl(user.getUserPic()) 20 | size = 21 | if prefs.autoplayGifs and user.userPic.endsWith("gif"): "" 22 | else: "_400x400" 23 | 24 | a(class="profile-card-avatar", href=url, target="_blank"): 25 | genImg(user.getUserPic(size)) 26 | 27 | tdiv(class="profile-card-tabs-name"): 28 | linkUser(user, class="profile-card-fullname") 29 | linkUser(user, class="profile-card-username") 30 | 31 | tdiv(class="profile-card-extra"): 32 | if user.bio.len > 0: 33 | tdiv(class="profile-bio"): 34 | p(dir="auto"): 35 | verbatim replaceUrls(user.bio, prefs) 36 | 37 | if user.location.len > 0: 38 | tdiv(class="profile-location"): 39 | span: icon "location" 40 | let (place, url) = getLocation(user) 41 | if url.len > 1: 42 | a(href=url): text place 43 | elif "://" in place: 44 | a(href=place): text place 45 | else: 46 | span: text place 47 | 48 | if user.website.len > 0: 49 | tdiv(class="profile-website"): 50 | span: 51 | let url = replaceUrls(user.website, prefs) 52 | icon "link" 53 | a(href=url): text url.shortLink 54 | 55 | tdiv(class="profile-joindate"): 56 | span(title=getJoinDateFull(user)): 57 | icon "calendar", getJoinDate(user) 58 | 59 | tdiv(class="profile-card-extra-links"): 60 | ul(class="profile-statlist"): 61 | renderStat(user.tweets, "posts", text="Tweets") 62 | renderStat(user.following, "following") 63 | renderStat(user.followers, "followers") 64 | renderStat(user.likes, "likes") 65 | 66 | proc renderPhotoRail(profile: Profile): VNode = 67 | let count = insertSep($profile.user.media, ',') 68 | buildHtml(tdiv(class="photo-rail-card")): 69 | tdiv(class="photo-rail-header"): 70 | a(href=(&"/{profile.user.username}/media")): 71 | icon "picture", count & " Photos and videos" 72 | 73 | input(id="photo-rail-grid-toggle", `type`="checkbox") 74 | label(`for`="photo-rail-grid-toggle", class="photo-rail-header-mobile"): 75 | icon "picture", count & " Photos and videos" 76 | icon "down" 77 | 78 | tdiv(class="photo-rail-grid"): 79 | for i, photo in profile.photoRail: 80 | if i == 16: break 81 | let photoSuffix = 82 | if "format" in photo.url or "placeholder" in photo.url: "" 83 | else: ":thumb" 84 | a(href=(&"/{profile.user.username}/status/{photo.tweetId}#m")): 85 | genImg(photo.url & photoSuffix) 86 | 87 | proc renderBanner(banner: string): VNode = 88 | buildHtml(): 89 | if banner.len == 0: 90 | a() 91 | elif banner.startsWith('#'): 92 | a(style={backgroundColor: banner}) 93 | else: 94 | a(href=getPicUrl(banner), target="_blank"): genImg(banner) 95 | 96 | proc renderProtected(username: string): VNode = 97 | buildHtml(tdiv(class="timeline-container")): 98 | tdiv(class="timeline-header timeline-protected"): 99 | h2: text "This account's tweets are protected." 100 | p: text &"Only confirmed followers have access to @{username}'s tweets." 101 | 102 | proc renderProfile*(profile: var Profile; cfg: Config; prefs: Prefs; path: string): VNode = 103 | profile.tweets.query.fromUser = @[profile.user.username] 104 | 105 | buildHtml(tdiv(class="profile-tabs")): 106 | if not prefs.hideBanner: 107 | tdiv(class="profile-banner"): 108 | renderBanner(profile.user.banner) 109 | 110 | let sticky = if prefs.stickyProfile: " sticky" else: "" 111 | tdiv(class=("profile-tab" & sticky)): 112 | renderUserCard(profile.user, prefs) 113 | if profile.photoRail.len > 0: 114 | renderPhotoRail(profile) 115 | 116 | if profile.user.protected: 117 | renderProtected(profile.user.username) 118 | else: 119 | renderTweetSearch(profile.tweets, cfg, prefs, path, profile.pinned) 120 | -------------------------------------------------------------------------------- /src/sass/tweet/_base.scss: -------------------------------------------------------------------------------- 1 | @import '_variables'; 2 | @import '_mixins'; 3 | @import 'thread'; 4 | @import 'media'; 5 | @import 'video'; 6 | @import 'embed'; 7 | @import 'card'; 8 | @import 'poll'; 9 | @import 'quote'; 10 | 11 | .tweet-body { 12 | flex: 1; 13 | min-width: 0; 14 | margin-left: 58px; 15 | pointer-events: none; 16 | z-index: 1; 17 | } 18 | 19 | .tweet-content { 20 | font-family: $font_3; 21 | line-height: 1.3em; 22 | pointer-events: all; 23 | display: inline; 24 | } 25 | 26 | .tweet-bidi { 27 | display: block !important; 28 | } 29 | 30 | .tweet-header { 31 | padding: 0; 32 | vertical-align: bottom; 33 | flex-basis: 100%; 34 | margin-bottom: .2em; 35 | 36 | a { 37 | display: inline-block; 38 | word-break: break-all; 39 | max-width: 100%; 40 | pointer-events: all; 41 | } 42 | } 43 | 44 | .tweet-name-row { 45 | padding: 0; 46 | display: flex; 47 | justify-content: space-between; 48 | } 49 | 50 | .fullname-and-username { 51 | display: flex; 52 | min-width: 0; 53 | } 54 | 55 | .fullname { 56 | @include ellipsis; 57 | flex-shrink: 2; 58 | max-width: 80%; 59 | font-size: 14px; 60 | font-weight: 700; 61 | color: var(--fg_color); 62 | } 63 | 64 | .username { 65 | @include ellipsis; 66 | min-width: 1.6em; 67 | margin-left: .4em; 68 | word-wrap: normal; 69 | } 70 | 71 | .tweet-date { 72 | display: flex; 73 | flex-shrink: 0; 74 | margin-left: 4px; 75 | } 76 | 77 | .tweet-date a, .username, .show-more a { 78 | color: var(--fg_dark); 79 | } 80 | 81 | .tweet-published { 82 | margin: 0; 83 | margin-top: 5px; 84 | color: var(--grey); 85 | pointer-events: all; 86 | } 87 | 88 | .tweet-avatar { 89 | display: contents !important; 90 | 91 | img { 92 | float: left; 93 | margin-top: 3px; 94 | margin-left: -58px; 95 | width: 48px; 96 | height: 48px; 97 | } 98 | } 99 | 100 | .avatar { 101 | &.round { 102 | border-radius: 50%; 103 | -webkit-user-select: none; 104 | } 105 | 106 | &.mini { 107 | position: unset; 108 | margin-right: 5px; 109 | margin-top: -1px; 110 | width: 20px; 111 | height: 20px; 112 | } 113 | } 114 | 115 | .tweet-embed { 116 | display: flex; 117 | flex-direction: column; 118 | justify-content: center; 119 | height: 100%; 120 | background-color: var(--bg_panel); 121 | 122 | .tweet-content { 123 | font-size: 18px; 124 | } 125 | 126 | .tweet-body { 127 | display: flex; 128 | flex-direction: column; 129 | max-height: calc(100vh - 0.75em * 2); 130 | } 131 | 132 | .card-image img { 133 | height: auto; 134 | } 135 | 136 | .avatar { 137 | position: absolute; 138 | } 139 | } 140 | 141 | .attribution { 142 | display: flex; 143 | pointer-events: all; 144 | margin: 5px 0; 145 | 146 | strong { 147 | color: var(--fg_color); 148 | } 149 | } 150 | 151 | .media-tag-block { 152 | padding-top: 5px; 153 | pointer-events: all; 154 | color: var(--fg_faded); 155 | 156 | .icon-container { 157 | padding-right: 2px; 158 | } 159 | 160 | .media-tag, .icon-container { 161 | color: var(--fg_faded); 162 | } 163 | } 164 | 165 | .timeline-container .media-tag-block { 166 | font-size: 13px; 167 | } 168 | 169 | .tweet-geo { 170 | color: var(--fg_faded); 171 | } 172 | 173 | .replying-to { 174 | color: var(--fg_faded); 175 | margin: -2px 0 4px; 176 | 177 | a { 178 | pointer-events: all; 179 | } 180 | } 181 | 182 | .retweet-header, .pinned, .tweet-stats { 183 | align-content: center; 184 | color: var(--grey); 185 | display: flex; 186 | flex-shrink: 0; 187 | flex-wrap: wrap; 188 | font-size: 14px; 189 | font-weight: 600; 190 | line-height: 22px; 191 | 192 | span { 193 | @include ellipsis; 194 | } 195 | } 196 | 197 | .retweet-header { 198 | margin-top: -5px !important; 199 | } 200 | 201 | .tweet-stats { 202 | margin-bottom: -3px; 203 | -webkit-user-select: none; 204 | } 205 | 206 | .tweet-stat { 207 | padding-top: 5px; 208 | min-width: 1em; 209 | margin-right: 0.8em; 210 | } 211 | 212 | .show-thread { 213 | display: block; 214 | pointer-events: all; 215 | padding-top: 2px; 216 | } 217 | 218 | .unavailable-box { 219 | width: 100%; 220 | height: 100%; 221 | padding: 12px; 222 | border: solid 1px var(--dark_grey); 223 | box-sizing: border-box; 224 | border-radius: 10px; 225 | background-color: var(--bg_color); 226 | z-index: 2; 227 | } 228 | 229 | .tweet-link { 230 | height: 100%; 231 | width: 100%; 232 | left: 0; 233 | top: 0; 234 | position: absolute; 235 | -webkit-user-select: none; 236 | 237 | &:hover { 238 | background-color: var(--bg_hover); 239 | } 240 | } 241 | -------------------------------------------------------------------------------- /src/views/timeline.nim: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: AGPL-3.0-only 2 | import strutils, strformat, sequtils, algorithm, uri, options 3 | import karax/[karaxdsl, vdom] 4 | 5 | import ".."/[types, query, formatters] 6 | import tweet, renderutils 7 | 8 | proc getQuery(query: Query): string = 9 | if query.kind != posts: 10 | result = genQueryUrl(query) 11 | if result.len > 0: 12 | result &= "&" 13 | 14 | proc renderToTop*(focus="#"): VNode = 15 | buildHtml(tdiv(class="top-ref")): 16 | icon "down", href=focus 17 | 18 | proc renderNewer*(query: Query; path: string; focus=""): VNode = 19 | let 20 | q = genQueryUrl(query) 21 | url = if q.len > 0: "?" & q else: "" 22 | p = if focus.len > 0: path.replace("#m", focus) else: path 23 | buildHtml(tdiv(class="timeline-item show-more")): 24 | a(href=(p & url)): 25 | text "Load newest" 26 | 27 | proc renderMore*(query: Query; cursor: string; focus=""): VNode = 28 | buildHtml(tdiv(class="show-more")): 29 | a(href=(&"?{getQuery(query)}cursor={encodeUrl(cursor, usePlus=false)}{focus}")): 30 | text "Load more" 31 | 32 | proc renderNoMore(): VNode = 33 | buildHtml(tdiv(class="timeline-footer")): 34 | h2(class="timeline-end"): 35 | text "No more items" 36 | 37 | proc renderNoneFound(): VNode = 38 | buildHtml(tdiv(class="timeline-header")): 39 | h2(class="timeline-none"): 40 | text "No items found" 41 | 42 | proc renderThread(thread: seq[Tweet]; prefs: Prefs; path: string): VNode = 43 | buildHtml(tdiv(class="thread-line")): 44 | let sortedThread = thread.sortedByIt(it.id) 45 | for i, tweet in sortedThread: 46 | let show = i == thread.high and sortedThread[0].id != tweet.threadId 47 | let header = if tweet.pinned or tweet.retweet.isSome: "with-header " else: "" 48 | renderTweet(tweet, prefs, path, class=(header & "thread"), 49 | index=i, last=(i == thread.high), showThread=show) 50 | 51 | proc threadFilter(tweets: openArray[Tweet]; threads: openArray[int64]; it: Tweet): seq[Tweet] = 52 | result = @[it] 53 | if it.retweet.isSome or it.replyId in threads: return 54 | for t in tweets: 55 | if t.id == result[0].replyId: 56 | result.insert t 57 | elif t.replyId == result[0].id: 58 | result.add t 59 | 60 | proc renderUser(user: User; prefs: Prefs): VNode = 61 | buildHtml(tdiv(class="timeline-item")): 62 | a(class="tweet-link", href=("/" & user.username)) 63 | tdiv(class="tweet-body profile-result"): 64 | tdiv(class="tweet-header"): 65 | a(class="tweet-avatar", href=("/" & user.username)): 66 | genImg(user.getUserPic("_bigger"), class=prefs.getAvatarClass) 67 | 68 | tdiv(class="tweet-name-row"): 69 | tdiv(class="fullname-and-username"): 70 | linkUser(user, class="fullname") 71 | linkUser(user, class="username") 72 | 73 | tdiv(class="tweet-content media-body", dir="auto"): 74 | verbatim replaceUrls(user.bio, prefs) 75 | 76 | proc renderTimelineUsers*(results: Result[User]; prefs: Prefs; path=""): VNode = 77 | buildHtml(tdiv(class="timeline")): 78 | if not results.beginning: 79 | renderNewer(results.query, path) 80 | 81 | if results.content.len > 0: 82 | for user in results.content: 83 | renderUser(user, prefs) 84 | if results.bottom.len > 0: 85 | renderMore(results.query, results.bottom) 86 | renderToTop() 87 | elif results.beginning: 88 | renderNoneFound() 89 | else: 90 | renderNoMore() 91 | 92 | proc renderTimelineTweets*(results: Result[Tweet]; prefs: Prefs; path: string; 93 | pinned=none(Tweet)): VNode = 94 | buildHtml(tdiv(class="timeline")): 95 | if not results.beginning: 96 | renderNewer(results.query, parseUri(path).path) 97 | 98 | if not prefs.hidePins and pinned.isSome: 99 | let tweet = get pinned 100 | renderTweet(tweet, prefs, path, showThread=tweet.hasThread) 101 | 102 | if results.content.len == 0: 103 | if not results.beginning: 104 | renderNoMore() 105 | else: 106 | renderNoneFound() 107 | else: 108 | var 109 | threads: seq[int64] 110 | retweets: seq[int64] 111 | 112 | for tweet in results.content: 113 | let rt = if tweet.retweet.isSome: get(tweet.retweet).id else: 0 114 | 115 | if tweet.id in threads or rt in retweets or tweet.id in retweets or 116 | tweet.pinned and prefs.hidePins: continue 117 | 118 | let thread = results.content.threadFilter(threads, tweet) 119 | if thread.len < 2: 120 | var hasThread = tweet.hasThread 121 | if rt != 0: 122 | retweets &= rt 123 | hasThread = get(tweet.retweet).hasThread 124 | renderTweet(tweet, prefs, path, showThread=hasThread) 125 | else: 126 | renderThread(thread, prefs, path) 127 | threads &= thread.mapIt(it.id) 128 | 129 | renderMore(results.query, results.bottom) 130 | renderToTop() 131 | -------------------------------------------------------------------------------- /src/apiutils.nim: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: AGPL-3.0-only 2 | import httpclient, asyncdispatch, options, strutils, uri 3 | import jsony, packedjson, zippy 4 | import types, tokens, consts, parserutils, http_pool 5 | import experimental/types/common 6 | import config 7 | 8 | const 9 | rlRemaining = "x-rate-limit-remaining" 10 | rlReset = "x-rate-limit-reset" 11 | 12 | var pool: HttpPool 13 | 14 | proc genParams*(pars: openArray[(string, string)] = @[]; cursor=""; 15 | count="20"; ext=true): seq[(string, string)] = 16 | result = timelineParams 17 | for p in pars: 18 | result &= p 19 | if ext: 20 | result &= ("ext", "mediaStats") 21 | result &= ("include_ext_alt_text", "true") 22 | result &= ("include_ext_media_availability", "true") 23 | if count.len > 0: 24 | result &= ("count", count) 25 | if cursor.len > 0: 26 | # The raw cursor often has plus signs, which sometimes get turned into spaces, 27 | # so we need to turn them back into a plus 28 | if " " in cursor: 29 | result &= ("cursor", cursor.replace(" ", "+")) 30 | else: 31 | result &= ("cursor", cursor) 32 | 33 | proc genHeaders*(token: Token = nil): HttpHeaders = 34 | result = newHttpHeaders({ 35 | "connection": "keep-alive", 36 | "authorization": auth, 37 | "content-type": "application/json", 38 | "x-guest-token": if token == nil: "" else: token.tok, 39 | "x-twitter-active-user": "yes", 40 | "authority": "api.twitter.com", 41 | "accept-encoding": "gzip", 42 | "accept-language": "en-US,en;q=0.9", 43 | "accept": "*/*", 44 | "DNT": "1" 45 | }) 46 | 47 | template updateToken() = 48 | if api != Api.search and resp.headers.hasKey(rlRemaining): 49 | let 50 | remaining = parseInt(resp.headers[rlRemaining]) 51 | reset = parseInt(resp.headers[rlReset]) 52 | token.setRateLimit(api, remaining, reset) 53 | 54 | template fetchImpl(result, additional_headers, fetchBody) {.dirty.} = 55 | once: 56 | pool = HttpPool() 57 | 58 | var token = await getToken(api) 59 | if token.tok.len == 0: 60 | raise rateLimitError() 61 | 62 | try: 63 | var resp: AsyncResponse 64 | var headers = genHeaders(token) 65 | for key, value in additional_headers.pairs(): 66 | headers.add(key, value) 67 | pool.use(headers): 68 | template getContent = 69 | resp = await c.get($url) 70 | result = await resp.body 71 | 72 | getContent() 73 | 74 | # Twitter randomly returns 401 errors with an empty body quite often. 75 | # Retrying the request usually works. 76 | if resp.status == "401 Unauthorized" and result.len == 0: 77 | getContent() 78 | 79 | if resp.status == $Http429: 80 | raise rateLimitError() 81 | 82 | if resp.status == $Http503: 83 | badClient = true 84 | raise newException(InternalError, result) 85 | 86 | if result.len > 0: 87 | if resp.headers.getOrDefault("content-encoding") == "gzip": 88 | result = uncompress(result, dfGzip) 89 | else: 90 | echo "non-gzip body, url: ", url, ", body: ", result 91 | 92 | fetchBody 93 | 94 | release(token, used=true) 95 | 96 | if resp.status == $Http400: 97 | raise newException(InternalError, $url) 98 | except InternalError as e: 99 | raise e 100 | except Exception as e: 101 | echo "error: ", e.name, ", msg: ", e.msg, ", token: ", token[], ", url: ", url 102 | if "length" notin e.msg and "descriptor" notin e.msg: 103 | release(token, invalid=true) 104 | raise rateLimitError() 105 | 106 | proc fetch*(url: Uri; api: Api; additional_headers: HttpHeaders = newHttpHeaders()): Future[JsonNode] {.async.} = 107 | 108 | if len(cfg.cookieHeader) != 0: 109 | additional_headers.add("Cookie", cfg.cookieHeader) 110 | if len(cfg.xCsrfToken) != 0: 111 | additional_headers.add("x-csrf-token", cfg.xCsrfToken) 112 | 113 | var body: string 114 | fetchImpl(body, additional_headers): 115 | if body.startsWith('{') or body.startsWith('['): 116 | result = parseJson(body) 117 | else: 118 | echo resp.status, ": ", body, " --- url: ", url 119 | result = newJNull() 120 | 121 | updateToken() 122 | 123 | let error = result.getError 124 | if error in {invalidToken, badToken}: 125 | echo "fetch error: ", result.getError 126 | release(token, invalid=true) 127 | raise rateLimitError() 128 | 129 | proc fetchRaw*(url: Uri; api: Api; additional_headers: HttpHeaders = newHttpHeaders()): Future[string] {.async.} = 130 | fetchImpl(result, additional_headers): 131 | if not (result.startsWith('{') or result.startsWith('[')): 132 | echo resp.status, ": ", result, " --- url: ", url 133 | result.setLen(0) 134 | 135 | updateToken() 136 | 137 | if result.startsWith("{\"errors"): 138 | let errors = result.fromJson(Errors) 139 | if errors in {invalidToken, badToken}: 140 | echo "fetch error: ", errors 141 | release(token, invalid=true) 142 | raise rateLimitError() 143 | -------------------------------------------------------------------------------- /src/views/search.nim: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: AGPL-3.0-only 2 | import strutils, strformat, sequtils, unicode, tables, options 3 | import karax/[karaxdsl, vdom] 4 | 5 | import renderutils, timeline 6 | import ".."/[types, query] 7 | 8 | const toggles = { 9 | "nativeretweets": "Retweets", 10 | "media": "Media", 11 | "videos": "Videos", 12 | "news": "News", 13 | "verified": "Verified", 14 | "native_video": "Native videos", 15 | "replies": "Replies", 16 | "links": "Links", 17 | "images": "Images", 18 | "safe": "Safe", 19 | "quote": "Quotes", 20 | "pro_video": "Pro videos" 21 | }.toOrderedTable 22 | 23 | proc renderSearch*(): VNode = 24 | buildHtml(tdiv(class="panel-container")): 25 | tdiv(class="search-bar"): 26 | form(`method`="get", action="/search", autocomplete="off"): 27 | hiddenField("f", "users") 28 | input(`type`="text", name="q", autofocus="", 29 | placeholder="Enter username...", dir="auto") 30 | button(`type`="submit"): icon "search" 31 | 32 | proc renderProfileTabs*(query: Query; username: string; cfg: Config): VNode = 33 | let link = "/" & username 34 | buildHtml(ul(class="tab")): 35 | li(class=query.getTabClass(posts)): 36 | a(href=link): text "Tweets" 37 | li(class=(query.getTabClass(replies) & " wide")): 38 | a(href=(link & "/with_replies")): text "Tweets & Replies" 39 | li(class=query.getTabClass(media)): 40 | a(href=(link & "/media")): text "Media" 41 | if len(cfg.xCsrfToken) != 0 and len(cfg.cookieHeader) != 0: 42 | li(class=query.getTabClass(favorites)): 43 | a(href=(link & "/favorites")): text "Likes" 44 | li(class=query.getTabClass(tweets)): 45 | a(href=(link & "/search")): text "Search" 46 | 47 | proc renderSearchTabs*(query: Query): VNode = 48 | var q = query 49 | buildHtml(ul(class="tab")): 50 | li(class=query.getTabClass(tweets)): 51 | q.kind = tweets 52 | a(href=("?" & genQueryUrl(q))): text "Tweets" 53 | li(class=query.getTabClass(users)): 54 | q.kind = users 55 | a(href=("?" & genQueryUrl(q))): text "Users" 56 | 57 | proc isPanelOpen(q: Query): bool = 58 | q.fromUser.len == 0 and (q.filters.len > 0 or q.excludes.len > 0 or 59 | @[q.near, q.until, q.since].anyIt(it.len > 0)) 60 | 61 | proc renderSearchPanel*(query: Query): VNode = 62 | let user = query.fromUser.join(",") 63 | let action = if user.len > 0: &"/{user}/search" else: "/search" 64 | buildHtml(form(`method`="get", action=action, 65 | class="search-field", autocomplete="off")): 66 | hiddenField("f", "tweets") 67 | genInput("q", "", query.text, "Enter search...", class="pref-inline") 68 | button(`type`="submit"): icon "search" 69 | if isPanelOpen(query): 70 | input(id="search-panel-toggle", `type`="checkbox", checked="") 71 | else: 72 | input(id="search-panel-toggle", `type`="checkbox") 73 | label(`for`="search-panel-toggle"): 74 | icon "down" 75 | tdiv(class="search-panel"): 76 | for f in @["filter", "exclude"]: 77 | span(class="search-title"): text capitalize(f) 78 | tdiv(class="search-toggles"): 79 | for k, v in toggles: 80 | let state = 81 | if f == "filter": k in query.filters 82 | else: k in query.excludes 83 | genCheckbox(&"{f[0]}-{k}", v, state) 84 | 85 | tdiv(class="search-row"): 86 | tdiv: 87 | span(class="search-title"): text "Time range" 88 | tdiv(class="date-range"): 89 | genDate("since", query.since) 90 | span(class="search-title"): text "-" 91 | genDate("until", query.until) 92 | tdiv: 93 | span(class="search-title"): text "Near" 94 | genInput("near", "", query.near, "Location...", autofocus=false) 95 | 96 | proc renderTweetSearch*(results: Result[Tweet]; cfg: Config; prefs: Prefs; path: string; 97 | pinned=none(Tweet)): VNode = 98 | let query = results.query 99 | buildHtml(tdiv(class="timeline-container")): 100 | if query.fromUser.len > 1: 101 | tdiv(class="timeline-header"): 102 | text query.fromUser.join(" | ") 103 | 104 | if query.fromUser.len > 0: 105 | renderProfileTabs(query, query.fromUser.join(","), cfg) 106 | 107 | if query.fromUser.len == 0 or query.kind == tweets: 108 | tdiv(class="timeline-header"): 109 | renderSearchPanel(query) 110 | 111 | if query.fromUser.len == 0: 112 | renderSearchTabs(query) 113 | 114 | renderTimelineTweets(results, prefs, path, pinned) 115 | 116 | proc renderUserSearch*(results: Result[User]; prefs: Prefs): VNode = 117 | buildHtml(tdiv(class="timeline-container")): 118 | tdiv(class="timeline-header"): 119 | form(`method`="get", action="/search", class="search-field", autocomplete="off"): 120 | hiddenField("f", "users") 121 | genInput("q", "", results.query.text, "Enter username...", class="pref-inline") 122 | button(`type`="submit"): icon "search" 123 | 124 | renderSearchTabs(results.query) 125 | renderTimelineUsers(results, prefs) 126 | -------------------------------------------------------------------------------- /tests/test_card.py: -------------------------------------------------------------------------------- 1 | from base import BaseTestCase, Card, Conversation 2 | from parameterized import parameterized 3 | 4 | 5 | card = [ 6 | ['Thom_Wolf/status/1122466524860702729', 7 | 'facebookresearch/fairseq', 8 | 'Facebook AI Research Sequence-to-Sequence Toolkit written in Python. - GitHub - facebookresearch/fairseq: Facebook AI Research Sequence-to-Sequence Toolkit written in Python.', 9 | 'github.com', True], 10 | 11 | ['nim_lang/status/1136652293510717440', 12 | 'Version 0.20.0 released', 13 | 'We are very proud to announce Nim version 0.20. This is a massive release, both literally and figuratively. It contains more than 1,000 commits and it marks our release candidate for version 1.0!', 14 | 'nim-lang.org', True], 15 | 16 | ['voidtarget/status/1094632512926605312', 17 | 'Basic OBS Studio plugin, written in nim, supporting C++ (C fine too)', 18 | 'Basic OBS Studio plugin, written in nim, supporting C++ (C fine too) - obsplugin.nim', 19 | 'gist.github.com', True], 20 | 21 | ['FluentAI/status/1116417904831029248', 22 | 'Amazon’s Alexa isn’t just AI — thousands of humans are listening', 23 | 'One of the only ways to improve Alexa is to have human beings check it for errors', 24 | 'theverge.com', True] 25 | ] 26 | 27 | no_thumb = [ 28 | ['brent_p/status/1088857328680488961', 29 | 'Hts Nim Sugar', 30 | 'hts-nim is a library that allows one to use htslib via the nim programming language. Nim is a garbage-collected language that compiles to C and often has similar performance. I have become very...', 31 | 'brentp.github.io'], 32 | 33 | ['voidtarget/status/1133028231672582145', 34 | 'sinkingsugar/nimqt-example', 35 | 'A sample of a Qt app written using mostly nim. Contribute to sinkingsugar/nimqt-example development by creating an account on GitHub.', 36 | 'github.com'], 37 | 38 | ['mobile_test/status/490378953744318464', 39 | 'Nantasket Beach', 40 | 'Explore this photo titled Nantasket Beach by Ben Sandofsky (@sandofsky) on 500px', 41 | '500px.com'], 42 | 43 | ['nim_lang/status/1082989146040340480', 44 | 'Nim in 2018: A short recap', 45 | 'Posted by u/miran1 - 36 votes and 46 comments', 46 | 'reddit.com'] 47 | ] 48 | 49 | playable = [ 50 | ['nim_lang/status/1118234460904919042', 51 | 'Nim development blog 2019-03', 52 | 'Arne (aka Krux02)* debugging: * improved nim-gdb, $ works, framefilter * alias for --debugger:native: -g* bugs: * forwarding of .pure. * sizeof union* fe...', 53 | 'youtube.com'], 54 | 55 | ['nim_lang/status/1121090879823986688', 56 | 'Nim - First natively compiled language w/ hot code-reloading at...', 57 | '#nim #c++ #ACCUConfNim is a statically typed systems and applications programming language which offers perhaps some of the most powerful metaprogramming cap...', 58 | 'youtube.com'] 59 | ] 60 | 61 | # promo = [ 62 | # ['BangOlufsen/status/1145698701517754368', 63 | # 'Upgrade your journey', '', 64 | # 'www.bang-olufsen.com'], 65 | 66 | # ['BangOlufsen/status/1154934429900406784', 67 | # 'Learn more about Beosound Shape', '', 68 | # 'www.bang-olufsen.com'] 69 | # ] 70 | 71 | 72 | class CardTest(BaseTestCase): 73 | @parameterized.expand(card) 74 | def test_card(self, tweet, title, description, destination, large): 75 | self.open_nitter(tweet) 76 | c = Card(Conversation.main + " ") 77 | self.assert_text(title, c.title) 78 | self.assert_text(destination, c.destination) 79 | self.assertIn('_img', self.get_image_url(c.image + ' img')) 80 | if len(description) > 0: 81 | self.assert_text(description, c.description) 82 | if large: 83 | self.assert_element_visible('.card.large') 84 | else: 85 | self.assert_element_not_visible('.card.large') 86 | 87 | @parameterized.expand(no_thumb) 88 | def test_card_no_thumb(self, tweet, title, description, destination): 89 | self.open_nitter(tweet) 90 | c = Card(Conversation.main + " ") 91 | self.assert_text(title, c.title) 92 | self.assert_text(destination, c.destination) 93 | if len(description) > 0: 94 | self.assert_text(description, c.description) 95 | 96 | @parameterized.expand(playable) 97 | def test_card_playable(self, tweet, title, description, destination): 98 | self.open_nitter(tweet) 99 | c = Card(Conversation.main + " ") 100 | self.assert_text(title, c.title) 101 | self.assert_text(destination, c.destination) 102 | self.assertIn('_img', self.get_image_url(c.image + ' img')) 103 | self.assert_element_visible('.card-overlay') 104 | if len(description) > 0: 105 | self.assert_text(description, c.description) 106 | 107 | # @parameterized.expand(promo) 108 | # def test_card_promo(self, tweet, title, description, destination): 109 | # self.open_nitter(tweet) 110 | # c = Card(Conversation.main + " ") 111 | # self.assert_text(title, c.title) 112 | # self.assert_text(destination, c.destination) 113 | # self.assert_element_visible('.video-overlay') 114 | # if len(description) > 0: 115 | # self.assert_text(description, c.description) 116 | -------------------------------------------------------------------------------- /src/tokens.nim: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: AGPL-3.0-only 2 | import asyncdispatch, httpclient, times, sequtils, json, random 3 | import strutils, tables 4 | import zippy 5 | import types, consts, http_pool 6 | 7 | const 8 | maxConcurrentReqs = 5 # max requests at a time per token, to avoid race conditions 9 | maxLastUse = 1.hours # if a token is unused for 60 minutes, it expires 10 | maxAge = 2.hours + 55.minutes # tokens expire after 3 hours 11 | failDelay = initDuration(minutes=30) 12 | 13 | var 14 | clientPool: HttpPool 15 | tokenPool: seq[Token] 16 | lastFailed: Time 17 | enableLogging = false 18 | 19 | template log(str) = 20 | if enableLogging: echo "[tokens] ", str 21 | 22 | proc getPoolJson*(): JsonNode = 23 | var 24 | list = newJObject() 25 | totalReqs = 0 26 | totalPending = 0 27 | reqsPerApi: Table[string, int] 28 | 29 | for token in tokenPool: 30 | totalPending.inc(token.pending) 31 | list[token.tok] = %*{ 32 | "apis": newJObject(), 33 | "pending": token.pending, 34 | "init": $token.init, 35 | "lastUse": $token.lastUse 36 | } 37 | 38 | for api in token.apis.keys: 39 | list[token.tok]["apis"][$api] = %token.apis[api] 40 | 41 | let 42 | maxReqs = 43 | case api 44 | of Api.listMembers, Api.listBySlug, Api.list, 45 | Api.userRestId, Api.userScreenName, Api.tweetDetail: 500 46 | of Api.timeline: 187 47 | else: 180 48 | reqs = maxReqs - token.apis[api].remaining 49 | 50 | reqsPerApi[$api] = reqsPerApi.getOrDefault($api, 0) + reqs 51 | totalReqs.inc(reqs) 52 | 53 | return %*{ 54 | "amount": tokenPool.len, 55 | "requests": totalReqs, 56 | "pending": totalPending, 57 | "apis": reqsPerApi, 58 | "tokens": list 59 | } 60 | 61 | proc rateLimitError*(): ref RateLimitError = 62 | newException(RateLimitError, "rate limited") 63 | 64 | proc fetchToken(): Future[Token] {.async.} = 65 | if getTime() - lastFailed < failDelay: 66 | raise rateLimitError() 67 | 68 | let headers = newHttpHeaders({ 69 | "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", 70 | "accept-encoding": "gzip", 71 | "accept-language": "en-US,en;q=0.5", 72 | "connection": "keep-alive", 73 | "authorization": auth 74 | }) 75 | 76 | try: 77 | let 78 | resp = clientPool.use(headers): await c.postContent(activate) 79 | tokNode = parseJson(uncompress(resp))["guest_token"] 80 | tok = tokNode.getStr($(tokNode.getInt)) 81 | time = getTime() 82 | 83 | return Token(tok: tok, init: time, lastUse: time) 84 | except Exception as e: 85 | echo "[tokens] fetching token failed: ", e.msg 86 | if "Try again" notin e.msg: 87 | echo "[tokens] fetching tokens paused, resuming in 30 minutes" 88 | lastFailed = getTime() 89 | 90 | proc expired(token: Token): bool = 91 | let time = getTime() 92 | token.init < time - maxAge or token.lastUse < time - maxLastUse 93 | 94 | proc isLimited(token: Token; api: Api): bool = 95 | if token.isNil or token.expired: 96 | return true 97 | 98 | if api in token.apis: 99 | let limit = token.apis[api] 100 | return (limit.remaining <= 10 and limit.reset > epochTime().int) 101 | else: 102 | return false 103 | 104 | proc isReady(token: Token; api: Api): bool = 105 | not (token.isNil or token.pending > maxConcurrentReqs or token.isLimited(api)) 106 | 107 | proc release*(token: Token; used=false; invalid=false) = 108 | if token.isNil: return 109 | if invalid or token.expired: 110 | if invalid: log "discarding invalid token" 111 | elif token.expired: log "discarding expired token" 112 | 113 | let idx = tokenPool.find(token) 114 | if idx > -1: tokenPool.delete(idx) 115 | elif used: 116 | dec token.pending 117 | token.lastUse = getTime() 118 | 119 | proc getToken*(api: Api): Future[Token] {.async.} = 120 | for i in 0 ..< tokenPool.len: 121 | if result.isReady(api): break 122 | release(result) 123 | result = tokenPool.sample() 124 | 125 | if not result.isReady(api): 126 | release(result) 127 | result = await fetchToken() 128 | log "added new token to pool" 129 | tokenPool.add result 130 | 131 | if not result.isNil: 132 | inc result.pending 133 | else: 134 | raise rateLimitError() 135 | 136 | proc setRateLimit*(token: Token; api: Api; remaining, reset: int) = 137 | # avoid undefined behavior in race conditions 138 | if api in token.apis: 139 | let limit = token.apis[api] 140 | if limit.reset >= reset and limit.remaining < remaining: 141 | return 142 | 143 | token.apis[api] = RateLimit(remaining: remaining, reset: reset) 144 | 145 | proc poolTokens*(amount: int) {.async.} = 146 | var futs: seq[Future[Token]] 147 | for i in 0 ..< amount: 148 | futs.add fetchToken() 149 | 150 | for token in futs: 151 | var newToken: Token 152 | 153 | try: newToken = await token 154 | except: discard 155 | 156 | if not newToken.isNil: 157 | log "added new token to pool" 158 | tokenPool.add newToken 159 | 160 | proc initTokenPool*(cfg: Config) {.async.} = 161 | clientPool = HttpPool() 162 | enableLogging = cfg.enableDebug 163 | 164 | while true: 165 | if tokenPool.countIt(not it.isLimited(Api.timeline)) < cfg.minTokens: 166 | await poolTokens(min(4, cfg.minTokens - tokenPool.len)) 167 | await sleepAsync(2000) 168 | -------------------------------------------------------------------------------- /tests/test_tweet.py: -------------------------------------------------------------------------------- 1 | from base import BaseTestCase, Tweet, get_timeline_tweet 2 | from parameterized import parameterized 3 | 4 | # image = tweet + 'div.attachments.media-body > div > div > a > div > img' 5 | # self.assert_true(self.get_image_url(image).split('/')[0] == 'http') 6 | 7 | timeline = [ 8 | [1, 'Test account', 'mobile_test', '10 Aug 2016', '763483571793174528', 9 | '.'], 10 | 11 | [3, 'Test account', 'mobile_test', '3 Mar 2016', '705522133443571712', 12 | 'LIVE on #Periscope pscp.tv/w/aadiTzF6dkVOTXZSbX…'], 13 | 14 | [6, 'mobile test 2', 'mobile_test_2', '1 Oct 2014', '517449200045277184', 15 | 'Testing. One two three four. Test.'] 16 | ] 17 | 18 | status = [ 19 | [20, 'jack', 'jack', '21 Mar 2006', 'just setting up my twttr'], 20 | [134849778302464000, 'The Twoffice', 'TheTwoffice', '11 Nov 2011', 'test'], 21 | [105685475985080322, 'The Twoffice', 'TheTwoffice', '22 Aug 2011', 'regular tweet'], 22 | [572593440719912960, 'Test account', 'mobile_test', '3 Mar 2015', 'testing test'] 23 | ] 24 | 25 | invalid = [ 26 | ['mobile_test/status/120938109238'], 27 | ['TheTwoffice/status/8931928312'] 28 | ] 29 | 30 | multiline = [ 31 | [400897186990284800, 'mobile_test_3', 32 | """ 33 | ♔ 34 | KEEP 35 | CALM 36 | AND 37 | CLICHÉ 38 | ON"""] 39 | ] 40 | 41 | link = [ 42 | ['nim_lang/status/1110499584852353024', [ 43 | 'nim-lang.org/araq/ownedrefs.…', 44 | 'news.ycombinator.com/item?id…', 45 | 'teddit.net/r/programming…' 46 | ]], 47 | ['nim_lang/status/1125887775151140864', [ 48 | 'en.wikipedia.org/wiki/Nim_(p…' 49 | ]], 50 | ['hiankun_taioan/status/1086916335215341570', [ 51 | '(hackernoon.com/interview-wit…)' 52 | ]], 53 | ['archillinks/status/1146302618223951873', [ 54 | 'flickr.com/photos/87101284@N…', 55 | 'hisafoto.tumblr.com/post/176…' 56 | ]], 57 | ['archillinks/status/1146292551936335873', [ 58 | 'flickr.com/photos/michaelrye…', 59 | 'furtho.tumblr.com/post/16618…' 60 | ]] 61 | ] 62 | 63 | username = [ 64 | ['Bountysource/status/1094803522053320705', ['nim_lang']], 65 | ['leereilly/status/1058464250098704385', ['godotengine', 'unity3d', 'nim_lang']] 66 | ] 67 | 68 | emoji = [ 69 | ['Tesla/status/1134850442511257600', '🌈❤️🧡💛💚💙💜'] 70 | ] 71 | 72 | retweet = [ 73 | [7, 'mobile_test_2', 'mobile test 2', 'Test account', '@mobile_test', '1234'], 74 | [3, 'mobile_test_8', 'mobile test 8', 'jack', '@jack', 'twttr'] 75 | ] 76 | 77 | reply = [ 78 | ['mobile_test/with_replies', 15] 79 | ] 80 | 81 | 82 | class TweetTest(BaseTestCase): 83 | @parameterized.expand(timeline) 84 | def test_timeline(self, index, fullname, username, date, tid, text): 85 | self.open_nitter(username) 86 | tweet = get_timeline_tweet(index) 87 | self.assert_exact_text(fullname, tweet.fullname) 88 | self.assert_exact_text('@' + username, tweet.username) 89 | self.assert_exact_text(date, tweet.date) 90 | self.assert_text(text, tweet.text) 91 | permalink = self.find_element(tweet.date + ' a') 92 | self.assertIn(tid, permalink.get_attribute('href')) 93 | 94 | @parameterized.expand(status) 95 | def test_status(self, tid, fullname, username, date, text): 96 | tweet = Tweet() 97 | self.open_nitter(f'{username}/status/{tid}') 98 | self.assert_exact_text(fullname, tweet.fullname) 99 | self.assert_exact_text('@' + username, tweet.username) 100 | self.assert_exact_text(date, tweet.date) 101 | self.assert_text(text, tweet.text) 102 | 103 | @parameterized.expand(multiline) 104 | def test_multiline_formatting(self, tid, username, text): 105 | self.open_nitter(f'{username}/status/{tid}') 106 | self.assert_text(text.strip('\n'), '.main-tweet') 107 | 108 | @parameterized.expand(emoji) 109 | def test_emoji(self, tweet, text): 110 | self.open_nitter(tweet) 111 | self.assert_text(text, '.main-tweet') 112 | 113 | @parameterized.expand(link) 114 | def test_link(self, tweet, links): 115 | self.open_nitter(tweet) 116 | for link in links: 117 | self.assert_text(link, '.main-tweet') 118 | 119 | @parameterized.expand(username) 120 | def test_username(self, tweet, usernames): 121 | self.open_nitter(tweet) 122 | for un in usernames: 123 | link = self.find_link_text(f'@{un}') 124 | self.assertIn(f'/{un}', link.get_property('href')) 125 | 126 | @parameterized.expand(retweet) 127 | def test_retweet(self, index, url, retweet_by, fullname, username, text): 128 | self.open_nitter(url) 129 | tweet = get_timeline_tweet(index) 130 | self.assert_text(f'{retweet_by} retweeted', tweet.retweet) 131 | self.assert_text(text, tweet.text) 132 | self.assert_exact_text(fullname, tweet.fullname) 133 | self.assert_exact_text(username, tweet.username) 134 | 135 | @parameterized.expand(invalid) 136 | def test_invalid_id(self, tweet): 137 | self.open_nitter(tweet) 138 | self.assert_text('Tweet not found', '.error-panel') 139 | 140 | @parameterized.expand(reply) 141 | def test_thread(self, tweet, num): 142 | self.open_nitter(tweet) 143 | thread = self.find_element(f'.timeline > div:nth-child({num})') 144 | self.assertIn(thread.get_attribute('class'), 'thread-line') 145 | -------------------------------------------------------------------------------- /src/routes/rss.nim: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: AGPL-3.0-only 2 | import asyncdispatch, tables, times, hashes, uri 3 | 4 | import jester 5 | 6 | import router_utils, timeline 7 | import ../query 8 | 9 | include "../views/rss.nimf" 10 | 11 | export times, hashes 12 | 13 | proc redisKey*(page, name, cursor: string): string = 14 | result = page & ":" & name 15 | if cursor.len > 0: 16 | result &= ":" & cursor 17 | 18 | proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.} = 19 | var profile: Profile 20 | let 21 | name = req.params.getOrDefault("name") 22 | after = getCursor(req) 23 | names = getNames(name) 24 | 25 | if names.len == 1: 26 | profile = await fetchProfile(after, query, cfg, skipRail=true, skipPinned=true) 27 | else: 28 | var q = query 29 | q.fromUser = names 30 | profile = Profile( 31 | tweets: await getSearch[Tweet](q, after), 32 | # this is kinda dumb 33 | user: User( 34 | username: name, 35 | fullname: names.join(" | "), 36 | userpic: "https://abs.twimg.com/sticky/default_profile_images/default_profile.png" 37 | ) 38 | ) 39 | 40 | if profile.user.suspended: 41 | return Rss(feed: profile.user.username, cursor: "suspended") 42 | 43 | if profile.user.fullname.len > 0: 44 | let rss = renderTimelineRss(profile, cfg, multi=(names.len > 1)) 45 | return Rss(feed: rss, cursor: profile.tweets.bottom) 46 | 47 | template respRss*(rss, page) = 48 | if rss.cursor.len == 0: 49 | let info = case page 50 | of "User": " \"" & @"name" & "\" " 51 | of "List": " \"" & @"id" & "\" " 52 | else: " " 53 | 54 | resp Http404, showError(page & info & "not found", cfg) 55 | elif rss.cursor.len == 9 and rss.cursor == "suspended": 56 | resp Http404, showError(getSuspended(@"name"), cfg) 57 | 58 | let headers = {"Content-Type": "application/rss+xml; charset=utf-8", 59 | "Min-Id": rss.cursor} 60 | resp Http200, headers, rss.feed 61 | 62 | proc createRssRouter*(cfg: Config) = 63 | router rss: 64 | get "/search/rss": 65 | cond cfg.enableRss 66 | if @"q".len > 200: 67 | resp Http400, showError("Search input too long.", cfg) 68 | 69 | let query = initQuery(params(request)) 70 | if query.kind != tweets: 71 | resp Http400, showError("Only Tweet searches are allowed for RSS feeds.", cfg) 72 | 73 | let 74 | cursor = getCursor() 75 | key = redisKey("search", $hash(genQueryUrl(query)), cursor) 76 | 77 | var rss = await getCachedRss(key) 78 | if rss.cursor.len > 0: 79 | respRss(rss, "Search") 80 | 81 | let tweets = await getSearch[Tweet](query, cursor) 82 | rss.cursor = tweets.bottom 83 | rss.feed = renderSearchRss(tweets.content, query.text, genQueryUrl(query), cfg) 84 | 85 | await cacheRss(key, rss) 86 | respRss(rss, "Search") 87 | 88 | get "/@name/rss": 89 | cond cfg.enableRss 90 | cond '.' notin @"name" 91 | let 92 | name = @"name" 93 | key = redisKey("twitter", name, getCursor()) 94 | 95 | var rss = await getCachedRss(key) 96 | if rss.cursor.len > 0: 97 | respRss(rss, "User") 98 | 99 | rss = await timelineRss(request, cfg, Query(fromUser: @[name])) 100 | 101 | await cacheRss(key, rss) 102 | respRss(rss, "User") 103 | 104 | get "/@name/@tab/rss": 105 | cond cfg.enableRss 106 | cond '.' notin @"name" 107 | cond @"tab" in ["with_replies", "media", "favorites", "search"] 108 | let 109 | name = @"name" 110 | tab = @"tab" 111 | query = 112 | case tab 113 | of "with_replies": getReplyQuery(name) 114 | of "media": getMediaQuery(name) 115 | of "favorites": getFavoritesQuery(name) 116 | of "search": initQuery(params(request), name=name) 117 | else: Query(fromUser: @[name]) 118 | 119 | let searchKey = if tab != "search": "" 120 | else: ":" & $hash(genQueryUrl(query)) 121 | 122 | let key = redisKey(tab, name & searchKey, getCursor()) 123 | 124 | var rss = await getCachedRss(key) 125 | if rss.cursor.len > 0: 126 | respRss(rss, "User") 127 | 128 | rss = await timelineRss(request, cfg, query) 129 | 130 | await cacheRss(key, rss) 131 | respRss(rss, "User") 132 | 133 | get "/@name/lists/@slug/rss": 134 | cond cfg.enableRss 135 | cond @"name" != "i" 136 | let 137 | slug = decodeUrl(@"slug") 138 | list = await getCachedList(@"name", slug) 139 | cursor = getCursor() 140 | 141 | if list.id.len == 0: 142 | resp Http404, showError("List \"" & @"slug" & "\" not found", cfg) 143 | 144 | let url = "/i/lists/" & list.id & "/rss" 145 | if cursor.len > 0: 146 | redirect(url & "?cursor=" & encodeUrl(cursor, false)) 147 | else: 148 | redirect(url) 149 | 150 | get "/i/lists/@id/rss": 151 | cond cfg.enableRss 152 | let 153 | id = @"id" 154 | cursor = getCursor() 155 | key = redisKey("lists", id, cursor) 156 | 157 | var rss = await getCachedRss(key) 158 | if rss.cursor.len > 0: 159 | respRss(rss, "List") 160 | 161 | let 162 | list = await getCachedList(id=id) 163 | timeline = await getListTimeline(list.id, cursor) 164 | rss.cursor = timeline.bottom 165 | rss.feed = renderListRss(timeline.content, list, cfg) 166 | 167 | await cacheRss(key, rss) 168 | respRss(rss, "List") 169 | -------------------------------------------------------------------------------- /src/routes/timeline.nim: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: AGPL-3.0-only 2 | import asyncdispatch, strutils, sequtils, uri, options, times 3 | import jester, karax/vdom 4 | 5 | import router_utils 6 | import ".."/[types, redis_cache, formatters, query, api] 7 | import ../views/[general, profile, timeline, status, search] 8 | 9 | export vdom 10 | export uri, sequtils 11 | export router_utils 12 | export redis_cache, formatters, query, api 13 | export profile, timeline, status 14 | 15 | proc getQuery*(request: Request; tab, name: string): Query = 16 | case tab 17 | of "with_replies": getReplyQuery(name) 18 | of "media": getMediaQuery(name) 19 | of "favorites": getFavoritesQuery(name) 20 | of "search": initQuery(params(request), name=name) 21 | else: Query(fromUser: @[name]) 22 | 23 | template skipIf[T](cond: bool; default; body: Future[T]): Future[T] = 24 | if cond: 25 | let fut = newFuture[T]() 26 | fut.complete(default) 27 | fut 28 | else: 29 | body 30 | 31 | proc fetchProfile*(after: string; query: Query; cfg: Config; skipRail=false; 32 | skipPinned=false): Future[Profile] {.async.} = 33 | let 34 | name = query.fromUser[0] 35 | userId = await getUserId(name) 36 | 37 | if userId.len == 0: 38 | return Profile(user: User(username: name)) 39 | elif userId == "suspended": 40 | return Profile(user: User(username: name, suspended: true)) 41 | 42 | # temporary fix to prevent errors from people browsing 43 | # timelines during/immediately after deployment 44 | var after = after 45 | if query.kind in {posts, replies} and after.startsWith("scroll"): 46 | after.setLen 0 47 | 48 | let 49 | timeline = 50 | case query.kind 51 | of posts: getTimeline(userId, after) 52 | of replies: getTimeline(userId, after, replies=true) 53 | of media: getMediaTimeline(userId, after) 54 | of favorites: getFavorites(userId, cfg, after) 55 | else: getSearch[Tweet](query, after) 56 | 57 | rail = 58 | skipIf(skipRail or query.kind == media, @[]): 59 | getCachedPhotoRail(name) 60 | 61 | user = await getCachedUser(name) 62 | 63 | var pinned: Option[Tweet] 64 | if not skipPinned and user.pinnedTweet > 0 and 65 | after.len == 0 and query.kind in {posts, replies}: 66 | let tweet = await getCachedTweet(user.pinnedTweet) 67 | if not tweet.isNil: 68 | tweet.pinned = true 69 | pinned = some tweet 70 | 71 | result = Profile( 72 | user: user, 73 | pinned: pinned, 74 | tweets: await timeline, 75 | photoRail: await rail 76 | ) 77 | 78 | if result.user.protected or result.user.suspended: 79 | return 80 | 81 | result.tweets.query = query 82 | 83 | proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs; 84 | rss, after: string): Future[string] {.async.} = 85 | if query.fromUser.len != 1: 86 | let 87 | timeline = await getSearch[Tweet](query, after) 88 | html = renderTweetSearch(timeline, cfg, prefs, getPath()) 89 | return renderMain(html, request, cfg, prefs, "Multi", rss=rss) 90 | 91 | var profile = await fetchProfile(after, query, cfg, skipPinned=prefs.hidePins) 92 | template u: untyped = profile.user 93 | 94 | if u.suspended: 95 | return showError(getSuspended(u.username), cfg) 96 | 97 | if profile.user.id.len == 0: return 98 | 99 | let pHtml = renderProfile(profile, cfg, prefs, getPath()) 100 | result = renderMain(pHtml, request, cfg, prefs, pageTitle(u), pageDesc(u), 101 | rss=rss, images = @[u.getUserPic("_400x400")], 102 | banner=u.banner) 103 | 104 | template respTimeline*(timeline: typed) = 105 | let t = timeline 106 | if t.len == 0: 107 | resp Http404, showError("User \"" & @"name" & "\" not found", cfg) 108 | resp t 109 | 110 | template respUserId*() = 111 | cond @"user_id".len > 0 112 | let username = await getCachedUsername(@"user_id") 113 | if username.len > 0: 114 | redirect("/" & username) 115 | else: 116 | resp Http404, showError("User not found", cfg) 117 | 118 | proc createTimelineRouter*(cfg: Config) = 119 | router timeline: 120 | get "/i/user/@user_id": 121 | respUserId() 122 | 123 | get "/intent/user": 124 | respUserId() 125 | 126 | get "/@name/?@tab?/?": 127 | cond '.' notin @"name" 128 | cond @"name" notin ["pic", "gif", "video"] 129 | cond @"tab" in ["with_replies", "media", "search", "favorites", ""] 130 | let 131 | prefs = cookiePrefs() 132 | after = getCursor() 133 | names = getNames(@"name") 134 | 135 | var query = request.getQuery(@"tab", @"name") 136 | if names.len != 1: 137 | query.fromUser = names 138 | 139 | # used for the infinite scroll feature 140 | if @"scroll".len > 0: 141 | if query.fromUser.len != 1: 142 | var timeline = await getSearch[Tweet](query, after) 143 | if timeline.content.len == 0: resp Http404 144 | timeline.beginning = true 145 | resp $renderTweetSearch(timeline, cfg, prefs, getPath()) 146 | else: 147 | var profile = await fetchProfile(after, query, cfg, skipRail=true) 148 | if profile.tweets.content.len == 0: resp Http404 149 | profile.tweets.beginning = true 150 | resp $renderTimelineTweets(profile.tweets, prefs, getPath()) 151 | 152 | let rss = 153 | if @"tab".len == 0: 154 | "/$1/rss" % @"name" 155 | elif @"tab" == "search": 156 | "/$1/search/rss?$2" % [@"name", genQueryUrl(query)] 157 | else: 158 | "/$1/$2/rss" % [@"name", @"tab"] 159 | 160 | respTimeline(await showTimeline(request, query, cfg, prefs, rss, after)) 161 | -------------------------------------------------------------------------------- /src/views/rss.nimf: -------------------------------------------------------------------------------- 1 | #? stdtmpl(subsChar = '$', metaChar = '#') 2 | ## SPDX-License-Identifier: AGPL-3.0-only 3 | #import strutils, xmltree, strformat, options, unicode 4 | #import ../types, ../utils, ../formatters, ../prefs 5 | # 6 | #proc getTitle(tweet: Tweet; retweet: string): string = 7 | #if tweet.pinned: result = "Pinned: " 8 | #elif retweet.len > 0: result = &"RT by @{retweet}: " 9 | #elif tweet.reply.len > 0: result = &"R to @{tweet.reply[0]}: " 10 | #end if 11 | #var text = stripHtml(tweet.text) 12 | ##if unicode.runeLen(text) > 32: 13 | ## text = unicode.runeSubStr(text, 0, 32) & "..." 14 | ##end if 15 | #result &= xmltree.escape(text) 16 | #if result.len > 0: return 17 | #end if 18 | #if tweet.photos.len > 0: 19 | # result &= "Image" 20 | #elif tweet.video.isSome: 21 | # result &= "Video" 22 | #elif tweet.gif.isSome: 23 | # result &= "Gif" 24 | #end if 25 | #end proc 26 | # 27 | #proc getDescription(desc: string; cfg: Config): string = 28 | Twitter feed for: ${desc}. Generated by ${cfg.hostname} 29 | #end proc 30 | # 31 | #proc renderRssTweet(tweet: Tweet; cfg: Config): string = 32 | #let tweet = tweet.retweet.get(tweet) 33 | #let urlPrefix = getUrlPrefix(cfg) 34 | #let text = replaceUrls(tweet.text, defaultPrefs, absolute=urlPrefix) 35 |
${text.replace("\n", "
\n")}