├── .gitignore
├── LICENSE
├── README.md
├── mastodon2html.py
├── requirements.txt
├── tweet2html.py
├── tweet2img.py
└── tweet2json.py


/.gitignore:
--------------------------------------------------------------------------------
1 | output
2 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2024 Terence Eden
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Tweet2Embed
 2 | 
 3 | Convert a public Tweet into either:
 4 | 
 5 | * Image & alt text
 6 | * Semantic HTML and CSS
 7 | 
 8 | Uses Selenium's Webdriver to launch a Firefox or Chrome instance and takes a screenshot. Uses the Twitter embed API to get a copy of the text and any alt text. An HTML representation is copied to the clipboard.
 9 | 
10 | ## Features
11 | 
12 | * 🗣 Avatars inlined as WebP
13 | * 📸 All attached photos inlined
14 | * 🎥 Video poster inline, <video> to original mp4
15 | * 🔗 Hyperlinks don't use t.co
16 | * #️⃣ Hashtags & @ mentions linked
17 | * 🔄 Includes reply threads & quote Tweets
18 | * 🕰 Semantic time
19 | * 🔍 Schema.org metadata
20 | * 🖼 Cards
21 | * 📊 Polls
22 | * ♥ , 🔁 & 🗨 counts
23 | * 📖 Autosubmit the Tweet to Archive.org
24 | 
25 | ## Usage
26 | 
27 | ### tweet2html
28 | * `python tweet2html.py 123` will get the Tweet with ID 123, create an embedded HTML, and copy it to the clipboard.
29 | * `--thread` to get a parent or quote tweet
30 | * `--css` if you want the CSS as well
31 | * `--pretty` for pretty-printed HTML
32 | * `--save` save the HTML to a file
33 | * `--schema` adds Schema.org metadata
34 | 
35 | #### Typical Output
36 | 
37 | Run `python tweet2html.py -mtp 671919410630819840` and receive:
38 | 
39 | ```html
40 | <blockquote class="tweet-embed" id="tweet-embed-671919410630819840" lang="en" itemscope itemtype="https://schema.org/SocialMediaPosting">
41 |     <header class="tweet-embed-header" itemprop="author" itemscope itemtype="https://schema.org/Person">
42 |         <a href="https://twitter.com/polls" class="tweet-embed-user" itemprop="url">
43 |             <img class="tweet-embed-avatar tweet-embed-avatar-circle" src="data:image/webp;base64,UklGRuwAAABXRUJQVlA4IOAAAABQBgCdASowADAAPrVWpEunJSOhqrqpWOAWiWUAxQaACJBCAEB6EJ7HdwZ7m9AsQTxW+yk80gC5I/REUAD+5Ij/FsUhuZ/jfEF7U+ofYABMBkF4Sc8d827tC2qwG95CN3fVuuFS/uqP/Fwucurp8KcurrXcBQpkUCdvp40Y29kx8lP8Y45C3t4IcJPYcIDFVl5+L1M3426aJn0CIdA27KAZjABt0TDw3lgHKggxpvOpEjEgBMnQHzq9rFumwbXgCzvqgOwsseDr6msoySerlXwDZWfNYqz4k58dV2tZoAAAAA==" alt="" itemprop="image">
44 |             <div class="tweet-embed-user-names">
45 |                 <p class="tweet-embed-user-names-name" itemprop="name">polls</p>@polls
46 |             </div>
47 |         </a>
48 |         <img class="tweet-embed-logo" alt="" src='data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciCmFyaWEtbGFiZWw9IlR3aXR0ZXIiIHJvbGU9ImltZyIKdmlld0JveD0iMCAwIDUxMiA1MTIiPjxwYXRoCmQ9Im0wIDBINTEyVjUxMkgwIgpmaWxsPSIjZmZmIi8+PHBhdGggZmlsbD0iIzFkOWJmMCIgZD0ibTQ1OCAxNDBxLTIzIDEwLTQ1IDEyIDI1LTE1IDM0LTQzLTI0IDE0LTUwIDE5YTc5IDc5IDAgMDAtMTM1IDcycS0xMDEtNy0xNjMtODNhODAgODAgMCAwMDI0IDEwNnEtMTcgMC0zNi0xMHMtMyA2MiA2NCA3OXEtMTkgNS0zNiAxczE1IDUzIDc0IDU1cS01MCA0MC0xMTcgMzNhMjI0IDIyNCAwIDAwMzQ2LTIwMHEyMy0xNiA0MC00MSIvPjwvc3ZnPg=='>
49 |     </header>
50 |     <section class="tweet-embed-text" itemprop="articleBody">
51 |         The Beatles or The Rolling Stones?
52 |         <hr class="tweet-embed-hr">
53 |         <label for="poll_1_count">The Beatles: (28,857)</label><br>
54 |         <meter class="tweet-embed-meter" id="poll_1_count" min="0" max="100" low="33" high="66" value="76.1">28857</meter><br>
55 |         <label for="poll_2_count">The Rolling Stones: (9,074)</label><br>
56 |         <meter class="tweet-embed-meter" id="poll_2_count" min="0" max="100" low="33" high="66" value="23.9">9074</meter><br>
57 |     </section>
58 |     <hr class="tweet-embed-hr">
59 |     <footer class="tweet-embed-footer">
60 |         <a href="https://twitter.com/polls/status/671919410630819840" aria-label="113 likes" class="tweet-embed-meta">❤️ 113</a>
61 |         <a href="https://twitter.com/polls/status/671919410630819840" aria-label="38 replies" class="tweet-embed-meta">💬 38</a>
62 |         <a href="https://twitter.com/polls/status/671919410630819840" aria-label="0 retweets" class="tweet-embed-meta">🔁 0</a>			
63 |         <a href="https://twitter.com/polls/status/671919410630819840"><time datetime="2015-12-02T05:10:45.000Z" itemprop="datePublished">05:10 - Wed 02 December 2015</time></a>
64 |     </footer>
65 | </blockquote>
66 | ```
67 | 
68 | ## tweet2img
69 | * `python tweet2img.py 123` will get the Tweet with ID 123, save a WebP screenshot, and print out the alt text.
70 | * `python tweet2img.py 123 --thread` as above, but will include the parent Tweet if this is a reply.
71 | * Screenshot and alt text are saved in the `output` directory.
72 | * Clipboard receives a copy of the HTML - including data-encoded image - ready to paste in.
73 |     * `<a href="https://twitter.com/edent/status/123"><img src="data:image/webp;base64,Ukl..." width="550" height="439" alt="Screenshot from Twitter. 2022-08-19T13:36:44.000Z. Description."/></a>`
74 | 
75 | 
76 | ##  Useful Examples
77 | * `1432768058028875791` Video
78 | * `1095659600420966400` Reply - parent has image
79 | * `909106648928718848` Multiple images
80 | * `1560621791470448642` Quote Tweet
81 | * `670060095972245504` Poll
82 | * `83659275024601088` Deleted Tweet
83 | * `1131218926493413377` Summary Card
84 | * `1485588404037648389` Reply to a quoted Tweet
85 | 
86 | ## Known bugs:
87 | 
88 | * Fractional Scaling may produce slightly fuzzy images (Wayland related?)
89 | * On tweet2img, the alt text contains t.co URls rather than the expanded ones (could use entities?)
90 | * Only some Twitter Cards are rendered in HTML (are there more?)
91 | * No Dark Mode (overkill?)
92 | * Many other things (probably?)
93 | 


--------------------------------------------------------------------------------
/mastodon2html.py:
--------------------------------------------------------------------------------
  1 | #   For command line
  2 | import argparse
  3 | 
  4 | #   File and Bits
  5 | import io
  6 | import os
  7 | import tempfile
  8 | 
  9 | #   Image Manipulation
 10 | from PIL import Image
 11 | 
 12 | #   Etc
 13 | import requests
 14 | import pyperclip
 15 | import base64
 16 | import html
 17 | 
 18 | #	Date manipulation
 19 | from dateutil import parser
 20 | 
 21 | #	Formatting
 22 | import locale
 23 | locale.setlocale(locale.LC_ALL, '')
 24 | 
 25 | #	URl manipulation
 26 | import urllib
 27 | from urllib.parse import urlparse
 28 | 
 29 | #   Command line options
 30 | arguments = argparse.ArgumentParser(
 31 | 	prog="mastodon2html",
 32 | 	description="Convert a Tweet ID to semantic HTML")
 33 | arguments.add_argument("id", type=str,                        help="URl of the Mastodon post")
 34 | arguments.add_argument("-t", "--thread", action="store_true", help="Show the thread (default false)", required=False)
 35 | arguments.add_argument("-c", "--css",    action="store_true", help="Copy the CSS (default false)",    required=False)
 36 | arguments.add_argument("-p", "--pretty", action="store_true", help="Pretty Print the output (default false)",    required=False)
 37 | arguments.add_argument("-s", "--save",   action="store_true", help="Save the output to a file (default false)",    required=False)
 38 | arguments.add_argument("-m", "--schema", action='store_true', help="Add Schema.org metadata (default false)",    required=False)
 39 | 
 40 | args = arguments.parse_args()
 41 | 
 42 | #	Get Mastodon information
 43 | mastodon_url   = args.id
 44 | mastodon_parts = urlparse( mastodon_url )
 45 | mastodon_host  = mastodon_parts.netloc
 46 | mastodon_path  = mastodon_parts.path
 47 | mastodon_id    = mastodon_path.split("/")[-1] #	Last element of /@example/123456
 48 | mastodon_api   = f"https://{mastodon_host}/api/v1/statuses/{mastodon_id}"
 49 | 
 50 | #	Get settings from arguments
 51 | thread_show  = True if args.thread else False
 52 | css_show     = True if args.css    else False
 53 | pretty_print = True if args.pretty else False
 54 | save_file    = True if args.save   else False
 55 | schema_org   = True if args.schema else False
 56 | 
 57 | def image_to_inline( url ) : 
 58 | 	#	Download the image
 59 | 	image_file = requests.get( url )
 60 | 	#	Convert to bytes
 61 | 	image_file = Image.open( io.BytesIO( image_file.content ) )
 62 | 	#	Temp file name with only alphanumeric characters
 63 | 	temp_file_name = "".join(x for x in url if x.isalnum())
 64 | 	#	Full path of the image
 65 | 	output_img = os.path.join( tempfile.gettempdir() , f"{temp_file_name}.webp" )
 66 | 	#	Save as a low quality WebP
 67 | 	image_file.save( output_img, 'webp', optimize=True, quality=60 )
 68 | 	#   Convert image to base64 data URl
 69 | 	#	Read the image from disk
 70 | 	binary_img      = open( output_img, 'rb' ).read()
 71 | 	#	Encode to Base64
 72 | 	base64_utf8_str = base64.b64encode( binary_img ).decode('utf-8')
 73 | 	#	Delete the temporary file
 74 | 	os.remove( output_img )
 75 | 	#	Return as data encoded suitable for an <img src="...">
 76 | 	return f'data:image/webp;base64,{base64_utf8_str}'
 77 | 
 78 | def mastodon_emojis( mastodon_text, emojis ) :
 79 | 	for emoji in emojis:
 80 | 		shortcode = emoji["shortcode"]
 81 | 		url	      = emoji["url"]
 82 | 		emoji_img = image_to_inline( url )
 83 | 		emoji_img_html = f'<img src="{emoji_img}" alt=":{shortcode}:" class="social-embed-emoji">'
 84 | 		mastodon_text = mastodon_text.replace( f":{shortcode}:", emoji_img_html )
 85 | 	return mastodon_text
 86 | 
 87 | 
 88 | def get_media( media_attachments) :
 89 | 	media_html = '<div class="social-embed-media-grid">'
 90 | 	#	Iterate through the attached media
 91 | 	for media in media_attachments :
 92 | 		media_type = media["type"]
 93 | 
 94 | 		#	Convert small version of media to embedded WebP
 95 | 		print( "Embedding media…" )
 96 | 		media_img = image_to_inline( media["preview_url"] )
 97 | 		
 98 | 		#	Find alt text
 99 | 		media_alt = ""
100 | 		if "description" in media :
101 | 			if media["description"] is not None:
102 | 				media_alt = html.escape( media["description"] )
103 | 	
104 | 		#	Is this a video or an image?
105 | 		if "video" == media_type :
106 | 			print( "Video poster…" )
107 | 			#	Embed the poster in the <video>, link to last video which should be highest quality
108 | 			#	TODO! Find a better way to get the best video
109 | 			media_html += f'''
110 | 			<video class='social-embed-video' controls src="{media["url"]}" poster="{media_img}" width="550"></video>
111 | 			'''
112 | 		if "image" == media_type :
113 | 			#	Embed the image
114 | 			media_html += f'''
115 | 			<a href="{media['url']}" class="social-embed-media-link"><img class="social-embed-media" alt="{media_alt}" src="{media_img}"></a>
116 | 			'''
117 | 	return media_html + "</div>"
118 | 
119 | def get_poll_html( poll_data ) :
120 | 	print( "Poll…")
121 | 	poll_html = "<hr class=\"social-embed-hr\">"
122 | 
123 | 	votes_count = poll_data["votes_count"]
124 | 
125 | 	option_counter = 0
126 | 
127 | 	for option in poll_data["options"] :
128 | 		option_title = option["title"]
129 | 		#	Calculate the percentages. Round to 1 decimal place.
130 | 		option_votes = option["votes_count"]
131 | 		if votes_count > 0 :
132 | 			option_percent = '{0:.1f}'.format( (option_votes / votes_count) * 100 )
133 | 		else :
134 | 			option_percent = "0"
135 | 
136 | 		#	Generate semantic HTML
137 | 		poll_html += f'''
138 | 			<label for="poll_{option_counter}">{option_title}: ({option_votes:n})</label><br>
139 | 			<meter class="social-embed-meter" id="poll_{option_counter}" min="0" max="100" low="33" high="66" value="{option_percent}">{option_votes}</meter><br>
140 | 		'''
141 | 
142 | 		option_counter += 1
143 | 	
144 | 	return poll_html
145 | 
146 | def get_card_html( card_data ) :
147 | 	card_type = card_data["type"] 
148 | 	print( "Card of type " + card_type )
149 | 
150 | 	#	Photo Card
151 | 	if "photo" == card_type or "link" == card_type :
152 | 		card_title            = ""
153 | 		card_title_html       = ""
154 | 		card_description      = ""
155 | 		card_description_html = ""
156 | 		card_thumbnail        = ""
157 | 		card_thumbnail_html   = ""
158 | 		card_thumbnail_alt    = ""
159 | 		card_url              = ""
160 | 		card_html             = ""
161 | 		
162 | 		if "provider_name" in card_data:
163 | 			card_provider = card_data["provider_name"]
164 | 			card_provider_html = f"{card_provider}<br>"
165 | 
166 | 		if "title" in card_data:
167 | 			card_title = card_data["title"]
168 | 			card_title_html = f"{card_title}<br>"
169 | 		
170 | 		if "description" in card_data :
171 | 			card_description = card_data["description"]
172 | 			card_description_html = f"{card_description}<br>"
173 | 		
174 | 		if "image_description" in card_data :
175 | 			card_thumbnail_alt = html.escape( card_data["image_description"] )
176 | 		
177 | 		if "image" in card_data and card_data['image'] is not None :
178 | 			print( "Converting card's thumbnail_image…" )
179 | 			card_thumbnail = card_data["image"]
180 | 			#   Convert  media to embedded WebP
181 | 			card_thumbnail = image_to_inline( card_thumbnail )
182 | 			card_thumbnail_html = f'''
183 | 				<div class="social-embed-media-grid">
184 | 					<img src="{card_thumbnail}" alt="{card_thumbnail_alt}" class="social-embed-media">
185 | 				</div>
186 | 				'''
187 | 		
188 | 		if "url" in card_data :
189 | 			card_url = card_data["url"]
190 | 
191 | 		card_html += f'''
192 | 			<a href="{card_url}" class="social-embed-card">
193 | 				{card_thumbnail_html}
194 | 				{card_provider_html}
195 | 				{card_title_html}
196 | 				{card_description_html}
197 | 			</a>
198 | 		'''
199 | 		return card_html
200 | 
201 | def mastodon_to_html( mastodon_data ) :
202 | 	#	Show the thread / quote?
203 | 	# tweet_parent = ""
204 | 	# tweet_quote  = ""
205 | 	# if thread_show :
206 | 	# 	if "parent" in tweet_data :
207 | 	# 		print( "Parent detected…" )
208 | 	# 		tweet_parent = tweet_to_html( tweet_data["parent"] )
209 | 	# 	if "quoted_tweet" in tweet_data :
210 | 	# 		print( "Quote detected…" )
211 | 	# 		tweet_quote = tweet_to_html( tweet_data["quoted_tweet"] )
212 | 
213 | 	#	Take the data from the API of a single Tweet (which might also be a quote or reply).
214 | 	#	Create a semantic HTML representation
215 | 	#   Post Information
216 | 	mastodon_id       = mastodon_data["id"]
217 | 	mastodon_url      = mastodon_data["url"]
218 | 	mastodon_text     = mastodon_data["content"]
219 | 	mastodon_date     = mastodon_data["created_at"] 
220 | 	mastodon_language = mastodon_data["language"]
221 | 	print( f"Detected language - {mastodon_language}" )
222 | 
223 | 	mastodon_likes    = (int)(mastodon_data.get("favourites_count", 0))#	Might not exist
224 | 	mastodon_replies  = (int)(mastodon_data.get("replies_count",    0))#	Might not exist
225 | 	mastodon_retweets = (int)(mastodon_data.get("reblogs_count",    0))#	Might not exist
226 | 
227 | 	#	User information
228 | 	user_name         = mastodon_data["account"]["username"]
229 | 	user_display      = mastodon_data["account"]["display_name"]
230 | 	user_avatar       = mastodon_data["account"]["avatar"]
231 | 	user_url          = mastodon_data["account"]["url"]
232 | 	user_bot          = (bool)(mastodon_data["account"].get("bot",  False))
233 | 
234 | 	#	User labels
235 | 	if user_bot :
236 | 		user_label     = "Automated"
237 | 		user_badge_img = "🤖"
238 | 		print( f"Bot found …")
239 | 		user_badge  = f'<br>{user_badge_img} {user_label}'
240 | 	else :
241 | 		user_badge = ""
242 | 
243 | 	#	Get the datetime
244 | 	mastodon_time = parser.parse( mastodon_date )
245 | 	mastodon_time = mastodon_time.strftime('%H:%M - %a %d %B %Y')
246 | 
247 | 	#	Is this a reply?
248 | 	# if "in_reply_to_screen_name" in tweet_data :
249 | 	# 	tweet_reply_to   = tweet_data["in_reply_to_screen_name"]
250 | 	# 	tweet_reply_id = tweet_data.get("in_reply_to_status_id_str","")	#	Doesn't exist on older tweets
251 | 	# 	if "" == tweet_reply_id :
252 | 	# 		tweet_reply_link = f'https://twitter.com/{tweet_reply_to}'
253 | 	# 	else :
254 | 	# 		tweet_reply_link = f'https://twitter.com/{tweet_reply_to}/status/{tweet_reply_id}'
255 | 	# 	tweet_reply = f'''
256 | 	# 		<small class="social-embed-reply"><a href="{tweet_reply_link}">Replying to @{tweet_reply_to}</a></small>
257 | 	# 	'''
258 | 	# else :
259 | 	# 	tweet_reply = ""
260 | 
261 | 	# #   Embed entities
262 | 	# tweet_text = tweet_entities_to_html( tweet_text, tweet_entities )
263 | 
264 | 	#	Add shortcode emoji to text
265 | 	if "emojis" in mastodon_data :
266 | 		if mastodon_data["emojis"] is not None :
267 | 			mastodon_text = mastodon_emojis( mastodon_text, mastodon_data["emojis"] )
268 | 
269 | 	#	Add media
270 | 	mastodon_media = ""
271 | 	if ( "media_attachments" in mastodon_data ) :
272 | 		if mastodon_data["media_attachments"] is not None :
273 | 			mastodon_media = get_media( mastodon_data["media_attachments"] )
274 | 
275 | 	#	Add card
276 | 	mastodon_card = ""
277 | 	if "card" in mastodon_data :
278 | 		if mastodon_data["card"] is not None :
279 | 			mastodon_card = get_card_html( mastodon_data["card"] )
280 | 	
281 | 	#	Add poll
282 | 	mastodon_poll = ""
283 | 	if "poll" in mastodon_data :
284 | 		if mastodon_data["poll"] is not None :
285 | 			mastodon_poll = get_poll_html( mastodon_data["poll"] )
286 | 
287 | 	#   Convert avatar to embedded WebP
288 | 	print( "Storing avatar…")
289 | 	mastodon_avatar = image_to_inline( user_avatar )
290 | 
291 | 	#	Schema.org metadata
292 | 	schema_post   = ' itemscope itemtype="https://schema.org/SocialMediaPosting"'       if schema_org else ""
293 | 	schema_body   = ' itemprop="articleBody"'   if schema_org else ""
294 | 	schema_time   = ' itemprop="datePublished"' if schema_org else ""
295 | 	schema_author = ' itemprop="author" itemscope itemtype="https://schema.org/Person"' if schema_org else ""
296 | 	schema_url    = ' itemprop="url"'   if schema_org else ""
297 | 	schema_image  = ' itemprop="image"' if schema_org else ""
298 | 	schema_name   = ' itemprop="name"'  if schema_org else ""
299 | 
300 | 	#   HTML
301 | 	mastodon_html = f'''
302 | 	<blockquote class="social-embed" id="social-embed-{mastodon_id}" lang="{mastodon_language}"{schema_post}>
303 | 		<header class="social-embed-header"{schema_author}>
304 | 			<a href="{user_url}" class="social-embed-user"{schema_url}>
305 | 				<img class="social-embed-avatar" src="{mastodon_avatar}" alt=""{schema_image}>
306 | 				<div class="social-embed-user-names">
307 | 					<p class="social-embed-user-names-name"{schema_name}>{user_display}</p>@{user_name}{user_badge}
308 | 				</div>
309 | 			</a>
310 | 			<img class="social-embed-logo" alt="Mastodon" src="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' aria-label='Mastodon' role='img' viewBox='0 0 512 512' fill='%23fff'%3E%3Cpath d='m0 0H512V512H0'/%3E%3ClinearGradient id='a' y2='1'%3E%3Cstop offset='0' stop-color='%236364ff'/%3E%3Cstop offset='1' stop-color='%23563acc'/%3E%3C/linearGradient%3E%3Cpath fill='url(%23a)' d='M317 381q-124 28-123-39 69 15 149 2 67-13 72-80 3-101-3-116-19-49-72-58-98-10-162 0-56 10-75 58-12 31-3 147 3 32 9 53 13 46 70 69 83 23 138-9'/%3E%3Cpath d='M360 293h-36v-93q-1-26-29-23-20 3-20 34v47h-36v-47q0-31-20-34-30-3-30 28v88h-36v-91q1-51 44-60 33-5 51 21l9 15 9-15q16-26 51-21 43 9 43 60'/%3E%3C/svg%3E" >
311 | 		</header>
312 | 		<section class="social-embed-text"{schema_body}>
313 | 			{mastodon_text}
314 | 			{mastodon_media}
315 | 			{mastodon_poll}
316 | 			{mastodon_card}
317 | 		</section>
318 | 		<hr class="social-embed-hr">
319 | 		<footer class="social-embed-footer">
320 | 			<a href="{mastodon_url}">
321 | 				<span aria-label="{mastodon_likes} likes" class="social-embed-meta">❤️ {mastodon_likes:n}</span>
322 | 				<span aria-label="{mastodon_replies} replies" class="social-embed-meta">💬 {mastodon_replies:n}</span>
323 | 				<span aria-label="{mastodon_retweets} reposts" class="social-embed-meta">🔁 {mastodon_retweets:n}</span>
324 | 				<time datetime="{mastodon_date}"{schema_time}>{mastodon_time}</time>
325 | 			</a>
326 | 		</footer>
327 | 	</blockquote>
328 | 	'''
329 | 	return mastodon_html
330 | 
331 | #   Get the data from the Mastodon API
332 | for _ in range(5):
333 | 	#	Lazy retry strategy
334 | 	try :
335 | 		print( f"Downloading {mastodon_api}" )
336 | 		response = requests.get( mastodon_api )
337 | 		data = response.json()
338 | 		break
339 | 	except :
340 | 		print( "Retrying…" )
341 | 		continue
342 | 
343 | #	If Post was deleted, exit.
344 | if "error" in data :
345 | 	print( "This post doesn't exist." )
346 | 	raise SystemExit
347 | 
348 | #	Turn the Tweet into HTML
349 | mastodon_html = mastodon_to_html( data )
350 | 
351 | #   Generate Content to be pasted
352 | 
353 | #	CSS
354 | mastodon_css = '''
355 | <style>
356 | .social-embed{
357 | 	all:unset;
358 | 	display:block;
359 | }
360 | .social-embed * {
361 | 	all:unset;
362 | 	display:revert;
363 | }
364 | .social-embed::after{
365 | 	all:unset;
366 | }
367 | .social-embed::before{
368 | 	all:unset;
369 | }
370 | blockquote:not(*){
371 | 	all:unset;
372 | }
373 | .social-embed a{
374 | 	cursor:pointer;
375 | }
376 | blockquote.social-embed{
377 | 	box-sizing:border-box;
378 | 	border:.5px solid;
379 | 	width:550px;
380 | 	max-width:100%;
381 | 	font-family:sans-serif;
382 | 	margin:auto;
383 | 	margin-bottom:.5em;
384 | 	padding:1em;
385 | 	border-radius:.5em;
386 | 	background-color:#FFF;
387 | 	color:#000;
388 | 	display:block;
389 | 	white-space: normal;
390 | }
391 | .social-embed-header{
392 | 	display:flex;
393 | 	justify-content:space-between;
394 | }
395 | .social-embed-user{
396 | 	display:flex;
397 | 	position:relative;
398 | 	align-items:center;
399 | 	text-decoration:none;
400 | 	color:inherit;
401 | }
402 | .social-embed-avatar{
403 | 	width:3em;
404 | 	height:3em;
405 | 	margin-right:.5em;
406 | }
407 | .social-embed-avatar-circle{
408 | 	border-radius:50%;
409 | }
410 | .social-embed-avatar-square{
411 | 	border-radius:5%;
412 | }
413 | .social-embed-user-names-name{
414 | 	display:flex;
415 | 	align-items:center;
416 | 	font-weight:bold;
417 | 	margin:0;
418 | }
419 | .social-embed-text{
420 | 	margin-top:.5em;
421 | }
422 | .social-embed-footer a{
423 | 	display:flex;
424 | 	align-items:center;
425 | 	justify-content:space-between;
426 | }
427 | .social-embed-logo{
428 | 	width:3em;
429 | }
430 | .social-embed-hr{
431 | 	border:.1px solid;
432 | 	margin:.5em 0 .5em 0;
433 | }
434 | .social-embed-meta{
435 | 	text-decoration:none !important;
436 | 	color:unset !important;
437 | }
438 | .social-embed-reply{
439 | 	display:block;
440 | }
441 | .social-embed-text a, .social-embed-footer time{
442 | 	color:blue;
443 | 	text-decoration:underline;
444 | }
445 | .social-embed-media-grid {
446 | 	display: flex;
447 | 	flex-wrap: wrap;
448 | }
449 | .social-embed-media-link {
450 | 	flex-grow: 1;
451 | 	width: 50%;
452 | }
453 | .social-embed-media, .social-embed-video {
454 | 	padding: .1em;
455 | 	width: 100%;
456 | 	border-radius:.5em;
457 | 	max-width:100%;
458 | 	object-fit: cover;
459 | 	height: 100%;
460 | }
461 | .social-embed-reply{
462 | 	font-size:.75em;
463 | 	display:block;
464 | }
465 | .social-embed-meter{
466 | 	width:100%;
467 | 	background:#0005;
468 | }
469 | .social-embed-card{
470 | 	text-decoration:none !important;
471 | 	color:unset !important;
472 | 	border:.5px solid;
473 | 	display:block;
474 | 	font-size:.85em;
475 | 	padding:.5em;
476 | 	border-radius:.5em;
477 | }
478 | .social-embed-badge{
479 | 	height:1em;
480 | 	vertical-align: text-top;
481 | }
482 | .social-embed-text p {
483 | 	margin-bottom:1em;
484 | }
485 | .social-embed-emoji{
486 | 	display:inline;
487 | 	width:1em;
488 | }
489 | </style>
490 | '''
491 | 
492 | #   Compact the output if necessary
493 | if not pretty_print :
494 | 	print( "Compacting…")
495 | 	mastodon_html = mastodon_html.replace("\n", "").replace("\t", "")
496 | 	mastodon_css  = mastodon_css.replace("\n", "").replace("\t", "")
497 | 	#	Inline CSS
498 | 	mastodon_css = '<link rel="stylesheet" type="text/css" href="data:text/css,' + urllib.parse.quote(mastodon_css) + '">'
499 | 
500 | #	Add the CSS to the output if requsted
501 | if css_show :
502 | 	mastodon_html = mastodon_css + mastodon_html
503 | 
504 | #   Copy to clipboard
505 | pyperclip.copy( mastodon_html )
506 | 
507 | #   Print to say we've finished
508 | print( f"Copied {mastodon_url}" )
509 | 
510 | if save_file :
511 | 	#	Save HTML
512 | 	#   Save directory
513 | 	output_directory = "output"
514 | 	os.makedirs(output_directory, exist_ok = True)
515 | 	#	Make URl filename safe
516 | 	mastodon_file_name = "".join(x for x in mastodon_url if x.isalnum())
517 | 	save_location = os.path.join( output_directory, f"{mastodon_file_name}.html" ) 
518 | 	#   Save as HTML file
519 | 	with open( save_location, 'w', encoding="utf-8" ) as html_file:
520 | 		html_file.write( mastodon_html )
521 | 	print( f"Saved to {save_location}" )
522 | 
523 | #	Submit the Tweet to Archive.org
524 | print( f"Archiving… {mastodon_url}" )
525 | requests.post( "https://web.archive.org/save/", data={"url": mastodon_url, "capture_all":"on"}, timeout=5 )
526 | 


--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | Pillow==10.4.0
2 | pyperclip==1.9.0
3 | python-dateutil==2.9.0.post0
4 | Requests==2.32.3
5 | selenium==4.23.1
6 | webdriver_manager==4.0.2
7 | 


--------------------------------------------------------------------------------
/tweet2html.py:
--------------------------------------------------------------------------------
  1 | #   For command line
  2 | import argparse
  3 | 
  4 | #   File and Bits
  5 | import io
  6 | import os
  7 | import tempfile
  8 | 
  9 | #   Image Manipulation
 10 | from PIL import Image
 11 | 
 12 | #   Etc
 13 | import requests
 14 | import pyperclip
 15 | import base64
 16 | import html
 17 | import random
 18 | 
 19 | #	Date manipulation
 20 | from datetime import datetime
 21 | from dateutil import parser
 22 | 
 23 | #	Formatting
 24 | import locale
 25 | locale.setlocale(locale.LC_ALL, '')
 26 | 
 27 | #   Command line options
 28 | arguments = argparse.ArgumentParser(
 29 | 	prog='tweet2html',
 30 | 	description='Convert a Tweet ID to semantic HTML')
 31 | arguments.add_argument("id", type=int,                        help="ID of the Tweet (integer)")
 32 | arguments.add_argument("-t", "--thread", action="store_true", help="Show the thread (default false)", required=False)
 33 | arguments.add_argument("-c", "--css",    action="store_true", help="Copy the CSS (default false)",    required=False)
 34 | arguments.add_argument("-p", "--pretty", action="store_true", help="Pretty Print the output (default false)",    required=False)
 35 | arguments.add_argument("-s", "--save",   action="store_true", help="Save the output to a file (default false)",    required=False)
 36 | arguments.add_argument("-m", "--schema", action='store_true', help="Add Schema.org metadata (default false)",    required=False)
 37 | 
 38 | args = arguments.parse_args()
 39 | tweet_id = args.id
 40 | 
 41 | #	Get settings from arguments
 42 | thread_show  = True if args.thread else False
 43 | css_show     = True if args.css    else False
 44 | pretty_print = True if args.pretty else False
 45 | save_file    = True if args.save   else False
 46 | schema_org   = True if args.schema else False
 47 | 
 48 | def image_to_inline( url ) : 
 49 | 	#	Download the image
 50 | 	image_file = requests.get( url )
 51 | 	#	Convert to bytes
 52 | 	image_file = Image.open( io.BytesIO( image_file.content ) )
 53 | 	#	Temp file name with only alphanumeric characters
 54 | 	temp_file_name = "".join(x for x in url if x.isalnum())
 55 | 	#	Full path of the image
 56 | 	output_img = os.path.join( tempfile.gettempdir() , f"{temp_file_name}.webp" )
 57 | 	#	Save as a low quality WebP
 58 | 	image_file.save( output_img, 'webp', optimize=True, quality=60 )
 59 | 	#   Convert image to base64 data URl
 60 | 	#	Read the image from disk
 61 | 	binary_img      = open( output_img, 'rb' ).read()
 62 | 	#	Encode to Base64
 63 | 	base64_utf8_str = base64.b64encode( binary_img ).decode('utf-8')
 64 | 	#	Delete the temporary file
 65 | 	os.remove( output_img )
 66 | 	#	Return as data encoded suitable for an <img src="...">
 67 | 	return f'data:image/webp;base64,{base64_utf8_str}'
 68 | 
 69 | def tweet_entities_to_html(text, entities):
 70 | 	#	Initialize a list to hold parts of the HTML output
 71 | 	html_parts = []
 72 | 
 73 | 	#	Current position in the text we are processing
 74 | 	last_index = 0
 75 | 
 76 | 	#	Combine all entities into one list and sort by the start index
 77 | 	all_entities = []
 78 | 
 79 | 	#	Process URls - show the display URl and link to the expanded URl, bypassing t.co
 80 | 	if 'urls' in entities:
 81 | 		all_entities.extend(
 82 | 			[
 83 | 				(
 84 | 					url['url'], 
 85 | 					f"<a href='{url['expanded_url']}'>{url['display_url']}</a>", 
 86 | 					url['indices'][0], 
 87 | 					url['indices'][1]
 88 | 				) for url in entities['urls']
 89 | 			]
 90 | 		)
 91 | 	
 92 | 	#	Link hashtags to Twitter
 93 | 	if 'hashtags' in entities:
 94 | 		all_entities.extend(
 95 | 			[
 96 | 				(
 97 | 					f"#{hashtag['text']}", 
 98 | 					f"<a href='https://twitter.com/hashtag/{hashtag['text']}'>#{hashtag['text']}</a>",
 99 | 					hashtag['indices'][0], 
100 | 					hashtag['indices'][1]
101 | 				) for hashtag in entities['hashtags']
102 | 			]
103 | 		)
104 | 	
105 | 	#	Link user mentions to Twitter
106 | 	if 'user_mentions' in entities:
107 | 		all_entities.extend(
108 | 			[
109 | 				(
110 | 					f"@{mention['screen_name']}", 
111 | 					f"<a href='https://twitter.com/{mention['screen_name']}'>@{mention['screen_name']}</a>", 
112 | 					mention['indices'][0], 
113 | 					mention['indices'][1]
114 | 				) for mention in entities['user_mentions']
115 | 			]
116 | 		)
117 | 
118 | 	#	Link media to Twitter (will also embed later on)
119 | 	if 'media' in entities:
120 | 		all_entities.extend(
121 | 			[
122 | 				(
123 | 					f"{media['url']}", 
124 | 					f"<a href='{media['expanded_url']}'>{media['display_url']}</a>", 
125 | 					media['indices'][0], 
126 | 					media['indices'][1]
127 | 				) for media in entities['media']
128 | 			]
129 | 		)
130 | 
131 | 	#	Sort entities by start index
132 | 	all_entities.sort(key=lambda e: e[2])
133 | 
134 | 	#	Iterate over entities to build HTML
135 | 	for entity_text, replacement, start_index, end_index in all_entities:
136 | 		#	Add text between the last processed entity and the current one
137 | 		html_parts.append(text[last_index:start_index])
138 | 		#	Add the HTML replacement for the current entity
139 | 		html_parts.append(replacement)
140 | 		#	Update the last index to the end of the current entity
141 | 		last_index = end_index
142 | 
143 | 	#	Add the remaining text after the last entity
144 | 	html_parts.append(text[last_index:])
145 | 
146 | 	#	Join all parts into a single HTML string
147 | 	return ''.join(html_parts)
148 | 
149 | def get_media( mediaDetails) :
150 | 	media_html = '<div class="social-embed-media-grid">'
151 | 	#	Iterate through the attached media
152 | 	for media in mediaDetails :
153 | 		#	Convert small version of media to embedded WebP
154 | 		print( "Embedding media…" )
155 | 		media_img = image_to_inline( media["media_url_https"] + ":small" )
156 | 		
157 | 		#	Find alt text
158 | 		media_alt = ""
159 | 		if "ext_alt_text" in media :
160 | 			media_alt = html.escape( media["ext_alt_text"] )
161 | 	
162 | 		#	Is this a video or an image?
163 | 		if "video_info" in media :
164 | 			print( "Video poster…" )
165 | 			#	Embed the poster in the <video>, link to last video which should be highest quality
166 | 			#	TODO! Find a better way to get the best video
167 | 			media_html += f'''
168 | 			<video class='social-embed-video' controls src="{media["video_info"]["variants"][-1]["url"]}" poster="{media_img}" width="550"></video>
169 | 			'''
170 | 		else:
171 | 			#	Embed the image
172 | 			media_html += f'''
173 | 			<a href="{media['media_url_https']}" class="social-embed-media-link"><img class="social-embed-media" alt="{media_alt}" src="{media_img}"></a>
174 | 			'''
175 | 	return media_html + "</div>"
176 | 
177 | def get_card_html( card_data ) :
178 | 	poll_html = ""
179 | 	#	As per https://github.com/igorbrigadir/twitter-advanced-search
180 | 	card_name = card_data["name"] 
181 | 	print( "Card of type " + card_name )
182 | 
183 | 	#	Poll
184 | 	if card_name == "poll2choice_text_only" or card_name == "poll3choice_text_only" or card_name == "poll4choice_text_only" :
185 | 		#	Default values
186 | 		poll_1_label = ""
187 | 		poll_2_label = ""
188 | 		poll_3_label = ""
189 | 		poll_4_label = ""
190 | 		poll_1_count = 0
191 | 		poll_2_count = 0
192 | 		poll_3_count = 0
193 | 		poll_4_count = 0
194 | 
195 | 		#	Get labels and counts
196 | 		if "choice1_label" in card_data["binding_values"] :
197 | 			poll_1_label = html.escape( card_data["binding_values"]["choice1_label"]["string_value"] )
198 | 			poll_1_count = (int) (card_data["binding_values"]["choice1_count"]["string_value"])
199 | 		if "choice2_label" in card_data["binding_values"] :
200 | 			poll_2_label = html.escape( card_data["binding_values"]["choice2_label"]["string_value"] )
201 | 			poll_2_count = (int) (card_data["binding_values"]["choice2_count"]["string_value"])
202 | 		if "choice3_label" in card_data["binding_values"] :
203 | 			poll_3_label = html.escape( card_data["binding_values"]["choice3_label"]["string_value"] )
204 | 			poll_3_count = (int) (card_data["binding_values"]["choice3_count"]["string_value"])
205 | 		if "choice4_label" in card_data["binding_values"] :
206 | 			poll_4_label = html.escape( card_data["binding_values"]["choice4_label"]["string_value"] )
207 | 			poll_4_count = (int) (card_data["binding_values"]["choice4_count"]["string_value"])
208 | 
209 | 		#	Calculate the percentages. Round to 1 decimal place.
210 | 		poll_total = poll_1_count + poll_2_count + poll_3_count + poll_4_count
211 | 		poll_1_percent = '{0:.1f}'.format( (poll_1_count / poll_total) * 100 )
212 | 		poll_2_percent = '{0:.1f}'.format( (poll_2_count / poll_total) * 100 )
213 | 		poll_3_percent = '{0:.1f}'.format( (poll_3_count / poll_total) * 100 )
214 | 		poll_4_percent = '{0:.1f}'.format( (poll_4_count / poll_total) * 100 )
215 | 
216 | 		#	Generate semantic HTML
217 | 		if poll_1_label != "" :
218 | 			poll_html += f'''
219 | 				<hr class="social-embed-hr">
220 | 				<label for="poll_1_count">{poll_1_label}: ({poll_1_count:n})</label><br>
221 | 				<meter class="social-embed-meter" id="poll_1_count" min="0" max="100" low="33" high="66" value="{poll_1_percent}">{poll_1_count}</meter><br>
222 | 			'''
223 | 		if poll_2_label != "" :
224 | 			poll_html += f'''
225 | 				<label for="poll_2_count">{poll_2_label}: ({poll_2_count:n})</label><br>
226 | 				<meter class="social-embed-meter" id="poll_2_count" min="0" max="100" low="33" high="66" value="{poll_2_percent}">{poll_2_count}</meter><br>
227 | 			'''
228 | 		if poll_3_label != "" :
229 | 			poll_html += f'''
230 | 				<label for="poll_3_count">{poll_3_label}: ({poll_3_count:n})</label><br>
231 | 				<meter class="social-embed-meter" id="poll_3_count" min="0" max="100" low="33" high="66" value="{poll_3_percent}">{poll_3_count}</meter><br>
232 | 			'''
233 | 		if poll_4_label != "" :
234 | 			poll_html += f'''
235 | 				<label for="poll_4_count">{poll_4_label}: ({poll_4_count:n})</label><br>
236 | 				<meter class="social-embed-meter" id="poll_4_count" min="0" max="100" low="33" high="66" value="{poll_4_percent}">{poll_4_count}</meter>
237 | 			'''
238 | 		return poll_html
239 | 
240 | 	#	Photo Card
241 | 	if "summary_large_image" == card_name :
242 | 		card_vanity           = ""
243 | 		card_vanity_html      = ""
244 | 		card_title            = ""
245 | 		card_title_html       = ""
246 | 		card_description      = ""
247 | 		card_description_html = ""
248 | 		card_thumbnail        = ""
249 | 		card_thumbnail_html   = ""
250 | 		card_thumbnail_alt    = ""
251 | 		card_url              = ""
252 | 		card_html             = ""
253 | 
254 | 		if "vanity_url" in card_data["binding_values"] :
255 | 			card_vanity = card_data["binding_values"]["vanity_url"]["string_value"]
256 | 			card_vanity_html = f"{card_vanity}<br>"
257 | 		
258 | 		if "title" in card_data["binding_values"] :
259 | 			card_title = card_data["binding_values"]["title"]["string_value"]
260 | 			card_title_html = f"{card_title}<br>"
261 | 		
262 | 		if "description" in card_data["binding_values"] :
263 | 			card_description = card_data["binding_values"]["description"]["string_value"]
264 | 			card_description_html = f"{card_description}<br>"
265 | 		
266 | 		if "summary_photo_image_alt_text" in card_data["binding_values"] :
267 | 			card_thumbnail_alt = html.escape( card_data["binding_values"]["summary_photo_image_alt_text"]["string_value"] )
268 | 		
269 | 		if "thumbnail_image" in card_data["binding_values"] :
270 | 			print( "Converting card's thumbnail_image…" )
271 | 			card_thumbnail = card_data["binding_values"]["thumbnail_image"]["image_value"]["url"]
272 | 			#   Convert  media to embedded WebP
273 | 			card_thumbnail = image_to_inline( card_thumbnail )
274 | 			card_thumbnail_html = f'''
275 | 				<img src="{card_thumbnail}" alt="{card_thumbnail_alt}" class="social-embed-media">
276 | 				'''
277 | 		
278 | 		if "url" in card_data :
279 | 			card_url = card_data["url"]
280 | 
281 | 		card_html += f'''
282 | 			<a href="{card_url}" class="social-embed-card">
283 | 				{card_thumbnail_html}
284 | 				{card_vanity_html}
285 | 				{card_title_html}
286 | 				{card_description_html}
287 | 			</a>
288 | 		'''
289 | 		return card_html
290 | 
291 | def tweet_to_html( tweet_data ) :
292 | 	#	Show the thread / quote?
293 | 	tweet_parent = ""
294 | 	tweet_quote  = ""
295 | 	if thread_show :
296 | 		if "parent" in tweet_data :
297 | 			print( "Parent detected…" )
298 | 			tweet_parent = tweet_to_html( tweet_data["parent"] )
299 | 		if "quoted_tweet" in tweet_data :
300 | 			print( "Quote detected…" )
301 | 			tweet_quote = tweet_to_html( tweet_data["quoted_tweet"] )
302 | 
303 | 	#	Take the data from the API of a single Tweet (which might also be a quote or reply).
304 | 	#	Create a semantic HTML representation
305 | 	#   Tweet Information
306 | 	tweet_id       = tweet_data["id_str"]
307 | 	tweet_name     = tweet_data["user"]["name"]
308 | 	tweet_user     = tweet_data["user"]["screen_name"]
309 | 	tweet_avatar   = tweet_data["user"]["profile_image_url_https"]
310 | 	tweet_shape    = tweet_data["user"].get("profile_image_shape", "Square")#	Some users don't have this
311 | 	tweet_text     = tweet_data["text"]
312 | 	tweet_date     = tweet_data["created_at"] 
313 | 	tweet_lang     = tweet_data["lang"]
314 | 	print( f"Detected language - {tweet_lang}" )
315 | 
316 | 	tweet_likes    = (int)(tweet_data.get("favorite_count",     0))#	Might not exist
317 | 	tweet_replies  = (int)(tweet_data.get("conversation_count", 0))#	Might not exist
318 | 	tweet_retweets = (int)(tweet_data.get("retweet_count",      0))#	Might not exist
319 | 	tweet_entities = tweet_data["entities"]
320 | 	global tweet_url
321 | 	tweet_url      = f"https://twitter.com/{tweet_user}/status/{tweet_id}"
322 | 
323 | 	#	User labels
324 | 	if "highlighted_label" in tweet_data["user"] :
325 | 		tweet_label     = html.escape( tweet_data["user"]["highlighted_label"]["description"] )
326 | 		tweet_badge_img = image_to_inline( tweet_data["user"]["highlighted_label"]["badge"]["url"] )
327 | 		print( f"Badge found '{tweet_label}'…")
328 | 		tweet_badge  = f'<br><img src="{tweet_badge_img}" alt="" class="social-embed-badge"> {tweet_label}'
329 | 	else :
330 | 		tweet_badge = ""
331 | 
332 | 	#	Get the datetime
333 | 	tweet_time = parser.parse( tweet_date )
334 | 	tweet_time = tweet_time.strftime('%H:%M - %a %d %B %Y')
335 | 
336 | 	#	Is this a reply?
337 | 	if "in_reply_to_screen_name" in tweet_data :
338 | 		tweet_reply_to   = tweet_data["in_reply_to_screen_name"]
339 | 		tweet_reply_id = tweet_data.get("in_reply_to_status_id_str","")	#	Doesn't exist on older tweets
340 | 		if "" == tweet_reply_id :
341 | 			tweet_reply_link = f'https://twitter.com/{tweet_reply_to}'
342 | 		else :
343 | 			tweet_reply_link = f'https://twitter.com/{tweet_reply_to}/status/{tweet_reply_id}'
344 | 		tweet_reply = f'''
345 | 			<small class="social-embed-reply"><a href="{tweet_reply_link}">Replying to @{tweet_reply_to}</a></small>
346 | 		'''
347 | 	else :
348 | 		tweet_reply = ""
349 | 
350 | 	#   Embed entities
351 | 	tweet_text = tweet_entities_to_html( tweet_text, tweet_entities )
352 | 
353 | 	#	Add media
354 | 	tweet_media = ""
355 | 	if ( "mediaDetails" in tweet_data ) :
356 | 		tweet_media = get_media( tweet_data["mediaDetails"] )
357 | 
358 | 	#	Add card
359 | 	tweet_card = ""
360 | 	if "card" in tweet_data :
361 | 		tweet_card = get_card_html( tweet_data["card"] )
362 | 
363 | 	#   Newlines to BR
364 | 	tweet_text = tweet_text.replace("\n","<br>")
365 | 
366 | 	#   Convert avatar to embedded WebP
367 | 	print( "Storing avatar…")
368 | 	tweet_avatar = image_to_inline( tweet_avatar )
369 | 
370 | 	#	Avatar shape
371 | 	if tweet_shape == "Circle" :
372 | 		avatar_shape = "social-embed-avatar-circle"
373 | 	elif tweet_shape == "Square" :
374 | 		avatar_shape = "social-embed-avatar-square"
375 | 	else :
376 | 		avatar_shape = "social-embed-avatar-circle"
377 | 
378 | 	#	Schema.org metadata
379 | 	schema_post   = ' itemscope itemtype="https://schema.org/SocialMediaPosting"'       if schema_org else ""
380 | 	schema_body   = ' itemprop="articleBody"'   if schema_org else ""
381 | 	schema_time   = ' itemprop="datePublished"' if schema_org else ""
382 | 	schema_author = ' itemprop="author" itemscope itemtype="https://schema.org/Person"' if schema_org else ""
383 | 	schema_url    = ' itemprop="url"'   if schema_org else ""
384 | 	schema_image  = ' itemprop="image"' if schema_org else ""
385 | 	schema_name   = ' itemprop="name"'  if schema_org else ""
386 | 
387 | 	#   HTML
388 | 	tweet_html = f'''
389 | 	<blockquote class="social-embed" id="social-embed-{tweet_id}" lang="{tweet_lang}"{schema_post}>
390 | 		{tweet_parent}
391 | 		<header class="social-embed-header"{schema_author}>
392 | 			<a href="https://twitter.com/{tweet_user}" class="social-embed-user"{schema_url}>
393 | 				<img class="social-embed-avatar {avatar_shape}" src="{tweet_avatar}" alt=""{schema_image}>
394 | 				<div class="social-embed-user-names">
395 | 					<p class="social-embed-user-names-name"{schema_name}>{tweet_name}</p>@{tweet_user}{tweet_badge}
396 | 				</div>
397 | 			</a>
398 | 			<img class="social-embed-logo" alt="Twitter" src='data:image/svg+xml,%3Csvg%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%0Aaria-label%3D%22Twitter%22%20role%3D%22img%22%0AviewBox%3D%220%200%20512%20512%22%3E%3Cpath%0Ad%3D%22m0%200H512V512H0%22%0Afill%3D%22%23fff%22%2F%3E%3Cpath%20fill%3D%22%231d9bf0%22%20d%3D%22m458%20140q-23%2010-45%2012%2025-15%2034-43-24%2014-50%2019a79%2079%200%2000-135%2072q-101-7-163-83a80%2080%200%200024%20106q-17%200-36-10s-3%2062%2064%2079q-19%205-36%201s15%2053%2074%2055q-50%2040-117%2033a224%20224%200%2000346-200q23-16%2040-41%22%2F%3E%3C%2Fsvg%3E'>
399 | 		</header>
400 | 		<section class="social-embed-text"{schema_body}>
401 | 			{tweet_reply}
402 | 			{tweet_text}
403 | 			{tweet_media}
404 | 			{tweet_card}
405 | 			{tweet_quote}
406 | 		</section>
407 | 		<hr class="social-embed-hr">
408 | 		<footer class="social-embed-footer">
409 | 			<a href="{tweet_url}">
410 | 				<span aria-label="{tweet_likes} likes" class="social-embed-meta">❤️ {tweet_likes:n}</span>
411 | 				<span aria-label="{tweet_replies} replies" class="social-embed-meta">💬 {tweet_replies:n}</span>
412 | 				<span aria-label="{tweet_retweets} reposts" class="social-embed-meta">🔁 {tweet_retweets:n}</span>
413 | 				<time datetime="{tweet_date}"{schema_time}>{tweet_time}</time>
414 | 			</a>
415 | 		</footer>
416 | 	</blockquote>
417 | 	'''
418 | 	return tweet_html
419 | 
420 | #   Get the data from the Twitter embed API
421 | for _ in range(5):
422 | 	#	Lazy retry strategy
423 | 	try :
424 | 		print( "Downloading data…" )
425 | 		token = random.randint(1,10000)
426 | 		json_url =  f"https://cdn.syndication.twimg.com/tweet-result?id={tweet_id}&lang=en&token={token}"
427 | 		response = requests.get(json_url)
428 | 		data = response.json()
429 | 		break
430 | 	except :
431 | 		print( "Retrying…" )
432 | 		continue
433 | 
434 | #	If Tweet was deleted, exit.
435 | if "TweetTombstone" == data["__typename"] :
436 | 	print( "This Post was deleted by the Post author." )
437 | 	raise SystemExit
438 | 
439 | #	Turn the Tweet into HTML
440 | tweet_html = tweet_to_html(data)
441 | 
442 | #   Generate Content to be pasted
443 | 
444 | #	CSS
445 | tweet_css = '''
446 | <style>
447 | .social-embed {
448 | 	all: unset;
449 | 	display: block;
450 | }
451 | 
452 | .social-embed * {
453 | 	all: unset;
454 | 	display: revert;
455 | }
456 | 
457 | .social-embed::after {
458 | 	all: unset;
459 | }
460 | 
461 | .social-embed::before {
462 | 	all: unset;
463 | }
464 | 
465 | blockquote:not(*) {
466 | 	all: unset;
467 | }
468 | 
469 | .social-embed a {
470 | 	cursor: pointer;
471 | }
472 | 
473 | blockquote.social-embed {
474 | 	box-sizing: border-box;
475 | 	border: .5px solid;
476 | 	width: 550px;
477 | 	max-width: 100%;
478 | 	font-family: sans-serif;
479 | 	margin: auto;
480 | 	margin-bottom: .5em;
481 | 	padding: 1em;
482 | 	border-radius: .5em;
483 | 	background-color: #FFF;
484 | 	color: #000;
485 | 	display: block;
486 | 	white-space: normal;
487 | }
488 | 
489 | .social-embed-header {
490 | 	display: flex;
491 | 	justify-content: space-between;
492 | }
493 | 
494 | .social-embed-user {
495 | 	display: flex;
496 | 	position: relative;
497 | 	align-items: center;
498 | 	text-decoration: none;
499 | 	color: inherit;
500 | }
501 | 
502 | .social-embed-avatar {
503 | 	width: 3em;
504 | 	height: 3em;
505 | 	margin-right: .5em;
506 | }
507 | 
508 | .social-embed-avatar-circle {
509 | 	border-radius: 50%;
510 | }
511 | 
512 | .social-embed-avatar-square {
513 | 	border-radius: 5%;
514 | }
515 | 
516 | .social-embed-user-names-name {
517 | 	display: flex;
518 | 	align-items: center;
519 | 	font-weight: bold;
520 | 	margin: 0;
521 | }
522 | 
523 | .social-embed-text {
524 | 	margin-top: .5em;
525 | }
526 | 
527 | .social-embed-footer a {
528 | 	display: flex;
529 | 	align-items: center;
530 | 	justify-content: space-between;
531 | }
532 | 
533 | .social-embed-logo {
534 | 	width: 3em;
535 | }
536 | 
537 | .social-embed-hr {
538 | 	border: .1px solid;
539 | 	margin: .5em 0 .5em 0;
540 | }
541 | 
542 | .social-embed-meta {
543 | 	text-decoration: none !important;
544 | 	color: unset !important;
545 | }
546 | 
547 | .social-embed-reply {
548 | 	display: block;
549 | }
550 | 
551 | .social-embed-text a, .social-embed-footer time {
552 | 	color: blue;
553 | 	text-decoration: underline;
554 | }
555 | 
556 | .social-embed-media-grid {
557 | 	display: flex;
558 | 	flex-wrap: wrap;
559 | }
560 | 
561 | .social-embed-media-link {
562 | 	flex-grow: 1;
563 | 	width: 50%;
564 | }
565 | 
566 | .social-embed-media, .social-embed-video {
567 | 	padding: .1em;
568 | 	width: 100%;
569 | 	border-radius: .5em;
570 | 	max-width: 100%;
571 | 	object-fit: cover;
572 | }
573 | 
574 | .social-embed-reply {
575 | 	font-size: .75em;
576 | 	display: block;
577 | }
578 | 
579 | .social-embed-meter {
580 | 	width: 100%;
581 | 	background: #0005;
582 | }
583 | 
584 | .social-embed-card {
585 | 	text-decoration: none !important;
586 | 	color: unset !important;
587 | 	border: .5px solid;
588 | 	display: block;
589 | 	font-size: .85em;
590 | 	padding: .5em;
591 | 	border-radius: .5em;
592 | }
593 | 
594 | .social-embed-badge {
595 | 	height: 1em;
596 | 	vertical-align: text-top;
597 | }
598 | 
599 | .social-embed-text p {
600 | 	margin-bottom: 1em;
601 | }
602 | 
603 | .social-embed-emoji {
604 | 	display: inline;
605 | 	width: 1em;
606 | }
607 | </style>
608 | '''
609 | 
610 | #	Add the CSS to the output if requsted
611 | if css_show :
612 | 	tweet_html = tweet_css + tweet_html
613 | 
614 | #   Compact the output if necessary
615 | if not pretty_print :
616 | 	print( "Compacting…")
617 | 	tweet_html = tweet_html.replace("\n", "")
618 | 	tweet_html = tweet_html.replace("\t", "")
619 | 
620 | #   Copy to clipboard
621 | pyperclip.copy( tweet_html )
622 | #   Print to say we've finished
623 | print( f"Copied {tweet_id}" )
624 | 
625 | if save_file :
626 | 	#	Save HTML
627 | 	#   Save directory
628 | 	output_directory = "output"
629 | 	os.makedirs(output_directory, exist_ok = True)
630 | 	save_location = os.path.join( output_directory, f"{tweet_id}.html" ) 
631 | 	#   Save as HTML file
632 | 	with open( save_location, 'w', encoding="utf-8" ) as html_file:
633 | 		html_file.write( tweet_html )
634 | 	print( f"Saved to {save_location}" )
635 | 
636 | #	Submit the Tweet to Archive.org
637 | print( f"Archiving… {tweet_url}" )
638 | requests.post( "https://web.archive.org/save/", data={"url": tweet_url, "capture_all":"on"}, timeout=5 )
639 | 


--------------------------------------------------------------------------------
/tweet2img.py:
--------------------------------------------------------------------------------
  1 | #   For command line
  2 | import argparse
  3 | 
  4 | #   Sleeping
  5 | import time
  6 | 
  7 | #   File and Bits
  8 | import io
  9 | import os
 10 | import tempfile
 11 | 
 12 | #   Image Manipulation
 13 | from PIL import Image
 14 | 
 15 | #   Selenium
 16 | from selenium import webdriver 
 17 | from selenium.webdriver.common.keys import Keys
 18 | from selenium.webdriver.common.by import By
 19 | 
 20 | #   Firefox specific
 21 | from selenium.webdriver.firefox.options import Options
 22 | from selenium.webdriver.firefox.service import Service
 23 | from webdriver_manager.firefox import GeckoDriverManager
 24 | 
 25 | #   Etc
 26 | import requests
 27 | import pyperclip
 28 | import base64
 29 | import html
 30 | 
 31 | #   If using Chrome
 32 | # from selenium.webdriver.chrome.options import Options
 33 | 
 34 | #   Command line options
 35 | parser = argparse.ArgumentParser(
 36 |     prog='tweet2img',
 37 |     description='Convert a Tweet ID to an image and alt text')
 38 | parser.add_argument('id',       type=int,            help='ID of the Tweet (integer)')
 39 | parser.add_argument('--thread', action='store_true', help='Show the thread (default false)', required=False)
 40 | 
 41 | args = parser.parse_args()
 42 | tweet_id = args.id
 43 | thread = args.thread
 44 | 
 45 | if ( True == thread ):
 46 |     hide_thread = "false"
 47 | else :
 48 |     hide_thread = "true"
 49 | 
 50 | # #   Chrome's Headless Options
 51 | # chrome_options = Options()
 52 | # chrome_options.add_argument('--headless=new')
 53 | # chrome_options.add_argument('--window-size=1920,2160')
 54 | 
 55 | # #   Turn off everything
 56 | # chrome_options.add_argument("--no-sandbox")
 57 | # chrome_options.add_argument("--disable-dev-shm-usage")
 58 | # chrome_options.add_argument("--disable-extensions")
 59 | # chrome_options.add_argument("--disable-infobars")
 60 | # chrome_options.add_argument("--disable-logging")
 61 | # chrome_options.add_argument("--log-level=3")
 62 | # chrome_options.add_argument('--force-device-scale-factor=1')
 63 | # chrome_options.add_argument('--high-dpi-support=1')
 64 | 
 65 | # #   Wayland to stop fuzzyness on fractional scaling
 66 | # chrome_options.add_argument("--enable-features=UseOzonePlatform")
 67 | # chrome_options.add_argument("--ozone-platform=wayland")
 68 | 
 69 | # #   Start Chrome
 70 | # driver = webdriver.Chrome(options=chrome_options)
 71 | 
 72 | #   Firefox's Headless Options
 73 | firefox_options = Options()
 74 | firefox_options.add_argument("--headless")
 75 | #   Start Firefox
 76 | driver = webdriver.Firefox( options=firefox_options )
 77 | 
 78 | #   Open the Tweet on the embed platform
 79 | driver.get(f"https://platform.twitter.com/embed/Tweet.html?hideCard=false&hideThread={hide_thread}&lang=en&theme=light&width=550px&id={tweet_id}")
 80 | 
 81 | #   Wait for page to fully render
 82 | time.sleep(3)
 83 | 
 84 | #   Get the Tweet
 85 | tweet = driver.find_element(By.TAG_NAME, "article")
 86 | #   Use the parent element for more padding
 87 | tweet = driver.execute_script("return arguments[0].parentNode;", tweet)
 88 | 
 89 | #   Get Screenshot
 90 | image_binary = tweet.screenshot_as_png
 91 | img = Image.open(io.BytesIO(image_binary))
 92 | width  = img.width
 93 | height = img.height
 94 | 
 95 | #   Resize to a maximum width (useful if on HiDPI screen)
 96 | max_width = 550
 97 | resize_factor = width // max_width
 98 | (width, height) = ( int(img.width // resize_factor), int(img.height // resize_factor))
 99 | img = img.resize( (width, height), Image.Resampling.LANCZOS )
100 | 
101 | #   Save directory
102 | output_directory = "output"
103 | os.makedirs(output_directory, exist_ok = True)
104 | 
105 | #   Optimal(ish) quality
106 | output_img = os.path.join(output_directory, f"{tweet_id}.webp")
107 | img.save( output_img, 'webp', optimize=True, quality=60 )
108 | 
109 | #   Kill the driver
110 | driver.quit()
111 | 
112 | #   Generate Alt Text
113 | tweet_alt  = ""
114 | ptweet_alt = ""
115 | qtweet_alt = ""
116 | 
117 | #   Get the data
118 | json_url =  f"https://cdn.syndication.twimg.com/tweet-result?id={tweet_id}&lang=en&token=1"
119 | response = requests.get(json_url)
120 | data = response.json()
121 | 
122 | #   Is this a thread?
123 | if ( "parent" in data and hide_thread == "false" ) :
124 |     ptweet_text = data["parent"]["text"]
125 |     ptweet_name = data["parent"]["user"]["name"] + " (@" + data["parent"]["user"]["screen_name"] + ")"
126 |     ptweet_date = data["parent"]["created_at"]
127 |     if "mediaDetails" in data["parent"]:
128 |         for media in data["parent"]["mediaDetails"] :
129 |             if "ext_alt_text" in media :
130 |                 ptweet_text += " . Image: " + media["ext_alt_text"]
131 |     ptweet_alt += f"{ptweet_date}. {ptweet_name}. {ptweet_text}. Reply "
132 | 
133 | #   Text of Tweet
134 | tweet_text = data["text"]
135 | tweet_name = data["user"]["name"] + " (@" + data["user"]["screen_name"] + ")"
136 | tweet_date = data["created_at"] 
137 | if "mediaDetails" in data:
138 |     for media in data["mediaDetails"] :
139 |         if "ext_alt_text" in media :
140 |             tweet_text += " . Image: " + media["ext_alt_text"]
141 | tweet_alt += f"{tweet_date}. {tweet_name}. {tweet_text}."
142 | 
143 | 
144 | #   Is this a quote Tweet?
145 | if ( "quoted_tweet" in data and hide_thread == "false" ) :
146 |     qtweet_text = data["quoted_tweet"]["text"]
147 |     qtweet_name = data["quoted_tweet"]["user"]["name"] + " (@" + data["quoted_tweet"]["user"]["screen_name"] + ")"
148 |     qtweet_date = data["quoted_tweet"]["created_at"]
149 |     if "mediaDetails" in data["quoted_tweet"]:
150 |         for media in data["quoted_tweet"]["mediaDetails"] :
151 |             if "ext_alt_text" in media :
152 |                 qtweet_text += " . Image: " + media["ext_alt_text"]
153 |     qtweet_alt += f" Quoting: {qtweet_date}. {qtweet_name}. {qtweet_text}."
154 | 
155 | #   Stick it all together
156 | tweet_alt = f"Screenshot from Twitter. {ptweet_alt}{tweet_alt}{qtweet_alt}".replace("\n", " ")
157 | 
158 | #   Save as a text file
159 | with open(  os.path.join( output_directory, f"{tweet_id}.txt" ) , 'w', encoding="utf-8" ) as text_file:
160 |     text_file.write( tweet_alt )
161 | 
162 | #   Generate HTML to be pasted
163 | 
164 | #   Link
165 | tweet_url = "https://twitter.com/" + data["user"]["screen_name"] + "/status/" + data["id_str"]
166 | 
167 | #   Convert image to base64 data URl
168 | binary_img      = open(output_img, 'rb').read()
169 | base64_utf8_str = base64.b64encode(binary_img).decode('utf-8')
170 | data_url = f'data:image/webp;base64,{base64_utf8_str}'
171 | 
172 | #   Ensure alt is sanitised
173 | tweet_alt = html.escape(tweet_alt)
174 | 
175 | #   HTML to be pasted
176 | tweet_html = f"<a href=\"{tweet_url}\"><img src=\"{data_url}\" width=\"{width}\" height=\"{height}\" alt=\"{tweet_alt}\"/></a>"
177 | 
178 | #   Copy to clipboard
179 | pyperclip.copy( tweet_html )
180 | 
181 | #   Print to say we've finished
182 | print( f"Copied {tweet_url}" )


--------------------------------------------------------------------------------
/tweet2json.py:
--------------------------------------------------------------------------------
 1 | #   For command line
 2 | import argparse
 3 | 
 4 | #   File and Bits
 5 | import os
 6 | import json
 7 | 
 8 | #   Etc
 9 | import requests
10 | import random
11 | from urllib.parse import urlparse
12 | 
13 | def is_valid_url(url):
14 | 	try:
15 | 		result = urlparse(url)
16 | 		return all([result.scheme, result.netloc, result.path])
17 | 	except ValueError:
18 | 		return False
19 | 
20 | #   Command line options
21 | arguments = argparse.ArgumentParser(
22 | 	prog='tweet2json',
23 | 	description='Download the JSON representation of any public Tweet')
24 | arguments.add_argument("id", type=str,                        help="ID of the Tweet (integer)")
25 | arguments.add_argument("-p", "--pretty", action="store_true", help="Pretty Print the output (default false)",    required=False)
26 | 
27 | args = arguments.parse_args()
28 | 
29 | if (is_valid_url(args.id)):
30 | 	url_parts = result = urlparse(args.id)
31 | 	tweet_id = url_parts.path.split("/")[-1]
32 | else :
33 | 	tweet_id = args.id
34 | 
35 | if (tweet_id.isdecimal() is False) :
36 | 	print( "No valid Tweet ID found." )
37 | 	raise SystemExit
38 | 
39 | pretty_print = True if args.pretty else False
40 | 
41 | #   Get the data from the Twitter embed API
42 | for _ in range(5):
43 | 	#	Lazy retry strategy
44 | 	try :
45 | 		print( "Downloading data…" )
46 | 		token = random.randint(1,10000)
47 | 		json_url =  f"https://cdn.syndication.twimg.com/tweet-result?id={tweet_id}&lang=en&token={token}"
48 | 		response = requests.get(json_url)
49 | 		data = response.json()
50 | 		break
51 | 	except :
52 | 		print( "Retrying…" )
53 | 		continue
54 | 
55 | #	If Tweet was deleted, exit.
56 | if "TweetTombstone" == data["__typename"] :
57 | 	print( "This Post was deleted by the Post author." )
58 | 	raise SystemExit
59 | 
60 | if (pretty_print) :
61 | 	twitter_json = json.dumps(data, indent=3)
62 | else :
63 | 	twitter_json = json.dumps(data)
64 | 
65 | #   Save directory
66 | output_directory = "output"
67 | os.makedirs(output_directory, exist_ok = True)
68 | save_location = os.path.join( output_directory, f"{tweet_id}.json" ) 
69 | 
70 | #   Save as JSON
71 | with open( save_location, 'w', encoding="utf-8" ) as json_file:
72 | 	json_file.write( twitter_json )
73 | print( f"Saved to {save_location}" )


--------------------------------------------------------------------------------