4 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 | <\/div>)',
60 | current_html
61 | )
62 | for post in posts:
63 | if post not in existing_posts_html:
64 | existing_posts_html.append(post)
65 | for comment in comments:
66 | if comment not in existing_comments_html:
67 | existing_comments_html.append(comment)
68 | return existing_ids, existing_posts_html, existing_comments_html
69 |
70 |
71 | def get_saved_posts(client):
72 | """Gets a list of posts that the user has saved."""
73 |
74 | return [
75 | saved for saved in client.user.me().saved(limit=None)
76 | if saved.__class__.__name__ == "Submission"
77 | ]
78 |
79 |
80 | def get_upvoted_posts(client):
81 | """Gets a list of posts that the user has upvoted."""
82 |
83 | return [
84 | upvoted for upvoted in client.user.me().upvoted(limit=None)
85 | if upvoted.__class__.__name__ == "Submission"
86 | ]
87 |
88 |
89 | def get_saved_comments(client):
90 | """Gets a list of comments that the user has saved."""
91 |
92 | return [
93 | saved for saved in client.user.me().saved(limit=None)
94 | if saved.__class__.__name__ != "Submission"
95 | ]
96 |
97 |
98 | def get_user_posts(client, username):
99 | """Gets a list of posts that the user has made."""
100 |
101 | return [
102 | post for post in client.redditor(username).submissions.new(limit=None)
103 | ]
104 |
105 |
106 | def get_user_comments(client, username):
107 | """Gets a list of comments that the user has made."""
108 |
109 | return [
110 | comment for comment in client.redditor(username).comments.new(limit=None)
111 | ]
112 |
113 |
114 | def get_post_html(post):
115 | """Takes a post object and creates a HTML for it - but not including the
116 | preview HTML."""
117 |
118 | with open(os.path.join("html", "post-div.html"), encoding="utf-8") as f:
119 | html = f.read()
120 | dt = datetime.utcfromtimestamp(post.created_utc)
121 | html = html.replace("", post.title)
122 | html = html.replace("", f"/r/{str(post.subreddit)}")
123 | html = html.replace("", f"/u/{post.author.name}" if post.author else "[deleted]")
124 | html = html.replace("", f"posts/{post.id}.html")
125 | html = html.replace("", f"https://reddit.com{post.permalink}")
126 | html = html.replace("", post.url)
127 | html = html.replace("", post.id)
128 | html = html.replace("", (post.selftext_html or "").replace(
129 | '", str(dt))
132 | html = html.replace("", dt.strftime("%d %B, %Y"))
133 | return html
134 |
135 |
136 | def save_media(post, location):
137 | """Takes a post object and tries to download any image/video it might be
138 | associated with. If it can, it will return the filename."""
139 |
140 | url = post.url
141 | stripped_url = url.split("?")[0]
142 | if url.endswith(post.permalink): return None
143 |
144 | # What is the key information?
145 | extension = stripped_url.split(".")[-1].lower()
146 | domain = ".".join(post.url.split("/")[2].split(".")[-2:])
147 | readable_name = list(filter(bool, post.permalink.split("/")))[-1]
148 |
149 | # If it's an imgur gallery, forget it
150 | if domain == "imgur.com" and "gallery" in url: return None
151 |
152 | # Can the media be obtained directly?
153 | if extension in IMAGE_EXTENSIONS + VIDEO_EXTENSIONS:
154 | filename = f"{readable_name}_{post.id}.{extension}"
155 | try:
156 | response = requests.get(post.url)
157 | except:
158 | return
159 | media_type = response.headers.get("Content-Type", "")
160 | if media_type.startswith("image") or media_type.startswith("video"):
161 | with open(os.path.join(location, "media", filename), "wb") as f:
162 | f.write(response.content)
163 | return filename
164 |
165 | # Is this a reddit gallery?
166 | if domain == "reddit.com" and "gallery" in url:
167 | json_url = url + ".json"
168 | resp = requests.get(json_url)
169 |
170 | sleep = 1
171 | while resp.status_code == 429:
172 | time.sleep(sleep)
173 | print(f"Rate limited, sleeping for {sleep} seconds")
174 | resp = requests.get(json_url)
175 | sleep *= 2
176 |
177 | data = resp.json()
178 | post_data = data[0]["data"]["children"][0]["data"]
179 | media = post_data.get("media_metadata")
180 | if not media: return None
181 | filenames = []
182 | for idx, data in enumerate(list(media.values()), 1):
183 | if "m" not in data: continue
184 | ext = data["m"].split("/")[-1]
185 | if "u" in data["s"]:
186 | base_url = data["s"]["u"].replace("&", "&") # unescape URL
187 | else:
188 | continue
189 | response = requests.get(base_url)
190 | if response.status_code == 200:
191 | filename = f"{readable_name}_{post.id}_{idx}.{ext}"
192 | with open(os.path.join(location, "media", filename), "wb") as f:
193 | f.write(response.content)
194 | filenames.append(filename)
195 | return filenames[0] if filenames else None
196 |
197 | # Is this a v.redd.it link?
198 | if domain == "redd.it":
199 | downloader = Downloader(max_q=True, log=False)
200 | downloader.url = url
201 | current = os.getcwd()
202 | try:
203 | name = downloader.download()
204 | extension = name.split(".")[-1]
205 | filename = f"{readable_name}_{post.id}.{extension}"
206 | os.rename(name, os.path.join(location, "media", filename))
207 | return filename
208 | except:
209 | os.chdir(current)
210 | return None
211 |
212 | # Is it a gfycat link that redirects? Update the URL if possible
213 | if domain == "gfycat.com":
214 | html = requests.get(post.url).content
215 | if len(html) < 50000:
216 | match = re.search(r"http([\dA-Za-z\+\:\/\.]+)\.mp4", html.decode())
217 | if match:
218 | url = match.group()
219 | else:
220 | return None
221 |
222 | # Is this an imgur image?
223 | if domain == "imgur.com" and extension != "gifv":
224 | for extension in IMAGE_EXTENSIONS:
225 | direct_url = f'https://i.{url[url.find("//") + 2:]}.{extension}'
226 | direct_url = direct_url.replace("i.imgur.com", "imgur.com")
227 | direct_url = direct_url.replace("m.imgur.com", "imgur.com")
228 | try:
229 | response = requests.get(direct_url)
230 | except: continue
231 | if response.status_code == 200:
232 | filename = f"{readable_name}_{post.id}.{extension}"
233 | with open(os.path.join(location, "media", filename), "wb") as f:
234 | f.write(response.content)
235 | return filename
236 |
237 | # Try to use youtube_dl if it's one of the possible domains
238 | if domain in PLATFORMS:
239 | options = {
240 | "nocheckcertificate": True, "quiet": True, "no_warnings": True,
241 | "ignoreerrors": True, "no-progress": True,
242 | "outtmpl": os.path.join(
243 | location, "media", f"{readable_name}_{post.id}" + ".%(ext)s"
244 | )
245 | }
246 | with yt_dlp.YoutubeDL(options) as ydl:
247 | try:
248 | ydl.download([url])
249 | except:
250 | os.chdir(current)
251 | return
252 | for f in os.listdir(os.path.join(location, "media")):
253 | if f.startswith(f"{readable_name}_{post.id}"):
254 | return f
255 |
256 |
257 | def add_media_preview_to_html(post_html, media):
258 | """Takes post HTML and returns a modified version with the preview
259 | inserted."""
260 |
261 | extension = media.split(".")[-1]
262 | location = "/".join(["media", media])
263 | if extension in IMAGE_EXTENSIONS:
264 | return post_html.replace(
265 | "",
266 | f'
'
267 | )
268 | if extension in VIDEO_EXTENSIONS:
269 | return post_html.replace(
270 | "",
271 | f''
272 | )
273 | return post_html
274 |
275 |
276 | def create_post_page_html(post, post_html):
277 | """Creates the HTML for a post's own page."""
278 |
279 | with open(os.path.join("html", "post.html"), encoding="utf-8") as f:
280 | html = f.read()
281 | html = html.replace("", post.title)
282 | html = html.replace("", post_html.replace("h2>", "h1>").replace(
283 | '
", f"")
290 | with open(os.path.join("html", "main.js"), encoding="utf-8") as f:
291 | html = html.replace("", f"")
292 | comments_html = []
293 | post.comments.replace_more(limit=0)
294 | for comment in post.comments:
295 | comments_html.append(get_comment_html(
296 | comment, op=post.author.name if post.author else None
297 | ))
298 | html = html.replace("", "\n".join(comments_html))
299 | return html
300 |
301 |
302 | def get_comment_html(comment, children=True, op=None):
303 | """Takes a post object and creates a HTML for it - it will get its children
304 | too unless you specify otherwise."""
305 |
306 | with open(os.path.join("html", "comment-div.html"), encoding="utf-8") as f:
307 | html = f.read()
308 | dt = datetime.utcfromtimestamp(comment.created_utc)
309 | author = "[deleted]"
310 | if comment.author:
311 | if comment.author == op:
312 | author = f'/u/{comment.author.name}'
313 | else:
314 | author = f"/u/{comment.author.name}"
315 | html = html.replace("", author)
316 | html = html.replace("", (comment.body_html or "").replace(
317 | '", str(comment.score))
320 | html = html.replace("", f"https://reddit.com{comment.permalink}")
321 | html = html.replace("", str(dt))
322 | html = html.replace("", comment.id)
323 | html = html.replace("", dt.strftime("%H:%M - %d %B, %Y"))
324 | if children:
325 | children_html = []
326 | for child in comment.replies:
327 | children_html.append(get_comment_html(child, children=False, op=op))
328 | html = html.replace("", "\n".join(children_html))
329 | return html
330 |
331 |
332 | def save_html(posts, comments, location, html_file, page, has_next, username=None):
333 | if username:
334 | with open(os.path.join("html", "username.html"), encoding="utf-8") as f:
335 | html = f.read().replace("[username]", username)
336 | else:
337 | with open(os.path.join("html", html_file), encoding="utf-8") as f:
338 | html = f.read()
339 | with open(os.path.join("html", "style.css"), encoding="utf-8") as f:
340 | html = html.replace("", f"")
341 | with open(os.path.join("html", "main.js"), encoding="utf-8") as f:
342 | html = html.replace("", f"")
343 | if page == 0 or page is None:
344 | html = html.replace("Previous", "")
345 | else:
346 | html = html.replace(".p.html", f".{page-1}.html")
347 | if not has_next or page is None:
348 | html = html.replace("Next", "")
349 | else:
350 | html = html.replace(".n.html", f".{page+1}.html")
351 | html = html.replace("", "\n".join(posts))
352 | html = html.replace("", "\n".join(comments))
353 | file_name = html_file if page is None else html_file.replace(".html", f".{page}.html")
354 | with open(os.path.join(location, file_name), "w", encoding="utf-8") as f:
355 | f.write(html)
356 |
--------------------------------------------------------------------------------
Comments
12 | 13 |