├── README.md ├── requirements.txt ├── config.ini ├── host_list.json ├── LICENSE ├── RedditMirrorBot.py └── PublicFreakout.py /README.md: -------------------------------------------------------------------------------- 1 | README will be written eventually 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | praw 2 | youtube_dl 3 | boto3 4 | requests==2.18.4 5 | python-magic 6 | simplejson 7 | -------------------------------------------------------------------------------- /config.ini: -------------------------------------------------------------------------------- 1 | [General] 2 | ; current options are pomf and s3 (for now only digitalocean) 3 | backend_host= 4 | 5 | [Reddit] 6 | client_id= 7 | client_secret= 8 | password= 9 | username= 10 | user_agent= 11 | host_account= 12 | sub_to_mirror= 13 | 14 | [DigitalOcean] 15 | access_id= 16 | secret_key= 17 | bucket_name= 18 | -------------------------------------------------------------------------------- /host_list.json: -------------------------------------------------------------------------------- 1 | [ 2 | ["https://mixtape.moe/", "https://mixtape.moe/upload.php", "mixtape.moe", 104857600], 3 | ["https://doko.moe/", "https://doko.moe/upload.php", "doko.moe", 2147483648], 4 | ["https://coka.la/", "https://coka.la/upload.php", "coka.la", 536870912], 5 | ["https://safe.moe/", "https://safe.moe/api/upload", "safe.moe", 314572800] 6 | ] 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Gprime 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /RedditMirrorBot.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | 3 | from configparser import ConfigParser 4 | from json import load, dump 5 | from os import getpid, listdir, remove, path, makedirs, rename 6 | from shutil import rmtree 7 | from prawcore.exceptions import RequestException, ServerError 8 | from time import sleep, ctime, time 9 | from requests import get 10 | from copy import copy 11 | 12 | import logging 13 | import praw 14 | import re 15 | import subprocess 16 | import youtube_dl 17 | import os 18 | import simplejson as json 19 | import requests 20 | import threading 21 | import urllib.parse 22 | 23 | # Initialize logger 24 | logger = logging.getLogger(__name__) 25 | logger.setLevel(logging.DEBUG) 26 | 27 | # Create log file handler 28 | handler = logging.FileHandler('mirrorbot.log') 29 | handler.setLevel(logging.DEBUG) 30 | 31 | # Create logging format 32 | formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') 33 | handler.setFormatter(formatter) 34 | logger.addHandler(handler) 35 | logger.debug('Logging initialized.') 36 | 37 | logger.debug("Opening config") 38 | config = ConfigParser() 39 | config.read("config.ini") 40 | logger.debug("Config opened") 41 | 42 | do_access_id = config["DigitalOcean"]["access_id"] 43 | do_secret_key = config["DigitalOcean"]["secret_key"] 44 | do_bucket_name = config["DigitalOcean"]["bucket_name"] 45 | specified_sub = config["Reddit"]["sub_to_mirror"] 46 | specified_host = config["General"]["backend_host"] 47 | 48 | # Logger class for youtube_dl 49 | class MyLogger(object): 50 | def debug(self, msg): 51 | logger.debug(msg) 52 | 53 | def warning(self, msg): 54 | logger.warning(msg) 55 | 56 | def error(self, msg): 57 | logger.error(msgs) 58 | 59 | ydl_opts = { 60 | 'format': 'best[ext=webm]/bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best', 61 | 'logger': MyLogger(), 62 | } 63 | 64 | def cleanup_init(): 65 | if not path.exists("Media/"): 66 | makedirs("Media/") 67 | logger.debug("Media path doesn't exist, created it.") 68 | return 69 | 70 | else: 71 | rmtree("Media/") 72 | makedirs("Media/") 73 | logger.debug("Cleaned Media path") 74 | return 75 | 76 | 77 | def cleanup(path): 78 | if path[-1] != "/": 79 | path += "/" 80 | 81 | logger.debug("Cleanup " + path) 82 | 83 | if not path.exists(path): 84 | logging.debug("Cleanup: " + path + " does not exist. Creating it.") 85 | makedirs(path) 86 | return 87 | 88 | else: 89 | rmtree(path) 90 | logging.debug("Cleanup: " + path + " successfully removed.") 91 | return 92 | 93 | # Returns true if a video for the submission already exists, false otherwise 94 | def is_submission_mirrored(submission): 95 | video_list = get_sub_video_list(do_bucket_name) 96 | return (submission.id in video_list) 97 | 98 | 99 | # Takes an s3 bucket object as input 100 | # Returns a list of reddit submission IDs for which videos exist 101 | def get_sub_video_list(bucket_name): 102 | bucket = get_bucket(bucket_name) 103 | logger.debug("get_video_list()") 104 | regex = re.compile('/\w*\.', re.IGNORECASE) 105 | raw_text = str([x for x in bucket.objects.all()]) 106 | video_list = regex.findall(raw_text) 107 | 108 | return [x[1:-1] for x in video_list] 109 | 110 | # Returns an object representing the specified bucket 111 | def get_bucket(bucket_name): 112 | logger.debug("Creating s3 resource object") 113 | resource = boto3.resource('s3', 114 | region_name='nyc3', 115 | endpoint_url="https://nyc3.digitaloceanspaces.com", 116 | aws_access_key_id=do_access_id, 117 | aws_secret_access_key=do_secret_key) 118 | 119 | return resource.Bucket(bucket_name) 120 | 121 | # Takes in a url and attempts to download the video 122 | # Returns true if successful 123 | def download_video(url, sub_id=None, working_dir): 124 | logger.info("Downloading " + url) 125 | output_filename = working_dir + "video.mp4" 126 | opts = copy(ydl_opts) 127 | opts['outtmpl'] = output_filename 128 | yt = youtube_dl.YoutubeDL(opts) 129 | # Twitter post 130 | if "twitter" in url: 131 | logger.info('Twitter video detected.') 132 | response = yt.extract_info(url, process=False) 133 | 134 | try: 135 | while response.get("url"): 136 | response = yt.extract_info(response["url"], process=False) 137 | 138 | except youtube_dl.utils.DownloadError: 139 | logger.error("Twitter Youtube-dl download error. Unable to download video from URL.") 140 | return False 141 | 142 | url = response["webpage_url"] 143 | 144 | # Reddit video, only entered if sub_id exists (won't be used for comment mirroring) 145 | elif "v.redd.it" in url and sub_id: 146 | if download_reddit_video(url, sub_id, working_dir, output_filename): 147 | logger.info(url + " successfully downloaded.") 148 | else: 149 | logger.error(url + " was unable to be downloaded.") 150 | 151 | # Otherwise, download normally 152 | else: 153 | try: 154 | yt.download([url]) 155 | except (youtube_dl.utils.DownloadError) as e: 156 | logger.error("Download error: " + str(e)) 157 | return False 158 | except (youtube_dl.utils.SameFileError) as e: 159 | logger.error("Download error: " + str(e)) 160 | return False 161 | except (UnicodeDecodeError) as e: 162 | logger.error("Download error: " + str(e)) 163 | return False 164 | 165 | return True 166 | 167 | # Attempts to download a reddit-hosted video. 168 | # Returns true if successful 169 | def download_reddit_video(url, sub_id, working_dir, output_filename): 170 | sub = reddit.submission(id=sub_id) 171 | if sub.media is None: 172 | if hasattr(sub, "crosspost_parent"): 173 | sub.media = reddit.submission(sub.crosspost_parent[3:]).media 174 | else: 175 | url = get(sub.url).url 176 | _id = praw.models.reddit.submission.Submission.id_from_url(url) 177 | sub.media = reddit.submission(_id).media 178 | 179 | # Time added to filename to avoid processes writing over each other 180 | video_url = sub.media["reddit_video"]["fallback_url"] 181 | video_name = working_dir + (time()) + "_video" 182 | download(video_name, video_url) 183 | 184 | audio_url = video_url.rsplit("/", 1)[0] + "/audio" 185 | audio_name = working_dir + str(time()) + "_audio" 186 | download(audio_name, audio_url) 187 | 188 | if sub.media["reddit_video"]["is_gif"]: 189 | logger.debug("Reddit video is a gif.") 190 | rename(video_name, output_filename) 191 | 192 | #if not gif but still no audio 193 | elif not 'octet-stream' in magic.Magic(mime=True,uncompress=True).from_file('Media/audio'): 194 | logger.debug("Reddit video has no audio.") 195 | rename(video_name, output_filename) 196 | 197 | #audio exists 198 | else: 199 | logger.debug("Running combine_media() on reddit video.") 200 | combine_media(video_name, audio_name, output_filename) 201 | logger.debug("Media combining complete.") 202 | 203 | return True 204 | 205 | def combine_media(video, audio, output_filename): 206 | output = str(time()) + "_output.mp4" 207 | command = [ 208 | "ffmpeg", 209 | "-v", "quiet", 210 | "-i", "Media/" + video, 211 | "-i", "Media/" + audio, 212 | "-c", "copy", 213 | "-f", "mp4", 214 | "Media/" + output, 215 | "-y" 216 | ] 217 | 218 | subprocess.run(command) 219 | 220 | 221 | def download(filename, url): 222 | with open("Media/" + filename, "wb") as file: 223 | file.write(get(url).content) 224 | 225 | #converts given file to mp4, and returns new filename 226 | def conv_to_mp4(file_name): 227 | 228 | vid_name = file_name[:-4] + ".mp4" 229 | 230 | ##check if file is mkv and convert to mp4 231 | if ".mkv" in file_name: 232 | ffmpeg_subproc = [ 233 | "ffmpeg", 234 | "-i", file_name, 235 | "-strict", "-2", #fixes opus experimental error 236 | "-vcodec", "copy", 237 | "-y", 238 | vid_name 239 | ] 240 | conv_process = subprocess.run(ffmpeg_subproc) 241 | return vid_name 242 | 243 | else: 244 | return file_name 245 | 246 | def upload_video(file_name, _id): 247 | file_name = conv_to_mp4(file_name) 248 | logger.debug("Uploading to DO...") 249 | save_file_size(file_name) 250 | logger.debug("Size:", str(os.path.getsize(file_name)/1024/1024) + "MB") 251 | client = session.client('s3', 252 | region_name='nyc3', 253 | endpoint_url="https://nyc3.digitaloceanspaces.com", 254 | aws_access_key_id=do_access_id, 255 | aws_secret_access_key=do_secret_key) 256 | key = "videos/" + str(_id) + ".mp4" 257 | 258 | try: 259 | client.upload_file(file_name, 'pf-mirror-1', key) 260 | except (boto3.S3UploadFailedError) as e: 261 | logger.error(_id + " failed to upload: " + str(e)) 262 | return None 263 | 264 | logger.debug(_id + " successfully uploaded.") 265 | 266 | resource = boto3.resource('s3', 267 | region_name='nyc3', 268 | endpoint_url="https://nyc3.digitaloceanspaces.com", 269 | aws_access_key_id=do_access_id, 270 | aws_secret_access_key=do_secret_key) 271 | 272 | logger.debug("DO key: " + key) 273 | client.put_object_acl(ACL='public-read', Bucket='pf-mirror-1', Key=key) 274 | key = "videos/" + key 275 | mirror_url = "https://pf-mirror-1.nyc3.digitaloceanspaces.com/" + key 276 | 277 | logger.info("Upload complete!") 278 | return str(mirror_url) 279 | 280 | 281 | # Takes in a submission object and attempts to mirror it 282 | # Returns true if mirror was successful 283 | def mirror_submission(submission): 284 | working_dir = "Media/" + str(time()) + "_dl/" 285 | cleanup(working_dir) 286 | download_video(submission.url, submission.id, working_dir) 287 | mirror_url = upload_video(working_dir + "video.mp4", submission.id) 288 | 289 | 290 | # Listens for POST requests to create new mirrors 291 | def network_listener(): 292 | 293 | 294 | # Watches for new subreddit submissions, and attempts to mirror them 295 | def sub_watcher(): 296 | reddit = praw.Reddit(**config["Reddit"]) 297 | stream = reddit.subreddit(sub_to_mirror).stream.submissions(pause_after=1) 298 | 299 | while True: 300 | cleanup_submissions() 301 | try: 302 | # Get next post 303 | submission = next(stream) 304 | logger.debug("Got next post: ", submission.title, " ", "https://reddit.com" + submission.permalink + "\n" + submission.url) 305 | 306 | except RequestException: 307 | # Client side error 308 | logger.error("RequestException in sub_watcher") 309 | sleep(60) 310 | except ServerError: 311 | # Reddit side error 312 | logger.error("ServerError in sub_watcher") 313 | sleep(60) 314 | except StopIteration: 315 | logger.error("StopIteration in sub_watcher") 316 | break 317 | else: 318 | if submission is None: 319 | logger.info("No new posts.") 320 | continue 321 | 322 | if submission.is_self: 323 | logger.info("Skipping self-post.") 324 | continue 325 | 326 | # Don't bother creating mirror for posts over a month old 327 | if submission.created_utc < time() - 3600 * 24 * 30: 328 | logger.info("Submission is too old.") 329 | continue 330 | 331 | if is_submission_mirrored(submission): 332 | logger.info("Submission already mirrored.") 333 | continue 334 | if mirror_submission(submission): 335 | logger.info(submission.id + " successfully mirrored!") 336 | continue 337 | else: 338 | logger.error("Something went wrong with " + submission.id + ". Check the logs for info.") 339 | continue 340 | cleanup_submissions() 341 | 342 | 343 | # Watches for new comments containing !mirror, and mirrors the parent comment's video if possible 344 | def comment_watcher(): 345 | 346 | 347 | # Manages POST listener, comment watcher, and sub watcher 348 | def main(): 349 | jobs = [] 350 | # sub_watcher 351 | jobs.append(multiprocessing.Process(name="Submissions:", target=sub_watcher)) 352 | # comment_watcher 353 | jobs.append(multiprocessing.Process(name="Comments:", target=comment_watcher)) 354 | # network listener 355 | jobs.append(multiprocessing.Process(name="Network:", target=network_listener)) 356 | 357 | # start threads 358 | for job in jobs: 359 | job.start() 360 | 361 | if __name__ == "__main__": 362 | cleanup_init() 363 | if path.exists("/usr/bin/ffmpeg"): 364 | print(main()) 365 | else: 366 | print("Needs ffmpeg") 367 | -------------------------------------------------------------------------------- /PublicFreakout.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | 3 | from configparser import ConfigParser 4 | from json import load, dump 5 | from os import getpid, listdir, remove, path, makedirs, rename 6 | from prawcore.exceptions import RequestException, ServerError 7 | from time import sleep, ctime, time 8 | from requests import get 9 | from boto3 import session 10 | from botocore.client import Config 11 | 12 | import praw.models 13 | import re 14 | import subprocess 15 | import youtube_dl 16 | import os 17 | import simplejson as json 18 | import requests 19 | import multiprocessing 20 | import urllib.parse 21 | import boto3 22 | import praw 23 | import magic 24 | 25 | print(getpid()) 26 | 27 | # Empty youtube logger 28 | class MyLogger(): 29 | def debug(self, msg): 30 | pass 31 | 32 | def warning(self, msg): 33 | pass 34 | 35 | def error(self, msg): 36 | pass 37 | 38 | config = ConfigParser() 39 | config.read("config.ini") 40 | 41 | reddit = praw.Reddit(**config["Reddit"]) 42 | do_access_id = config["DigitalOcean"]["access_id"] 43 | do_secret_key = config["DigitalOcean"]["secret_key"] 44 | 45 | ydl_opts = { 46 | 'format': 'best[ext=mp4]/bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best', 47 | 'logger': MyLogger(), 48 | 'outtmpl': "Media/%(id)s.mp4", 49 | } 50 | 51 | yt = youtube_dl.YoutubeDL(ydl_opts) 52 | session = session.Session() 53 | 54 | try: 55 | with open("saved_links.txt") as file: 56 | # Load hash file and only keep most recent 28 days 57 | saved_links = [n for n in load(file) if n["created"] > time() - 3600 * 24 * 28] 58 | except FileNotFoundError: 59 | with open("saved_links.txt", "w") as file: 60 | saved_links = [] 61 | dump(saved_links, file) 62 | 63 | 64 | def check_links(submission): 65 | mirror_list = [] 66 | 67 | if saved_links: 68 | while saved_links[0]["created"] < time() - 3600 * 24 * 28: 69 | saved_links.pop(0) 70 | 71 | for data in saved_links: 72 | if data["video_url"] == submission.url: 73 | mirror_url = data["mirror_url"] 74 | 75 | 76 | if data["mirror_url"]: 77 | if not ("error" or "supplied") in str(mirror_url): 78 | for x in mirror_url.split(): 79 | mirror_list.append(x.replace("'","").replace("[","").replace("]","").replace(",","")) 80 | 81 | reply_reddit(submission, mirror_list) 82 | 83 | return save("Repost", submission, mirror_list) 84 | 85 | def cleanup(): 86 | print("Cleanup") 87 | if not path.exists("Media"): 88 | makedirs("Media") 89 | 90 | for file in listdir("Media"): 91 | remove("Media/" + file) 92 | 93 | def combine_media(): 94 | command = [ 95 | "ffmpeg", 96 | "-v", "quiet", 97 | "-i", "Media/video", 98 | "-i", "Media/audio", 99 | "-c", "copy", 100 | "-f", "mp4", 101 | "Media/output.mp4", 102 | "-y" 103 | ] 104 | 105 | subprocess.run(command) 106 | 107 | def download(filename, url): 108 | with open("Media/" + filename, "wb") as file: 109 | file.write(get(url).content) 110 | 111 | def process(submission): 112 | print("process() entered") 113 | print("Got next post: ", submission.title, " ", "https://reddit.com" + submission.permalink) 114 | #Don't wanna mirror nazi propaganda 115 | if "Antifa" in submission.title or "antifa" in submission.title: 116 | return 117 | mirror_url = None 118 | 119 | 120 | print("VIDEO URL: "+ submission.url) 121 | 122 | # Twitter post 123 | if "twitter" in submission.url: 124 | print("TWEETER VIDEO") 125 | 126 | try: 127 | response = yt.extract_info(submission.url, process=False) 128 | while response.get("url"): 129 | response = yt.extract_info(response["url"], process=False) 130 | 131 | except youtube_dl.utils.DownloadError: 132 | return save("Twitter download error", submission) 133 | 134 | 135 | submission.url = response["webpage_url"] 136 | 137 | # Reddit hosted video 138 | if submission.domain == "v.redd.it": 139 | print("LEDDIT VIDEO") 140 | # If post is crosspost, set submission to linked post 141 | if submission.media is None: 142 | if hasattr(submission, "crosspost_parent"): 143 | submission.media = reddit.submission(submission.crosspost_parent[3:]).media 144 | else: 145 | url = get(submission.url).url 146 | _id = praw.models.reddit.submission.Submission.id_from_url(url) 147 | submission.media = reddit.submission(_id).media 148 | 149 | video_url = submission.media["reddit_video"]["fallback_url"] 150 | download("video", video_url) 151 | 152 | audio_url = video_url.rsplit("/", 1)[0] + "/audio" 153 | download("audio", audio_url) 154 | 155 | if submission.media["reddit_video"]["is_gif"]: 156 | mirror_url = upload("Media/video", submission.id) 157 | status = "Complete" 158 | print("Mirror url: " + str(mirror_url)) 159 | 160 | #if not gif but still no audio 161 | elif not 'octet-stream' in magic.Magic(mime=True,uncompress=True).from_file('Media/audio'): 162 | mirror_url = upload("Media/video", submission.id) 163 | status = "Complete" 164 | print("Mirror url: " + str(mirror_url)) 165 | 166 | #audio exists 167 | else: 168 | combine_media() 169 | 170 | mirror_url = upload("Media/output.mp4", submission.id) 171 | status = "Complete" 172 | print("Mirror url: " + str(mirror_url)) 173 | 174 | 175 | 176 | if status == "Complete": 177 | reply_reddit(submission, mirror_url) 178 | return save(status, submission, mirror_url) 179 | 180 | #download video 181 | try: 182 | yt.download([submission.url]) 183 | except (youtube_dl.utils.DownloadError) as e: 184 | print(str(e)) 185 | return save(str(e), submission, "Download error") 186 | except (youtube_dl.utils.SameFileError) as e: 187 | print(str(e)) 188 | return save(str(e), submission, "Same file error") 189 | except (UnicodeDecodeError) as e: 190 | print(str(e)) 191 | return save(str(e), submission, "UnicodeDecodeError") 192 | except (TypeError) as e: 193 | return save(str(e), submission, "TypeError") 194 | 195 | if listdir("Media") == []: 196 | return save("Download failed", submission, "Media folder empty") 197 | 198 | file = [i for i in listdir("Media")][0] 199 | file = "Media/" + str(file) 200 | mirror_url = upload(file, submission.id) 201 | if "NOT_HTTP: " in mirror_url: 202 | print("NOT HTTP") 203 | return 204 | else: 205 | status = "Complete" 206 | print("Mirror url: " + str(mirror_url)) 207 | reply_reddit(submission, mirror_url) 208 | return save(status, submission, mirror_url) 209 | 210 | # Should never be called 211 | save("End", submission) 212 | 213 | def reply_reddit(submission, mirror_url): 214 | print("Submitting comment...") 215 | while True: 216 | if not mirror_url: 217 | return 218 | try: 219 | mirror_text = "" 220 | mirror_text += "[Mirror](http://mirrorbot.ga" + submission.permalink + ") \n\n".format(urllib.parse.quote(str(mirror_url), safe='')) 221 | comment = submission.reply(" | ".join([ 222 | mirror_text + " \nI am a bot", 223 | "[Feedback](https://www.reddit.com/message/compose/?to={[Reddit][host_account]}&subject=PublicFreakout%20Mirror%20Bot)".format(config), 224 | "[Github](https://github.com/dopeslothe/PublicFreakout-Mirror-Bot) " 225 | ])) 226 | comment.mod.approve() 227 | comment.mod.distinguish(how='yes',sticky=True) 228 | break 229 | 230 | except praw.exceptions.APIException: 231 | print("Rate limit exception") 232 | sleep(60) 233 | continue 234 | 235 | def run(): 236 | while True: 237 | stream = reddit.subreddit("PublicFreakout").stream.submissions(pause_after=1) 238 | 239 | try: 240 | checked = [n._extract_submission_id() for n in reddit.user.me().comments.new()] 241 | except RequestException: 242 | sleep(60) 243 | continue 244 | 245 | while True: 246 | cleanup() 247 | 248 | try: 249 | # Get next post 250 | submission = next(stream) 251 | except RequestException: 252 | # Client side error 253 | sleep(60) 254 | except ServerError: 255 | # Reddit side error 256 | sleep(60) 257 | except StopIteration: 258 | break 259 | else: 260 | if submission is None: 261 | print("No new posts.") 262 | continue 263 | 264 | if submission.is_self: 265 | print("Skipping self-post.") 266 | continue 267 | 268 | # Don't bother creating mirror for posts over a day old 269 | if submission.created_utc < time() - 3600 * 24 * 1: 270 | print("Submission is too old") 271 | continue 272 | 273 | if submission in checked: 274 | print("Submission already mirrored") 275 | continue 276 | 277 | try: 278 | process(submission) 279 | except PermissionError: 280 | return "Permission denied" 281 | 282 | cleanup() 283 | 284 | def save_file_size(file_name): 285 | with open("file_sizes.txt", "a") as file: 286 | out = str(ctime()) + ": " + str(os.path.getsize(file_name)) + "\n" 287 | file.write(out) 288 | 289 | 290 | def save(status, submission, mirror_url=None): 291 | if not mirror_url: 292 | print("Unable to save to file: No url supplied") 293 | return 294 | text = "{:<19} | " + ctime() + " | https://www.reddit.com{:<85} | {}\n" 295 | permalink = submission.permalink.encode("ascii", "ignore").decode() 296 | 297 | with open("mirror_bot_log.txt", "a") as file: 298 | file.write(text.format(status, permalink, " | " + str(mirror_url))) 299 | 300 | saved_links.append({ 301 | "created": int(submission.created_utc), 302 | "reddit": "https://www.reddit.com" + permalink, 303 | "video_url": submission.url, 304 | "mirror_url": str(mirror_url) 305 | }) 306 | 307 | while saved_links[0]["created"] < time() - 3600 * 24 * 28: 308 | saved_links.pop(0) 309 | 310 | with open("saved_links.txt", "w") as file: 311 | dump(saved_links, file, indent=4, sort_keys=True) 312 | 313 | return True 314 | 315 | #new upload function, doesn't use limf 316 | #uploads to given pomf.se clone 317 | 318 | def upload(file_name, submission_id): 319 | file_name = conv_to_mp4(file_name) 320 | save_file_size(file_name) 321 | print("Size:", str(os.path.getsize(file_name)/1024/1024) + "MB") 322 | output_file = "/var/www/html/media/" + str(submission_id) + ".mp4" 323 | rename(file_name, output_file) 324 | 325 | mirror_url = "https://mirrorbot.ga/media/" + str(submission_id) + ".mp4" 326 | 327 | print("Upload complete!") 328 | return str(mirror_url) 329 | 330 | 331 | 332 | #old upload 333 | #def upload(file_name, submission_id): 334 | #file_name = conv_to_mp4(file_name) 335 | #print("Uploading to DO...") 336 | #save_file_size(file_name) 337 | #print("Size:", str(os.path.getsize(file_name)/1024/1024) + "MB") 338 | #client = session.client('s3', 339 | #region_name='nyc3', 340 | #endpoint_url="https://pf-mirror-1.nyc3.digitaloceanspaces.com", 341 | #aws_access_key_id=do_access_id, 342 | #aws_secret_access_key=do_secret_key) 343 | #key = str(submission_id) + ".mp4" 344 | 345 | #client.upload_file(file_name, 'videos', key) 346 | 347 | #resource = boto3.resource('s3', 348 | #region_name='nyc3', 349 | #endpoint_url="https://pf-mirror-1.nyc3.digitaloceanspaces.com", 350 | #aws_access_key_id=do_access_id, 351 | #aws_secret_access_key=do_secret_key) 352 | 353 | #print(key) 354 | #client.put_object_acl(ACL='public-read', Bucket='videos', Key=key) 355 | #key = "videos/" + key 356 | #mirror_url = "https://pf-mirror-1.nyc3.digitaloceanspaces.com/" + key 357 | 358 | #print("Upload complete!") 359 | #return str(mirror_url) 360 | 361 | #converts given file to mp4, and returns new filename 362 | def conv_to_mp4(file_name): 363 | 364 | vid_name = file_name[:-4] + ".mp4" 365 | 366 | ##check if file is mkv and convert to mp4 367 | if ".mkv" in file_name: 368 | ffmpeg_subproc = [ 369 | "ffmpeg", 370 | "-i", file_name, 371 | "-strict", "-2", #fixes opus experimental error 372 | "-vcodec", "copy", 373 | "-y", 374 | vid_name 375 | ] 376 | conv_process = subprocess.run(ffmpeg_subproc) 377 | return vid_name 378 | 379 | else: 380 | return file_name 381 | 382 | if __name__ == "__main__": 383 | if path.exists("/usr/bin/ffmpeg"): 384 | print(run()) 385 | else: 386 | print("Needs ffmpeg") 387 | --------------------------------------------------------------------------------