├── .gitignore
├── check_command.py
├── check_valid.py
├── config.py
├── download_page.py
├── functions
    ├── __init__.py
    ├── archive.py
    └── periodical_job.py
├── irc_bot.py
├── irc_message.py
├── main.py
├── periodical_jobs
    ├── __init__.py
    └── check_temp_perjob.py
├── refresh.py
├── services
    ├── __init__.py
    ├── dl__facebook_com.py
    ├── dl__ignores__twitter_com
    ├── dl__liveleak_com.py
    ├── dl__periscope_tv.py
    ├── dl__twitter_com.py
    ├── dl__vine_co.py
    ├── video__facebook_com.py
    ├── video__liveleak_com.py
    ├── video__periscope_tv.py
    ├── video__twitter_com.py
    ├── video__twitter_com_hashtag.py
    ├── video__twitter_com_user.py
    ├── video__video_stream.py
    ├── video__vine_co.py
    └── video__webpage.py
├── temp_perjobs
    └── __init__.py
├── upload.py
└── url.py


/.gitignore:
--------------------------------------------------------------------------------
1 | *~
2 | *.pyc
3 | keys
4 | to_be_uploaded/
5 | irclog
6 | 


--------------------------------------------------------------------------------
/check_command.py:
--------------------------------------------------------------------------------
 1 | import refresh
 2 | 
 3 | def find_command_service(command):
 4 |     services_list = refresh.services_list
 5 |     for service in services_list:
 6 |         if command in service[2]:
 7 |             return service[0]
 8 |     else:
 9 |         return None
10 | 


--------------------------------------------------------------------------------
/check_valid.py:
--------------------------------------------------------------------------------
 1 | import re
 2 | 
 3 | def check_valid_url(url):
 4 |     if re.search(r'^http:\/\/[^\/]+?\.[^\/]+', url):
 5 |         return 'http'
 6 |     elif re.search(r'^https:\/\/[^\/]+?\.[^\/]+', url):
 7 |         return 'https'
 8 |     elif re.search(r'^rtsp:\/\/[^\/]+?\.[^\/]+', url):
 9 |         return 'rtsp'
10 |     elif re.search(r'^rtmp:\/\/[^\/]+?\.[^\/]+', url):
11 |         return 'rtmp'
12 |     elif re.search(r'^mms:\/\/[^\/]+?\.[^\/]+', url):
13 |         return 'mms'
14 |     elif re.search(r'^mmsh:\/\/[^\/]+?\.[^\/]+', url):
15 |         return 'mmsh'
16 |     else:
17 |         return False
18 | 
19 | def check_num(string):
20 |     try:
21 |         int(string)
22 |         return True
23 |     except:
24 |         return False
25 | 


--------------------------------------------------------------------------------
/config.py:
--------------------------------------------------------------------------------
 1 | version = 20170916.01
 2 | 
 3 | irc_server = 'irc.underworld.no'
 4 | irc_port = 6667
 5 | irc_channel = '#videobot'
 6 | irc_nick = 'video'
 7 | 
 8 | github = 'https://github.com/ArchiveTeam/VideoBot'
 9 | 
10 | periodical_job_open_time = 172800
11 | max_warc_item_size = 5368709120
12 | 
13 | with open('keys', 'r') as file:
14 |     ia_access_key, ia_secret_key = file.read().replace('\n', '').replace('\r', '').replace(' ', '').split(',')
15 |     print(ia_access_key, ia_secret_key)
16 | 


--------------------------------------------------------------------------------
/download_page.py:
--------------------------------------------------------------------------------
 1 | import requests
 2 | import re
 3 | 
 4 | def extract_info(regexes, url):
 5 |     html = requests.get(url)
 6 |     if type(regexes) is str:
 7 |         regexes = [regexes]
 8 |     extracted = []
 9 |     for regex in regexes:
10 |         if re.search(regex, html.text):
11 |             extracted.append(re.findall(regex, html.text))
12 |         else:
13 |             extracted.append('')
14 |     if len(extracted) == 1:
15 |         extracted = extracted[0]
16 |     return extracted
17 |                 
18 | 


--------------------------------------------------------------------------------
/functions/__init__.py:
--------------------------------------------------------------------------------
1 | import archive
2 | import periodical_job
3 | 


--------------------------------------------------------------------------------
/functions/archive.py:
--------------------------------------------------------------------------------
 1 | from config import irc_channel, irc_nick
 2 | import irc_bot
 3 | import refresh
 4 | import threading
 5 | import re
 6 | import url
 7 | import services
 8 | 
 9 | def irc_bot_print(irc_channel, irc_bot_message):
10 |     irc_bot.irc_bot_print(irc_channel, irc_bot_message)
11 | 
12 | def find_url_service(url_):
13 |     return url.find_url_service(url_)
14 | 
15 | def main(command, user):
16 |     if len(command) == 1:
17 |         irc_bot_print(irc_channel, user + ': Please specify an URL.')
18 |     elif command[1].startswith('http://') or command[1].startswith('https://'):
19 |         print(command)
20 |         threading.Thread(target = process_url, args = (command, user)).start()
21 |     else:
22 |         irc_bot_print(irc_channel, user + ': I can only handle http:// and https://')
23 | 
24 | def process_url(command, user):
25 |     url_service = find_url_service(command[1])
26 |     for irc_bot_message in eval('services.' + url_service + '.process(url_service, command, user)'):
27 |         if not irc_bot_message.startswith(irc_nick + ':'):
28 |             irc_bot_print(irc_channel, irc_bot_message)


--------------------------------------------------------------------------------
/functions/periodical_job.py:
--------------------------------------------------------------------------------
  1 | from config import irc_channel
  2 | from config import periodical_job_open_time
  3 | import periodical_jobs
  4 | import check_valid
  5 | import check_command
  6 | import irc_bot
  7 | import refresh
  8 | import threading
  9 | import re
 10 | import os
 11 | import random
 12 | import string
 13 | import url
 14 | import services
 15 | import subprocess
 16 | import time
 17 | import sys
 18 | 
 19 | irc_bot_print = lambda irc_channel, irc_bot_message: irc_bot.irc_bot_print(irc_channel, irc_bot_message)
 20 | check_temp_perjob_variable = lambda ticket_id, var: periodical_jobs.check_temp_perjob.check_temp_perjob_variable(ticket_id, var)
 21 | get_temp_perjob_variables = lambda ticket_id: periodical_jobs.check_temp_perjob.get_temp_perjob_variables(ticket_id)
 22 | check_valid_url = lambda url: check_valid.check_valid_url(url)
 23 | find_url_service = lambda url_: url.find_url_service(url_)
 24 | find_url_title = lambda url_service, url_: url.find_url_title(url_service, url_)
 25 | find_url_service_name = lambda url_service: url.find_url_service_name(url_service)
 26 | find_url_id = lambda url_service, url_: url.find_url_id(url_service, url_)
 27 | find_command_service = lambda command: check_command.find_command_service(command)
 28 | 
 29 | command_url = ''
 30 | 
 31 | def main(message, user):
 32 |     '''Create a periodical job.
 33 | 
 34 |     Usage:
 35 |     !perjob
 36 |         to create a new periodical job.
 37 |     !perjob <command> <ticket ID> <command dependent>
 38 |         to specify a command for the job.
 39 |     
 40 |     With commands and command dependent variables:
 41 |     --service-url and <URL>
 42 |         to periodically scrape a service.
 43 |     --scrape-url and <URL>
 44 |         to periodically scrape links from a webpage.
 45 |     '''
 46 |     if len(message) == 1:
 47 |         ticket_id = ''.join(random.choice(string.ascii_lowercase) for num in range(10))
 48 |         with open('temp_perjobs/'+ticket_id+'.py', 'a') as file:
 49 |             file.write('user = \'' + user + '\'\n')
 50 |         open_time_hours = int(periodical_job_open_time/3600)
 51 |         irc_bot_print(irc_channel, user + ': Your ticket ID is \'' + ticket_id + '\'. The ticket is open for ' + str(open_time_hours) + ' hours without edits.')
 52 |         irc_bot_print(irc_channel, user + ': Configure your new periodical job using \'!perjob <command> <ticket ID> <command dependent>\'.')
 53 |         irc_bot_print(irc_channel, user + ': See ' + command_url + ' for available commands.')
 54 |     elif message[1] == '--service-url':
 55 |         if len(message) != 4:
 56 |             irc_bot_print(irc_channel, user + ': I don\'t understand your command. Please review it.')
 57 |         else:
 58 |             ticket_id = message[2]
 59 |             if not os.path.isfile('temp_perjobs/'+ticket_id+'.py'):
 60 |                 irc_bot_print(irc_channel, user + ': Ticket ID \'' + ticket_id + '\' does not exist.')
 61 |             else:
 62 |                 url = message[3]
 63 |                 if check_temp_perjob_variable(ticket_id, 'url') != 'var not found':
 64 |                     irc_bot_print(irc_channel, user + ': You already provided an URL for ticket ID \'' + ticket_id + '\'.')
 65 |                     irc_bot_print(irc_channel, user + ': Create a new ticket ID to use a new URL.')
 66 |                 elif check_valid_url(url) == False:
 67 |                     irc_bot_print(irc_channel, user + ': URL \'' + url + '\' doesn\'t seem to be valid.')
 68 |                 else:
 69 |                     url_service = find_url_service(url)
 70 |                     if url_service != None:
 71 |                         service_name = find_url_service_name(url_service)
 72 |                         url_id = find_url_id(url_service, url)
 73 |                         url_title = find_url_title(url_service, url)
 74 |                         if url_id != None:
 75 |                             url_id_string =  'with ID \'' + url_id + '\''
 76 |                         else:
 77 |                             url_id_string = 'with URL \'' + url + '\''
 78 |                         if url_title != None:
 79 |                             url_title_string =  '\'' + url_title + '\' '
 80 |                         else:
 81 |                             url_title_string = ''
 82 |                         irc_bot_print(irc_channel, user + ': Found ' + service_name + ' ' + url_title_string + url_id_string + '.')
 83 |                         threading.Thread(target = process_messages, args = ('add_url', url, ticket_id, user, ticket_id, url_service)).start()
 84 |                     else:
 85 |                         irc_bot_print(irc_channel, user + ': URL \'' + message[3] + '\' is currently not supported.')
 86 |     elif message[1] == '--scrape-url':
 87 |         if len(message) != 4:
 88 |             irc_bot_print(irc_channel, user + ': I don\'t understand your command. Please review it.')
 89 |         else:
 90 |             ticket_id = message[2]
 91 |             if not os.path.isfile('temp_perjobs/'+ticket_id+'.py'):
 92 |                 irc_bot_print(irc_channel, user + ': Ticket ID \'' + ticket_id + '\' does not exist.')
 93 |             else:
 94 |                 url = message[3]
 95 |                 if check_temp_perjob_variable(ticket_id, 'url') != 'var not found':
 96 |                     irc_bot_print(irc_channel, user + ': You already provided an URL for ticket ID \'' + ticket_id + '\'.')
 97 |                 elif check_valid_url(url) == False:
 98 |                     irc_bot_print(irc_channel, user + ': URL \'' + url + '\' doesn\'t seem to be valid.')
 99 |                 else:
100 |                     url_service = 'video__webpage'
101 |                     service_name = find_url_service_name(url_service)
102 |                     irc_bot_print(irc_channel, user + ': Found ' + service_name + ' \'' + url + '\'.')
103 |                     threading.Thread(target = process_messages, args = ('add_url', url, ticket_id, user, ticket_id, url_service)).start()
104 |     elif message[1] == '--edit':
105 |         if len(message) != 3:
106 |             irc_bot_print(irc_channel, user + ': I don\'t understand your command. Please review it.')
107 |         else:
108 |             ticket_id = message[2]
109 |             if not os.path.isfile('periodical_jobs/' + ticket_id + '.py'):
110 |                 irc_bot_print(irc_channel, user + ': Ticket ID \'' + ticket_id + '\' does not exist.')
111 |             else:
112 |                 os.rename('periodical_jobs/' + ticket_id + '.py', 'temp_perjobs/' + ticket_id + '.py')
113 |                 irc_bot_print(irc_channel, user + ': Ticket ID \'' + ticket_id + '\' is reopened for editing.')
114 |     elif message[1] == '--remove':
115 |         if len(message) != 3:
116 |             irc_bot_print(irc_channel, user + ': I don\'t understand your command. Please review it.')
117 |         else:
118 |             ticket_id = message[2]
119 |             if os.path.isfile('periodical_jobs/' + ticket_id + '.py'):
120 |                 os.remove('periodical_jobs/' + ticket_id + '.py')
121 |                 irc_bot_print(irc_channel, user + ': Periodical job with ticket ID ' + ticket_id + ' is removed.')
122 |             elif os.path.isfile('temp_perjobs/' + ticket_id + '.py'):
123 |                 os.remove('temp_perjobs/' + ticket_id + '.py')
124 |                 irc_bot_print(irc_channel, user + ': Periodical job with ticket ID ' + ticket_id + ' is removed.')
125 |             if os.path.isfile('periodical_jobs/' + ticket_id + '.pyc'):
126 |                 os.remove('periodical_jobs/' + ticket_id + '.pyc')
127 |             elif os.path.isfile('temp_perjobs/' + ticket_id + '.pyc'):
128 |                 os.remove('temp_perjobs/' + ticket_id + '.pyc')
129 |             else:
130 |                 irc_bot_print(irc_channel, user + ': Ticket ID \'' + ticket_id + '\' does not exist.')
131 |     elif message[1] in ('--info', '--information'):
132 |         if len(message) == 3:
133 |             ticket_id = message[2]
134 |             variables = get_temp_perjob_variables(ticket_id)
135 |             if variables == None:
136 |                 irc_bot_print(irc_channel, user + ': Periodical job with ticket ID ' + ticket_id + ' does not exist.')
137 |             else:
138 |                 irc_bot_print(irc_channel, user + ': Periodical job with ticket ID ' + ticket_id + ' has variables ' + ', '.join(variables) + '.')
139 |         elif len(message) == 4:
140 |             ticket_id = message[2]
141 |             variable = message[3]
142 |             variable_content = check_temp_perjob_variable(ticket_id, variable)
143 |             if variable_content == 'var not found':
144 |                 irc_bot_print(irc_channel, user + ': Periodical job with ticket ID ' + ticked_id + '')
145 |         else:
146 |             irc_bot_print(irc_channel, user + ': I don\'t understand your command. Please review it.')
147 |     else:
148 |         ticket_id = message[2]
149 |         if not os.path.isfile('temp_perjobs/'+ticket_id+'.py'):
150 |             irc_bot_print(irc_channel, user + ': Ticket ID \'' + ticket_id + '\' does not exist.')
151 |         else:
152 |             perjob_commands = check_temp_perjob_variable(ticket_id, 'type')
153 |             if perjob_commands == 'var not found':
154 |                 irc_bot_print(irc_channel, user + ': Please provide an URL first for ticket ID \'' + ticket_id + '\'.')
155 |             else:
156 |                 service = find_command_service(perjob_commands[0])
157 |                 if service != None:
158 |                     threading.Thread(target = process_messages, args = ('periodical_job', service, message, user, ticket_id, service)).start()
159 |                 else:
160 |                     irc_bot_print(irc_channel, user + ': Command \'' + message[0] + '\' was removed. Please create a new periodical job.')
161 | 
162 | def process_messages(name, a, b, c, ticket_id, service):
163 |     for service_message in eval('services.' + service + '.' + name + '(a, b, c)'):
164 |         if service_message[0] == 'add':
165 |             filelines = []
166 |             with open('temp_perjobs/'+ticket_id+'.py', 'r') as file:
167 |                 added = False
168 |                 for line in file:
169 |                     if not line.startswith(service_message[1]):
170 |                         filelines.append(line)
171 |                     else:
172 |                         filelines.append(service_message[1] + ' = ' + str(service_message[2]))
173 |                         added = True
174 |                 if not added:
175 |                     filelines.append(service_message[1] + ' = ' + str(service_message[2]))
176 |             with open('temp_perjobs/'+ticket_id+'.py', 'w') as file:
177 |                 file.write('\n'.join([fileline for fileline in filelines if not fileline == '']))
178 |         elif service_message[0] == 'message':
179 |             irc_bot_print(irc_channel, str(service_message[1]))
180 |         elif service_message[0] == 'finish':
181 |             required_commands = service_message[1]
182 |             default_commands = service_message[2]
183 |             user = service_message[3]
184 |             for required_command in required_commands + default_commands:
185 |                 if check_temp_perjob_variable(b[2], required_command) == 'var not found':
186 |                     irc_bot_print(irc_channel, user + ': You are missing \'' + required_command + '\'.')
187 |                     break
188 |             else:
189 |                 os.rename('temp_perjobs/' + ticket_id + '.py', 'periodical_jobs/' + ticket_id + '.py')
190 |                 irc_bot_print(irc_channel, user + ': Periodical job with ticket ID \'' + b[2] + '\' is finished.')
191 |         elif service_message[0] == 'execute':
192 |             os.system(service_message[1])
193 |         elif service_message[0] == 'bad_command':
194 |             bad_command = service_message[1]
195 |             user = service_message[2]
196 |             irc_bot_print(irc_channel, user + ': I don\'t understand command \'' + bad_command + '\'.')
197 |         elif service_message[0] == 'write_metadata':
198 |             ia_metadata = service_message[1]
199 |             fulldir = service_message[2]
200 |             if not os.path.isdir(fulldir):
201 |                 os.makedirs(fulldir)
202 |             for a, b in ia_metadata.items():
203 |                 with open(fulldir + 'ia_metadata.py', 'a') as file:
204 |                     if type(b) is list:
205 |                         content_string = str(b)
206 |                     else:
207 |                         content_string = '\'' + str(b).replace('\'', '\\\'') + '\''
208 |                     file.write(str(a) + ' = ' + content_string + '\n')
209 |         elif service_message[0] == 'help':
210 |             required_commands = service_message[1]
211 |             optional_commands = service_message[2]
212 |             user = service_message[3]
213 |             irc_bot_print(irc_channel, user + ': The required commands are ' + ', '.join(required_commands) + '.')
214 |             irc_bot_print(irc_channel, user + ': The optional commands are ' + ', '.join(optional_commands) + '.')
215 |             irc_bot_print(irc_channel, user + ': Set a command using \'!perjob <command> <ticket ID> <command option>\'.')
216 |         elif service_message[0] == 'execute_timeout':
217 |             # Do not use for grab-site processes
218 |             command = service_message[1].split(' ')
219 |             timeout = int(service_message[2])
220 |             dir_ = service_message[3]
221 |             with open(dir_ + 'no_upload', 'w') as file:
222 |                 pass
223 |             process = subprocess.Popen(command)
224 |             time.sleep(timeout)
225 |             os.remove(dir_ + 'no_upload')
226 |             if process.poll() is None:
227 |                 process.terminate()
228 |                 exit_code = -1
229 |             else:
230 |                 exit_code = process.poll()
231 | 
232 | def process_url(url, user):
233 |     services_list = refresh.services_list
234 |     for service in services_list:
235 |         print(service)
236 |         if re.search(service[1], url):
237 |             for irc_bot_message in eval('services.' + service[0] + '.process(service[0].replace("video__", ""), url, user)'):
238 |                 irc_bot_print(irc_channel, irc_bot_message)
239 | 
240 | def periodical_job_start(filename, type_, user):
241 |     service = find_command_service(type_[0])
242 |     if service != None:
243 |         threading.Thread(target = process_messages, args = ('periodical_job_start', filename, user, None, None, service)).start()
244 | 
245 | def periodical_job_auto_remove():
246 |     while True:
247 |         for temp_periodical_job in [name for name in os.listdir('./temp_perjobs/') if name.endswith('.py') and not name == '__init__.py']:
248 |             creation_date = os.path.getctime('./temp_perjobs/' + temp_periodical_job)
249 |             ticket_id = temp_periodical_job[:-3]
250 |             user = check_temp_perjob_variable(ticket_id, 'user')
251 |             if int(creation_date) + periodical_job_open_time < int(time.time()):
252 |                 os.remove('./temp_perjobs/' + temp_periodical_job)
253 |                 if os.path.isfile('./temp_perjobs/' + temp_periodical_job + 'c'):
254 |                     os.remove('./temp_perjobs/' + temp_periodical_job + 'c')
255 |                 irc_bot_print(irc_channel, user + ': Unfinished periodical job with ticket ID ' + ticket_id + ' is expired.')
256 |         time.sleep(3600)
257 | 


--------------------------------------------------------------------------------
/irc_bot.py:
--------------------------------------------------------------------------------
 1 | from config import irc_channel, irc_port, irc_server, irc_nick, github, version
 2 | import functions
 3 | import socket
 4 | import re
 5 | import check_command
 6 | import services
 7 | import refresh
 8 | import threading
 9 | 
10 | irc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
11 | irc.connect((irc_server, irc_port))
12 | 
13 | def archive(command, user):
14 |     functions.archive.main(command, user)
15 | 
16 | def periodical_job(command, user):
17 |     functions.periodical_job.main(command, user)
18 | 
19 | def find_command_service(command):
20 |     return check_command.find_command_service(command)
21 | 
22 | def new_socket():
23 |     global irc
24 |     irc.close()
25 |     irc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
26 |     irc.connect((irc_server, irc_port))
27 |     irc_bot_join()
28 |     threading.Thread(target = irc_bot_listener).start()
29 | 
30 | def irc_bot_join():
31 |     irc.send('USER ' + irc_nick + ' ' + irc_nick + ' ' + irc_nick + ' :This is the bot for ' + irc_channel + '. ' + github + '.\n')
32 |     irc.send('NICK ' + irc_nick + '\n')
33 |     irc.send('JOIN ' + irc_channel + '\n')
34 | 
35 | def irc_bot_print(channel, message):
36 |     try:
37 |         message = ''.join([character if ord(character) < 128 else '_' for character in message])
38 |         irc.send("PRIVMSG " + channel + " :" + message + "\n")
39 |     except Exception as exception:
40 |         with open('exceptions', 'a') as exceptions:
41 |             print(str(version) + '\n' + str(exception) + '\n\n')
42 |             exceptions.write(str(version) + '\n' + str(exception) + '\n\n')
43 |         new_socket()
44 |     print("IRC BOT: " + message)
45 | 
46 | def irc_bot_listener():
47 |     while True:
48 |         irc_message = irc.recv(2048)
49 |         with open('irclog', 'a') as file:
50 |             file.write(irc_message)
51 |         if irc_message.startswith('PING :'):
52 |             message = re.search(r'^[^:]+:(.*)$', irc_message).group(1)
53 |             irc.send('PONG :' + message + '\n')
54 |         elif re.search(r'^:.+PRIVMSG[^:]+:!.*', irc_message):
55 |             command = re.search(r'^:.+PRIVMSG[^:]+:(!.*)', irc_message).group(1).replace('\r', '').replace('\n', '').split(' ')
56 |             user = re.search(r'^:([^!]+)!', irc_message).group(1)
57 |             if command[0] in ('!a', '!archive'):
58 |                 archive(command, user)
59 |             elif command[0] in ('!perjob', '!periodical-job'):
60 |                 periodical_job(command, user)
61 |             elif command[0] == '!version':
62 |                 irc_bot_print(irc_channel, user + ': Current version of videobot is ' + str(version) + '.')
63 |             elif command[0] in ('!update-services', '!us'):
64 |                 irc_bot_print(irc_channel, user + ': Services are resfreshing.')
65 |                 threading.Thread(target = refresh.refresh_services).start()
66 |             else:
67 |                 command_short = command[0].replace('!', '')
68 |                 service = find_command_service(command_short)
69 |                 if service != None:
70 |                     for irc_bot_message in eval('services.' + service + '.process(service, command, user)'):
71 |                         irc_bot_print(irc_channel, irc_bot_message)
72 |                 else:
73 |                     irc_bot_print(irc_channel, user + ': Command \'' + command[0] + '\' does not exist.')
74 | 
75 | 


--------------------------------------------------------------------------------
/irc_message.py:
--------------------------------------------------------------------------------
 1 | def job_finished(user, name, title, id=None):
 2 |     if id:
 3 |         return user + ': Your job for ' + name + ' \'' + title + '\' with ID ' + id + ' is finished.'
 4 |     return user + ': Your job for ' + name + ' \'' + title + '\' is finished.'
 5 | 
 6 | def job_failed(user, name, title, id=None):
 7 |     if id:
 8 |         return user + ': Your job for ' + name + ' \'' + title + '\' with ID ' + id + ' failed.'
 9 |     return user + ': Your job for ' + name + ' \'' + title + '\' failed.'
10 | 
11 | def job_added(user, name, title, id=None):
12 |     if id:
13 |         return user + ': Your job for ' + name + ' \'' + title + '\' with ID ' + id + ' is added.'
14 |     return user + ': Your job for ' + name + ' \'' + title + '\' is added.'
15 | 
16 | def job_aborted(user, name, id=None):
17 |     if id:
18 |         return user + ': Your job for ' + name + ' with ID ' + id + ' is aborted.'
19 |     return user + ': Your job for ' + name + ' is aborted.'
20 | 
21 | def failed_extraction(user, name, extract, id=None):
22 |     if id:
23 |         return user + ': Failed to extract ' + extract + ' from ' + name + ' with ID ' + id + '.'
24 |     return user + ': Failed to extract ' + extract + ' from ' + name + '.'


--------------------------------------------------------------------------------
/main.py:
--------------------------------------------------------------------------------
 1 | #if os.path.isdir('./services'):
 2 | #    shutil.rmtree('./services')
 3 | #    os.makedirs('./services')
 4 | #if not os.path.isfile('./services/__init__.py'):
 5 | #    open('./services/__init__.py', 'w').close()
 6 | import os
 7 | import glob
 8 | import threading
 9 | import irc_bot
10 | import refresh
11 | import upload
12 | import time
13 | import functions
14 | 
15 | def dashboard():
16 |     os.system('~/.local/bin/gs-server')
17 | 
18 | def irc_bot_listener():
19 |     irc_bot.irc_bot_listener()
20 | 
21 | def irc_bot_join():
22 |     irc_bot.irc_bot_join()
23 | 
24 | def refresh_services():
25 |     refresh.refresh_services()
26 | 
27 | def process_warcs():
28 |     upload.firstrun()
29 |     while True:
30 |         try:
31 |             upload.move_warcs()
32 |             threading.Thread(target = upload.upload_items).start()
33 |         except:
34 |             pass #for now
35 |         time.sleep(60)
36 | 
37 | def remove_old_files():
38 |     for file in glob.glob('to_be_uploaded/ia_items/*/no_upload') + glob.glob('to_be_uploaded/ia_items/*/*.upload'):
39 |         os.remove(file)
40 |     while True:
41 |         for file in glob.glob('to_be_uploaded/ia_items/*/*.upload'):
42 |             os.remove(file)
43 |         time.sleep(21600)
44 | 
45 | def main():
46 |     if not os.path.isdir('./to_be_uploaded/ia_items'):
47 |         os.makedirs('./to_be_uploaded/ia_items')
48 |     if not os.path.isdir('./to_be_uploaded/ia_warcs'):
49 |         os.makedirs('./to_be_uploaded/ia_warcs')
50 |     irc_bot_join()
51 |     refresh.refresh_services()
52 |     threading.Thread(target = remove_old_files).start()
53 |     threading.Thread(target = refresh.refresh_periodical_jobs).start()
54 |     threading.Thread(target = refresh.refresh_periodical_jobs_start).start()
55 |     threading.Thread(target = irc_bot_listener).start()
56 |     threading.Thread(target = functions.periodical_job.periodical_job_auto_remove).start()
57 |     threading.Thread(target = dashboard).start()
58 |     threading.Thread(target = process_warcs).start()
59 | 
60 | if __name__ == '__main__':
61 | 	main()


--------------------------------------------------------------------------------
/periodical_jobs/__init__.py:
--------------------------------------------------------------------------------
1 | import os
2 | 
3 | for module in os.listdir(os.path.dirname(__file__)):
4 |     if module == '__init__.py' or module[-3:] != '.py':
5 |         continue
6 |     __import__(module[:-3], locals(), globals())
7 | del module
8 | 


--------------------------------------------------------------------------------
/periodical_jobs/check_temp_perjob.py:
--------------------------------------------------------------------------------
 1 | import temp_perjobs
 2 | import os
 3 | import random
 4 | import string
 5 | 
 6 | def check_temp_perjob_variable(ticket_id, var):
 7 |     ticket_id_ = ''.join(random.choice(string.ascii_lowercase) for num in range(10))
 8 |     os.rename('temp_perjobs/' + ticket_id + '.py', 'temp_perjobs/' + ticket_id + ticket_id_ + '.py')
 9 |     reload(temp_perjobs)
10 |     try:
11 |         variable = eval('temp_perjobs.' + ticket_id + ticket_id_ + '.' + var)
12 |         os.rename('temp_perjobs/' + ticket_id + ticket_id_ + '.py', 'temp_perjobs/' + ticket_id + '.py')
13 |         os.remove('temp_perjobs/' + ticket_id + ticket_id_ + '.pyc')
14 |         return variable
15 |     except:
16 |         os.rename('temp_perjobs/' + ticket_id + ticket_id_ + '.py', 'temp_perjobs/' + ticket_id + '.py')
17 |         os.remove('temp_perjobs/' + ticket_id + ticket_id_ + '.pyc')
18 |         return 'var not found'
19 | 
20 | def get_temp_perjob_variables(ticket_id):
21 |     file_location = None
22 |     if os.path.isfile('temp_perjobs/' + ticket_id + '.py'):
23 |         file_location = 'temp_perjobs/' + ticket_id + '.py'
24 |     elif os.path.isfile('periodical_jobs/' + ticket_id + '.py'):
25 |         file_location = 'periodical_jobs/' + ticket_id + '.py'
26 |     if file_location != None:
27 |         variables = []
28 |         with open('temp_perjobs/' + ticket_id + '.py', 'r') as file:
29 |             for line in file:
30 |                 line = line.replace('\n', '').replace('\r', '')
31 |                 if ' = ' in line:
32 |                     variables.append(line.split(' = ')[0])
33 |         return variables
34 |     else:
35 |         return None
36 | 


--------------------------------------------------------------------------------
/refresh.py:
--------------------------------------------------------------------------------
  1 | from config import irc_channel
  2 | from config import github
  3 | import os
  4 | import services
  5 | import time
  6 | import re
  7 | import irc_bot
  8 | import periodical_jobs
  9 | import random
 10 | import string
 11 | import threading
 12 | import functions
 13 | 
 14 | periodical_job_start = lambda filename, type_, user: functions.periodical_job.periodical_job_start(filename, type_, user)
 15 | 
 16 | services_count = 0
 17 | services_list = [['service_name', 'service_regex', ['service_commands']]]
 18 | periodical_jobs_list = [['perjob_name', 'refreshtime']]
 19 | periodical_jobs_start = {}
 20 | 
 21 | def irc_bot_print(channel, message):
 22 |     irc_bot.irc_bot_print(channel, message)
 23 | 
 24 | def refresh_services():
 25 |     global services_list
 26 |     global services_count
 27 |     services_list = [['service_name', 'service_regex', ['service_commands']]]
 28 |     new_services = 0
 29 |     #if os.path.isdir('./services'):
 30 |     #    shutil.rmtree('./services')
 31 |     #os.system('git clone ' + github + '.git')
 32 |     #repository_name = re.search(r'([^\/]+)\/?$', github).group(1)
 33 |     #shutil.copytree('./' + repository_name + '/services', './services')
 34 |     #shutil.rmtree('./' + repository_name)
 35 |     reload(services)
 36 |     for root, dirs, files in os.walk("./services"):
 37 |         for service in files:
 38 |             if service.startswith("video__") and service.endswith(".py"):
 39 |                 if service[:-3] in services_list:
 40 |                     break
 41 |                 else:
 42 |                     try:
 43 |                         url_regex = eval('services.' + service[:-3] + '.url_regex')
 44 |                     except AttributeError:
 45 |                         url_regex = None
 46 |                     service_commands = eval('services.' + service[:-3] + '.service_commands')
 47 |                     services_list.append([service[:-3], url_regex, service_commands])
 48 |                     new_services += 1
 49 |                     print('Found service ' + service[:-3] + '.')
 50 |     new_count = new_services-services_count
 51 |     services_count = new_services
 52 |     if new_count == 1:
 53 |         irc_bot_print(irc_channel, 'Found and updated ' + str(new_count) + ' service.')
 54 |     elif new_count != 0:
 55 |         irc_bot_print(irc_channel, 'Found and updated ' + str(new_count) + ' services.')
 56 | 
 57 | def refresh_periodical_jobs():
 58 |     global periodical_jobs_list
 59 |     while True:
 60 |         periodical_jobs_list_ = [['perjob_name', 'refreshtime']]
 61 |         random_string = ''.join(random.choice(string.ascii_lowercase) for num in range(10))
 62 |         for filename in os.listdir('periodical_jobs'):
 63 |             if filename.endswith('.py') and filename not in ('check_temp_perjob.py', '__init__.py'):
 64 |                 filename_ = filename.replace('.py', random_string + '.py')
 65 |                 os.rename('periodical_jobs/' + filename, 'periodical_jobs/' + filename_)
 66 |         reload(periodical_jobs)
 67 |         time.sleep(10)
 68 |         for filename in os.listdir('periodical_jobs'):
 69 |             if filename.endswith(random_string + '.py'):
 70 |                 filename_ = filename.replace(random_string + '.py', '.py')
 71 |                 os.rename('periodical_jobs/' + filename, 'periodical_jobs/' + filename_)
 72 |                 for periodical_job_list_ in periodical_jobs_list_:
 73 |                     if filename[:-3] in periodical_job_list_:
 74 |                         break
 75 |                 else:
 76 |                     periodical_jobs_list_.append([filename[:-3], eval('periodical_jobs.' + filename[:-3] + '.refreshtime')])
 77 |                     print('Found periodical job ' + filename[:-13] + '.')
 78 |                 os.remove('periodical_jobs/' + filename + 'c')
 79 |         periodical_jobs_list = list(periodical_jobs_list_)
 80 |         time.sleep(300)
 81 | 
 82 | def refresh_periodical_jobs_start():
 83 |     global periodical_jobs_list
 84 |     global periodical_jobs_start
 85 |     while True:
 86 |         for periodical_job_list in periodical_jobs_list:
 87 |             if periodical_job_list[0] != 'perjob_name':
 88 |                 periodical_job_name = periodical_job_list[0][:-10]
 89 |                 if periodical_job_name in periodical_jobs_start:
 90 |                     last_start = periodical_jobs_start[periodical_job_name]
 91 |                 else:
 92 |                     last_start = 0
 93 |                 current_time = int(time.time())
 94 |                 if last_start + periodical_job_list[1] <= current_time:
 95 |                     periodical_jobs_start[periodical_job_name] = current_time
 96 |                     threading.Thread(target = periodical_job_start, args = (periodical_job_list[0], eval('periodical_jobs.' + periodical_job_list[0] + '.type'), eval('periodical_jobs.' + periodical_job_list[0] + '.user'),)).start()
 97 |         time.sleep(1)
 98 | 
 99 | def periodical_job_args(filename, args):
100 |     args_ = []
101 |     for arg in args:
102 |         try:
103 |             variable_content = eval('periodical_jobs.' + filename + '.' + arg)
104 |         except AttributeError:
105 |             variable_content = ''
106 |         args_.append(variable_content)
107 |     return args_
108 | 


--------------------------------------------------------------------------------
/services/__init__.py:
--------------------------------------------------------------------------------
1 | import os
2 | 
3 | for module in os.listdir(os.path.dirname(__file__)):
4 |     if module.startswith('dl__') or module == '__init__.py' or module[-3:] != '.py':
5 |         continue
6 |     __import__(module[:-3], locals(), globals())
7 | del module
8 | 


--------------------------------------------------------------------------------
/services/dl__facebook_com.py:
--------------------------------------------------------------------------------
  1 | import sys
  2 | import time
  3 | import os
  4 | import re
  5 | import shutil
  6 | import time
  7 | import json
  8 | import html
  9 | import datetime
 10 | 
 11 | wpull_hook = globals().get('wpull_hook')
 12 | 
 13 | firsturl = ''
 14 | ia_metadata = {'identifier': '', 'files': [], 'title': '', 'description': '', 'mediatype': 'movies', 'collection': 'archiveteam_videobot', 'date': '', 'original_url': '', 'creator': '', 'subject': ''}
 15 | added_to_list = []
 16 | tries = {}
 17 | video_file = None
 18 | item_id = None
 19 | 
 20 | def accept_url(url_info, record_info, verdict, reasons):
 21 |     global added_to_list
 22 |     if (firsturl == '' or url_info["url"] in added_to_list) and not '\\' in url_info["url"]:
 23 |         return True
 24 |     return False
 25 | 
 26 | def get_urls(filename, url_info, document_info):
 27 |     global firsturl
 28 |     global item_id
 29 |     global ia_metadata
 30 |     global added_to_list
 31 |     global video_file
 32 | 
 33 |     newurls = []
 34 | 
 35 |     def url_allowed(url, parent_url=None):
 36 |         return True
 37 | 
 38 |     def add_url(url, parent_url=None):
 39 |         if url in added_to_list:
 40 |             return None
 41 |         if url_allowed(url, parent_url):
 42 |             added_to_list.append(url)
 43 |             newurls.append({'url': url})
 44 | 
 45 |     if video_file is not None and video_file in url_info["url"]:
 46 |         if not os.path.isdir('../ia_item'):
 47 |             os.makedirs('../ia_item')
 48 |         shutil.copyfile(filename, '../ia_item/' + video_file)
 49 |         ia_metadata['files'].append(video_file)
 50 | 
 51 |     if firsturl == '':
 52 |         with open(filename, 'r', encoding='utf-8') as file:
 53 |             content = file.read()
 54 |             firsturl = url_info['url']
 55 |             for url in extract_urls(content, url_info['url']):
 56 |                 add_url(url)
 57 |             url_name = 'hd_src_no_ratelimit'
 58 |             if 'hd_src_no_ratelimit' not in content:
 59 |                 url_name = 'sd_src_no_ratelimit'
 60 |             video_file = re.search(url_name + ':"https?://[^/]+/v/[^/]+/([a-zA-Z0-9-_]+\.mp4)', content).group(1)
 61 |             item_id = re.search('video_id:"([0-9]+)"', content).group(1)
 62 |             item_name = re.search('ownerName:"([^"]+)"', content).group(1)
 63 |             ia_metadata['identifier'] = 'archiveteam_videobot_facebook_com_' + item_id
 64 |             ia_metadata['title'] = re.search('<title\s+id="pageTitle">([^<]+(?:\.\.\.))\s+-\s+[^<]+</title>', content).group(1)
 65 |             ia_metadata['description'] = html.unescape(re.search('<div\s+class="[^"]+userContent"[^>]+>(.+?)</div>', content).group(1))
 66 |             ia_metadata['date'] = datetime.datetime.fromtimestamp(int(re.search('data-utime="([0-9]+)"', content).group(1))).strftime('%Y-%m-%d %H:%M:%S')
 67 |             ia_metadata['original_url'] = firsturl
 68 |             ia_metadata['creator'] = item_name
 69 |             ia_metadata['creator_id'] = re.search('^https?://[^/]+/([^/]+)/videos/', url_info["url"]).group(1)
 70 |             ia_metadata['video_id'] = item_id
 71 |             ia_metadata['subject'] = ['videobot', 'archiveteam', 'facebook', 'facebook.com', item_id, item_name]
 72 | 
 73 |     for newurl in newurls:
 74 |         added_to_list.append(newurl['url'])
 75 | 
 76 |     return newurls
 77 | 
 78 | def exit_status(exit_code):
 79 |     global ia_metadata
 80 | 
 81 |     if os.path.isdir('../ia_item'):
 82 |         item_identifier = ia_metadata['identifier']
 83 |         for a, b in ia_metadata.items():
 84 |             with open('../ia_item/ia_metadata.py', 'a') as file:
 85 |                 if type(b) is list:
 86 |                     content_string = str(b)
 87 |                 else:
 88 |                     content_string = '\'' + str(b).replace('\'', '\\\'').replace('\n', '\\n').replace('\r', '\\r') + '\''
 89 |                 file.write(str(a) + ' = ' + content_string + '\n')
 90 | 
 91 |         if len(os.listdir('../ia_item')) > 1:
 92 |             print(ia_metadata['files'])
 93 |             os.rename('../ia_item', '../../to_be_uploaded/ia_items/ia_item_' + item_identifier + '_' + str(int(time.time())))
 94 | 
 95 |     return exit_code
 96 | 
 97 | handle_response_grabsite = wpull_hook.callbacks.handle_response
 98 | def handle_response(url_info, record_info, response_info):
 99 |     global tries
100 | 
101 |     if not url_info["url"] in tries:
102 |         tries[url_info["url"]] = 0
103 |     elif tries[url_info["url"]] > 5:
104 |         return wpull_hook.actions.FINISH        
105 | 
106 |     tries[url_info["url"]] += 1
107 | 
108 |     return handle_response_grabsite(url_info, record_info, response_info)
109 | 
110 | wpull_hook.callbacks.get_urls = get_urls
111 | wpull_hook.callbacks.exit_status = exit_status
112 | wpull_hook.callbacks.accept_url = accept_url
113 | wpull_hook.callbacks.handle_response = handle_response
114 | 
115 | def extract_urls(file, url):
116 |     extractedurls = []
117 |     for extractedurl in re.findall('((?:....=)?(?P<quote>[\'"]).*?(?P=quote))', file, re.I):
118 |         extractedstart = ''
119 |         if re.search('^....=[\'"](.*?)[\'"]$', extractedurl[0], re.I):
120 |             extractedstart = re.search(r'^(....)', extractedurl[0], re.I).group(1)
121 |             extractedurl = re.search('^....=[\'"](.*?)[\'"]$', extractedurl[0], re.I).group(1)
122 |         else:
123 |             extractedurl = extractedurl[0][1:-1]
124 |         extractedurl = re.search(r'^([^#]*)', extractedurl, re.I).group(1)
125 |         extractedurl = extractedurl.replace('%3A', ':').replace('%2F', '/')
126 |         if extractedurl.startswith('http:\/\/') or extractedurl.startswith('https:\/\/') or extractedurl.startswith('HTTP:\/\/') or extractedurl.startswith('HTTPS:\/\/'):
127 |             extractedurl = extractedurl.replace('\/', '/')
128 |         if extractedurl.startswith('//'):
129 |             extractedurls.append("http:" + extractedurl)
130 |         elif extractedurl.startswith('/'):
131 |             extractedurls.append(re.search(r'^(https?:\/\/[^\/]+)', url, re.I).group(1) + extractedurl)
132 |         elif re.search(r'^https?:?\/\/?', extractedurl, re.I):
133 |             extractedurls.append(extractedurl.replace(re.search(r'^(https?:?\/\/?)', extractedurl, re.I).group(1), re.search(r'^(https?)', extractedurl, re.I).group(1) + '://'))
134 |         elif extractedurl.startswith('?'):
135 |             extractedurls.append(re.search(r'^(https?:\/\/[^\?]+)', url, re.I).group(1) + extractedurl)
136 |         elif extractedurl.startswith('./'):
137 |             if re.search(r'^https?:\/\/.*\/', url, re.I):
138 |                 extractedurls.append(re.search(r'^(https?:\/\/.*)\/', url, re.I).group(1) + '/' + re.search(r'^\.\/(.*)', extractedurl, re.I).group(1))
139 |             else:
140 |                 extractedurls.append(re.search(r'^(https?:\/\/.*)', url, re.I).group(1) + '/' + re.search(r'^\.\/(.*)', extractedurl, re.I).group(1))
141 |         elif extractedurl.startswith('../'):
142 |             tempurl = url
143 |             tempextractedurl = extractedurl
144 |             while tempextractedurl.startswith('../'):
145 |                 if not re.search(r'^https?://[^\/]+\/$', tempurl, re.I):
146 |                     tempurl = re.search(r'^(.*\/)[^\/]*\/', tempurl, re.I).group(1)
147 |                 tempextractedurl = re.search(r'^\.\.\/(.*)', tempextractedurl).group(1)
148 |             extractedurls.append(tempurl + tempextractedurl)
149 |         elif extractedstart == 'href':
150 |             if re.search(r'^https?:\/\/.*\/', url, re.I):
151 |                 extractedurls.append(re.search(r'^(https?:\/\/.*)\/', url, re.I).group(1) + '/' + extractedurl)
152 |             else:
153 |                 extractedurls.append(re.search(r'^(https?:\/\/.*)', url, re.I).group(1) + '/' + extractedurl)
154 |     for extractedurl in re.findall(r'>[^<a-zA-Z0-9]*(https?:?//?[^<]+)<', file, re.I):
155 |         extractedurl = re.search(r'^([^#]*)', extractedurl, re.I).group(1)
156 |         extractedurls.append(extractedurl.replace(re.search(r'^(https?:?\/\/?)', extractedurl, re.I).group(1), re.search(r'^(https?)', extractedurl, re.I).group(1) + '://'))
157 |     for extractedurl in re.findall(r'\[[^<a-zA-Z0-9]*(https?:?//?[^\]]+)\]', file, re.I):
158 |         extractedurl = re.search(r'^([^#]*)', extractedurl, re.I).group(1)
159 |         extractedurls.append(extractedurl.replace(re.search(r'^(https?:?\/\/?)', extractedurl, re.I).group(1), re.search(r'^(https?)', extractedurl, re.I).group(1) + '://'))
160 |     return [extractedurl.replace('&amp;', '&').replace('&amp;', '&') for extractedurl in extractedurls]
161 | 


--------------------------------------------------------------------------------
/services/dl__ignores__twitter_com:
--------------------------------------------------------------------------------
 1 | https://abs.twimg.com/a/1478980555/css/t1/twitter_core.bundle.css
 2 | https://abs.twimg.com/a/1478980555/css/t1/twitter_more_1.bundle.css
 3 | https://abs.twimg.com/a/1478980555/css/t1/twitter_more_2.bundle.css
 4 | https://abs.twimg.com/c/swift/en/init.9a91b42242b76640a1a7b4612bece0745a36d412.js
 5 | https://abs.twimg.com/c/swift/en/bundle/timeline.be5487227341749cb7eb301264c5745abf5d4730.js
 6 | https://abs.twimg.com/c/swift/en/bundle/boot.d2bccebb539fb1eb89f5051eb7a5f178dc869214.js
 7 | http://abs.twimg.com/favicons/win8-tile-144.png
 8 | https://abs.twimg.com/a/1478980555/img/t1/favicon.svg
 9 | http://abs.twimg.com/favicons/favicon.ico
10 | https://abs.twimg.com/icons/apple-touch-icon-192x192.png
11 | https://twitter.com/manifest.json
12 | https://twitter.com/opensearch.xml
13 | https://abs.twimg.com/c/swift/en
14 | https://twitter.com/
15 | https://twitter.com/about
16 | https://twitter.com/search
17 | https://twitter.com/sessions/change_locale
18 | https://twitter.com/login
19 | https://twitter.com/sessions
20 | https://twitter.com/login
21 | https://twitter.com/account/begin_password_reset
22 | https://twitter.com/signup
23 | https://support.twitter.com/articles/20170514
24 | https://support.twitter.com/articles/20174632
25 | https://twitter.com/tos
26 | https://twitter.com/privacy
27 | http://support.twitter.com/articles/20170514
28 | https://support.twitter.com/articles/20170514
29 | http://support.twitter.com/articles/20170451
30 | http://support.twitter.com/forums/26810/entries/78525
31 | http://dev.twitter.com/docs/embedded-tweets
32 | http://support.twitter.com/articles/14226-how-to-find-your-twitter-short-code-or-long-code
33 | https://abs.twimg.com/web-video-player/926f18062272ee82cb150784d63e419001e2c8e8/styles/main.css
34 | https://abs.twimg.com/web-video-player/926f18062272ee82cb150784d63e419001e2c8e8/styles/img/media_equalizer.gif
35 | https://t.lv.twimg.com/live_video_cookies


--------------------------------------------------------------------------------
/services/dl__liveleak_com.py:
--------------------------------------------------------------------------------
  1 | import sys
  2 | import time
  3 | import os
  4 | import re
  5 | import shutil
  6 | import time
  7 | import json
  8 | import html
  9 | 
 10 | wpull_hook = globals().get('wpull_hook')  # silence code checkers
 11 | 
 12 | counter = 0
 13 | firsturl = ''
 14 | ia_metadata = {'identifier': '', 'files': [], 'title': '', 'description': '', 'mediatype': 'movies', 'collection': 'archiveteam_videobot', 'date': '', 'original_url': '', 'creator': '', 'subject': ''}
 15 | orig_video = ''
 16 | added_to_list = []
 17 | item_id = ''
 18 | tempfiles = {}
 19 | 
 20 | def accept_url(url_info, record_info, verdict, reasons):
 21 |     global added_to_list
 22 |     if (firsturl == '' or url_info["url"] in added_to_list) and not '?lang=' in url_info["url"]:
 23 |         return True
 24 |     return False
 25 | 
 26 | def get_urls(filename, url_info, document_info):
 27 |     global counter
 28 |     global firsturl
 29 |     global ia_metadata
 30 |     global orig_video
 31 |     global added_to_list
 32 |     global item_id
 33 |     global tempfiles
 34 |     newurls = []
 35 |     if url_info["url"] in orig_video:
 36 |         filename_new = re.search(r'^https?://cdn\.liveleak\.com/.+?([^/]+\.[0-9a-zA-Z]+)\?', url_info["url"]).group(1)
 37 |         if not os.path.isdir('../ia_item'):
 38 |             os.makedirs('../ia_item')
 39 |         if not os.path.isfile('../ia_item/' + filename_new):
 40 |             shutil.copyfile(filename, '../ia_item/' + filename_new)
 41 |             ia_metadata['files'].append(filename_new)
 42 |     elif re.search('^https?://(?:www\.)?liveleak\.com/view\?i=[a-z0-9_]+', url_info["url"]) and firsturl == '':
 43 |         with open(filename, 'r', encoding='utf-8') as file:
 44 |             content = file.read()
 45 |             months = {'Jan': '01', 'Feb': '02', 'Mar': '03', 'Apr': '04', 'May': '05', 'Jun': '06', 'Jul': '07', 'Aug': '08', 'Sep': '09', 'Oct': '10', 'Nov': '11', 'Dec': '12'}
 46 |             item_description = re.sub('(?:<a\s+href=[^>]+>|</a>|<p>)', '', re.search(r'<div\s+id="body_text">(.+(?:\n.+)+)<style>', content).group(1).strip())
 47 |             item_title = re.search(r'<meta\s+property="og:title"\s+content="([^"]+)"/>', content).group(1).replace('LiveLeak.com - ', '')
 48 |             item_id = re.search(r'^https?://(?:www\.)?liveleak\.com/view\?i=([a-z0-9_]+)', url_info["url"]).group(1)
 49 |             item_name = re.search(r'By:.*?([0-9a-zA-Z-_]+)<\/a>', content).group(1)
 50 |             item_tags = [tag.replace('browse&q=', '') for tag in re.findall(r'"browse\?q=([^"]+)"', content)]
 51 |             item_date_ = re.search(r'/thumbs/([0-9]+/[A-Z][a-z]+/[0-9]+)/', content).group(1).replace('/', '-')
 52 |             item_date = item_date_.replace(re.search(r'-([A-Z][a-z]+)-', item_date_).group(1), months[re.search(r'-([A-Z][a-z]+)-', item_date_).group(1)])
 53 |             ia_metadata['title'] = item_title
 54 |             ia_metadata['identifier'] = 'archiveteam_videobot_liveleak_com_' + item_id
 55 |             ia_metadata['description'] = item_description
 56 |             ia_metadata['date'] = item_date
 57 |             ia_metadata['original_url'] = url_info["url"]
 58 |             ia_metadata['creator'] = item_name
 59 |             ia_metadata['subject'] = ';'.join(['videobot', 'archiveteam', 'liveleak', 'liveleak.com', item_id, item_name] + item_tags)
 60 |             orig_video = [''.join(parts) for parts in re.findall(r'file: "([^"]+)\.[^\.]+\.mp4(\?[^"]+)"', content)]
 61 |             print(orig_video)
 62 |             firsturl = url_info["url"]
 63 |             newurls += [{'url': url} for url in orig_video]
 64 |     if item_id in url_info["url"] and not re.search(r'^https?://cdn.liveleak.com', url_info["url"]):
 65 |         with open(filename, 'r', encoding='utf-8') as file:
 66 |             content = file.read()
 67 |             newurls += [{'url': re.search(r'^(https?://.+/)', url_info["url"]).group(1) + url.replace('&amp;', '&')} for url in re.findall(r"\.load\('([^']+)'\)", content)]
 68 |             newurls += [{'url': url} for url in extract_urls(content, url_info["url"]) if (re.search(r'^https?://(?:www\.)?liveleak\.com/view\?i=' + item_id, url) or not (re.search(r'^https?://(?:www\.)?liveleak\.com/view\?i=[a-z0-9_]+', url) or re.search(r'^https?://(?:www\.)?liveleak\.com/c/', url))) and not url in added_to_list]
 69 |     newurls += [{'url': newurl["url"] + '&ec_seek=0'} for newurl in newurls if re.search(r'ec_rate=[0-9]+$', newurl["url"])]
 70 |     for newurl in newurls:
 71 |         added_to_list.append(newurl["url"])
 72 |     return newurls + [{'url': newurl["url"] + '&ec_seek=0'} for newurl in newurls if re.search(r'ec_rate=[0-9]+$', newurl["url"])]
 73 | 
 74 | def exit_status(exit_code):
 75 |     global ia_metadata
 76 |     global tempfiles
 77 |     if os.path.isdir('../ia_item'):
 78 |         item_identifier = ia_metadata['identifier']
 79 |         for a, b in ia_metadata.items():
 80 |             with open('../ia_item/ia_metadata.py', 'a') as file:
 81 |                 if type(b) is list:
 82 |                     content_string = str(b)
 83 |                 else:
 84 |                     content_string = '\'' + str(b).replace('\'', '\\\'').replace('\n', '\\n').replace('\r', '\\r') + '\''
 85 |                 file.write(str(a) + ' = ' + content_string + '\n')
 86 |         os.rename('../ia_item', '../../to_be_uploaded/ia_items/ia_item_' + item_identifier + '_' + str(int(time.time())))
 87 |     return exit_code
 88 | 
 89 | wpull_hook.callbacks.get_urls = get_urls
 90 | wpull_hook.callbacks.exit_status = exit_status
 91 | wpull_hook.callbacks.accept_url = accept_url
 92 | 
 93 | def extract_urls(file, url):
 94 |     extractedurls = []
 95 |     for extractedurl in re.findall('((?:....=)?(?P<quote>[\'"]).*?(?P=quote))', file, re.I):
 96 |         extractedstart = ''
 97 |         if re.search('^....=[\'"](.*?)[\'"]$', extractedurl[0], re.I):
 98 |             extractedstart = re.search(r'^(....)', extractedurl[0], re.I).group(1)
 99 |             extractedurl = re.search('^....=[\'"](.*?)[\'"]$', extractedurl[0], re.I).group(1)
100 |         else:
101 |             extractedurl = extractedurl[0][1:-1]
102 |         extractedurl = re.search(r'^([^#]*)', extractedurl, re.I).group(1)
103 |         extractedurl = extractedurl.replace('%3A', ':').replace('%2F', '/')
104 |         if extractedurl.startswith('http:\/\/') or extractedurl.startswith('https:\/\/') or extractedurl.startswith('HTTP:\/\/') or extractedurl.startswith('HTTPS:\/\/'):
105 |             extractedurl = extractedurl.replace('\/', '/')
106 |         if extractedurl.startswith('//'):
107 |             extractedurls.append("http:" + extractedurl)
108 |         elif extractedurl.startswith('/'):
109 |             extractedurls.append(re.search(r'^(https?:\/\/[^\/]+)', url, re.I).group(1) + extractedurl)
110 |         elif re.search(r'^https?:?\/\/?', extractedurl, re.I):
111 |             extractedurls.append(extractedurl.replace(re.search(r'^(https?:?\/\/?)', extractedurl, re.I).group(1), re.search(r'^(https?)', extractedurl, re.I).group(1) + '://'))
112 |         elif extractedurl.startswith('?'):
113 |             extractedurls.append(re.search(r'^(https?:\/\/[^\?]+)', url, re.I).group(1) + extractedurl)
114 |         elif extractedurl.startswith('./'):
115 |             if re.search(r'^https?:\/\/.*\/', url, re.I):
116 |                 extractedurls.append(re.search(r'^(https?:\/\/.*)\/', url, re.I).group(1) + '/' + re.search(r'^\.\/(.*)', extractedurl, re.I).group(1))
117 |             else:
118 |                 extractedurls.append(re.search(r'^(https?:\/\/.*)', url, re.I).group(1) + '/' + re.search(r'^\.\/(.*)', extractedurl, re.I).group(1))
119 |         elif extractedurl.startswith('../'):
120 |             tempurl = url
121 |             tempextractedurl = extractedurl
122 |             while tempextractedurl.startswith('../'):
123 |                 if not re.search(r'^https?://[^\/]+\/$', tempurl, re.I):
124 |                     tempurl = re.search(r'^(.*\/)[^\/]*\/', tempurl, re.I).group(1)
125 |                 tempextractedurl = re.search(r'^\.\.\/(.*)', tempextractedurl).group(1)
126 |             extractedurls.append(tempurl + tempextractedurl)
127 |         elif extractedstart == 'href':
128 |             if re.search(r'^https?:\/\/.*\/', url, re.I):
129 |                 extractedurls.append(re.search(r'^(https?:\/\/.*)\/', url, re.I).group(1) + '/' + extractedurl)
130 |             else:
131 |                 extractedurls.append(re.search(r'^(https?:\/\/.*)', url, re.I).group(1) + '/' + extractedurl)
132 |     for extractedurl in re.findall(r'>[^<a-zA-Z0-9]*(https?:?//?[^<]+)<', file, re.I):
133 |         extractedurl = re.search(r'^([^#]*)', extractedurl, re.I).group(1)
134 |         extractedurls.append(extractedurl.replace(re.search(r'^(https?:?\/\/?)', extractedurl, re.I).group(1), re.search(r'^(https?)', extractedurl, re.I).group(1) + '://'))
135 |     for extractedurl in re.findall(r'\[[^<a-zA-Z0-9]*(https?:?//?[^\]]+)\]', file, re.I):
136 |         extractedurl = re.search(r'^([^#]*)', extractedurl, re.I).group(1)
137 |         extractedurls.append(extractedurl.replace(re.search(r'^(https?:?\/\/?)', extractedurl, re.I).group(1), re.search(r'^(https?)', extractedurl, re.I).group(1) + '://'))
138 |     return [extractedurl.replace('&amp;', '&').replace('&amp;', '&') for extractedurl in extractedurls]
139 | 


--------------------------------------------------------------------------------
/services/dl__periscope_tv.py:
--------------------------------------------------------------------------------
  1 | import sys
  2 | import time
  3 | import os
  4 | import re
  5 | import shutil
  6 | import time
  7 | import json
  8 | import html
  9 | 
 10 | wpull_hook = globals().get('wpull_hook')  # silence code checkers
 11 | 
 12 | counter = 0
 13 | firsturl = ''
 14 | ia_metadata = {'identifier': '', 'files': [], 'title': '', 'description': '', 'mediatype': 'movies', 'collection': 'archiveteam_videobot', 'date': '', 'original_url': '', 'creator': '', 'subject': ''}
 15 | video_id = ''
 16 | added_to_list = []
 17 | tempfiles = []
 18 | tries = {}
 19 | 
 20 | def accept_url(url_info, record_info, verdict, reasons):
 21 |     global added_to_list
 22 |     global firsturl
 23 |     if firsturl == '' or url_info["url"] in added_to_list:
 24 |         return True
 25 |     return False
 26 | 
 27 | def get_urls(filename, url_info, document_info):
 28 |     global counter
 29 |     global firsturl
 30 |     global ia_metadata
 31 |     global video_id
 32 |     global added_to_list
 33 |     global tempfiles
 34 |     newurls = []
 35 | 
 36 |     if firsturl == '':
 37 |         firsturl = url_info["url"]
 38 | 
 39 |     if re.search(r'^https?://[^\.]+\.periscope\.tv/.+/playlist[_0-9]*\.m3u8$', url_info["url"]):
 40 |         with open(filename, 'r', encoding='utf-8') as file:
 41 |             for line in file:
 42 |                 line = line.replace('\r', '').replace('\n', '')
 43 |                 if line.endswith('.ts'):
 44 |                     newurls.append({'url': re.search(r'^(https?://[^\.]+\.periscope\.tv/.+/)playlist[_0-9]*\.m3u8$', url_info["url"]).group(1) + line})
 45 |                     tempfiles.append(line)
 46 | 
 47 |     elif re.search(r'^https?://[^\.]+\.periscope\.tv/.+/chunk_[0-9_a-z]+\.ts$', url_info["url"]):
 48 |         filename_new = re.search(r'^https?://[^\.]+\.periscope\.tv/.+/(chunk_[0-9_a-z]+\.ts)$', url_info["url"]).group(1)
 49 |         if not os.path.isdir('../ia_item'):
 50 |             os.makedirs('../ia_item')
 51 |         if not os.path.isfile('../ia_item/' + filename_new):
 52 |             shutil.copyfile(filename, '../ia_item/' + filename_new)
 53 | 
 54 |     elif re.search(r'^https?://(?:www\.)?periscope\.tv/w/[0-9a-zA-Z]+$', url_info["url"]):
 55 |         with open(filename, 'r', encoding='utf-8') as file:
 56 |             content = file.read()
 57 |             content_json = html.unescape(re.search(r'data-store="({[^"]+})"', content).group(1))
 58 |             json_ = json.loads(content_json)
 59 |             api_session = json_['SessionToken']['public']['thumbnailPlaylist']['token']['session_id']
 60 |             item_url_id = re.search(r'^https?://(?:www\.)?periscope\.tv/w/([0-9a-zA-Z]+)$', url_info["url"]).group(1)
 61 |             item_name_id = json_['BroadcastCache']['broadcasts'][item_url_id]['broadcast']['user_id']
 62 |             item_description = json_['BroadcastCache']['broadcasts'][item_url_id]['broadcast']['status']
 63 |             item_id = json_['BroadcastCache']['broadcasts'][item_url_id]['broadcast']['id']
 64 |             item_location_city = json_['BroadcastCache']['broadcasts'][item_url_id]['broadcast']['city']
 65 |             item_location_country = json_['BroadcastCache']['broadcasts'][item_url_id]['broadcast']['country']
 66 |             item_location_country_state = json_['BroadcastCache']['broadcasts'][item_url_id]['broadcast']['country_state']
 67 |             item_location = (item_location_city + (', ' if item_location_country != '' else '') if item_location_city != '' else '') + item_location_country if item_location_city + item_location_country != '' else ''
 68 |             item_language = json_['BroadcastCache']['broadcasts'][item_url_id]['broadcast']['language']
 69 |             item_name_description = json_['UserCache']['users'][item_name_id]['user']['description']
 70 |             item_name = json_['UserCache']['users'][item_name_id]['user']['display_name']
 71 |             item_username = json_['UserCache']['users'][item_name_id]['user']['username']
 72 |             item_twitter = json_['UserCache']['users'][item_name_id]['user']['twitter_screen_name']
 73 |             item_date = json_['BroadcastCache']['broadcasts'][item_url_id]['broadcast']['created_at'].split('.')[0].replace('T', ' ')
 74 |             ia_metadata['identifier'] = 'archiveteam_videobot_periscope_tv_' + item_id
 75 |             ia_metadata['description'] = item_description + '\n\n' + item_name_description
 76 |             ia_metadata['date'] = item_date
 77 |             ia_metadata['original_url'] = url_info["url"]
 78 |             ia_metadata['twitter'] = item_twitter
 79 |             ia_metadata['creator'] = item_name
 80 |             ia_metadata['creator_username'] = item_username
 81 |             ia_metadata['creator_id'] = item_name_id
 82 |             ia_metadata['title'] = item_description + (' — ' + item_location if item_location != '' else '')
 83 |             ia_metadata['language'] = item_language
 84 |             ia_metadata['creator_description'] = item_name_description
 85 |             ia_metadata['city'] = item_location_city
 86 |             ia_metadata['country'] = item_location_country
 87 |             ia_metadata['country_state'] = item_location_country_state
 88 |             ia_metadata['location'] = item_location
 89 |             ia_metadata['subject'] = ';'.join(['videobot', 'archiveteam', 'periscope', 'periscope.tv', item_id, item_name])
 90 |             if not os.path.isdir('../ia_item'):
 91 |                 os.makedirs('../ia_item')
 92 |             json.dump(json_, open('../ia_item/data.json', 'w'), indent = 4, ensure_ascii = False)
 93 |             ia_metadata['files'].append('data.json')
 94 |             newurls.append({'url': 'https://api.periscope.tv/api/v2/accessVideoPublic?broadcast_id=' + item_id})
 95 |             newurls.append({'url': 'https://api.periscope.tv/api/v2/publicReplayThumbnailPlaylist?broadcast_id=' + item_id + '&session_id=' + api_session})
 96 |             newurls.append({'url': 'https://api.periscope.tv/api/v2/getBroadcastPublic?broadcast_id=' + item_id})
 97 |             newurls += [{'url': url} for url in extract_urls(content, url_info["url"]) if ((re.search(r'^https?://(?:www\.)?periscope\.tv/w/[0-9a-zA-Z]+', url) and item_id in url) or not re.search(r'^https?://(?:www\.)?periscope\.tv/w/[0-9a-zA-Z]+', url)) and not url in added_to_list]
 98 |             newurls += [{'url': url} for url in extract_urls(content_json, url_info["url"]) if not url in added_to_list]
 99 | 
100 |     elif re.search(r'^https://api\.periscope\.tv/api/v2/publicReplayThumbnailPlaylist', url_info["url"]) or re.search(r'^https://api\.periscope\.tv/api/v2/accessVideoPublic', url_info["url"]) or re.search(r'^https?://(?:www\.)?periscope\.tv/w/[0-9a-zA-Z]+', url_info["url"]):
101 |         with open(filename, 'r', encoding='utf-8') as file:
102 |             content = html.unescape(file.read())
103 |             newurls += [{'url': url} for url in extract_urls(content, url_info["url"]) if not url in added_to_list]
104 | 
105 |     for newurl in newurls:
106 |         added_to_list.append(newurl["url"])
107 | 
108 |     return newurls
109 | 
110 | def exit_status(exit_code):
111 |     global ia_metadata
112 |     global tempfiles
113 | 
114 |     tempfiles_ = list(tempfiles)
115 | 
116 |     if os.path.isdir('../ia_item'):
117 |         lists = []
118 |         listsnames = []
119 | 
120 |         while len(tempfiles) > 0:
121 |             if len(tempfiles) < 500:
122 |                 lists.append(list(tempfiles))
123 |                 tempfiles = []
124 |             else:
125 |                 lists.append(list(tempfiles[:500]))
126 |                 tempfiles = list(tempfiles[500:])
127 | 
128 |         for i in range(len(lists)):
129 |             os.system('ffmpeg -i "concat:' + '|'.join(['../ia_item/' + file for file in lists[i]]) + '" -c copy ../ia_item/video' + str(i) + '.ts')
130 |             listsnames.append('video' + str(i) + '.ts')
131 | 
132 |         os.system('ffmpeg -i "concat:' + '|'.join(['../ia_item/' + file for file in listsnames]) + '" -c copy ../ia_item/video.ts')
133 | 
134 |         print(lists)
135 |         print(listsnames)
136 | 
137 |         if os.path.isfile('../ia_item/video.ts'):
138 |             ia_metadata['files'].append('video.ts')
139 | 
140 |         for filename in ['../ia_item/' + file for file in tempfiles_] + ['../ia_item/' + file for file in listsnames]:
141 |             os.remove(filename)
142 | 
143 |         item_identifier = ia_metadata['identifier']
144 | 
145 |         for a, b in ia_metadata.items():
146 |             with open('../ia_item/ia_metadata.py', 'a') as file:
147 |                 if type(b) is list:
148 |                     content_string = str(b)
149 |                 else:
150 |                     content_string = '\'' + str(b).replace('\'', '\\\'').replace('\n', '\\n').replace('\r', '\\r') + '\''
151 |                 file.write(str(a) + ' = ' + content_string + '\n')
152 |         os.rename('../ia_item', '../../to_be_uploaded/ia_items/ia_item_' + item_identifier + '_' + str(int(time.time())))
153 | 
154 |     return exit_code
155 | 
156 | handle_response_grabsite = wpull_hook.callbacks.handle_response
157 | def handle_response(url_info, record_info, response_info):
158 |     global tries
159 | 
160 |     if not url_info["url"] in tries:
161 |         tries[url_info["url"]] = 0
162 |     elif tries[url_info["url"]] > 5:
163 |         return wpull_hook.actions.FINISH        
164 | 
165 |     tries[url_info["url"]] += 1
166 | 
167 |     return handle_response_grabsite(url_info, record_info, response_info)
168 | 
169 | wpull_hook.callbacks.get_urls = get_urls
170 | wpull_hook.callbacks.exit_status = exit_status
171 | wpull_hook.callbacks.accept_url = accept_url
172 | wpull_hook.callbacks.handle_response = handle_response
173 | 
174 | def extract_urls(file, url):
175 |     extractedurls = []
176 |     for extractedurl in re.findall('((?:....=)?(?P<quote>[\'"]).*?(?P=quote))', file, re.I):
177 |         extractedstart = ''
178 |         if re.search('^....=[\'"](.*?)[\'"]$', extractedurl[0], re.I):
179 |             extractedstart = re.search(r'^(....)', extractedurl[0], re.I).group(1)
180 |             extractedurl = re.search('^....=[\'"](.*?)[\'"]$', extractedurl[0], re.I).group(1)
181 |         else:
182 |             extractedurl = extractedurl[0][1:-1]
183 |         extractedurl = re.search(r'^([^#]*)', extractedurl, re.I).group(1)
184 |         extractedurl = extractedurl.replace('%3A', ':').replace('%2F', '/')
185 |         if extractedurl.startswith('http:\/\/') or extractedurl.startswith('https:\/\/') or extractedurl.startswith('HTTP:\/\/') or extractedurl.startswith('HTTPS:\/\/'):
186 |             extractedurl = extractedurl.replace('\/', '/')
187 |         if extractedurl.startswith('//'):
188 |             extractedurls.append("http:" + extractedurl)
189 |         elif extractedurl.startswith('/'):
190 |             extractedurls.append(re.search(r'^(https?:\/\/[^\/]+)', url, re.I).group(1) + extractedurl)
191 |         elif re.search(r'^https?:?\/\/?', extractedurl, re.I):
192 |             extractedurls.append(extractedurl.replace(re.search(r'^(https?:?\/\/?)', extractedurl, re.I).group(1), re.search(r'^(https?)', extractedurl, re.I).group(1) + '://'))
193 |         elif extractedurl.startswith('?'):
194 |             extractedurls.append(re.search(r'^(https?:\/\/[^\?]+)', url, re.I).group(1) + extractedurl)
195 |         elif extractedurl.startswith('./'):
196 |             if re.search(r'^https?:\/\/.*\/', url, re.I):
197 |                 extractedurls.append(re.search(r'^(https?:\/\/.*)\/', url, re.I).group(1) + '/' + re.search(r'^\.\/(.*)', extractedurl, re.I).group(1))
198 |             else:
199 |                 extractedurls.append(re.search(r'^(https?:\/\/.*)', url, re.I).group(1) + '/' + re.search(r'^\.\/(.*)', extractedurl, re.I).group(1))
200 |         elif extractedurl.startswith('../'):
201 |             tempurl = url
202 |             tempextractedurl = extractedurl
203 |             while tempextractedurl.startswith('../'):
204 |                 if not re.search(r'^https?://[^\/]+\/$', tempurl, re.I):
205 |                     tempurl = re.search(r'^(.*\/)[^\/]*\/', tempurl, re.I).group(1)
206 |                 tempextractedurl = re.search(r'^\.\.\/(.*)', tempextractedurl).group(1)
207 |             extractedurls.append(tempurl + tempextractedurl)
208 |         elif extractedstart == 'href':
209 |             if re.search(r'^https?:\/\/.*\/', url, re.I):
210 |                 extractedurls.append(re.search(r'^(https?:\/\/.*)\/', url, re.I).group(1) + '/' + extractedurl)
211 |             else:
212 |                 extractedurls.append(re.search(r'^(https?:\/\/.*)', url, re.I).group(1) + '/' + extractedurl)
213 |     for extractedurl in re.findall(r'>[^<a-zA-Z0-9]*(https?:?//?[^<]+)<', file, re.I):
214 |         extractedurl = re.search(r'^([^#]*)', extractedurl, re.I).group(1)
215 |         extractedurls.append(extractedurl.replace(re.search(r'^(https?:?\/\/?)', extractedurl, re.I).group(1), re.search(r'^(https?)', extractedurl, re.I).group(1) + '://'))
216 |     for extractedurl in re.findall(r'\[[^<a-zA-Z0-9]*(https?:?//?[^\]]+)\]', file, re.I):
217 |         extractedurl = re.search(r'^([^#]*)', extractedurl, re.I).group(1)
218 |         extractedurls.append(extractedurl.replace(re.search(r'^(https?:?\/\/?)', extractedurl, re.I).group(1), re.search(r'^(https?)', extractedurl, re.I).group(1) + '://'))
219 |     return [extractedurl.replace('&amp;', '&').replace(r'\u0026', '&') for extractedurl in extractedurls]


--------------------------------------------------------------------------------
/services/dl__twitter_com.py:
--------------------------------------------------------------------------------
  1 | import sys
  2 | import time
  3 | import os
  4 | import re
  5 | import shutil
  6 | import time
  7 | import json
  8 | import html
  9 | 
 10 | wpull_hook = globals().get('wpull_hook')  # silence code checkers
 11 | 
 12 | counter = 0
 13 | firsturl = ''
 14 | ia_metadata = {'identifier': '', 'files': [], 'title': '', 'description': '', 'mediatype': 'movies', 'collection': 'archiveteam_videobot', 'date': '', 'original_url': '', 'creator': '', 'subject': ''}
 15 | video_id = ''
 16 | video_title = ''
 17 | added_to_list = []
 18 | vmap = ''
 19 | tempfiles = {}
 20 | tries = {}
 21 | 
 22 | ignored_urls = []
 23 | 
 24 | with open(os.path.join('..', '..', 'services', 'dl__ignores__twitter_com'), 'r') as f:
 25 |     for line in f:
 26 |         ignored_urls.append(line.strip())
 27 | 
 28 | def accept_url(url_info, record_info, verdict, reasons):
 29 |     global added_to_list
 30 |     if (firsturl == '' or url_info["url"] in added_to_list) and not ('?lang=' in url_info["url"] or '\\' in url_info["url"]):
 31 |         return True
 32 |     return False
 33 | 
 34 | def get_urls(filename, url_info, document_info):
 35 |     global counter
 36 |     global firsturl
 37 |     global ia_metadata
 38 |     global video_id
 39 |     global added_to_list
 40 |     global vmap
 41 |     global tempfiles
 42 |     global video_title
 43 | 
 44 |     newurls = []
 45 | 
 46 |     def url_allowed(url, parent_url=None):
 47 |         if re.search(r'^https?://(?:www\.)?twitter\.com/[^/]+/status/' + video_id, url):
 48 |             return True
 49 | 
 50 |         elif not re.search(r'^https?://(?:www\.)?twitter\.com/[^/]+/status/[0-9]+', url):
 51 |             video_user = re.search(r'^https?://(?:www\.)?twitter\.com/([^/]+)/status/[0-9]+', firsturl).group(1)
 52 |             if re.search(r'^https?://(?:www\.)?twitter\.com/[^/]+(?:/status/)?$', url) and not video_user in url:
 53 |                 return False
 54 |             return True
 55 | 
 56 |         return False
 57 | 
 58 |     def add_url(url, parent_url=None):
 59 |         if url in added_to_list or url in ignored_urls:
 60 |             return None
 61 |         if url_allowed(url, parent_url):
 62 |             added_to_list.append(url)
 63 |             newurls.append({'url': url})
 64 | 
 65 |     if re.search(r'^https?://video\.twimg\.com.+/[0-9a-zA-Z_-]+\.mp4', url_info["url"]):
 66 |         if re.search(r'^https?://video\.twimg\.com.+/[0-9]+x[0-9]+/[0-9a-zA-Z_-]+\.mp4', url_info["url"]):
 67 |             filename_new = re.search(r'^https?://video\.twimg\.com.+/([0-9]+x[0-9]+)/[0-9a-zA-Z_-]+\.mp4', url_info["url"]).group(1) + '.mp4'
 68 |         else:
 69 |             filename_new = re.search(r'^https?://video\.twimg\.com.+/([0-9a-zA-Z_-]+\.mp4)', url_info["url"]).group(1)
 70 | 
 71 |         if not os.path.isdir('../ia_item'):
 72 |             os.makedirs('../ia_item')
 73 |         if not os.path.isfile('../ia_item/' + filename_new):
 74 |             shutil.copyfile(filename, '../ia_item/' + filename_new)
 75 |             ia_metadata['files'].append(filename_new)
 76 | 
 77 |     # Rename .mp4 video from akamaihd.
 78 |     elif re.search(r'^https?://(?:snappytv[^\.]+\.akamaihd\.net|amp\.twimg\.com).+/[^/]+\.mp4', url_info["url"]):
 79 |         filename_new = re.search(r'^https?://(?:snappytv[^\.]+\.akamaihd\.net|amp\.twimg\.com).+/([^/]+\.mp4)', url_info["url"]).group(1)
 80 | 
 81 |         if not os.path.isdir('../ia_item'):
 82 |             os.makedirs('../ia_item')
 83 |         if not os.path.isfile('../ia_item/' + filename_new):
 84 |             shutil.copyfile(filename, '../ia_item/' + filename_new)
 85 |             ia_metadata['files'].append(filename_new)
 86 | 
 87 |     # Queue videos from .m3u8 playlists.
 88 |     elif re.search(r'^https?://video\.twimg\.com/(?:ext_tw_video|amplify_video).+/[0-9a-zA-Z_-]+\.m3u8', url_info["url"]):
 89 |         with open(filename, 'r', encoding='utf-8') as file:
 90 |             for line in file:
 91 |                 part = re.search(r'^(https://video\.twimg\.com)', url_info["url"]).group(1)
 92 | 
 93 |                 if (line.startswith('/ext_tw_video') or line.startswith('/amplify_video')) and line.strip().endswith('.m3u8'):
 94 |                     print(part + line.strip())
 95 |                     newurls.append({'url': part + line.strip()})
 96 | 
 97 |                 elif (line.startswith('/ext_tw_video') or line.startswith('/amplify_video')) and line.strip().endswith('.ts'):
 98 |                     if re.search(r'/[0-9]+x[0-9]+/[0-9a-zA-Z_-]+\.ts', line):
 99 |                         newurl = part + line.strip()
100 |                         size = re.search(r'/([0-9]+x[0-9]+)/[0-9a-zA-Z_-]+\.ts', line).group(1)
101 |                         if not size in tempfiles:
102 |                             tempfiles[size] = []
103 |                         tempfiles[size].append(re.search(r'/([0-9a-zA-Z_-]+\.ts)', line).group(1))
104 |                         newurls.append({'url': newurl})
105 | 
106 |     # Prepare .ts videos from .m3u8 playlist for merging.
107 |     elif re.search(r'^https://video\.twimg\.com/(?:ext_tw_video|amplify_video).+/[0-9]+x[0-9]+/[0-9a-zA-Z_-]+\.ts', url_info["url"]):
108 |         filename_new = re.search(r'/([0-9a-zA-Z_-]+\.ts)', url_info["url"]).group(1)
109 |         if not os.path.isdir('../ia_item'):
110 |             os.makedirs('../ia_item')
111 |         if not os.path.isfile('../ia_item/' + filename_new):
112 |             shutil.copyfile(filename, '../ia_item/' + filename_new)
113 | 
114 |     # The vmap URL contains videos.
115 |     elif url_info["url"] == vmap:
116 |         with open(filename, 'r', encoding='utf-8') as file:
117 |             content = file.read()
118 |             newurls += [{'url': url} for url in extract_urls(content, url_info["url"]) if not url in added_to_list]
119 | 
120 |     # Prepare the metadata and queue new URLs.
121 |     elif re.search('^https://twitter\.com/i/videos/tweet/[0-9]+\?embed_source=clientlib&player_id=0&rpc_init=1', url_info["url"]):
122 |         with open(filename, 'r', encoding='utf-8') as file:
123 |             months = {'Jan': 1, 'Feb': 2, 'Mar': 3, 'Apr': 4, 'May': 5, 'Jun': 6, 'Jul': 7, 'Aug': 8, 'Sep': 9, 'Oct': 10, 'Nov': 11, 'Dec': 12}
124 |             content = file.read()
125 |             content_json = html.unescape(re.search(r'data-config="([^"]+)"', content).group(1))
126 |             json_ = json.loads(content_json)
127 | 
128 |             if 'vmap_url' in json_:
129 |                 vmap = json_['vmap_url']
130 | 
131 |             item_description = video_title
132 | 
133 |             if json_['videoInfo']['title']:
134 |                 item_description += '\n\n' + str(json_['videoInfo']['title'])
135 | 
136 |             if json_['videoInfo']['description']:
137 |                 item_description += '\n\n' + str(json_['videoInfo']['description'])
138 | 
139 |             item_id = json_['status']['id_str']
140 |             item_name = json_['videoInfo']['publisher']['name']
141 |             item_url_t_co = json_['cardUrl']
142 |             item_date_ = json_['status']['created_at'].replace('T', ' ')
143 |             item_date = item_date_[-4:] + '-' + str(months[item_date_[4:7]]).zfill(2) + '-' + item_date_[8:10] + ' ' + item_date_[11:19]
144 |             ia_metadata['identifier'] = 'archiveteam_videobot_twitter_com_' + item_id
145 |             ia_metadata['title'] = video_title
146 |             ia_metadata['description'] = item_description
147 |             ia_metadata['date'] = item_date
148 |             ia_metadata['original_url'] = firsturl
149 |             ia_metadata['url_t_co'] = item_url_t_co
150 |             ia_metadata['user_name'] = json_['user']['name']
151 |             ia_metadata['user_screen_name'] = json_['user']['screen_name']
152 |             ia_metadata['creator'] = item_name
153 |             ia_metadata['tweet_id'] = video_id
154 |             ia_metadata['subject'] = ';'.join(['videobot', 'archiveteam', 'twitter', 'twitter.com', item_id, item_name]
155 |                 + re.findall(r'(#[^#\s]+)', ia_metadata['title'])
156 |                 + re.findall(r'#([^#\s]+)', ia_metadata['title']))
157 | 
158 |             if ia_metadata['user_name'] != ia_metadata['creator']:
159 |                 ia_metadata['creator'] = [item_name, ia_metadata['user_name']]
160 | 
161 |             if not os.path.isdir('../ia_item'):
162 |                 os.makedirs('../ia_item')
163 |             json.dump(json_, open('../ia_item/data_video.json', 'w'), indent = 4, ensure_ascii = False)
164 |             ia_metadata['files'].append('data_video.json')
165 | 
166 |             for url in extract_urls(' '.join([content, content_json]), url_info["url"]):
167 |                 add_url(url)
168 | 
169 |     # Queue first-URL new urls.
170 |     if re.search('^https?://(?:www\.)?twitter\.com/[^/]+/status/[0-9]+', url_info["url"]) and video_id == '':
171 |         with open(filename, 'r', encoding='utf-8') as file:
172 |             content = file.read()
173 | 
174 |             if not os.path.isdir('../ia_item'):
175 |                 os.makedirs('../ia_item')
176 |             json.dump(json.loads(html.unescape(re.search(r'class="json-data"\s+value="([^"]+)"', content).group(1))), open('../ia_item/data.json', 'w'), indent = 4, ensure_ascii = False)
177 |             ia_metadata['files'].append('data.json')
178 | 
179 |             video_title = html.unescape(re.search(r'<meta\s+property="og:description"\s+content=".([^"]+).">', content).group(1))
180 | 
181 |         video_id = re.search('^https?://(?:www\.)?twitter\.com/[^/]+/status/([0-9]+)', url_info["url"]).group(1)
182 |         if not 'https://twitter.com/i/videos/tweet/' + video_id + '?embed_source=clientlib&player_id=0&rpc_init=1' in added_to_list:
183 |             newurls.append({'url': 'https://twitter.com/i/videos/tweet/' + video_id + '?embed_source=clientlib&player_id=0&rpc_init=1'})
184 |         if not 'https://twitter.com/i/videos/' + video_id in added_to_list:
185 |             newurls.append({'url': 'https://twitter.com/i/videos/' + video_id})
186 |         if not 'https://twitter.com/i/videos/' + video_id + '?embed_source=facebook' in added_to_list:
187 |             newurls.append({'url': 'https://twitter.com/i/videos/' + video_id + '?embed_source=facebook'})
188 | 
189 |     if firsturl == '':
190 |         with open(filename, 'r', encoding='utf-8') as file:
191 |             content = file.read()
192 |             firsturl = url_info["url"]
193 |             for url in extract_urls(content, url_info["url"]):
194 |                 add_url(url)
195 | 
196 |     for newurl in newurls:
197 |         added_to_list.append(newurl["url"])
198 | 
199 |     return [newurl for newurl in newurls if not '?lang=' in newurl['url']]
200 | 
201 | def exit_status(exit_code):
202 |     global ia_metadata
203 |     global tempfiles
204 |     if os.path.isdir('../ia_item'):
205 |         item_identifier = ia_metadata['identifier']
206 |         print(tempfiles)
207 | 
208 |         if len(tempfiles) > 0:
209 |             for size, files in tempfiles.items():
210 |                 os.system('ffmpeg -i "concat:' + '|'.join(['../ia_item/' + file for file in files]) + '" -c copy ../ia_item/' + size + '.ts')
211 |                 ia_metadata['files'].append(size + '.ts')
212 |                 for file in ['../ia_item/' + file for file in files]:
213 |                     os.remove(file)
214 | 
215 |         for a, b in ia_metadata.items():
216 |             with open('../ia_item/ia_metadata.py', 'a') as file:
217 |                 if type(b) is list:
218 |                     content_string = str(b)
219 |                 else:
220 |                     content_string = '\'' + str(b).replace('\'', '\\\'').replace('\n', '\\n').replace('\r', '\\r') + '\''
221 |                 file.write(str(a) + ' = ' + content_string + '\n')
222 | 
223 |         if len(os.listdir('../ia_item')) > 3:
224 |             print(ia_metadata['files'])
225 |             os.rename('../ia_item', '../../to_be_uploaded/ia_items/ia_item_' + item_identifier + '_' + str(int(time.time())))
226 | 
227 |     return exit_code
228 | 
229 | handle_response_grabsite = wpull_hook.callbacks.handle_response
230 | def handle_response(url_info, record_info, response_info):
231 |     global tries
232 | 
233 |     if not url_info["url"] in tries:
234 |         tries[url_info["url"]] = 0
235 |     elif tries[url_info["url"]] > 5:
236 |         return wpull_hook.actions.FINISH        
237 | 
238 |     tries[url_info["url"]] += 1
239 | 
240 |     return handle_response_grabsite(url_info, record_info, response_info)
241 | 
242 | wpull_hook.callbacks.get_urls = get_urls
243 | wpull_hook.callbacks.exit_status = exit_status
244 | wpull_hook.callbacks.accept_url = accept_url
245 | wpull_hook.callbacks.handle_response = handle_response
246 | 
247 | def extract_urls(file, url):
248 |     extractedurls = []
249 |     for extractedurl in re.findall('((?:....=)?(?P<quote>[\'"]).*?(?P=quote))', file, re.I):
250 |         extractedstart = ''
251 |         if re.search('^....=[\'"](.*?)[\'"]$', extractedurl[0], re.I):
252 |             extractedstart = re.search(r'^(....)', extractedurl[0], re.I).group(1)
253 |             extractedurl = re.search('^....=[\'"](.*?)[\'"]$', extractedurl[0], re.I).group(1)
254 |         else:
255 |             extractedurl = extractedurl[0][1:-1]
256 |         extractedurl = re.search(r'^([^#]*)', extractedurl, re.I).group(1)
257 |         extractedurl = extractedurl.replace('%3A', ':').replace('%2F', '/')
258 |         if extractedurl.startswith('http:\/\/') or extractedurl.startswith('https:\/\/') or extractedurl.startswith('HTTP:\/\/') or extractedurl.startswith('HTTPS:\/\/'):
259 |             extractedurl = extractedurl.replace('\/', '/')
260 |         if extractedurl.startswith('//'):
261 |             extractedurls.append("http:" + extractedurl)
262 |         elif extractedurl.startswith('/'):
263 |             extractedurls.append(re.search(r'^(https?:\/\/[^\/]+)', url, re.I).group(1) + extractedurl)
264 |         elif re.search(r'^https?:?\/\/?', extractedurl, re.I):
265 |             extractedurls.append(extractedurl.replace(re.search(r'^(https?:?\/\/?)', extractedurl, re.I).group(1), re.search(r'^(https?)', extractedurl, re.I).group(1) + '://'))
266 |         elif extractedurl.startswith('?'):
267 |             extractedurls.append(re.search(r'^(https?:\/\/[^\?]+)', url, re.I).group(1) + extractedurl)
268 |         elif extractedurl.startswith('./'):
269 |             if re.search(r'^https?:\/\/.*\/', url, re.I):
270 |                 extractedurls.append(re.search(r'^(https?:\/\/.*)\/', url, re.I).group(1) + '/' + re.search(r'^\.\/(.*)', extractedurl, re.I).group(1))
271 |             else:
272 |                 extractedurls.append(re.search(r'^(https?:\/\/.*)', url, re.I).group(1) + '/' + re.search(r'^\.\/(.*)', extractedurl, re.I).group(1))
273 |         elif extractedurl.startswith('../'):
274 |             tempurl = url
275 |             tempextractedurl = extractedurl
276 |             while tempextractedurl.startswith('../'):
277 |                 if not re.search(r'^https?://[^\/]+\/$', tempurl, re.I):
278 |                     tempurl = re.search(r'^(.*\/)[^\/]*\/', tempurl, re.I).group(1)
279 |                 tempextractedurl = re.search(r'^\.\.\/(.*)', tempextractedurl).group(1)
280 |             extractedurls.append(tempurl + tempextractedurl)
281 |         elif extractedstart == 'href':
282 |             if re.search(r'^https?:\/\/.*\/', url, re.I):
283 |                 extractedurls.append(re.search(r'^(https?:\/\/.*)\/', url, re.I).group(1) + '/' + extractedurl)
284 |             else:
285 |                 extractedurls.append(re.search(r'^(https?:\/\/.*)', url, re.I).group(1) + '/' + extractedurl)
286 |     for extractedurl in re.findall(r'>[^<a-zA-Z0-9]*(https?:?//?[^<]+)<', file, re.I):
287 |         extractedurl = re.search(r'^([^#]*)', extractedurl, re.I).group(1)
288 |         extractedurls.append(extractedurl.replace(re.search(r'^(https?:?\/\/?)', extractedurl, re.I).group(1), re.search(r'^(https?)', extractedurl, re.I).group(1) + '://'))
289 |     for extractedurl in re.findall(r'\[[^<a-zA-Z0-9]*(https?:?//?[^\]]+)\]', file, re.I):
290 |         extractedurl = re.search(r'^([^#]*)', extractedurl, re.I).group(1)
291 |         extractedurls.append(extractedurl.replace(re.search(r'^(https?:?\/\/?)', extractedurl, re.I).group(1), re.search(r'^(https?)', extractedurl, re.I).group(1) + '://'))
292 |     return [extractedurl.replace('&amp;', '&').replace('&amp;', '&') for extractedurl in extractedurls]


--------------------------------------------------------------------------------
/services/dl__vine_co.py:
--------------------------------------------------------------------------------
  1 | import sys
  2 | import time
  3 | import os
  4 | import re
  5 | import shutil
  6 | import time
  7 | import json
  8 | 
  9 | wpull_hook = globals().get('wpull_hook')  # silence code checkers
 10 | 
 11 | counter = 0
 12 | firsturl = True
 13 | ia_metadata = {'identifier': '', 'files': [], 'title': '', 'description': '', 'mediatype': 'movies', 'collection': 'archiveteam_videobot', 'date': '', 'original_url': '', 'creator': '', 'creator_url': '', 'subject': '', 'url': ''}
 14 | 
 15 | def get_urls(filename, url_info, document_info):
 16 |     global counter
 17 |     global firsturl
 18 |     global ia_metadata
 19 |     newurls = []
 20 |     if re.search(r'^https?://[^/]+\.vine\.co/r/video.+\.mp4\?', url_info["url"]):
 21 |         filename_new = re.search(r'^https?://[^/]+\.vine\.co/r/([^/]+)', url_info["url"]).group(1) + '.mp4'
 22 |         if not os.path.isdir('../ia_item'):
 23 |             os.makedirs('../ia_item')
 24 |         if not os.path.isfile('../ia_item/' + filename_new):
 25 |             shutil.copyfile(filename, '../ia_item/' + filename_new)
 26 |             ia_metadata['files'].append(filename_new)
 27 |     if firsturl:
 28 |         with open(filename, 'r', encoding='utf-8') as file:
 29 |             content = file.read()
 30 |             content_json = re.sub('(?P<string>(?P<quote>["\']).*?(?P=quote)\s*:\s*),', '\g<string>"",', re.search(r'<script\s+type="application\/ld\+json">\s+({[^<]+})\s+<\/script>', content).group(1))
 31 |             json_ = json.loads(content_json)
 32 |             item_id = re.search(r'^https?://(?:www\.)?vine\.co/v/([0-9a-zA-Z]+)', url_info["url"]).group(1)
 33 |             item_name = json_['author']['name']
 34 |             item_name_url = json_['author']['url']
 35 |             item_url = json_['url']
 36 |             item_date = json_['datePublished'].replace('T', ' ')
 37 |             item_description = json_['articleBody']
 38 |             ia_metadata['identifier'] = 'archiveteam_videobot_vine_co_' + item_id
 39 |             ia_metadata['title'] = item_description
 40 |             ia_metadata['description'] = item_description
 41 |             ia_metadata['date'] = item_date
 42 |             ia_metadata['original_url'] = url_info["url"]
 43 |             ia_metadata['url'] = item_url
 44 |             ia_metadata['creator'] = item_name
 45 |             ia_metadata['creator_url'] = item_name_url
 46 |             ia_metadata['creator'] = item_name
 47 |             ia_metadata['subject'] = ';'.join(['videobot', 'archiveteam', 'vine', 'vine.co', item_id, item_name])
 48 |             newurls = [{'url': url} for url in extract_urls(content, url_info["url"]) if not re.match(r'^https?://(?:www\.)?vine\.co/v/', url)]
 49 |             firsturl = False
 50 |     return newurls
 51 | 
 52 | def exit_status(exit_code):
 53 |     global ia_metadata
 54 |     if os.path.isdir('ia_item'):
 55 |         item_identifier = ia_metadata['identifier']
 56 |         for a, b in ia_metadata.items():
 57 |             with open('../ia_item/ia_metadata.py', 'a') as file:
 58 |                 if type(b) is list:
 59 |                     content_string = str(b)
 60 |                 else:
 61 |                     content_string = '\'' + str(b).replace('\'', '\\\'').replace('\n', '\\n').replace('\r', '\\r') + '\''
 62 |                 file.write(str(a) + ' = ' + content_string + '\n')
 63 |         os.rename('../ia_item', '../../to_be_uploaded/ia_items/ia_item_' + item_identifier + '_' + str(int(time.time())))
 64 |     return exit_code
 65 | 
 66 | wpull_hook.callbacks.get_urls = get_urls
 67 | wpull_hook.callbacks.exit_status = exit_status
 68 | 
 69 | def extract_urls(file, url):
 70 |     extractedurls = []
 71 |     for extractedurl in re.findall('((?:....=)?(?P<quote>[\'"]).*?(?P=quote))', file, re.I):
 72 |         extractedstart = ''
 73 |         if re.search('^....=[\'"](.*?)[\'"]$', extractedurl[0], re.I):
 74 |             extractedstart = re.search(r'^(....)', extractedurl[0], re.I).group(1)
 75 |             extractedurl = re.search('^....=[\'"](.*?)[\'"]$', extractedurl[0], re.I).group(1)
 76 |         else:
 77 |             extractedurl = extractedurl[0][1:-1]
 78 |         extractedurl = re.search(r'^([^#]*)', extractedurl, re.I).group(1)
 79 |         extractedurl = extractedurl.replace('%3A', ':').replace('%2F', '/')
 80 |         if extractedurl.startswith('http:\/\/') or extractedurl.startswith('https:\/\/') or extractedurl.startswith('HTTP:\/\/') or extractedurl.startswith('HTTPS:\/\/'):
 81 |             extractedurl = extractedurl.replace('\/', '/')
 82 |         if extractedurl.startswith('//'):
 83 |             extractedurls.append("http:" + extractedurl)
 84 |         elif extractedurl.startswith('/'):
 85 |             extractedurls.append(re.search(r'^(https?:\/\/[^\/]+)', url, re.I).group(1) + extractedurl)
 86 |         elif re.search(r'^https?:?\/\/?', extractedurl, re.I):
 87 |             extractedurls.append(extractedurl.replace(re.search(r'^(https?:?\/\/?)', extractedurl, re.I).group(1), re.search(r'^(https?)', extractedurl, re.I).group(1) + '://'))
 88 |         elif extractedurl.startswith('?'):
 89 |             extractedurls.append(re.search(r'^(https?:\/\/[^\?]+)', url, re.I).group(1) + extractedurl)
 90 |         elif extractedurl.startswith('./'):
 91 |             if re.search(r'^https?:\/\/.*\/', url, re.I):
 92 |                 extractedurls.append(re.search(r'^(https?:\/\/.*)\/', url, re.I).group(1) + '/' + re.search(r'^\.\/(.*)', extractedurl, re.I).group(1))
 93 |             else:
 94 |                 extractedurls.append(re.search(r'^(https?:\/\/.*)', url, re.I).group(1) + '/' + re.search(r'^\.\/(.*)', extractedurl, re.I).group(1))
 95 |         elif extractedurl.startswith('../'):
 96 |             tempurl = url
 97 |             tempextractedurl = extractedurl
 98 |             while tempextractedurl.startswith('../'):
 99 |                 if not re.search(r'^https?://[^\/]+\/$', tempurl, re.I):
100 |                     tempurl = re.search(r'^(.*\/)[^\/]*\/', tempurl, re.I).group(1)
101 |                 tempextractedurl = re.search(r'^\.\.\/(.*)', tempextractedurl).group(1)
102 |             extractedurls.append(tempurl + tempextractedurl)
103 |         elif extractedstart == 'href':
104 |             if re.search(r'^https?:\/\/.*\/', url, re.I):
105 |                 extractedurls.append(re.search(r'^(https?:\/\/.*)\/', url, re.I).group(1) + '/' + extractedurl)
106 |             else:
107 |                 extractedurls.append(re.search(r'^(https?:\/\/.*)', url, re.I).group(1) + '/' + extractedurl)
108 |     for extractedurl in re.findall(r'>[^<a-zA-Z0-9]*(https?:?//?[^<]+)<', file, re.I):
109 |         extractedurl = re.search(r'^([^#]*)', extractedurl, re.I).group(1)
110 |         extractedurls.append(extractedurl.replace(re.search(r'^(https?:?\/\/?)', extractedurl, re.I).group(1), re.search(r'^(https?)', extractedurl, re.I).group(1) + '://'))
111 |     return extractedurls
112 | 


--------------------------------------------------------------------------------
/services/video__facebook_com.py:
--------------------------------------------------------------------------------
 1 | import re
 2 | import os
 3 | import sys
 4 | import download_page
 5 | import url
 6 | import irc_message
 7 | import json
 8 | import requests
 9 | 
10 | extract_info = lambda regexes, url: download_page.extract_info(regexes, url)
11 | job_finished = lambda user, name, title, id: irc_message.job_finished(user, name, title, id)
12 | job_failed = lambda user, name, title, id: irc_message.job_failed(user, name, title, id)
13 | job_added = lambda user, name, title, id: irc_message.job_added(user, name, title, id)
14 | job_aborted = lambda user, name, id: irc_message.job_added(user, name, id)
15 | failed_extraction = lambda user, name, id: irc_message.failed_extraction(user, name, id)
16 | check_create_url = lambda url_, prefix, suffix: url.check_create_url(url_, prefix, suffix)
17 | 
18 | service_name = 'Facebook media'
19 | service_commands = ['facebook']
20 | url_regex = r'^https?://(?:www\.)?facebook\.com/[^/]+/videos/[0-9]+/?'
21 | url_prefix = 'https://www.facebook.com/video/video.php?v='
22 | url_suffix = ''
23 | url_user = lambda url: re.search(r'^https?://(?:www\.)?facebook\.com/([^/]+)/videos/[0-9]+/?', url).group(1)
24 | url_id = lambda url: re.search(r'^https?://(?:www\.)?facebook\.com/[^/]+/videos/([0-9]+)/?', url).group(1)
25 | item_title = lambda url: extract_info(r'<title\s+id="pageTitle">([^<]+(?:\.\.\.)?)\s+-\s+[^<]+</title>', url)[0]
26 | 
27 | def process(service_file_name, command, user):
28 |     url = check_create_url(command[1], url_prefix, url_suffix)
29 |     videotitle = item_title(url)
30 |     videoid = url_id(url)
31 | 
32 |     if requests.get('https://archive.org/details/archiveteam_videobot_facebook_com_' + url_id(url)).status_code == 200:
33 |         yield(job_finished(user, service_name, videotitle, url_id(url)))
34 |     elif videotitle != None:
35 |         yield(job_added(user, service_name, videotitle, url_id(url)))
36 |         if grab(url) in [0, 4, 8]:
37 |             yield(job_finished(user, service_name, videotitle, url_id(url)))
38 |         else:
39 |             yield(job_failed(user, service_name, videotitle, url_id(url)))
40 |     else:
41 |         yield(failed_extraction(user, service_name, url_id(url), 'title'))
42 |         yield(job_aborted(user, service_name, url_id(url)))
43 | 
44 | def grab(url):
45 |     exit_code = os.system('~/.local/bin/grab-site ' + url + ' --level=0 --custom-hooks=services/dl__facebook_com.py --ua="Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1" --no-sitemaps --concurrency=1 --warc-max-size=524288000 --wpull-args="--no-check-certificate --timeout=300"')
46 |     return exit_code
47 | 
48 | def add_url(url, ticket_id, user):
49 |     yield(['message', user + ': You cannot create a periodical job for a ' + name + '.'])
50 | 


--------------------------------------------------------------------------------
/services/video__liveleak_com.py:
--------------------------------------------------------------------------------
 1 | import re
 2 | import os
 3 | import sys
 4 | import download_page
 5 | import url
 6 | import irc_message
 7 | import json
 8 | 
 9 | extract_info = lambda regexes, url: download_page.extract_info(regexes, url)
10 | job_finished = lambda user, name, title, id: irc_message.job_finished(user, name, title, id)
11 | job_failed = lambda user, name, title, id: irc_message.job_failed(user, name, title, id)
12 | job_added = lambda user, name, title, id: irc_message.job_added(user, name, title, id)
13 | job_aborted = lambda user, name, id: irc_message.job_added(user, name, id)
14 | failed_extraction = lambda user, name, id: irc_message.failed_extraction(user, name, id)
15 | check_create_url = lambda url_, prefix, suffix: url.check_create_url(url_, prefix, suffix)
16 | 
17 | service_name = 'LiveLeak video'
18 | service_commands = ['liveleak']
19 | url_regex = r'^https?://(?:www\.)?liveleak\.com/view\?i=[a-z0-9_]+'
20 | url_prefix = 'http://www.liveleak.com/view?i='
21 | url_suffix = ''
22 | url_id = lambda url: re.search(r'^https?://(?:www\.)?liveleak\.com/view\?i=([a-z0-9_]+)', url).group(1)
23 | item_title = lambda url: extract_info(r'<meta\s+property="og:title"\s+content="([^"]+)"/>', url)[0].replace('LiveLeak.com - ', '').strip()
24 | 
25 | def process(service_file_name, command, user):
26 |     url = check_create_url(command[1], url_prefix, url_suffix)
27 |     videotitle = item_title(url)
28 |     videoid = url_id(url)
29 |     if videotitle != None:
30 |         yield(job_added(user, service_name, videotitle, url_id(url)))
31 |         if grab(url) == 0:
32 |             yield(job_finished(user, service_name, videotitle, url_id(url)))
33 |         else:
34 |             yield(job_failed(user, service_name, videotitle, url_id(url)))
35 |     else:
36 |         yield(failed_extraction(user, service_name, url_id(url), 'title'))
37 |         yield(job_aborted(user, service_name, url_id(url)))
38 | 
39 | def grab(url):
40 |     exit_code = os.system('~/.local/bin/grab-site ' + url + ' --level=0 --custom-hooks=services/dl__liveleak_com.py --no-sitemaps --concurrency=1 --warc-max-size=524288000 --wpull-args="--no-check-certificate --timeout=300"')
41 |     return exit_code
42 | 
43 | def add_url(url, ticket_id, user):
44 |     yield(['message', user + ': You cannot create a periodical job for a ' + name + '.'])
45 | 


--------------------------------------------------------------------------------
/services/video__periscope_tv.py:
--------------------------------------------------------------------------------
 1 | import re
 2 | import os
 3 | import sys
 4 | import download_page
 5 | import url
 6 | import irc_message
 7 | import json
 8 | 
 9 | extract_info = lambda regexes, url: download_page.extract_info(regexes, url)
10 | job_finished = lambda user, name, title, id: irc_message.job_finished(user, name, title, id)
11 | job_failed = lambda user, name, title, id: irc_message.job_failed(user, name, title, id)
12 | job_added = lambda user, name, title, id: irc_message.job_added(user, name, title, id)
13 | job_aborted = lambda user, name, id: irc_message.job_added(user, name, id)
14 | failed_extraction = lambda user, name, id: irc_message.failed_extraction(user, name, id)
15 | check_create_url = lambda url_, prefix, suffix: url.check_create_url(url_, prefix, suffix)
16 | 
17 | service_name = 'Periscope'
18 | service_commands = ['periscope']
19 | url_regex = r'^https?://(?:www\.)?periscope\.tv/w/[0-9a-zA-Z]+'
20 | url_prefix = 'https://www.periscope.tv/w/'
21 | url_suffix = ''
22 | url_id = lambda url: re.search(r'^https?://(?:www\.)?periscope\.tv/w/([0-9a-zA-Z]+)', url).group(1)
23 | item_title = lambda url: extract_info(r'<meta[^>]+property="og:description"\s+content="([^"]+)"', url)[0]
24 | 
25 | def process(service_file_name, command, user):
26 |     url = check_create_url(command[1], url_prefix, url_suffix)
27 |     videotitle = item_title(url)
28 |     videoid = url_id(url)
29 |     if videotitle != None:
30 |         yield(job_added(user, service_name, videotitle, url_id(url)))
31 |         if grab(url) == 0:
32 |             yield(job_finished(user, service_name, videotitle, url_id(url)))
33 |         else:
34 |             yield(job_failed(user, service_name, videotitle, url_id(url)))
35 |     else:
36 |         yield(failed_extraction(user, service_name, url_id(url), 'title'))
37 |         yield(job_aborted(user, service_name, url_id(url)))
38 | 
39 | def grab(url):
40 |     exit_code = os.system('~/.local/bin/grab-site ' + url + ' --level=0 --custom-hooks=services/dl__periscope_tv.py --ua="ArchiveTeam; Googlebot/2.1" --no-sitemaps --concurrency=1 --warc-max-size=524288000 --wpull-args="--no-check-certificate --timeout=300"')
41 |     return exit_code
42 | 
43 | def add_url(url, ticket_id, user):
44 |     yield(['message', user + ': You cannot create a periodical job for a ' + name + '.'])


--------------------------------------------------------------------------------
/services/video__twitter_com.py:
--------------------------------------------------------------------------------
 1 | import re
 2 | import os
 3 | import sys
 4 | import download_page
 5 | import url
 6 | import irc_message
 7 | import json
 8 | import requests
 9 | 
10 | extract_info = lambda regexes, url: download_page.extract_info(regexes, url)
11 | job_finished = lambda user, name, title, id: irc_message.job_finished(user, name, title, id)
12 | job_failed = lambda user, name, title, id: irc_message.job_failed(user, name, title, id)
13 | job_added = lambda user, name, title, id: irc_message.job_added(user, name, title, id)
14 | job_aborted = lambda user, name, id: irc_message.job_added(user, name, id)
15 | failed_extraction = lambda user, name, id: irc_message.failed_extraction(user, name, id)
16 | check_create_url = lambda url_, prefix, suffix: url.check_create_url(url_, prefix, suffix)
17 | 
18 | service_name = 'Twitter media'
19 | service_commands = ['twitter']
20 | url_regex = r'^https?://(?:www\.)?twitter\.com/[^/]+/status/[0-9]+'
21 | url_prefix = 'https://twitter.com/'
22 | url_suffix = ''
23 | url_user = lambda url: re.search(r'^https?://(?:www\.)?twitter\.com/([^/]+)/status/[0-9]+', url).group(1)
24 | url_id = lambda url: re.search(r'^https?://(?:www\.)?twitter\.com/[^/]+/status/([0-9]+)', url).group(1)
25 | item_title = lambda url: re.sub('&quot;', '"', extract_info(r'<meta\s+property="og:description"\s+content="([^"]+)">', url)[0])
26 | 
27 | def process(service_file_name, command, user):
28 |     url = check_create_url(command[1], url_prefix, url_suffix)
29 |     videotitle = item_title(url)
30 |     videoid = url_id(url)
31 | 
32 |     if requests.get('https://archive.org/details/archiveteam_videobot_twitter_com_' + url_id(url)).status_code == 200:
33 |         yield(job_finished(user, service_name, videotitle, url_id(url)))
34 |     elif videotitle != None:
35 |         yield(job_added(user, service_name, videotitle, url_id(url)))
36 |         if grab(url) == 0:
37 |             yield(job_finished(user, service_name, videotitle, url_id(url)))
38 |         else:
39 |             yield(job_failed(user, service_name, videotitle, url_id(url)))
40 |     else:
41 |         yield(failed_extraction(user, service_name, url_id(url), 'title'))
42 |         yield(job_aborted(user, service_name, url_id(url)))
43 | 
44 | def grab(url):
45 |     exit_code = os.system('~/.local/bin/grab-site ' + url + ' --level=0 --custom-hooks=services/dl__twitter_com.py --ua="ArchiveTeam; Googlebot/2.1" --no-sitemaps --concurrency=1 --warc-max-size=524288000 --wpull-args="--no-check-certificate --timeout=300"')
46 |     return exit_code
47 | 
48 | def add_url(url, ticket_id, user):
49 |     yield(['message', user + ': You cannot create a periodical job for a ' + name + '.'])


--------------------------------------------------------------------------------
/services/video__twitter_com_hashtag.py:
--------------------------------------------------------------------------------
 1 | import re
 2 | import os
 3 | import sys
 4 | import download_page
 5 | import url
 6 | import irc_message
 7 | import json
 8 | import functions
 9 | import requests
10 | import config
11 | import time
12 | 
13 | extract_info = lambda regexes, url: download_page.extract_info(regexes, url)
14 | job_finished = lambda user, name, title, id: irc_message.job_finished(user, name, title, id)
15 | job_failed = lambda user, name, title, id: irc_message.job_failed(user, name, title, id)
16 | job_added = lambda user, name, title, id: irc_message.job_added(user, name, title, id)
17 | job_aborted = lambda user, name, id: irc_message.job_added(user, name, id)
18 | failed_extraction = lambda user, name, id: irc_message.failed_extraction(user, name, id)
19 | check_create_url = lambda url_, prefix, suffix: url.check_create_url(url_, prefix, suffix)
20 | 
21 | service_name = 'Twitter hashtag'
22 | service_commands = ['twitterhashtag', 'twitter_hashtag', 'twitter-hashtag', 'twittertag', 'twitter_tag', 'twitter-tag', 'ta', 'tt']
23 | url_regex = r'^https?://(?:www\.)?twitter\.com/hashtag/[^/]+$'
24 | url_prefix = 'https://twitter.com/hashtag/'
25 | url_suffix = '?f=videos'
26 | url_user = lambda url: re.search(r'^https?://(?:www\.)?twitter\.com/hashtag/([^/]+)', url).group(1)
27 | 
28 | def process(service_file_name, command, user):
29 |     url = check_create_url(command[1] + url_suffix, url_prefix, url_suffix)
30 |     videotitle = '#' + url_user(url)
31 | 
32 |     yield(job_added(user, service_name, videotitle))
33 | 
34 |     response = requests.get(url)
35 |     for tweet in re.findall(r'data-permalink-path="([^"]+)"', response.text):
36 |         functions.archive.main(['!a', 'https://twitter.com' + tweet], config.irc_nick)
37 |         time.sleep(10)
38 |     if 'data-min-position' in response.text:
39 |         max_position = re.search(r'data-min-position="([^"]+)"', response.text).group(1)
40 |         while 'data-min-position' in response.text or response_json['min_position']:
41 |             response = requests.get('https://twitter.com/i/search/timeline?f=videos&vertical=default&q=%23'
42 |                 + url_user(url) + '&include_available_features=1&include_entities=1&max_position='
43 |                 + max_position + '&reset_error_state=false')
44 |             if response.status_code == 200:
45 |                 response_json = json.loads(response.text)
46 |                 max_position = response_json['min_position']
47 |                 for tweet in re.findall(r'data-permalink-path="([^"]+)"', response_json['items_html']):
48 |                     functions.archive.main(['!a', 'https://twitter.com' + tweet], config.irc_nick)
49 |                     time.sleep(10)
50 |             else:
51 |                 yield(failed_extraction(user, service_name, videoid, 'nextpage'))
52 |                 break
53 |         else:
54 |             yield(job_finished(user, service_name, videotitle, videoid))
55 | 
56 | def add_url(url, ticket_id, user):
57 |     yield(['message', user + ': You cannot create a periodical job for a ' + name + '.'])


--------------------------------------------------------------------------------
/services/video__twitter_com_user.py:
--------------------------------------------------------------------------------
 1 | import re
 2 | import os
 3 | import sys
 4 | import download_page
 5 | import url
 6 | import irc_message
 7 | import json
 8 | import functions
 9 | import requests
10 | import config
11 | import time
12 | 
13 | extract_info = lambda regexes, url: download_page.extract_info(regexes, url)
14 | job_finished = lambda user, name, title, id: irc_message.job_finished(user, name, title, id)
15 | job_failed = lambda user, name, title, id: irc_message.job_failed(user, name, title, id)
16 | job_added = lambda user, name, title, id: irc_message.job_added(user, name, title, id)
17 | job_aborted = lambda user, name, id: irc_message.job_added(user, name, id)
18 | failed_extraction = lambda user, name, id: irc_message.failed_extraction(user, name, id)
19 | check_create_url = lambda url_, prefix, suffix: url.check_create_url(url_, prefix, suffix)
20 | 
21 | service_name = 'Twitter account'
22 | service_commands = ['twitteraccount', 'twitter_account', 'twitter-account', 'twitteruser', 'twitter_user', 'twitter-user', 'ta', 'tu']
23 | url_regex = r'^https?://(?:www\.)?twitter\.com/[^/]+$'
24 | url_prefix = 'https://twitter.com/'
25 | url_suffix = '/media'
26 | url_user = lambda url: re.search(r'^https?://(?:www\.)?twitter\.com/([^/]+)', url).group(1)
27 | url_id = lambda url: re.sub('&quot;', '"', extract_info(r'data-user-id="([0-9]+)"', url)[0])
28 | item_title = lambda url: re.sub('&quot;', '"', extract_info(r'data-name="([^"]+)"', url)[0])
29 | 
30 | def process(service_file_name, command, user):
31 |     url = check_create_url(command[1] + '/media', url_prefix, url_suffix)
32 |     videotitle = item_title(url)
33 |     videoid = url_id(url)
34 | 
35 |     yield(job_added(user, service_name, videotitle, videoid))
36 | 
37 |     response = requests.get(url)
38 |     for tweet in re.findall(r'data-permalink-path="([^"]+)"', response.text):
39 |         functions.archive.main(['!a', 'https://twitter.com' + tweet], config.irc_nick)
40 |         time.sleep(10)
41 |     if 'data-min-position' in response.text:
42 |         max_position = re.search(r'data-min-position="([^"]+)"', response.text).group(1)
43 |         while 'data-min-position' in response.text or response_json['min_position']:
44 |             response = requests.get('https://twitter.com/i/profiles/show/'
45 |                 + url_user(url) + '/media_timeline?include_available_features=1&include_entities=1&max_position='
46 |                 + max_position + '&reset_error_state=false')
47 |             if response.status_code == 200:
48 |                 response_json = json.loads(response.text)
49 |                 max_position = response_json['min_position']
50 |                 for tweet in re.findall(r'data-permalink-path="([^"]+)"', response_json['items_html']):
51 |                     functions.archive.main(['!a', 'https://twitter.com' + tweet], config.irc_nick)
52 |                     time.sleep(10)
53 |             else:
54 |                 yield(failed_extraction(user, service_name, videoid, 'nextpage'))
55 |                 break
56 |         else:
57 |             yield(job_finished(user, service_name, videotitle, videoid))
58 | 
59 | def add_url(url, ticket_id, user):
60 |     yield(['message', user + ': You cannot create a periodical job for a ' + name + '.'])


--------------------------------------------------------------------------------
/services/video__video_stream.py:
--------------------------------------------------------------------------------
  1 | import re
  2 | import os
  3 | import sys
  4 | import time
  5 | import download_page
  6 | import url
  7 | import irc_message
  8 | import check_valid
  9 | import periodical_jobs
 10 | import refresh
 11 | 
 12 | extract_info = lambda regexes, url: download_page.extract_info(regexes, url)
 13 | job_finished = lambda user, name, title, id: irc_message.job_finished(user, name, title, id)
 14 | job_failed = lambda user, name, title, id: irc_message.job_failed(user, name, title, id)
 15 | job_added = lambda user, name, title, id: irc_message.job_added(user, name, title, id)
 16 | job_aborted = lambda user, name, id: irc_message.job_added(user, name, id)
 17 | failed_extraction = lambda user, name, id: irc_message.failed_extraction(user, name, id)
 18 | check_create_url = lambda url_, prefix, suffix: url.check_create_url(url_, prefix, suffix)
 19 | check_num = lambda string: check_valid.check_num(string)
 20 | check_temp_perjob_variable = lambda ticket, command: periodical_jobs.check_temp_perjob.check_temp_perjob_variable(ticket, command)
 21 | periodical_job_args = lambda filename, args: refresh.periodical_job_args(filename, args)
 22 | long_string = lambda command: ' '.join(command[3:]).replace('\'', '\\\'')
 23 | 
 24 | service_name = 'stream'
 25 | service_commands = ['mms', 'mmsh', 'rtsp', 'rtmp', 'MMS', 'MMSH', 'RTSP', 'RTMP']
 26 | url_regex = r'^(?:mms|mmsh|rtsp|rtmp|MMS|MMSH|RTSP|RTMP):\/\/'
 27 | 
 28 | def add_url(url, ticket_id, user):
 29 |     print('bla' + url)
 30 |     yield(['add', 'url', '\'' + url + '\''])
 31 |     yield(['add', 'type', service_commands])
 32 |     yield(['message', user + ': Added URL \'' + url + '\' to ticket ID \'' + ticket_id + '\'.'])
 33 |     yield(['message', user + ': Set the commands. For help use \'!perjob help <Ticket ID>\'. To finish ticket ID use command \'finish\'.'])
 34 | 
 35 | def periodical_job(service_name, command, user):
 36 |     default_commands = ['url', 'type']
 37 |     required_commands = ['piecelength', 'title', 'description']
 38 |     optional_commands = ['creator', 'subject', 'licenseurl', 'notes', 'rights', 'publisher', 'language', 'coverage', 'credits']
 39 |     if command[1] == 'piecelength':
 40 |         if check_num(command[3]):
 41 |             yield(['add', 'piecelength', command[3]])
 42 |             yield(['add', 'refreshtime', command[3]])
 43 |             yield(['message', user + ': Added piecelength ' + command[3] + ' to ticket ID \'' + command[2] + '\'.'])
 44 |         else:
 45 |             yield(['message', user + ': Piecelength should be a number for a ' + service_name + '.'])
 46 |     elif command[1] == 'title':
 47 |         title = long_string(command)
 48 |         yield(['add', 'title', '\'' + title + '\''])
 49 |         yield(['message', user + ': Added title \'' + title + '\' to ticket ID \'' + command[2] + '\'.'])
 50 |     elif command[1] == 'description':
 51 |         description = long_string(command)
 52 |         yield(['add', 'description', '\'' + description + '\''])
 53 |         yield(['message', user + ': Added description \'' + description + '\' to ticket ID \'' + command[2] + '\'.'])
 54 |     elif command[1] == 'creator':
 55 |         creator = long_string(command)
 56 |         yield(['add', 'creator', '\'' + creator + '\''])
 57 |         yield(['message', user + ': Added creator \'' + creator + '\' to ticket ID \'' + command[2] + '\'.'])
 58 |     elif command[1] == 'subject':
 59 |         subject = long_string(command)
 60 |         yield(['add', 'subject', '\'' + subject + '\''])
 61 |         yield(['message', user + ': Added subject \'' + subject + '\' to ticket ID \'' + command[2] + '\'.'])
 62 |     elif command[1] == 'licenseurl':
 63 |         licenseurl = long_string(command)
 64 |         yield(['add', 'licenseurl', '\'' + licenseurl + '\''])
 65 |         yield(['message', user + ': Added licenseurl \'' + licenseurl + '\' to ticket ID \'' + command[2] + '\'.'])
 66 |     elif command[1] == 'notes':
 67 |         notes = long_string(command)
 68 |         yield(['add', 'notes', '\'' + notes + '\''])
 69 |         yield(['message', user + ': Added notes \'' + notes + '\' to ticket ID \'' + command[2] + '\'.'])
 70 |     elif command[1] == 'rights':
 71 |         rights = long_string(command)
 72 |         yield(['add', 'rights', '\'' + rights + '\''])
 73 |         yield(['message', user + ': Added rights \'' + rights + '\' to ticket ID \'' + command[2] + '\'.'])
 74 |     elif command[1] == 'publisher':
 75 |         publisher = long_string(command)
 76 |         yield(['add', 'publisher', '\'' + publisher + '\''])
 77 |         yield(['message', user + ': Added publisher \'' + publisher + '\' to ticket ID \'' + command[2] + '\'.'])
 78 |     elif command[1] == 'language':
 79 |         language = long_string(command)
 80 |         yield(['add', 'language', '\'' + language + '\''])
 81 |         yield(['message', user + ': Added language \'' + language + '\' to ticket ID \'' + command[2] + '\'.'])
 82 |     elif command[1] == 'coverage':
 83 |         coverage = long_string(command)
 84 |         yield(['add', 'coverage', '\'' + coverage + '\''])
 85 |         yield(['message', user + ': Added coverage \'' + coverage + '\' to ticket ID \'' + command[2] + '\'.'])
 86 |     elif command[1] == 'credits':
 87 |         credits = long_string(command)
 88 |         yield(['add', 'credits', '\'' + credits + '\''])
 89 |         yield(['message', user + ': Added credits \'' + credits + '\' to ticket ID \'' + command[2] + '\'.'])
 90 | 
 91 |     # Do not change this.
 92 |     elif command[1] == 'help':
 93 |         yield(['help', required_commands, optional_commands, user])
 94 |     elif command[1] == 'finish':
 95 |         yield(['finish', required_commands, default_commands, user])
 96 |     else:
 97 |         yield(['bad_command', command[1], user])
 98 | 
 99 | def periodical_job_start(filename_, user, _):
100 |     optional_commands = ['creator', 'subject', 'licenseurl', 'notes', 'rights', 'publisher', 'language', 'coverage', 'credits']
101 |     title, url, piecelength, description = periodical_job_args(filename_, ['title', 'url', 'piecelength', 'description'])
102 |     piecelength = int(piecelength) + 120 # an overlap of 120 seconds.
103 |     date = time.strftime('%Y%m%d_%H%M')
104 |     metadata_date = date[:4] + '-' + date[4:6] + '-' + date[6:11].replace('_', ' ') + ':' + date[11:]
105 |     itemdir = 'archiveteam_videobot_' + re.sub(r'[^0-9a-zA-Z]', r'_', title) + '_' + date
106 |     fulldir = 'to_be_uploaded/ia_items/' + itemdir + '/'
107 |     filename = re.sub(r'[^0-9a-zA-Z]', r'_', title) + '_' + date + '.asf'
108 |     ia_metadata = {'identifier': itemdir, 'title': title + ' ' + metadata_date, 'date': metadata_date, 'url': url, 'collection': 'archiveteam_videobot', 'mediatype': 'movies', 'files': [filename], 'description': description, 'subject': 'videobot;archiveteam'}
109 |     for command in optional_commands:
110 |         content = periodical_job_args(filename_, [command])[0]
111 |         if command == 'subject':
112 |             ia_metadata['subject'] += ';' + content
113 |         else:
114 |             ia_metadata[command] = content
115 |     yield(['write_metadata', ia_metadata, fulldir])
116 |     yield(['execute_timeout', 'mplayer -dumpstream ' + url + ' -dumpfile ' + fulldir + filename, piecelength, fulldir])
117 | 


--------------------------------------------------------------------------------
/services/video__vine_co.py:
--------------------------------------------------------------------------------
 1 | import re
 2 | import os
 3 | import sys
 4 | import download_page
 5 | import url
 6 | import irc_message
 7 | import json
 8 | 
 9 | extract_info = lambda regexes, url: download_page.extract_info(regexes, url)
10 | job_finished = lambda user, name, title, id: irc_message.job_finished(user, name, title, id)
11 | job_failed = lambda user, name, title, id: irc_message.job_failed(user, name, title, id)
12 | job_added = lambda user, name, title, id: irc_message.job_added(user, name, title, id)
13 | job_aborted = lambda user, name, id: irc_message.job_added(user, name, id)
14 | failed_extraction = lambda user, name, id: irc_message.failed_extraction(user, name, id)
15 | check_create_url = lambda url_, prefix, suffix: url.check_create_url(url_, prefix, suffix)
16 | 
17 | service_name = 'Vine video'
18 | service_commands = ['vine']
19 | url_regex = r'^https?://(?:www\.)?vine\.co/v/[0-9a-zA-Z]+'
20 | url_prefix = 'https://vine.co/v/'
21 | url_suffix = ''
22 | url_id = lambda url: re.search(r'^https?://(?:www\.)?vine\.co/v/([0-9a-zA-Z]+)', url).group(1)
23 | item_title = lambda url: json.loads(re.sub('(?P<string>(?P<quote>["\']).*?(?P=quote)\s*:\s*),', '\g<string>"",', extract_info(r'<script\s+type="application\/ld\+json">\s+({[^<]+})\s+<\/script>', url)[0]))['articleBody']
24 | 
25 | 
26 | def process(service_file_name, command, user):
27 |     url = check_create_url(command[1], url_prefix, url_suffix)
28 |     videotitle = item_title(url)
29 |     videoid = url_id(url)
30 |     if videotitle != None:
31 |         yield(job_added(user, service_name, videotitle, url_id(url)))
32 |         if grab(url) == 0:
33 |             yield(job_finished(user, service_name, videotitle, url_id(url)))
34 |         else:
35 |             yield(job_failed(user, service_name, videotitle, url_id(url)))
36 |     else:
37 |         yield(failed_extraction(user, service_name, url_id(url), 'title'))
38 |         yield(job_aborted(user, service_name, url_id(url)))
39 | 
40 | def grab(url):
41 |     exit_code = os.system('~/.local/bin/grab-site ' + url + ' --level=0 --custom-hooks=services/dl__vine_co.py --ua="ArchiveTeam; Googlebot/2.1" --no-sitemaps --concurrency=1 --warc-max-size=524288000 --wpull-args="--no-check-certificate --timeout=300" > /dev/null 2>&1')
42 |     return exit_code
43 | 
44 | def add_url(url, ticket_id, user):
45 |     yield(['message', user + ': You cannot create a periodical job for a ' + name + '.'])
46 | 


--------------------------------------------------------------------------------
/services/video__webpage.py:
--------------------------------------------------------------------------------
 1 | import re
 2 | import os
 3 | import sys
 4 | import download_page
 5 | import url
 6 | import irc_message
 7 | import check_valid
 8 | import periodical_jobs
 9 | import refresh
10 | 
11 | extract_info = lambda regexes, url: download_page.extract_info(regexes, url)
12 | job_finished = lambda user, name, title, id: irc_message.job_finished(user, name, title, id)
13 | job_failed = lambda user, name, title, id: irc_message.job_failed(user, name, title, id)
14 | job_added = lambda user, name, title, id: irc_message.job_added(user, name, title, id)
15 | job_aborted = lambda user, name, id: irc_message.job_added(user, name, id)
16 | failed_extraction = lambda user, name, id: irc_message.failed_extraction(user, name, id)
17 | check_create_url = lambda url_, prefix, suffix: url.check_create_url(url_, prefix, suffix)
18 | check_num = lambda string: check_valid.check_num(string)
19 | check_temp_perjob_variable = lambda ticket, command: periodical_jobs.check_temp_perjob.check_temp_perjob_variable(ticket, command)
20 | periodical_job_args = lambda filename, args: refresh.periodical_job_args(filename, args)
21 | 
22 | service_name = 'website or webpage'
23 | service_commands = ['webpage']
24 | 
25 | def add_url(url, ticket_id, user):
26 |     yield(['add', 'url', '\'' + url + '\''])
27 |     yield(['add', 'type', service_commands])
28 |     yield(['message', user + ': Added URL \'' + url + '\' to ticket ID \'' + ticket_id + '\'.'])
29 |     yield(['message', user + ': Set the commands. For help use \'!perjob help <Ticket ID>\'. To finish ticket ID use command \'finish\'.'])
30 | 
31 | def periodical_job(service_name, command, user):
32 |     default_commands = ['url', 'type']
33 |     required_commands = ['refreshtime', 'depth', 'description']
34 |     optional_commands = []
35 |     if command[1] == 'refreshtime':
36 |         if check_num(command[3]):
37 |             yield(['add', 'refreshtime', command[3]])
38 |             yield(['message', user + ': Added refreshtime ' + command[3] + ' to ticket ID \'' + command[2] + '\'.'])
39 |         else:
40 |             yield(['message', user + ': Refreshtime should be a number for a ' + service_name + '.'])
41 |     elif command[1] == 'depth':
42 |         if check_num(command[3]):
43 |             yield(['add', 'depth', command[3]])
44 |             yield(['message', user + ': Added crawl depth ' + command[3] + ' to ticket ID \'' + command[2] + '\'.'])
45 |         else:
46 |             yield(['message', user + ': Crawl depth should be a number for a ' + service_name + '.'])
47 |     elif command[1] == 'description':
48 |         description = ' '.join(command[3:]).replace('\'', '\\\'')
49 |         yield(['add', 'description', '\'' + description + '\''])
50 |         yield(['message', user + ': Added description \'' + description + '\' to ticket ID \'' + command[2] + '\'.'])
51 | 
52 |     # Do not change this.
53 |     elif command[1] == 'help':
54 |         yield(['help', required_commands, optional_commands, user])
55 |     elif command[1] == 'finish':
56 |         yield(['finish', required_commands, default_commands, user])
57 |     else:
58 |         yield(['bad_command', command[1], user])
59 | 
60 | def periodical_job_start(filename, user, _):
61 |     depth, url = periodical_job_args(filename, ['depth', 'url'])
62 |     yield(['execute', '~/.local/bin/grab-site ' + url + ' --level=' + str(depth) + ' --ua="ArchiveTeam; Googlebot/2.1" --concurrency=1 --warc-max-size=524288000 --wpull-args="--no-check-certificate --timeout=300" > /dev/null 2>&1'])
63 | 


--------------------------------------------------------------------------------
/temp_perjobs/__init__.py:
--------------------------------------------------------------------------------
1 | import os
2 | 
3 | for module in os.listdir(os.path.dirname(__file__)):
4 |     if module.startswith('dl__') or module == '__init__.py' or module[-3:] != '.py':
5 |         continue
6 |     __import__(module[:-3], locals(), globals())
7 | del module
8 | 


--------------------------------------------------------------------------------
/upload.py:
--------------------------------------------------------------------------------
  1 | from config import ia_access_key, ia_secret_key, max_warc_item_size
  2 | import os
  3 | import shutil
  4 | import threading
  5 | import internetarchive
  6 | import time
  7 | import codecs
  8 | 
  9 | def upload(ia_args, ia_files, ia_identifier):
 10 |     ia_files_new = []
 11 |     for filename in ia_files:
 12 |         if not os.path.isfile(filename + '.upload'):
 13 |             ia_files_new.append(filename)
 14 |     ia_files = list(ia_files_new)
 15 |     if len(ia_files) > 0:
 16 |         for filename in ia_files:
 17 |             with open(filename + '.upload', 'w') as file:
 18 |                 pass
 19 |             upload_response = internetarchive.upload(ia_identifier, filename, metadata = ia_args, access_key = ia_access_key, secret_key = ia_secret_key, queue_derive = True, verify = True, verbose = True, delete = True, retries = 5, retries_sleep = 300)
 20 |             os.remove(filename + '.upload')
 21 | 
 22 | def upload_items():
 23 |     for folder in [name for name in os.listdir(os.path.join('to_be_uploaded', 'ia_items')) if os.path.isdir(os.path.join('to_be_uploaded', 'ia_items', name))]:
 24 |         files = os.listdir(os.path.join('to_be_uploaded', 'ia_items', folder))
 25 |         if 'ia_metadata.py' in files and len(files) > 1 and not 'no_upload' in files:
 26 |             if len(files) == 1:
 27 |                 continue
 28 |             with codecs.open(os.path.join('to_be_uploaded', 'ia_items', folder, 'ia_metadata.py'), 'r') as file:
 29 |                 args = file.read().splitlines()
 30 |                 ia_args = {}
 31 |                 ia_files = []
 32 |                 ia_identifier = None
 33 |                 for arg in args:
 34 |                     a, b = arg.split(' = ', 1)
 35 |                     if a == 'files':
 36 |                         for filename in eval(b):
 37 |                             if os.path.isfile(os.path.join('to_be_uploaded', 'ia_items', folder, filename)) and not os.path.isfile(os.path.join('to_be_uploaded', 'ia_items', folder, filename + '.upload')):
 38 |                                 ia_files.append(os.path.join('to_be_uploaded', 'ia_items', folder, filename))
 39 |                     elif a == 'identifier':
 40 |                         ia_identifier = eval(b)
 41 |                     else:
 42 |                         ia_args[a] = eval(b)
 43 |                 threading.Thread(target = upload, args = (ia_args, ia_files, ia_identifier)).start()
 44 |         elif len(files) == 1 and 'ia_metadata.py' in files:
 45 |             shutil.rmtree(os.path.join('to_be_uploaded', 'ia_items', folder))
 46 | 
 47 | def firstrun():
 48 |     for folder in [name for name in os.listdir(os.path.join('to_be_uploaded', 'ia_items')) if os.path.isdir(os.path.join('to_be_uploaded', 'ia_items', name))]:
 49 |         for filename in os.listdir(os.path.join('to_be_uploaded', 'ia_items', folder)):
 50 |             if filename.endswith('.upload'):
 51 |                 os.remove(os.path.join('to_be_uploaded', 'ia_items', folder, filename))
 52 | 
 53 | def warcs_items():
 54 |     max_item_size = max_warc_item_size
 55 |     item_size = 0
 56 |     new_item_dir = os.path.join('to_be_uploaded', 'ia_warcs', 'new_item')
 57 |     new_item_files = []
 58 |     if not os.path.isdir(new_item_dir):
 59 |         os.makedirs(new_item_dir)
 60 |     for file in os.listdir(new_item_dir):
 61 |         new_item_files.append(file)
 62 |         item_size += os.path.getsize(os.path.join(new_item_dir, file))
 63 |     for file in [name for name in os.listdir(os.path.join('to_be_uploaded', 'ia_warcs')) if not name == 'new_item']:
 64 |         file_path = os.path.join('to_be_uploaded', 'ia_warcs', file)
 65 |         file_path_item = os.path.join('to_be_uploaded', 'ia_warcs', 'new_item', file)
 66 |         file_size = os.path.getsize(file_path)
 67 |         item_size += file_size
 68 |         new_item_files.append(file)
 69 |         os.rename(file_path, file_path_item)
 70 |         if item_size >= max_item_size:
 71 |             timestamp = time.strftime('%Y%m%d%H%M%S')
 72 |             date = timestamp[:4] + '-' + timestamp[4:6] + '-' + timestamp[6:11]
 73 |             ia_metadata = {'identifier': 'archiveteam_videobot_web_' + timestamp, 'title': 'Archive Team VideoBot Crawls: ' + timestamp, 'date': date, 'collection': 'archiveteam_videobot', 'mediatype': 'web', 'files': new_item_files, 'description': 'Crawls by VideoBot.', 'subject': 'videobot;archiveteam'}
 74 |             new_item_dir_ready = os.path.join('to_be_uploaded', 'ia_items', 'archiveteam_videobot_web_' + timestamp)
 75 |             for a, b in ia_metadata.items():
 76 |                 print(a, b)
 77 |                 with open(os.path.join(new_item_dir, 'ia_metadata.py'), 'a') as meta_file:
 78 |                     if type(b) is list:
 79 |                         content_string = str(b)
 80 |                     else:
 81 |                         content_string = '\'' + str(b).replace('\'', '\\\'') + '\''
 82 |                     meta_file.write(str(a) + ' = ' + content_string + '\n')
 83 |             os.rename(new_item_dir, new_item_dir_ready)
 84 |             os.makedirs(new_item_dir)
 85 |             item_size = 0
 86 |             new_item_files = []
 87 |             time.sleep(1)
 88 | 
 89 | def move_warcs():
 90 |     list = []
 91 |     done = True
 92 |     maindir = '.'
 93 |     for folder in [name for name in os.listdir(maindir) if os.path.isdir(os.path.join(maindir, name))]:
 94 |         files = [name for name in os.listdir(os.path.join(maindir, folder)) if (os.path.isfile(os.path.join(maindir, folder, name)) and name.endswith('.warc.gz'))]
 95 |         grab_finished = False
 96 |         for file in files:
 97 |             if file.endswith('-meta.warc.gz'):
 98 |                 grab_finished = True
 99 |         for file in files:
100 |             file_location = os.path.join(maindir, folder, file)
101 |             if grab_finished:
102 |                 os.rename(file_location, os.path.join('to_be_uploaded', 'ia_warcs', file))
103 |             else:
104 |                 warc_num = int(file[-13:-8])
105 |                 warc_num_second = (5-len(str(warc_num + 1))) * '0' + str(warc_num + 1)
106 |                 if file[:-13] + warc_num_second + '.warc.gz' in files:
107 |                     os.rename(file_location, os.path.join('to_be_uploaded', 'ia_warcs', file))
108 |         if grab_finished:
109 |            shutil.rmtree(os.path.join(maindir, folder))
110 |     warcs_items()


--------------------------------------------------------------------------------
/url.py:
--------------------------------------------------------------------------------
 1 | from config import irc_channel
 2 | import refresh
 3 | import re
 4 | import irc_bot
 5 | import services
 6 | 
 7 | def irc_bot_print(channel, message):
 8 |     irc_bot.irc_bot_print(channel, message)
 9 | 
10 | def find_url_service(url):
11 |     services_list = refresh.services_list
12 |     for service in services_list:
13 |         if service[1] != None:
14 |             if re.search(service[1], url):
15 |                 return service[0]
16 |     else:
17 |         return None
18 | 
19 | def find_url_title(url_service, url):
20 |     try:
21 |         content = eval('services.' + url_service + '.item_title(url)')
22 |     except AttributeError:
23 |         content = None
24 |     return content
25 | 
26 | def find_url_service_name(url_service):
27 |     print('services.' + url_service + '.service_name')
28 |     return eval('services.' + url_service + '.service_name')
29 | 
30 | def find_url_id(url_service, url):
31 |     try:
32 |         content = eval('services.' + url_service + '.url_id(url)')
33 |     except AttributeError:
34 |         content = None
35 |     return content
36 | 
37 | def check_create_url(url, prefix, suffix):
38 |     if url.startswith('https://') or url.startswith('http://'):
39 |         return url
40 |     else:
41 |         return prefix + url + suffix
42 | 


--------------------------------------------------------------------------------