',
32 | type=str,
33 | default='http://localhost:9020',
34 | help='The Shepherd server url',
35 | )
36 | @click.option(
37 | '-q',
38 | '--quiet',
39 | is_flag=True,
40 | default=False,
41 | type=bool,
42 | help='quiet mode: print only crawl ids if success',
43 | )
44 | def cli(server, quiet, shepherd):
45 | settings.server_prefix = server
46 |
47 | settings.shepherd_prefix = shepherd
48 | settings.view_browsers_prefix = shepherd + '/attach/'
49 |
50 | settings.sesh = requests.session()
51 |
52 | settings.quiet_mode = quiet
53 |
54 |
55 | # ============================================================================
56 | def is_quiet():
57 | return settings.quiet_mode
58 |
59 |
60 | # ============================================================================
61 | def ensure_success(res, exit=True):
62 | """ Ensure API response is successful
63 | print error and exit if not
64 |
65 | :param res: Response from requests
66 | :param exit: Exit on any error
67 | :return: parsed JSON response as dict
68 | """
69 | if res.status_code == 200:
70 | json = res.json()
71 | return json
72 |
73 | if not is_quiet():
74 | print('Error response from API server')
75 | print('{0}: {1}'.format(res.status_code, res.text))
76 |
77 | if exit:
78 | sys.exit(1)
79 |
80 |
81 | # ============================================================================
82 | def conn_error_exit(url):
83 | if not is_quiet():
84 | print(
85 | 'Unable to connect to {0}. Is Browsertrix container running in Docker?'.format(
86 | url
87 | )
88 | )
89 | sys.exit(2)
90 |
91 |
92 | # ============================================================================
93 | def sesh_get(url, prefix=None):
94 | url = (prefix or settings.server_prefix) + url
95 | try:
96 | res = settings.sesh.get(url)
97 | return ensure_success(res)
98 | except requests.exceptions.ConnectionError:
99 | conn_error_exit(url)
100 |
101 |
102 | # ============================================================================
103 | def sesh_post(url, json=None, prefix=None):
104 | url = (prefix or settings.server_prefix) + url
105 | try:
106 | res = settings.sesh.post(url, json=json)
107 | return ensure_success(res)
108 | except requests.exceptions.ConnectionError:
109 | conn_error_exit(url)
110 |
111 |
112 | # ============================================================================
113 | def sesh_delete(url, prefix=None):
114 | url = (prefix or settings.server_prefix) + url
115 | try:
116 | res = settings.sesh.delete(url)
117 | return ensure_success(res, exit=False)
118 | except requests.exceptions.ConnectionError:
119 | conn_error_exit(url)
120 |
--------------------------------------------------------------------------------
/browsertrix_cli/crawl.py:
--------------------------------------------------------------------------------
1 | import click
2 | import datetime
3 | import docker
4 | import sys
5 | import time
6 | import yaml
7 | import webbrowser
8 |
9 | from collections import defaultdict
10 |
11 |
12 | from browsertrix_cli.basecli import (
13 | cli,
14 | is_quiet,
15 | sesh_get,
16 | sesh_post,
17 | sesh_delete,
18 | settings,
19 | )
20 | from browsertrix_cli.profile import get_profile_image
21 |
22 |
23 | COLUMNS = [
24 | ('id', 'CRAWL ID', 12),
25 | ('name', 'NAME', 12),
26 | ('start_time', 'STARTED', 12),
27 | ('finish_time', 'DURATION', 12),
28 | ('status', 'STATUS', 7),
29 | ('crawl_type', 'CRAWL TYPE', 12),
30 | ('coll', 'COLL', 16),
31 | ('mode', 'MODE', 8),
32 | ('num_queue', 'TO CRAWL', 8),
33 | ('num_pending', 'PENDING', 8),
34 | ('num_seen', 'SEEN', 8),
35 | ('num_browsers', 'BROWSERS', 9),
36 | ('num_tabs', 'TABS', 3),
37 | ]
38 |
39 |
40 | # ============================================================================
41 | @cli.group(help='Commands for working with crawls')
42 | def crawl():
43 | pass
44 |
45 |
46 | # ============================================================================
47 | def format_duration(start_time, finish_time):
48 | """ Format duration of crawl
49 |
50 | :param start_time: start time of crawl
51 | :param finish_time: finish time of crawl
52 | :return: string text for time elapsed since timestr
53 | """
54 | try:
55 | if start_time == 0:
56 | return '-'
57 |
58 | if not finish_time:
59 | finish = datetime.datetime.now()
60 | else:
61 | finish = datetime.datetime.fromtimestamp(int(finish_time))
62 |
63 | start = datetime.datetime.fromtimestamp(int(start_time))
64 | elapsed = finish - start
65 | return str(elapsed).split('.', 1)[0]
66 | except Exception:
67 | return start_time
68 |
69 |
70 | # ============================================================================
71 | def print_container_log(
72 | docker_api, reqid, name='autobrowser-', follow=False, wait=False
73 | ):
74 |
75 | full_name = name + reqid
76 | while True:
77 | try:
78 | container = docker_api.containers.get(full_name)
79 | break
80 | except docker.errors.NotFound:
81 | if not wait:
82 | return False
83 |
84 | print('Waiting for Logs...')
85 | time.sleep(0.25)
86 | continue
87 |
88 | print('---- Logs for Crawl {0}: {1} ----'.format(reqid, full_name))
89 | res = container.logs(follow=follow, stream=True)
90 | for line in res:
91 | sys.stdout.write(line.decode('utf-8'))
92 |
93 | print('-----------------------------------')
94 | print('')
95 | print('')
96 |
97 | return True
98 |
99 |
100 | # ============================================================================
101 | def print_logs(browsers, follow=False, wait=False, all_containers=False):
102 | docker_api = docker.from_env(version='auto')
103 |
104 | if follow is None:
105 | follow = False
106 |
107 | for reqid in browsers:
108 | if all_containers:
109 | print_container_log(
110 | docker_api, reqid, wait=False, follow=False, name='browser-'
111 | )
112 |
113 | print_container_log(
114 | docker_api, reqid, wait=False, follow=False, name='xserver-'
115 | )
116 |
117 | print_container_log(docker_api, reqid, wait=wait, follow=follow)
118 |
119 |
120 | # ============================================================================
121 | def open_browsers(browsers, crawl_id, tabs_done=None, num_tabs=-1):
122 | count = 1
123 | for reqid in browsers:
124 | skip = False
125 | if not tabs_done or tabs_done.get(reqid) != num_tabs:
126 | msg = 'Opening Browser {0} of {1} ({2}) for crawl {3}'
127 | else:
128 | msg = 'Skipping Finished Browser {0} of {1}, ({2}) for crawl {3}'
129 | skip = True
130 |
131 | if not is_quiet():
132 | print(msg.format(count, len(browsers), reqid, crawl_id))
133 |
134 | if not skip:
135 | webbrowser.open(settings.view_browsers_prefix + reqid)
136 | count += 1
137 |
138 |
139 | # ============================================================================
140 | @crawl.command(name='list', help='List all crawls')
141 | def list_crawls():
142 | """ List all available crawls
143 | """
144 | res = sesh_get('/crawls')
145 |
146 | sorted_list = sorted(res['crawls'], key=lambda x: x['start_time'], reverse=True)
147 |
148 | if is_quiet():
149 | for crawl in sorted_list:
150 | print(crawl['id'])
151 |
152 | return
153 |
154 | format_str = '{value: <{size}} '
155 |
156 | for _, text, size in COLUMNS:
157 | sys.stdout.write(format_str.format(value=text, size=size))
158 | print()
159 |
160 | for crawl in sorted_list:
161 | for field, _, size in COLUMNS:
162 | value = crawl[field]
163 | if field == 'start_time':
164 | value = format_duration(value, None) + ' ago'
165 | elif field == 'finish_time':
166 | value = format_duration(crawl['start_time'], value)
167 |
168 | sys.stdout.write(format_str.format(value=value, size=size))
169 | print()
170 | print()
171 |
172 |
173 | # ============================================================================
174 | @crawl.command(
175 | name='create', help='Create (and optionally start) new crawl from yaml crawl spec'
176 | )
177 | @click.option(
178 | '--start/--no-start',
179 | default=True,
180 | help="Start/Don't start crawl immediately after creation",
181 | )
182 | @click.option(
183 | '--browser',
184 | type=str,
185 | default=None,
186 | help='Browser Docker image to use for crawling, (overrides setting in spec)',
187 | )
188 | @click.option(
189 | '--profile',
190 | type=str,
191 | default=None,
192 | help='Browser Profile Docker image to use for crawling (overrides "browser" option)',
193 | )
194 | @click.option(
195 | '--coll',
196 | type=str,
197 | default=None,
198 | help='Set the collection (overrides setting in spec)',
199 | )
200 | @click.option(
201 | '--mode',
202 | type=str,
203 | default=None,
204 | help='Set the capture mode (overrides setting in spec)',
205 | )
206 | @click.option(
207 | '--screenshot_coll',
208 | type=str,
209 | default=None,
210 | help='Set the collection to save screenshots (overrides setting in spec)',
211 | )
212 | @click.option(
213 | '--headless',
214 | type=bool,
215 | is_flag=True,
216 | help='Use headless mode. Browsers can not be opened for watching the crawl',
217 | )
218 | @click.option(
219 | '--behavior-time',
220 | default=None,
221 | type=int,
222 | help='Max duration to run each in-page behavior',
223 | )
224 | @click.option(
225 | '--watch',
226 | is_flag=True,
227 | default=False,
228 | type=bool,
229 | help='Watch all started browsers in a local browser (only if starting crawl)',
230 | )
231 | @click.option(
232 | '--log',
233 | is_flag=True,
234 | default=False,
235 | type=bool,
236 | help='Tail the log for the browser crawler',
237 | )
238 | @click.argument('crawl_spec_file', type=click.File('rt'))
239 | def create_crawl(
240 | crawl_spec_file,
241 | start,
242 | browser,
243 | profile,
244 | coll,
245 | mode,
246 | screenshot_coll,
247 | headless,
248 | behavior_time,
249 | watch,
250 | log,
251 | ):
252 | """ Create a new crawl!
253 |
254 | :param crawl_spec_file: YAML file with one or more crawls in 'crawls' key
255 | :param start: If true, start crawl immediately after creation
256 | :param browser: Browser Docker image to use for crawling (overrides setting in spec)
257 | :param profile: Browser Profile Docker image to use for crawling (overrides "browser" setting)
258 | :param coll: Set the collection (overrides setting in spec)
259 | :param mode: Set the capture mode (overrides setting in spec)
260 | :param screenshot_coll: Set the collection to save screenshots (overrides setting in spec)
261 | :param headless: Use headless mode. Browsers can not be opened for watching the crawl
262 | :param behavior_time: Max duration (in seconds) to run each in-page behavior
263 | :param watch: Watch all started browsers in a local browser (only if starting crawl)
264 |
265 | """
266 | root = yaml.load(crawl_spec_file, Loader=yaml.Loader)
267 |
268 | for crawl_spec in root['crawls']:
269 | if not start:
270 | msg = 'Created'
271 | else:
272 | msg = 'Created and Started'
273 |
274 | if headless is not None:
275 | crawl_spec['headless'] = headless
276 |
277 | if behavior_time is not None:
278 | crawl_spec['behavior_time'] = behavior_time
279 |
280 | if profile is not None:
281 | crawl_spec['profile'] = profile
282 |
283 | if 'profile' in crawl_spec:
284 | browser = get_profile_image(crawl_spec.get('profile', ''))
285 |
286 | if browser is not None:
287 | crawl_spec['browser'] = browser
288 |
289 | if coll is not None:
290 | crawl_spec['coll'] = coll
291 |
292 | if mode is not None:
293 | crawl_spec['mode'] = mode
294 |
295 | if screenshot_coll is not None:
296 | crawl_spec['screenshot_coll'] = screenshot_coll
297 |
298 | if not is_quiet():
299 | print('Creating New Crawl, Please Wait...')
300 |
301 | res = sesh_post('/crawls', json=crawl_spec)
302 |
303 | if is_quiet():
304 | print(res['id'])
305 | else:
306 | print('Crawl {0}: {1}'.format(msg, res['id']))
307 | print('Status: {0}'.format(res['status']))
308 |
309 | if watch:
310 | if not start:
311 | if not is_quiet():
312 | print("Can't watch, crawl not started")
313 |
314 | elif headless:
315 | if not is_quiet():
316 | print("Can't watch, crawl is running in headless mode")
317 |
318 | else:
319 | open_browsers(res['browsers'], res['id'])
320 |
321 | if log:
322 | print_logs(res['browsers'], follow=True, wait=True)
323 |
324 |
325 | # ============================================================================
326 | @crawl.command(name='start', help='Start an existing crawl')
327 | @click.argument('crawl_id', nargs=-1)
328 | def start_crawl(crawl_id, browser, headless, behavior_time):
329 | """ Start an existing crawl
330 |
331 | :param crawl_id: list of crawl ids to start
332 | """
333 | for id_ in crawl_id:
334 | res = sesh_post('/crawl/{0}/start'.format(id_))
335 |
336 | if is_quiet():
337 | print(res['id'])
338 | else:
339 | print('Started Crawl: {0}'.format(res['id']))
340 |
341 |
342 | # ============================================================================
343 | @crawl.command(name='info', help='Get info on an existing crawl(s)')
344 | @click.argument('crawl_id', nargs=-1)
345 | @click.option(
346 | '--urls/--no-urls',
347 | default=False,
348 | help='Get detailed info on crawl, listing all urls',
349 | )
350 | def get_info(crawl_id, urls):
351 | """ Get info on existing crawl(s)
352 |
353 | :param crawl_id: list of crawl ids to get info on
354 | :param urls: Get detailed info on crawl, listing all urls
355 | """
356 | for id_ in crawl_id:
357 | if urls:
358 | res = sesh_get('/crawl/{0}/info'.format(id_))
359 | else:
360 | res = sesh_get('/crawl/{0}'.format(id_))
361 |
362 | print(yaml.dump(res))
363 |
364 |
365 | # ============================================================================
366 | @crawl.command(name='watch', help='Watch crawling browsers in local browser')
367 | @click.argument('crawl_id', nargs=-1)
368 | def watch_crawl(crawl_id):
369 | """ Watch crawling browsers in local browser
370 |
371 | :param crawl_id: list of crawl ids to watch
372 | """
373 | for id_ in crawl_id:
374 | res = sesh_get('/crawl/{0}'.format(id_))
375 |
376 | if res.get('headless'):
377 | if not is_quiet():
378 | print("Can not watch, crawl is running in headless mode")
379 | continue
380 |
381 | if res.get('status') != 'running':
382 | if not is_quiet():
383 | print('Crawl not running: {0}'.format(id_))
384 | continue
385 |
386 | browsers = res['browsers']
387 |
388 | done_count = defaultdict(int)
389 |
390 | for info in res.get('tabs_done'):
391 | done_count[info['id']] += 1
392 |
393 | if not browsers:
394 | if not is_quiet():
395 | print('No Browsers')
396 | continue
397 |
398 | open_browsers(browsers, id_, done_count, res['num_tabs'])
399 |
400 |
401 | # ============================================================================
402 | @crawl.command(name='stop', help='Stop one or more existing crawls')
403 | @click.argument('crawl_id', nargs=-1)
404 | def stop_crawl(crawl_id):
405 | """ Stop one or more existing crawls
406 |
407 | :param crawl_id: list of crawl ids to stop
408 | """
409 | for id_ in crawl_id:
410 | res = sesh_post('/crawl/{0}/stop'.format(id_))
411 |
412 | if not res.get('success'):
413 | print('Error stopping: ' + res)
414 | return
415 |
416 | if is_quiet():
417 | print(id_)
418 | else:
419 | print('Stopped Crawl: {0}'.format(id_))
420 |
421 |
422 | # ============================================================================
423 | @crawl.command(name='remove', help='Remove one or more existing crawls')
424 | @click.argument('crawl_id', nargs=-1)
425 | def remove_crawl(crawl_id):
426 | """ Remove one or more existing crawls
427 |
428 | :param crawl_id: list of crawl ids to stop
429 | """
430 | for id_ in crawl_id:
431 | res = sesh_delete('/crawl/{0}'.format(id_))
432 |
433 | if not res.get('success'):
434 | print('Error removing: ' + res)
435 | return
436 |
437 | if is_quiet():
438 | print(id_)
439 | else:
440 | print('Removed Crawl: {0}'.format(id_))
441 |
442 |
443 | # ============================================================================
444 | @crawl.command(name='remove-all', help='Stop and remove all crawls')
445 | def remove_all():
446 | """ Stop and remove all crawls
447 | """
448 | res = sesh_get('/crawls')
449 |
450 | crawls = res['crawls']
451 |
452 | for crawl in crawls:
453 | id_ = crawl['id']
454 | res = sesh_delete('/crawl/{0}'.format(id_))
455 | if not is_quiet():
456 | print('Removed Crawl: {0}'.format(id_))
457 |
458 |
459 | # ============================================================================
460 | @crawl.command(name='logs', help='View crawl logs for one or all crawlers')
461 | @click.argument('crawl_id', nargs=1)
462 | @click.option(
463 | '-b',
464 | '--browser',
465 | type=int,
466 | default=0,
467 | help='1-based index of browser to show logs for, or 0 for all (default)',
468 | )
469 | @click.option(
470 | '-f',
471 | '--follow',
472 | type=bool,
473 | default=False,
474 | is_flag=True,
475 | help='follow crawl log in real-time',
476 | )
477 | @click.option(
478 | '-a',
479 | '--all-containers',
480 | type=bool,
481 | default=False,
482 | is_flag=True,
483 | help='include logs from all containers, not just crawler',
484 | )
485 | def logs(crawl_id, browser, follow, all_containers):
486 | """ View crawl logs for one or all crawlers
487 | :param crawl_id: The crawl_id to view logs for
488 | :param browser: 1-based index of browser to show logs for, or 0 for all (default)
489 | :param follow: follow crawl log in real-time (for one browser only)
490 | :param all_containers: include logs from all containers, not just crawler
491 | """
492 | res = sesh_get('/crawl/{0}'.format(crawl_id))
493 |
494 | num_browsers = len(res['browsers'])
495 | if browser <= 0:
496 | print_logs(res['browsers'], follow=follow, all_containers=all_containers)
497 | elif browser > num_browsers:
498 | print(
499 | 'Crawl has {0} browsers. Index must be 1 to {0}'.format(
500 | num_browsers, num_browsers
501 | )
502 | )
503 | else:
504 | print_logs(
505 | [res['browsers'][browser - 1]], follow=follow, all_containers=all_containers
506 | )
507 |
--------------------------------------------------------------------------------
/browsertrix_cli/main.py:
--------------------------------------------------------------------------------
1 | from browsertrix_cli.basecli import cli
2 | import browsertrix_cli.profile
3 | import browsertrix_cli.crawl
4 |
5 |
6 | # ============================================================================
7 | if __name__ == '__main__':
8 | cli()
9 |
--------------------------------------------------------------------------------
/browsertrix_cli/profile.py:
--------------------------------------------------------------------------------
1 | import click
2 | import docker
3 | import sys
4 | import time
5 | import webbrowser
6 |
7 |
8 | from browsertrix_cli.basecli import cli, is_quiet, sesh_get, settings
9 |
10 |
11 | # ============================================================================
12 | docker_api = None
13 |
14 | PROFILE_PREFIX = 'oldwebtoday/profile:'
15 |
16 | LABEL_BROWSERPROFILE = 'wr.browserprofile'
17 | LABEL_BASEBROWSER = 'wr.basebrowser'
18 |
19 |
20 | # ============================================================================
21 | def get_profile_image(profile):
22 | try:
23 | global docker_api
24 | if not docker_api:
25 | docker_api = docker.from_env(version='auto')
26 |
27 | image_name = PROFILE_PREFIX + profile
28 | image = docker_api.images.get(image_name)
29 | assert image.labels.get(LABEL_BROWSERPROFILE) == profile
30 | return 'profile:' + profile
31 |
32 | except (docker.errors.ImageNotFound, AssertionError):
33 | if not is_quiet():
34 | print('Profile "{0}" not found'.format(profile))
35 | sys.exit(1)
36 |
37 |
38 | # ============================================================================
39 | @cli.group(help='Commands for creating/removing browser profiles')
40 | def profile():
41 | global docker_api
42 | docker_api = docker.from_env(version='auto')
43 |
44 |
45 | # ============================================================================
46 | @profile.command(name='list', help='List Profiles')
47 | def list_profiles():
48 | res = docker_api.images.list(filters={'label': LABEL_BROWSERPROFILE})
49 |
50 | format_str = '{profile: <16} {base}'
51 | if not is_quiet():
52 | print(format_str.format(profile='PROFILE', base='BASE BROWSER'))
53 |
54 | for image in res:
55 | if not image.tags:
56 | continue
57 |
58 | if not image.tags[0].startswith(PROFILE_PREFIX):
59 | continue
60 |
61 | profile = image.tags[0][len(PROFILE_PREFIX) :]
62 | base_browser = image.labels.get(LABEL_BASEBROWSER, '(unknown)')
63 |
64 | if not is_quiet():
65 | print(format_str.format(profile=profile, base=base_browser))
66 | else:
67 | print(profile)
68 |
69 | if not is_quiet():
70 | print()
71 |
72 |
73 | # ============================================================================
74 | @profile.command(name='remove', help='Remove Profile')
75 | @click.argument('profile', type=str)
76 | def remove_profile(profile):
77 | full_tag = PROFILE_PREFIX + profile
78 |
79 | try:
80 | docker_api.images.remove(full_tag, force=True, noprune=False)
81 | if not is_quiet():
82 | print('Removed profile "{0}"!'.format(profile))
83 |
84 | except docker.errors.ImageNotFound:
85 | if not is_quiet():
86 | print('Profile "{0}" not found'.format(profile))
87 | sys.exit(1)
88 |
89 |
90 | # ============================================================================
91 | @profile.command(name='create', help='Create Profile')
92 | @click.option(
93 | '--browser', default='chrome:73', type=str, help='Base Browser Image to Extend'
94 | )
95 | def create_profile(browser):
96 | res = sesh_get(
97 | '/api/request/{0}/about:blank'.format(browser), prefix=settings.shepherd_prefix
98 | )
99 |
100 | reqid = res.get('reqid')
101 |
102 | curr_browser = None
103 |
104 | webbrowser.open(settings.view_browsers_prefix + reqid)
105 |
106 | print('A new browser window should have been opened')
107 | print(
108 | 'You can use the browser to log-in to accounts or otherwise prepare the browser profile'
109 | )
110 | print('(The content will not be recorded to WARC)')
111 |
112 | while True:
113 | profile_name = click.prompt(
114 | 'When done, please enter a new name to save the browser profile', type=str
115 | )
116 |
117 | if not curr_browser:
118 | curr_browser = docker_api.containers.get('browser-' + reqid)
119 |
120 | # exit_code, output = curr_browser.exec_run('/app/prep-commit.sh')
121 | exit_code, output = curr_browser.exec_run('pkill -f "/usr/bin/google-chrome"')
122 | if not is_quiet():
123 | print('Killed Chrome to Save Profile for Commit')
124 | print('Result: {0}'.format(exit_code))
125 | print(output.decode('utf-8'))
126 |
127 | time.sleep(1.5)
128 |
129 | conf = {
130 | 'Labels': {LABEL_BROWSERPROFILE: profile_name, LABEL_BASEBROWSER: browser}
131 | }
132 |
133 | res = curr_browser.commit(
134 | repository=PROFILE_PREFIX[:-1],
135 | tag=profile_name,
136 | message='Browser Profile',
137 | conf=conf,
138 | )
139 |
140 | if not is_quiet():
141 | print('Created Image: {0} ({1})'.format(res.tags[0], res.short_id))
142 |
143 | print('The browser should have restarted to about:blank')
144 | if not click.confirm('Continue browsing to create another profile?'):
145 | break
146 |
--------------------------------------------------------------------------------
/cli-requirements.txt:
--------------------------------------------------------------------------------
1 | click
2 | docker
3 | pyyaml
4 | requests
5 |
--------------------------------------------------------------------------------
/dev-requirements.txt:
--------------------------------------------------------------------------------
1 | flake8
2 | flake8-bugbear
3 | flake8-mypy
4 | mypy
5 | pytest
6 | pytest-asyncio
7 | black
8 | mock
9 | requests
10 | fakeredis
11 | PyYAML
12 |
--------------------------------------------------------------------------------
/docker-compose.yml:
--------------------------------------------------------------------------------
1 | version: '3.5'
2 |
3 | services:
4 | browsertrix:
5 | image: webrecorder/browsertrix
6 | build:
7 | context: .
8 | environment:
9 | - REDIS_URL=redis://redis/0
10 | - DEFAULT_POOL=auto-pool
11 | - SCREENSHOT_API_URL=http://pywb:8080/api/screenshot/{coll}
12 | - EXTRACTED_RAW_DOM_API_URL=http://pywb:8080/api/dom/{coll}
13 | - PROXY_HOST=pywb
14 |
15 | depends_on:
16 | - redis
17 | - pywb
18 |
19 | ports:
20 | - 8000:8000
21 |
22 | #volumes:
23 | # - ./:/app/
24 |
25 | pywb:
26 | build: ./pywb/
27 |
28 | environment:
29 | - REDIS_URL=redis://redis/0
30 |
31 | volumes:
32 | - ./webarchive:/webarchive
33 |
34 | networks:
35 | - default
36 | - browsers
37 |
38 | ports:
39 | - 8180:8080
40 |
41 | depends_on:
42 | - redis
43 |
44 | solr:
45 | image: solr
46 |
47 | volumes:
48 | - ./solr:/var/solr
49 |
50 | entrypoint:
51 | - docker-entrypoint.sh
52 | - solr-precreate
53 | - browsertrix
54 |
55 | ports:
56 | - 8983:8983
57 |
58 | shepherd:
59 | image: oldwebtoday/shepherd:1.1.0-dev
60 |
61 | environment:
62 | - BROWSER_NET=browsertrix_browsers
63 | - MAIN_NET=browsertrix_default
64 |
65 | - PROXY_HOST=pywb
66 | - PROXY_PORT=8080
67 |
68 | depends_on:
69 | - redis
70 |
71 | volumes:
72 | - /var/run/docker.sock:/var/run/docker.sock
73 | - ./flocks:/app/flocks
74 | - ./pool_config.yaml:/app/pool_config.yaml
75 |
76 | ports:
77 | - 9020:9020
78 |
79 | redis:
80 | image: redis:3.2.4
81 |
82 | behaviors:
83 | image: webrecorder/behaviors
84 |
85 | ports:
86 | - 3030:3030
87 |
88 | networks:
89 | default:
90 | driver: bridge
91 |
92 | browsers:
93 | driver: bridge
94 |
95 |
--------------------------------------------------------------------------------
/flocks/browsers-headless.yaml:
--------------------------------------------------------------------------------
1 | name: browsers-headless
2 | auto_remove: false
3 |
4 | volumes:
5 | tmpdir: '/tmp/.X11-unix'
6 |
7 | containers:
8 | - name: browser
9 | image: oldwebtoday/base-browser
10 | set_user_params: true
11 | external_network: '${BROWSER_NET}'
12 |
13 | environment:
14 | DISPLAY: ':99'
15 | PULSE_SERVER: '/tmp/.X11-unix/pulse-socket'
16 |
17 | PROXY_HOST: 'pywb'
18 | PROXY_PORT: '8080'
19 | PROXY_CA_URL: 'http://wsgiprox/download/pem'
20 | PROXY_CA_FILE: '/tmp/proxy-ca.pem'
21 |
22 | - name: autobrowser
23 | image: webrecorder/autobrowser
24 |
25 | external_network: '${MAIN_NET}'
26 |
27 | environment:
28 | BROWSER_HOST: 'browser'
29 | REDIS_URL: 'redis://redis:6379/0'
30 | TAB_TYPE: 'CrawlerTab'
31 |
32 |
33 |
--------------------------------------------------------------------------------
/flocks/browsers.yaml:
--------------------------------------------------------------------------------
1 | name: browsers
2 | auto_remove: false
3 |
4 | volumes:
5 | tmpdir: '/tmp/.X11-unix'
6 |
7 | containers:
8 | - name: xserver
9 | image: oldwebtoday/vnc-webrtc-audio
10 | ports:
11 | vnc_port: 6080
12 | cmd_port: 6082
13 | ice_tcp_port: 10235
14 | ice_udp_port: '10235/udp'
15 |
16 | environment:
17 | IDLE_TIMEOUT: '${IDLE_TIMEOUT}'
18 |
19 | - name: browser
20 | image: oldwebtoday/base-browser
21 | image_label: wr.name
22 | set_user_params: true
23 | external_network: '${BROWSER_NET}'
24 |
25 | environment:
26 | DISPLAY: ':99'
27 | PULSE_SERVER: '/tmp/.X11-unix/pulse-socket'
28 |
29 | #default to no proxy, set by crawler
30 | #PROXY_HOST: 'pywb'
31 | PROXY_PORT: '8080'
32 | PROXY_CA_URL: 'http://wsgiprox/download/pem'
33 | PROXY_CA_FILE: '/tmp/proxy-ca.pem'
34 |
35 | - name: autobrowser
36 | image: webrecorder/autobrowser
37 |
38 | external_network: '${MAIN_NET}'
39 |
40 | environment:
41 | BROWSER_HOST: 'browser'
42 | REDIS_URL: 'redis://redis:6379/0'
43 | TAB_TYPE: 'CrawlerTab'
44 |
45 |
46 |
--------------------------------------------------------------------------------
/frontend/.eslintrc.js:
--------------------------------------------------------------------------------
1 | module.exports = {
2 | extends: ['plugin:prettier/recommended', 'prettier/react'],
3 | parser: 'babel-eslint',
4 | parserOptions: {
5 | ecmaVersion: 10
6 | },
7 | env: {
8 | browser: true,
9 | node: true
10 | }
11 | };
12 |
--------------------------------------------------------------------------------
/frontend/.prettierrc:
--------------------------------------------------------------------------------
1 | {
2 | "singleQuote": true,
3 | "jsxSingleQuote": true
4 | }
5 |
--------------------------------------------------------------------------------
/frontend/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "crawlmanager-frontend",
3 | "version": "1.0.0",
4 | "main": "index.js",
5 | "license": "Appache-2.0",
6 | "dependencies": {
7 | "@hot-loader/react-dom": "16.8.6",
8 | "fstream": "^1.0.12",
9 | "immutable": "^4.0.0-rc.12",
10 | "js-yaml": "^3.13.1",
11 | "plur": "^3.1.1",
12 | "prop-types": "^15.7.2",
13 | "react": "^16.8.6",
14 | "react-dom": "^16.8.6",
15 | "react-redux": "^6.0.1",
16 | "react-router": "^5.0.0",
17 | "react-router-dom": "^5.0.0",
18 | "react-table": "^6.9.2",
19 | "react-toastify": "^5.0.0-rc.3",
20 | "react-virtualized": "^9.21.0",
21 | "redux": "^4.0.1",
22 | "redux-actions": "^2.6.5",
23 | "redux-batched-actions": "^0.4.1",
24 | "redux-devtools-extension": "^2.13.8",
25 | "redux-form": "^8.1.0",
26 | "redux-immutable": "^4.0.0",
27 | "redux-promise": "^0.6.0",
28 | "redux-thunk": "^2.3.0",
29 | "uikit": "^3.0.3",
30 | "url-regex": "^4.1.1"
31 | },
32 | "devDependencies": {
33 | "@babel/cli": "^7.4.3",
34 | "@babel/core": "^7.4.3",
35 | "@babel/plugin-proposal-class-properties": "^7.4.0",
36 | "@babel/plugin-proposal-decorators": "^7.4.0",
37 | "@babel/preset-env": "^7.4.3",
38 | "@babel/preset-react": "^7.0.0",
39 | "@types/react": "^16.8.13",
40 | "babel-eslint": "^10.0.1",
41 | "babel-loader": "^8.0.5",
42 | "babel-plugin-add-module-exports": "^1.0.0",
43 | "babel-plugin-transform-react-remove-prop-types": "^0.4.24",
44 | "babel-preset-env": "^1.7.0",
45 | "css-loader": "^2.1.1",
46 | "eslint": "^5.16.0",
47 | "eslint-config-prettier": "^4.1.0",
48 | "eslint-plugin-prettier": "^3.0.1",
49 | "eslint-plugin-react": "^7.12.4",
50 | "fastify": "^2.2.0",
51 | "fastify-static": "^2.4.0",
52 | "fastify-webpack-hmr": "^2.0.1",
53 | "file-loader": "^3.0.1",
54 | "html-webpack-plugin": "^3.2.0",
55 | "node-sass": "^4.11.0",
56 | "prettier": "^1.16.4",
57 | "react-hot-loader": "^4.8.3",
58 | "sass-loader": "^7.1.0",
59 | "style-loader": "^0.23.1",
60 | "url-loader": "^1.1.2",
61 | "webpack": "^4.29.6",
62 | "webpack-cli": "^3.3.0",
63 | "webpack-dev-middleware": "^3.6.2",
64 | "webpack-hot-middleware": "^2.24.3"
65 | },
66 | "scripts": {
67 | "develop": "node ./webpack/development-server.js",
68 | "build-prod": "NODE_ENV=production webpack --config ./webpack/webpack.config.js"
69 | }
70 | }
71 |
--------------------------------------------------------------------------------
/frontend/public/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | HI
6 |
7 |
8 |
9 |
10 |
11 |
--------------------------------------------------------------------------------
/frontend/src/actions/crawls.js:
--------------------------------------------------------------------------------
1 | import { toast } from 'react-toastify';
2 | import { makeHTTPRequest } from './httpRequests';
3 | import { EndpointRequests } from '../utils';
4 |
5 | export const ActionTypes = {
6 | getAll: Symbol('crawl-get-all'),
7 | gotAll: Symbol('crawl-got-all'),
8 | gotAllInit: Symbol('crawl-got-all-init'),
9 | create: Symbol('crawl-create'),
10 | urls: Symbol('crawl-get-urls'),
11 | addURLs: Symbol('crawl-add-urls'),
12 | updateURLInfo: Symbol('crawl-update-url-info'),
13 | info: Symbol('crawl-get-info'),
14 | stop: Symbol('crawl-stop'),
15 | start: Symbol('crawl-start'),
16 | isDone: Symbol('crawl-is-done'),
17 | deleteCrawl: Symbol('crawl-delete'),
18 | updateInfo: Symbol('crawl-update-info')
19 | };
20 |
21 | export function getAllCrawls(init = false) {
22 | const request = EndpointRequests.retrieveAllCrawls();
23 | return makeHTTPRequest(request, {
24 | onError({ error }) {
25 | toast(`Failed to retrieve info about all crawls: ${error}`, {
26 | type: toast.TYPE.ERROR
27 | });
28 | },
29 | async onResponse({ response }) {
30 | if (!response.ok) {
31 | toast(
32 | `Failed to retrieve info about all crawls: Details
33 | HTTP ${response.status}`,
34 | {
35 | type: toast.TYPE.ERROR
36 | }
37 | );
38 | return;
39 | }
40 | return {
41 | type: init ? ActionTypes.gotAllInit : ActionTypes.gotAll,
42 | payload: await response.json()
43 | };
44 | }
45 | });
46 | }
47 |
48 | export function getCrawlInfo(id) {
49 | const request = EndpointRequests.crawlInfo(id);
50 | return makeHTTPRequest(request, {
51 | onError({ error }) {
52 | toast(`Failed to retrieve the info for crawl - ${id}: ${error}`, {
53 | type: toast.TYPE.ERROR
54 | });
55 | },
56 | async onResponse({ response }) {
57 | const json = await response.json();
58 | if (!response.ok) {
59 | toast(
60 | `Failed to get the crawl info - ${id}: Details
61 | ${json.detail}`,
62 | {
63 | type: toast.TYPE.ERROR
64 | }
65 | );
66 | return;
67 | }
68 | return {
69 | type: ActionTypes.info,
70 | payload: json
71 | };
72 | }
73 | });
74 | }
75 |
76 | export function addCrawlURLs(id, urls) {
77 | const { request } = EndpointRequests.addCrawlURLs(id, urls);
78 | return makeHTTPRequest(request, {
79 | onError({ error }) {
80 | toast(`Failed to add the urls to the crawl - ${id}: ${error}`, {
81 | type: toast.TYPE.ERROR
82 | });
83 | },
84 | async onResponse({ response }) {
85 | const json = await response.json();
86 | if (!response.ok) {
87 | toast(
88 | `Failed to add urls to the crawl - ${id}: Details
89 | ${json.detail}`,
90 | {
91 | type: toast.TYPE.ERROR
92 | }
93 | );
94 | return;
95 | }
96 | return {
97 | type: ActionTypes.addURLs,
98 | payload: {
99 | id,
100 | urls
101 | }
102 | };
103 | }
104 | });
105 | }
106 |
107 | export function getCrawlURLs(id) {
108 | const { request } = EndpointRequests.getCrawlURLs(id);
109 | return makeHTTPRequest(request, {
110 | onError({ error }) {
111 | toast(`Failed to retrieve the crawls URLs - ${id}: ${error}`, {
112 | type: toast.TYPE.ERROR
113 | });
114 | },
115 | async onResponse({ response }) {
116 | const json = await response.json();
117 | if (!response.ok) {
118 | toast(
119 | `Failed to retrieve the crawls URLs - ${id}: Details
120 | ${json.detail}`,
121 | {
122 | type: toast.TYPE.ERROR
123 | }
124 | );
125 | return;
126 | }
127 | return {
128 | type: ActionTypes.updateURLInfo,
129 | payload: Object.assign(
130 | {
131 | id
132 | },
133 | json
134 | )
135 | };
136 | }
137 | });
138 | }
139 |
140 | /**
141 | *
142 | * @param {Object} [newCrawlConfig]
143 | */
144 | export function createCrawl(newCrawlConfig) {
145 | const { body, request } = EndpointRequests.createNewCrawl(newCrawlConfig);
146 | return makeHTTPRequest(request, {
147 | onError({ error }) {
148 | toast(`Failed to create the new crawl ${error}`, {
149 | type: toast.TYPE.ERROR
150 | });
151 | },
152 | async onResponse({ dispatch, response }) {
153 | const json = await response.json();
154 | if (!response.ok) {
155 | toast(
156 | `Failed to create the crawl - ${json.id}: Details
157 | ${json.detail}`,
158 | {
159 | type: toast.TYPE.ERROR
160 | }
161 | );
162 | return;
163 | }
164 | const request = EndpointRequests.crawlInfo(json.id);
165 | const infoResponse = await fetch(request);
166 | const result = infoResponse.ok
167 | ? await infoResponse.json()
168 | : Object.assign(
169 | {
170 | id: json.id
171 | },
172 | body,
173 | newCrawlConfig.crawlRunInfo
174 | );
175 |
176 | console.log('newly created crawl info', result);
177 | return {
178 | type: ActionTypes.create,
179 | payload: result
180 | };
181 | }
182 | });
183 | }
184 |
185 | /**
186 | *
187 | * @param {string} id
188 | * @param {Object} [startConfig]
189 | */
190 | export function startCrawl(id, startConfig) {
191 | const { body, request } = EndpointRequests.startCrawl(id, startConfig);
192 | return makeHTTPRequest(request, {
193 | onError({ error }) {
194 | toast(`Failed to start the crawl - ${id}: ${error}`, {
195 | type: toast.TYPE.ERROR
196 | });
197 | },
198 | async onResponse({ response }) {
199 | const json = await response.json();
200 | if (!response.ok) {
201 | toast(
202 | `Failed to start the crawl - ${id}: Details
203 | ${json.detail}`,
204 | {
205 | type: toast.TYPE.ERROR
206 | }
207 | );
208 | return;
209 | }
210 | const result = {
211 | id,
212 | ...body
213 | };
214 | console.log('started crawl info', result);
215 | return {
216 | type: ActionTypes.start,
217 | payload: result
218 | };
219 | }
220 | });
221 | }
222 |
223 | export function stopCrawl(id) {
224 | const request = EndpointRequests.stopCrawl(id);
225 | return makeHTTPRequest(request, {
226 | onError({ error }) {
227 | toast(`Failed to remove the crawl - ${id}: ${error}`, {
228 | type: toast.TYPE.ERROR
229 | });
230 | },
231 | async onResponse({ dispatch, response }) {
232 | const json = await response.json();
233 | if (!response.ok) {
234 | toast(
235 | `Failed to stop the crawl - ${id}: Details
236 | ${json.detail}`,
237 | {
238 | type: toast.TYPE.ERROR
239 | }
240 | );
241 | return;
242 | }
243 | return {
244 | type: ActionTypes.stop,
245 | payload: { id }
246 | };
247 | }
248 | });
249 | }
250 |
251 | export function removeCrawl(id) {
252 | const request = EndpointRequests.removeCrawl(id);
253 | return makeHTTPRequest(request, {
254 | onError({ error }) {
255 | toast(`Failed to remove the crawl - ${id}: ${error}`, {
256 | type: toast.TYPE.ERROR
257 | });
258 | },
259 | async onResponse({ dispatch, response }) {
260 | const json = await response.json();
261 | if (!response.ok) {
262 | toast(
263 | `Failed to remove the crawl - ${id}: Details
264 | ${json.detail}`,
265 | {
266 | type: toast.TYPE.ERROR
267 | }
268 | );
269 | return;
270 | }
271 | return {
272 | type: ActionTypes.deleteCrawl,
273 | payload: { id }
274 | };
275 | }
276 | });
277 | }
278 |
--------------------------------------------------------------------------------
/frontend/src/actions/httpRequests.js:
--------------------------------------------------------------------------------
1 | export const HTTPRequestAction = Symbol('http-request-maker');
2 |
3 | export const FetchStates = {
4 | preflight: Symbol('http-request-preflight'),
5 | inflight: Symbol('http-request-inflight'),
6 | done: Symbol('http-request-done'),
7 | error: Symbol('http-request-errored')
8 | };
9 |
10 | export function requestErrorAction({ error, payload }) {
11 | return {
12 | type: HTTPRequestAction,
13 | payload: Object.assign({ error }, payload)
14 | };
15 | }
16 |
17 | /**
18 | * @typedef {Object} MakeHTTPRequestInit
19 | * @property {function ({dispatch: Function, response: Response}): *} onResponse
20 | * @property {function ({dispatch: Function, error: Error}): *} onError
21 | */
22 |
23 | function requestComplete(nextAction, wasError, url) {
24 | nextAction.meta = nextAction.meta || {};
25 | nextAction.meta.httpRequest = {
26 | url,
27 | state: wasError ? FetchStates.error : FetchStates.done
28 | };
29 | return nextAction;
30 | }
31 |
32 | /**
33 | *
34 | * @param {Request} request
35 | * @param {MakeHTTPRequestInit} init
36 | */
37 | export function makeHTTPRequest(request, { onResponse, onError }) {
38 | return dispatch => {
39 | const init = {
40 | type: HTTPRequestAction,
41 | meta: {
42 | httpRequest: {
43 | state: FetchStates.preflight,
44 | url: request.url
45 | }
46 | }
47 | };
48 | if (!dispatch(init)) return; // no op, this is a duplicate request
49 | let wasError = false;
50 | dispatch(
51 | fetch(request)
52 | .then(response => onResponse({ dispatch, response }))
53 | .catch(error => {
54 | wasError = true;
55 | return onError({ dispatch, error });
56 | })
57 | .then(requestFinished =>
58 | requestComplete(
59 | requestFinished || { type: HTTPRequestAction },
60 | wasError,
61 | request.url
62 | )
63 | )
64 | );
65 | };
66 | }
67 |
--------------------------------------------------------------------------------
/frontend/src/actions/index.js:
--------------------------------------------------------------------------------
1 | export * from './crawls';
2 |
--------------------------------------------------------------------------------
/frontend/src/components/Crawl/Control.js:
--------------------------------------------------------------------------------
1 | import React, { Component } from 'react';
2 | import * as PropTypes from 'prop-types';
3 | import { CrawlRecord } from '../../reducers/crawls';
4 |
5 | export default class Control extends Component {
6 | static propTypes = {
7 | crawl: PropTypes.instanceOf(CrawlRecord).isRequired,
8 | getCrawlInfo: PropTypes.func.isRequired,
9 | startCrawl: PropTypes.func.isRequired,
10 | stopCrawl: PropTypes.func.isRequired,
11 | removeCrawl: PropTypes.func.isRequired
12 | };
13 |
14 | startCrawl() {
15 | const { crawl } = this.props;
16 | this.props.startCrawl(crawl.startCrawlConfig());
17 | }
18 |
19 | constructor(props) {
20 | super(props);
21 | this.startCrawl = this.startCrawl.bind(this);
22 | }
23 |
24 | render() {
25 | const { crawl, getCrawlInfo, removeCrawl, stopCrawl } = this.props;
26 | return (
27 |
28 |
29 |
36 |
37 |
38 |
45 |
46 |
47 |
53 |
54 |
55 |
58 |
59 |
19 | {qinfo.get('url')} @ depth {qinfo.get('depth')}
20 |
21 |
22 | );
23 | }
24 | return (
25 | <>
26 | Queue
27 |
28 |
29 | );
30 | }
31 |
32 | renderBrowsers() {
33 | const crawlId = this.props.crawl.id;
34 | const browsers = this.props.crawl.browsers;
35 | const b = new Array(browsers.length);
36 | for (let i = 0; i < browsers.length; i++) {
37 | b[i] = (
38 |
39 |
43 | View {browsers[i]}
44 |
45 |
46 | );
47 | }
48 | return (
49 | <>
50 | View Running Crawl
51 |
52 |
53 | );
54 | }
55 |
56 | render() {
57 | const { crawl } = this.props;
58 | console.log('Viewing crawl info', crawl.toJS());
59 | return (
60 | <>
61 |
62 |
63 |
64 | | Type |
65 | Status |
66 | Num Browsers |
67 | Num Tabs |
68 | Crawl Depth |
69 |
70 |
71 |
72 |
73 | | {crawl.crawl_type} |
74 | {crawl.status || 'new'} |
75 | {crawl.num_browsers} |
76 | {crawl.num_tabs} |
77 | {crawl.crawl_depth} |
78 |
79 |
80 |
81 | {crawl.browsers.length > 0 && this.renderBrowsers()}
82 | {crawl.queue.size > 0 && this.renderQueue()}
83 |
84 | );
85 | }
86 | }
87 |
--------------------------------------------------------------------------------
/frontend/src/components/Crawl/index.js:
--------------------------------------------------------------------------------
1 | import React, { Component } from 'react';
2 | import * as PropTypes from 'prop-types';
3 | import { connect } from 'react-redux';
4 | import { Redirect } from 'react-router-dom';
5 | import { CrawlRecord } from '../../reducers/crawls';
6 | import {
7 | getCrawlInfo,
8 | removeCrawl,
9 | startCrawl,
10 | stopCrawl
11 | } from '../../actions';
12 | import Info from './Info';
13 | import Control from './Control';
14 |
15 | class Crawl extends Component {
16 | static propTypes = {
17 | location: PropTypes.object.isRequired,
18 | match: PropTypes.object.isRequired,
19 | crawl: PropTypes.instanceOf(CrawlRecord),
20 | getCrawlInfo: PropTypes.func.isRequired,
21 | startCrawl: PropTypes.func.isRequired,
22 | stopCrawl: PropTypes.func.isRequired,
23 | removeCrawl: PropTypes.func.isRequired
24 | };
25 |
26 | constructor(props) {
27 | super(props);
28 |
29 | this.handle = null;
30 | }
31 |
32 | componentDidMount() {
33 | const { crawl } = this.props;
34 |
35 | if (crawl.get('status') === 'running') {
36 | this.autoUpdate();
37 | }
38 | }
39 |
40 | componentDidUpdate(prevProps) {
41 | const { crawl } = this.props;
42 |
43 | if (
44 | crawl &&
45 | crawl.get('status') === 'running' &&
46 | prevProps.crawl.get('status') !== 'running'
47 | ) {
48 | this.autoUpdate();
49 | } else if (
50 | !crawl ||
51 | (crawl.get('status') !== 'running' && prevProps.crawl.get('status') === 'running')
52 | ) {
53 | clearInterval(this.handle);
54 | }
55 | }
56 |
57 | componentWillUnmount() {
58 | clearInterval(this.handle);
59 | }
60 |
61 | autoUpdate = () => {
62 | this.handle = setInterval(this.props.getCrawlInfo, 1000);
63 | }
64 |
65 | render() {
66 | if (this.props.crawl == null) return ;
67 | const {
68 | crawl,
69 | getCrawlInfo,
70 | removeCrawl,
71 | stopCrawl,
72 | startCrawl
73 | } = this.props;
74 | return (
75 |
76 |
80 |
81 |
Crawl Id - {crawl.id}
82 |
83 |
84 |
85 | {crawl.running ? '' : 'Not'} Running
86 |
87 |
88 |
89 |
96 |
97 |
11 |
14 |
17 |
18 | );
19 | }
20 |
21 | export function CrawlConfigInputField({ disabled, label, min, type, input, meta, title }) {
22 | const id = `${input.name}-${type}`;
23 | const className = `uk-input ${meta.valid ? '' : 'uk-form-danger'}`;
24 | return (
25 |
26 |
29 |
30 |
31 | );
32 | }
33 |
34 | const isURLRe = urlRegx({ exact: true, strict: false });
35 |
36 | const isURLTest = url => (isURLRe.test(url) ? null : 'Not a URL');
37 |
38 | class URLToCrawl extends Component {
39 | static propTypes = {
40 | idx: PropTypes.number.isRequired,
41 | remove: PropTypes.func.isRequired
42 | };
43 |
44 | constructor(props) {
45 | super(props);
46 | this.renderURL = this.renderURL.bind(this);
47 | this.remove = this.remove.bind(this);
48 | }
49 |
50 | renderURL({ input, meta }) {
51 | const className = `uk-input ${meta.valid ? '' : 'uk-form-danger'}`;
52 | return (
53 |
63 | );
64 | }
65 |
66 | remove() {
67 | this.props.remove(this.props.idx);
68 | }
69 |
70 | render() {
71 | return (
72 |
85 | );
86 | }
87 | }
88 |
89 | export class BulkURLInput extends Component {
90 | static propTypes = {
91 | addURL: PropTypes.func.isRequired
92 | };
93 |
94 | constructor(props) {
95 | super(props);
96 | this.textAreaRef = React.createRef();
97 | this.addURLs = this.addURLs.bind(this);
98 | this.close = this.close.bind(this);
99 | }
100 |
101 | componentWillUnmount() {
102 | UIKit.modal('#bulk-seed-input').$destroy(true);
103 | }
104 |
105 | addURLs() {
106 | const value = this.textAreaRef.current.value;
107 | if (value) {
108 | const rawValues = value.split('\n');
109 | let added = false;
110 | for (let i = 0; i < rawValues.length; i++) {
111 | const value = rawValues[i].trim();
112 | if (isURLRe.test(value)) {
113 | added = true;
114 | this.props.addURL(value);
115 | }
116 | }
117 | if (added) {
118 | this.close();
119 | }
120 | }
121 | }
122 |
123 | close() {
124 | this.textAreaRef.current.value = '';
125 | UIKit.modal('#bulk-seed-input').hide();
126 | }
127 |
128 | render() {
129 | return (
130 |
161 | );
162 | }
163 | }
164 |
165 | export class URLFields extends Component {
166 | static propTypes = {
167 | fields: PropTypes.object.isRequired
168 | };
169 |
170 | constructor(props) {
171 | super(props);
172 | this.renderURLs = this.renderURLs.bind(this);
173 | this.addURL = this.addURL.bind(this);
174 | this.removeURL = this.removeURL.bind(this);
175 | }
176 |
177 | addURL(maybeURL) {
178 | this.props.fields.push(typeof maybeURL === 'string' ? maybeURL : '');
179 | }
180 |
181 | removeURL(idx) {
182 | this.props.fields.remove(idx);
183 | }
184 |
185 | renderURLs() {
186 | const { fields } = this.props;
187 | const urls = new Array(fields.length);
188 | for (let i = 0; i < fields.length; i++) {
189 | urls[i] = (
190 |
195 | );
196 | }
197 | return urls;
198 | }
199 |
200 | render() {
201 | const haveURLS = this.props.fields.length >= 1;
202 | return (
203 | <>
204 |
208 |
209 |
215 |
216 |
217 |
223 |
224 |
230 |
234 | Seed URLs Required
235 |
236 | )}
237 |
238 |
239 |
{haveURLS && this.renderURLs()}
240 |
New Crawl
13 |
17 |
18 | );
19 | }
20 |
21 | CrawlCreator.propTypes = {
22 | createCrawl: PropTypes.func.isRequired
23 | };
24 |
25 | const selector = formValueSelector('CreateCrawl');
26 | const mapStateToProps = state => {
27 | return {
28 | crawlType: selector(state, 'crawlInfo.crawl_type')
29 | };
30 | };
31 |
32 | const mapDispatchToProps = (dispatch, ownProps) => ({
33 | createCrawl(crawlInfo) {
34 | ownProps.history.push('/');
35 | dispatch(createCrawl(crawlInfo.toJS()));
36 | }
37 | });
38 |
39 | const ConnectedCrawlCreator = connect(
40 | mapStateToProps,
41 | mapDispatchToProps
42 | )(CrawlCreator);
43 |
44 | export default withRouter(ConnectedCrawlCreator);
45 |
--------------------------------------------------------------------------------
/frontend/src/components/CrawlCreator/validate.js:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webrecorder/browsertrix-old/726578cfec61c8e0e074393a6373591697ff3bc0/frontend/src/components/CrawlCreator/validate.js
--------------------------------------------------------------------------------
/frontend/src/components/Crawls/LoadingCrawls.js:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 |
3 | export default function LoadingCrawls() {
4 | return (
5 |
6 |
Retrieving Crawl Info
7 |
8 |
9 | );
10 | }
11 |
--------------------------------------------------------------------------------
/frontend/src/components/Crawls/SelectCrawl.js:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 | import ReactTable from "react-table";
3 | import * as PropTypes from 'prop-types';
4 | import { List, Map } from 'immutable';
5 | import { Link } from 'react-router-dom';
6 |
7 | export default function SelectCrawls({ crawls, removeCrawl }) {
8 |
9 |
10 | function doRemove(id) {
11 | //this.props.removeCrawl(this.props.crawlId);
12 | console.log(id);
13 | removeCrawl(id);
14 | }
15 |
16 | function formatTimeDiff(from, to) {
17 | to = to ? new Date(to * 1000) : new Date();
18 | from = new Date(from * 1000);
19 |
20 | var delta = Math.abs(to - from) / 1000;
21 |
22 | var hours = Math.floor(delta / 3600);
23 | delta -= hours * 3600;
24 |
25 | var minutes = Math.floor(delta / 60) % 60;
26 | delta -= minutes * 60;
27 |
28 | var seconds = Math.floor(delta % 60);
29 |
30 | hours = hours.toString().padStart(2, '0');
31 | minutes = minutes.toString().padStart(2, '0');
32 | seconds = seconds.toString().padStart(2, '0');
33 |
34 | return `${hours}:${minutes}:${seconds}`;
35 | }
36 |
37 |
38 | const columns = [
39 | {
40 | Header: 'Name (Id)',
41 | accessor: 'id',
42 | Cell: props => {props.original.name ? props.original.name + ' (' + props.value + ')': props.value}
43 | },
44 | {
45 | Header: 'Started',
46 | accessor: 'start_time',
47 | Cell: props => {`${formatTimeDiff(props.value)} ago`}
48 | },
49 | {
50 | Header: 'Duration',
51 | accessor: 'finish_time',
52 | Cell: props => {`${formatTimeDiff(props.original.start_time, props.value)}`}
53 | },
54 | {
55 | Header: 'Status',
56 | accessor: 'status'
57 | },
58 | {
59 | Header: 'Crawl Type',
60 | accessor: 'crawl_type'
61 | },
62 | {
63 | Header: 'Collection',
64 | accessor: 'coll',
65 | Cell: props => {props.value}
66 | },
67 | {
68 | Header: 'Mode',
69 | accessor: 'mode'
70 | },
71 | {
72 | Header: 'To Crawl',
73 | accessor: 'num_queue'
74 | },
75 | {
76 | Header: 'Have Crawled',
77 | accessor: 'num_seen'
78 | },
79 | {
80 | Header: 'Remove Crawl',
81 | id: 'remove_crawl',
82 | Cell: props => ()
86 | },
87 | ];
88 |
89 | function resolveData(data) {
90 | return data.valueSeq().toArray();
91 | }
92 |
93 | function getTdProps(state, rowInfo, column, instance) {
94 | return {
95 | onClick: (e, handleOriginal) => {
96 | if (column.id === "remove_crawl") {
97 | doRemove(rowInfo.row.id);
98 | }
99 | }
100 | };
101 | }
102 |
103 | const crawlTable = ();
108 |
109 | return (
110 |
111 |
All Crawls
112 | {crawlTable}
113 |
114 | );
115 | }
116 |
117 | SelectCrawls.propTypes = {
118 | crawls: PropTypes.instanceOf(Map).isRequired,
119 | removeCrawl: PropTypes.func.isRequired
120 | };
121 |
122 |
123 |
--------------------------------------------------------------------------------
/frontend/src/components/Crawls/index.js:
--------------------------------------------------------------------------------
1 | import React, { Component } from 'react';
2 | import { connect } from 'react-redux';
3 | import * as PropTypes from 'prop-types';
4 | import { List, Map } from 'immutable';
5 | import { getAllCrawls, removeCrawl } from '../../actions/crawls';
6 | import LoadingCrawls from './LoadingCrawls';
7 | import SelectCrawl from './SelectCrawl';
8 | import CrawlCreator from '../CrawlCreator';
9 |
10 | class Crawls extends Component {
11 | static propTypes = {
12 | crawls: PropTypes.instanceOf(Map).isRequired,
13 | crawlsFetched: PropTypes.bool.isRequired,
14 | init: PropTypes.func.isRequired,
15 | removeCrawl: PropTypes.func.isRequired
16 | };
17 |
18 | constructor(props) {
19 | super(props);
20 |
21 | this.handle = null;
22 | }
23 |
24 | componentDidMount() {
25 | if (!this.props.crawlsFetched) {
26 | this.props.loadCrawls(true);
27 | }
28 |
29 | if (this.props.crawls.size > 0) {
30 | this.autoUpdate();
31 | }
32 | }
33 |
34 | componentDidUpdate() {
35 | if (this.props.crawls.size > 0 && !this.handle) {
36 | this.autoUpdate();
37 | } else if (this.props.crawls.size === 0 && this.handle) {
38 | clearInterval(this.handle);
39 | this.handle = null;
40 | }
41 | }
42 |
43 | componentWillUnmount() {
44 | clearInterval(this.handle);
45 | }
46 |
47 | autoUpdate = () => {
48 | this.handle = setInterval(this.props.loadCrawls, 1000);
49 | }
50 |
51 | render() {
52 | let component;
53 | if (!this.props.crawlsFetched) {
54 | component = ;
55 | } else {
56 | component = (
57 |
61 | );
62 | }
63 | return {component}
;
64 | }
65 | }
66 |
67 | const mapDispatchToProps = (dispatch, ownProps) => ({
68 | loadCrawls(init = false) {
69 | dispatch(getAllCrawls(init));
70 | },
71 | removeCrawl(id) {
72 | dispatch(removeCrawl(id));
73 | }
74 | });
75 |
76 | const mapStateToProps = (state, ownProps) => ({
77 | crawls: state.get('crawls'),
78 | crawlsFetched: state.get('crawlsFetched')
79 | });
80 |
81 | const ConnectedCrawls = connect(
82 | mapStateToProps,
83 | mapDispatchToProps
84 | )(Crawls);
85 |
86 | export default ConnectedCrawls;
87 |
--------------------------------------------------------------------------------
/frontend/src/components/Header/HeaderLink.js:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 | import * as PropTypes from 'prop-types';
3 | import { Link } from 'react-router-dom';
4 |
5 | export default function HeaderLink({ to, location, children, button }) {
6 | if (button) {
7 | return (
8 |
9 | {children}
10 |
11 | );
12 | }
13 | return (
14 |
15 | {children}
16 |
17 | );
18 | }
19 |
20 | HeaderLink.propTypes = {
21 | button: PropTypes.bool,
22 | to: PropTypes.string.isRequired
23 | };
24 |
25 | HeaderLink.defaultProps = {
26 | button: false
27 | };
28 |
--------------------------------------------------------------------------------
/frontend/src/components/Header/index.js:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 | import * as PropTypes from 'prop-types';
3 | import { withRouter } from 'react-router-dom';
4 | import HeaderLink from './HeaderLink';
5 |
6 | function Header(props) {
7 | return (
8 |
9 |
10 |
22 |
16 |
17 |
18 | );
19 | }
20 |
21 | function viewCrawl(props) {
22 | return ;
23 | }
24 |
25 | function App() {
26 | return (
27 | <>
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 | );
38 | }
39 |
40 | let ExportedApp = App;
41 |
42 | if (process.env.NODE_ENV === 'development') {
43 | ExportedApp = hot(App);
44 | }
45 |
46 | export default ExportedApp;
47 |
--------------------------------------------------------------------------------
/frontend/src/reducers/crawls.js:
--------------------------------------------------------------------------------
1 | import { fromJS, List, Map, Record } from 'immutable';
2 | import { ActionTypes } from '../actions/crawls';
3 |
4 | export class CrawlRecord extends Record({
5 | behavior_max_time: 60,
6 | browser: 'chrome:67',
7 | browsers: [],
8 | browsersDone: [],
9 | coll: '',
10 | crawl_type: '',
11 | crawl_depth: 0,
12 | finish_time: 0,
13 | id: '',
14 | mode: '',
15 | num_browsers: 0,
16 | num_queue: 0,
17 | num_pending: 0,
18 | num_seen: 0,
19 | num_tabs: 0,
20 | name: '',
21 | pending: [],
22 | queue: [],
23 | running: false,
24 | scopes: [],
25 | seen: [],
26 | start_time: 0,
27 | status: 'new'
28 | }) {
29 | updateBrowsers(browsers) {
30 | const existingBrowsers = this.browsers;
31 | return this.set('browsers', existingBrowsers.concat(browsers));
32 | }
33 |
34 | crawlRunning(browsers) {
35 | const existingBrowsers = this.browsers;
36 | return this.merge({
37 | browsers: existingBrowsers.concat(browsers),
38 | running: true,
39 | status: 'running'
40 | });
41 | }
42 |
43 | crawlStopped() {
44 | return this.merge({
45 | running: false,
46 | status: 'stopped'
47 | });
48 | }
49 |
50 | startCrawlConfig() {
51 | return {
52 | browser: this.browser,
53 | behavior_max_time: this.behavior_max_time,
54 | headless: this.headless
55 | };
56 | }
57 | }
58 |
59 | export function crawlsFetchedReducer(state = false, action) {
60 | if (action.type === ActionTypes.gotAllInit) {
61 | return true;
62 | }
63 | return state;
64 | }
65 |
66 | export function crawlIds(state = List([]), { type, payload, meta }) {
67 |
68 | switch (type) {
69 | case ActionTypes.deleteCrawl:
70 | const idx = state.indexOf(payload.id);
71 | return state.delete(idx);
72 | case ActionTypes.gotAllInit:
73 | case ActionTypes.gotAll:
74 | return List().withMutations(mutable => {
75 | const { crawls } = payload;
76 | for (let i = 0; i < crawls.length; i++) {
77 | mutable.push(crawls[i].id);
78 | }
79 | return mutable;
80 | });
81 | case ActionTypes.create:
82 | return state.push(payload.id);
83 | }
84 | return state;
85 | }
86 |
87 | export function crawlsReducer(state = Map({}), { type, payload, meta }) {
88 | switch (type) {
89 | case ActionTypes.start:
90 | if (!state.has(payload.id)) return state;
91 | return state.updateIn([payload.id], crawl =>
92 | crawl.crawlRunning(payload.browsers)
93 | );
94 | case ActionTypes.stop:
95 | if (!state.has(payload.id)) return state;
96 | return state.updateIn([payload.id], crawl =>
97 | crawl.crawlStopped()
98 | );
99 | case ActionTypes.deleteCrawl:
100 | return state.delete(payload.id);
101 | case ActionTypes.addURLs:
102 | if (!state.has(payload.id)) return state;
103 | return state.updateIn([payload.id], crawl => crawl.merge(payload));
104 | case ActionTypes.gotAllInit:
105 | case ActionTypes.gotAll:
106 | return state.withMutations(mutable => {
107 | const { crawls } = payload;
108 | for (let i = 0; i < crawls.length; i++) {
109 | const rawCrawl = crawls[i];
110 | const crec = mutable.get(rawCrawl.id);
111 | if (!crec) {
112 | mutable.set(rawCrawl.id, new CrawlRecord(rawCrawl));
113 | } else {
114 | mutable.set(rawCrawl.id, crec.merge(rawCrawl));
115 | }
116 | }
117 | return mutable;
118 | });
119 | case ActionTypes.create:
120 | return state.set(payload.id, new CrawlRecord(payload));
121 | case ActionTypes.updateURLInfo:
122 | if (!state.has(payload.id)) return state;
123 | return state.updateIn([payload.id], crawl => crawl.mergeDeep(payload));
124 | case ActionTypes.info:
125 | if (!state.has(payload.id)) return state;
126 | return state.updateIn([payload.id], crawl => crawl.merge(fromJS(payload)));
127 | }
128 | return state;
129 | }
130 |
--------------------------------------------------------------------------------
/frontend/src/reducers/index.js:
--------------------------------------------------------------------------------
1 | import { combineReducers } from 'redux-immutable';
2 | import { reducer as reduxFormReducer } from 'redux-form/immutable';
3 | import { crawlIds, crawlsFetchedReducer, crawlsReducer } from './crawls';
4 |
5 | const rootReducer = combineReducers({
6 | crawlIds,
7 | crawls: crawlsReducer,
8 | crawlsFetched: crawlsFetchedReducer,
9 | form: reduxFormReducer
10 | });
11 |
12 | export default rootReducer;
13 |
--------------------------------------------------------------------------------
/frontend/src/root.js:
--------------------------------------------------------------------------------
1 | import './styles/global.scss';
2 | import './utils/rhlConfig';
3 | import React from 'react';
4 | import ReactDOM from 'react-dom';
5 | import { Provider } from 'react-redux';
6 | import { HashRouter } from 'react-router-dom';
7 | import Icons from 'uikit/dist/js/uikit-icons';
8 | import Uikit from 'uikit';
9 | import createStore from './store';
10 | import App from './containers/App';
11 |
12 | window.Uikit = Uikit;
13 | Uikit.use(Icons);
14 | const store = createStore();
15 |
16 | ReactDOM.render(
17 |
18 |
19 |
20 |
21 | ,
22 | document.getElementById('mount')
23 | );
24 |
--------------------------------------------------------------------------------
/frontend/src/store/dev.js:
--------------------------------------------------------------------------------
1 | import { createStore } from 'redux';
2 | import { composeWithDevTools } from 'redux-devtools-extension';
3 | import Immutable from 'immutable';
4 | import * as actionCreators from '../actions';
5 | import rootReducer from '../reducers';
6 | import middleWare from './middleware';
7 |
8 | const composer = composeWithDevTools({
9 | actionCreators,
10 | serialize: {
11 | immutable: Immutable
12 | }
13 | });
14 |
15 | export default function configureStore() {
16 | return createStore(rootReducer, Immutable.Map({}), composer(middleWare));
17 | }
18 |
--------------------------------------------------------------------------------
/frontend/src/store/index.js:
--------------------------------------------------------------------------------
1 | module.exports =
2 | process.env.NODE_ENV === 'production'
3 | ? require('./prod')
4 | : require('./dev');
5 |
--------------------------------------------------------------------------------
/frontend/src/store/middleware.js:
--------------------------------------------------------------------------------
1 | import { applyMiddleware } from 'redux';
2 | import thunkMiddleware from 'redux-thunk';
3 | import promiseMiddleware from 'redux-promise';
4 | import { FetchStates } from '../actions/httpRequests';
5 |
6 | function getRequestTracker() {
7 | if (typeof window !== 'undefined') {
8 | if (window.__requestTrack == null) {
9 | window.__requestTrack = {};
10 | }
11 | return window.__requestTrack;
12 | }
13 | return {};
14 | }
15 |
16 | function isURLTracked(url) {
17 | const requests = getRequestTracker();
18 | return requests[url] != null;
19 | }
20 |
21 | function trackURL(url) {
22 | const requests = getRequestTracker();
23 | requests[url] = true;
24 | }
25 |
26 | function untrackURL(url) {
27 | const requests = getRequestTracker();
28 | delete requests[url];
29 | }
30 |
31 | function createRequestMiddleware() {
32 | return store => next => action => {
33 | if (action.meta && action.meta.httpRequest) {
34 | const httpRequest = action.meta.httpRequest;
35 | switch (httpRequest.state) {
36 | case FetchStates.preflight:
37 | if (isURLTracked(action.meta.httpRequest.url)) return;
38 | trackURL(action.meta.httpRequest.url);
39 | break;
40 | case FetchStates.done:
41 | case FetchStates.error:
42 | untrackURL(action.meta.httpRequest.url);
43 | break;
44 | }
45 | }
46 | return next(action);
47 | };
48 | }
49 |
50 | export default applyMiddleware(
51 | thunkMiddleware,
52 | createRequestMiddleware(),
53 | promiseMiddleware
54 | );
55 |
--------------------------------------------------------------------------------
/frontend/src/store/prod.js:
--------------------------------------------------------------------------------
1 | import { createStore } from 'redux';
2 | import rootReducer from '../reducers';
3 | import middleWare from './middleware';
4 | import { Map } from 'immutable';
5 |
6 | export default function configureStore() {
7 | return createStore(rootReducer, Map({}), middleWare);
8 | }
9 |
--------------------------------------------------------------------------------
/frontend/src/styles/global.scss:
--------------------------------------------------------------------------------
1 | @charset "UTF-8";
2 |
3 |
4 | @import "../../node_modules/react-table/react-table.css";
5 |
6 | @import "../../node_modules/uikit/src/scss/variables-theme";
7 | @import "../../node_modules/uikit/src/scss/mixins-theme";
8 |
9 | @mixin hook-base-body() {
10 | -webkit-font-smoothing: antialiased;
11 | -moz-osx-font-smoothing: grayscale;
12 | text-rendering: optimizeLegibility;
13 | }
14 |
15 | @import "../../node_modules/uikit/src/scss/uikit-theme";
16 | @import "../../node_modules/react-toastify/scss/main";
17 |
18 | html {
19 | overflow-y: scroll;
20 | }
21 |
22 | body {
23 | overflow: hidden;
24 | }
25 |
26 | .uk-tab a {
27 | font-size: 1.1rem;
28 | font-weight: bold;
29 | }
30 |
31 |
32 |
33 | nav {
34 | border-bottom: 1px solid #e8e6ea;
35 | transition: background-color 250ms ease;
36 | background-color: rgba(255, 255, 255, 1) !important;
37 |
38 | .uk-navbar-center {
39 | transition: max-height 1.5s ease;
40 | max-height: 90px;
41 | }
42 |
43 | &.uk-active {
44 | backface-visibility: hidden;
45 | -webkit-backface-visibility: hidden;
46 | background-color: rgba(255, 255, 255, 0.75) !important;
47 | z-index: 2;
48 | @extend .uk-box-shadow-small;
49 |
50 | .uk-navbar-center {
51 | max-height: 60px;
52 | }
53 |
54 | &:hover {
55 | background-color: rgba(255, 255, 255, 1) !important;
56 | }
57 | }
58 | }
59 |
60 |
61 | .form-border {
62 | position: relative;
63 | color: #212529;
64 | padding: 1rem;
65 | margin-right: 0;
66 | margin-left: 0;
67 | border: .2rem solid #ececec;
68 | border-radius: 8px;
69 | }
70 |
71 | .create-crawl-seedlist {
72 | height: 300px;
73 | max-height: 300px;
74 | overflow: auto;
75 | margin-top: 1rem !important;
76 | }
77 |
78 | .route-container {
79 | @extend .uk-section;
80 | @extend .uk-section-default;
81 | padding-top: 30px;
82 |
83 | }
84 |
85 | .removeCrawlFromListIcon {
86 | cursor: pointer;
87 | color: red;
88 | }
89 |
90 | .logo {
91 | height: auto;
92 | width: 250px;
93 | margin-top: 10px;
94 | margin-bottom: 10px;
95 | }
96 |
97 |
--------------------------------------------------------------------------------
/frontend/src/utils/bootstrap.js:
--------------------------------------------------------------------------------
1 | import Uikit from 'uikit';
2 |
--------------------------------------------------------------------------------
/frontend/src/utils/endpoints.js:
--------------------------------------------------------------------------------
1 | function getEndpointConfig() {
2 | if (typeof window !== 'undefined' && window.__crawlmanEndpoints != null) {
3 | return window.__crawlmanEndpoints;
4 | }
5 | return {
6 | crawls: {
7 | ep: window.location.origin + '/crawls',
8 | retrieve: { method: 'GET' },
9 | create: {
10 | defaults: { crawl_type: 'single-page', num_browsers: 2, num_tabs: 1, start: true },
11 | method: 'POST'
12 | }
13 | },
14 | crawl: {
15 | ep: window.location.origin + '/crawl/',
16 | info: { path: '/info', method: 'GET' },
17 | remove: { method: 'DELETE' },
18 | start: {
19 | defaults: {
20 | browser: 'chrome:67',
21 | behavior_max_time: 60,
22 | cache: 'always',
23 | headless: false
24 | },
25 | method: 'POST',
26 | path: '/start'
27 | },
28 | stop: { method: 'POST', path: '/stop' },
29 | done: { method: 'GET', path: '/done' },
30 | retrieveURLS: { method: 'GET', path: '/urls' },
31 | addURLS: { method: 'POST', path: '/urls' }
32 | }
33 | };
34 | }
35 |
36 | class Endpoints {
37 | constructor({ crawls, crawl }) {
38 | this.crawls = crawls;
39 | this.crawl = crawl;
40 | }
41 |
42 | /**
43 | * @param {Object} [newCrawlConfig]
44 | * @return {{body: Object, request: Request}}
45 | */
46 | createNewCrawl(newCrawlConfig) {
47 | const { defaults = {}, method } = this.crawls.create;
48 | const body = Object.assign(
49 | {},
50 | defaults,
51 | newCrawlConfig.crawlInfo,
52 | newCrawlConfig.crawlRunInfo
53 | );
54 |
55 | return {
56 | body,
57 | request: new Request(this.crawls.ep, {
58 | method,
59 | body: JSON.stringify(body)
60 | })
61 | };
62 | }
63 |
64 | /**
65 | * @return {Request}
66 | */
67 | retrieveAllCrawls() {
68 | const { method } = this.crawls.retrieve;
69 | return new Request(this.crawls.ep, { method });
70 | }
71 |
72 | /**
73 | * @param {string} id
74 | * @return {Request}
75 | */
76 | crawlInfo(id) {
77 | const { path = '', method } = this.crawl.info;
78 | const url = `${this.crawl.ep}${id}${path}`;
79 | return new Request(url, { method });
80 | }
81 |
82 | /**
83 | * @param {string} id
84 | * @return {Request}
85 | */
86 | crawlDone(id) {
87 | const { path = '', method } = this.crawl.done;
88 | const url = `${this.crawl.ep}${id}${path}`;
89 | return new Request(url, { method });
90 | }
91 |
92 | /**
93 | * @param {string} id
94 | * @return {Request}
95 | */
96 | getCrawlURLs(id) {
97 | const { path = '', method } = this.crawl.retrieveURLS;
98 | const url = `${this.crawl.ep}${id}${path}`;
99 | return new Request(url, { method });
100 | }
101 |
102 | /**
103 | * @param {string} id
104 | * @param {Array} urls
105 | * @return {{body: Object, request: Request}}
106 | */
107 | addCrawlURLs(id, urls) {
108 | const { path = '', method } = this.crawl.addURLS;
109 | const url = `${this.crawl.ep}${id}${path}`;
110 | const body = { urls };
111 | return {
112 | body,
113 | request: new Request(url, {
114 | method,
115 | body: JSON.stringify(body)
116 | })
117 | };
118 | }
119 |
120 | /**
121 | * @param {string} id
122 | * @return {Request}
123 | */
124 | removeCrawl(id) {
125 | const { path = '', method } = this.crawl.remove;
126 | const url = `${this.crawl.ep}${id}${path}`;
127 | return new Request(url, { method });
128 | }
129 |
130 | /**
131 | * @param {string} id
132 | * @param {Object} [config]
133 | * @return {{body: Object, request: Request}}
134 | */
135 | startCrawl(id, config) {
136 | const { path = '', method, defaults = {} } = this.crawl.start;
137 | const url = `${this.crawl.ep}${id}${path}`;
138 | const body = Object.assign({}, defaults, config);
139 | return {
140 | body,
141 | request: new Request(url, {
142 | method,
143 | body: JSON.stringify(body)
144 | })
145 | };
146 | }
147 |
148 | /**
149 | * @param {string} id
150 | * @return {Request}
151 | */
152 | stopCrawl(id) {
153 | const { path = '', method } = this.crawl.stop;
154 | const url = `${this.crawl.ep}${id}${path}`;
155 | return new Request(url, { method });
156 | }
157 | }
158 |
159 | export const EndpointRequests = new Endpoints(getEndpointConfig());
160 |
--------------------------------------------------------------------------------
/frontend/src/utils/index.js:
--------------------------------------------------------------------------------
1 | export { EndpointRequests } from './endpoints';
2 |
--------------------------------------------------------------------------------
/frontend/src/utils/rhlConfig.js:
--------------------------------------------------------------------------------
1 | import { setConfig, hot } from 'react-hot-loader';
2 | if (process.env.NODE_ENV === 'development') {
3 | setConfig({
4 | logLevel: 'debug'
5 | // ignoreSFC: true, // RHL will be __completely__ disabled for SFC
6 | // pureRender: true // RHL will not change render method
7 | });
8 | }
9 |
--------------------------------------------------------------------------------
/frontend/src/wrap-with-provider.js:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 | import { Provider } from 'react-redux';
3 | import * as PropTypes from 'prop-types';
4 | import createStore from './store';
5 |
6 | export default function WrapWithProvider({ element }) {
7 | const store = createStore();
8 | return {element};
9 | };
10 |
11 | WrapWithProvider.propTypes = {
12 | element: PropTypes.element.isRequired
13 | };
14 |
--------------------------------------------------------------------------------
/frontend/webpack/development-server.js:
--------------------------------------------------------------------------------
1 | const path = require('path');
2 | const createServer = require('fastify');
3 | const webpack = require('webpack');
4 | const HMR = require('fastify-webpack-hmr');
5 | const config = require('./webpack.config');
6 |
7 | const fastify = createServer({
8 | logger: false
9 | });
10 |
11 | const staticPath = path.join(__dirname, '..', 'public');
12 |
13 | fastify
14 | .register(HMR, {
15 | compiler: webpack(config),
16 | webpackDev: {
17 | logLevel: 'debug'
18 | }
19 | })
20 | .register(require('fastify-static'), {
21 | root: staticPath,
22 | prefix: '/'
23 | });
24 |
25 | async function run() {
26 | const listeningOn = await fastify.listen(8001);
27 | console.log(
28 | `Dev server listening on\n${
29 | listeningOn.startsWith('http://127.0.0.1')
30 | ? listeningOn.replace('http://127.0.0.1', 'http://localhost')
31 | : listeningOn
32 | }`
33 | );
34 | console.log(fastify.printRoutes());
35 | }
36 |
37 | run().catch(error => {
38 | console.error(error);
39 | });
40 |
--------------------------------------------------------------------------------
/frontend/webpack/webpack.config.js:
--------------------------------------------------------------------------------
1 | const path = require('path');
2 | const webpack = require('webpack');
3 |
4 | const rootPath = path.join(__dirname, '..');
5 | const staticPath = path.join(path.join(rootPath, '..'), 'static');
6 | const srcPath = path.join(rootPath, 'src');
7 | const entryPath = path.join(srcPath, 'root.js');
8 | const hotConf =
9 | 'webpack-hot-middleware/client?path=__webpack_hmr&reload=true&overlay=false';
10 |
11 | const mode = process.env.NODE_ENV || 'development';
12 | const production = mode === 'production';
13 |
14 | const plugins = [
15 | new webpack.NamedModulesPlugin(),
16 | new webpack.IgnorePlugin({
17 | contextRegExp: /moment$/
18 | }),
19 | new webpack.DefinePlugin({
20 | 'process.env.NODE_ENV': JSON.stringify(mode),
21 | __DEV__: JSON.stringify(!production)
22 | }),
23 | new webpack.NoEmitOnErrorsPlugin()
24 | ];
25 |
26 | const jsLoader = {
27 | loader: 'babel-loader',
28 | options: {
29 | cacheDirectory: true,
30 | babelrc: false,
31 | presets: [
32 | '@babel/preset-react',
33 | [
34 | '@babel/preset-env',
35 | {
36 | loose: true,
37 | debug: false,
38 | modules: false,
39 | useBuiltIns: false,
40 | targets: {
41 | chrome: '70',
42 | firefox: '66'
43 | }
44 | }
45 | ]
46 | ],
47 | plugins: [['@babel/plugin-proposal-class-properties', { loose: true }]]
48 | }
49 | };
50 |
51 | if (mode === 'development') {
52 | plugins.push(new webpack.HotModuleReplacementPlugin());
53 | jsLoader.options.plugins.unshift('react-hot-loader/babel');
54 | } else {
55 | jsLoader.options.plugins.push([
56 | 'transform-react-remove-prop-types',
57 | {
58 | ignoreFilenames: ['node_modules']
59 | }
60 | ]);
61 | }
62 |
63 | module.exports = {
64 | mode: mode,
65 | context: rootPath,
66 | entry: production ? entryPath : [entryPath, hotConf],
67 | output: {
68 | path: production ? staticPath : path.join(rootPath, 'public'),
69 | filename: 'app.js',
70 | publicPath: '/'
71 | },
72 | module: {
73 | rules: [
74 | {
75 | test: /\.jsx?$/,
76 | exclude: /node_modules/,
77 | use: [jsLoader]
78 | },
79 | {
80 | test: /\.s[ac]ss$/,
81 | use: [
82 | {
83 | loader: 'style-loader',
84 | options: {
85 | hmr: true
86 | }
87 | },
88 | {
89 | loader: 'css-loader'
90 | },
91 | {
92 | loader: 'sass-loader',
93 | options: {
94 | includePaths: [
95 | 'node_modules/uikit/src/scss',
96 | 'node_modules/react-toastify/scss/main'
97 | ],
98 | precision: 8
99 | }
100 | }
101 | ]
102 | },
103 | {
104 | test: /\.(eot|otf|ttf|woff(2)?)(\?.*)?$/,
105 | use: [
106 | {
107 | loader: 'url-loader'
108 | }
109 | ]
110 | }
111 | ]
112 | },
113 | resolve: {
114 | extensions: ['.mjs', '.js', '.jsx', '.wasm', '.json', '.ts', '.tsx'],
115 | alias: {
116 | 'react-dom': require.resolve('@hot-loader/react-dom')
117 | }
118 | },
119 | plugins
120 | };
121 |
--------------------------------------------------------------------------------
/install-browsers.sh:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 |
3 | # Current Browser
4 | docker pull oldwebtoday/chrome:73
5 |
6 | # Base Browser
7 | docker pull oldwebtoday/base-browser
8 |
9 | # Borwser automation driver
10 | docker pull webrecorder/autobrowser
11 |
12 | if [ "$1" != "--headless" ]; then
13 |
14 | # Required for non-headless mode
15 | docker pull oldwebtoday/vnc-webrtc-audio
16 | docker pull oldwebtoday/base-displayaudio
17 | fi
18 |
19 |
--------------------------------------------------------------------------------
/mypy.ini:
--------------------------------------------------------------------------------
1 | [mypy]
2 | # Specify the target platform details in config, so your developers are
3 | # free to run mypy on Windows, Linux, or macOS and get consistent
4 | # results.
5 | python_version=3.7
6 | platform=linux
7 |
8 | # flake8-mypy expects the two following for sensible formatting
9 | show_column_numbers=True
10 | show_error_context=True
11 | show_traceback=True
12 |
13 | # show error messages from unrelated files
14 | follow_imports=normal
15 |
16 | # suppress errors about unsatisfied imports
17 | ignore_missing_imports=True
18 |
19 | # no strict optionals
20 | strict_optional=False
21 |
22 | # be strict
23 | disallow_untyped_calls=False
24 | warn_return_any=True
25 | warn_no_return=True
26 | warn_redundant_casts=True
27 | warn_unused_ignores=False
28 |
29 | # The following are off by default. Flip them on if you feel
30 | # adventurous.
31 | check_untyped_defs=True
32 | # verbosity=10000
--------------------------------------------------------------------------------
/pool_config.yaml:
--------------------------------------------------------------------------------
1 | default_pool: fixed-pool
2 |
3 | pools:
4 | - name: fixed-pool
5 | type: fixed
6 | duration: 300
7 | max_size: 10
8 | expire_check: 30
9 | number_ttl: 30
10 | wait_ping_ttl: 30
11 |
12 | - name: auto-pool
13 | type: persist
14 | duration: 300
15 | max_size: 10
16 | expire_check: 30
17 | grace_time: 300
18 | stop_on_pause: false
19 |
20 |
21 |
--------------------------------------------------------------------------------
/pyproject.toml:
--------------------------------------------------------------------------------
1 | [tool.black]
2 | line-length = 88
3 | target-version = ['py36', 'py37', 'py38']
4 | include = '\.pyi?$'
5 | exclude = '''
6 | /(
7 | \.eggs
8 | | \.git
9 | | \.hg
10 | | \.mypy_cache
11 | | \.tox
12 | | \.venv
13 | | _build
14 | | buck-out
15 | | build
16 | | dist
17 | | venv
18 | )/
19 | '''
20 |
21 |
--------------------------------------------------------------------------------
/pytest.ini:
--------------------------------------------------------------------------------
1 | [pytest]
2 | testpaths = tests
3 | addopts=-vv -s
--------------------------------------------------------------------------------
/pywb/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM webrecorder/pywb:new-wombat
2 |
3 | WORKDIR /app
4 |
5 | COPY config.yaml ./
6 | COPY crawlapp.py ./
7 |
8 | COPY uwsgi.ini /uwsgi/
9 |
10 | COPY ./templates ./templates
11 | COPY ./static ./static
12 |
13 | #WORKDIR /webarchive
14 |
15 |
--------------------------------------------------------------------------------
/pywb/config.yaml:
--------------------------------------------------------------------------------
1 | debug: true
2 |
3 | proxy:
4 | coll: coll
5 | recording: true
6 |
7 | enable_banner: false
8 | enable_wombat: true
9 |
10 | recorder: live
11 |
12 | autoindex: 10
13 |
14 | enable_auto_fetch: true
15 |
16 | collections:
17 | live: $live
18 |
19 | collections_root: /webarchive/collections
20 |
21 |
--------------------------------------------------------------------------------
/pywb/crawlapp.py:
--------------------------------------------------------------------------------
1 | # fmt: off
2 | from gevent.monkey import patch_all; patch_all()
3 | # fmt: on
4 | import base64
5 | import hashlib
6 | import json
7 | import os
8 | import traceback
9 | from itertools import chain
10 | from urllib.parse import parse_qsl
11 |
12 | import redis
13 | import requests
14 | from pywb.apps.frontendapp import FrontEndApp
15 | from pywb.apps.wbrequestresponse import WbResponse
16 | from pywb.manager.manager import CollectionsManager
17 | from pywb.rewrite.templateview import BaseInsertView
18 | from warcio.timeutils import timestamp_now, timestamp_to_iso_date
19 | from werkzeug.routing import Rule
20 |
21 | EMPTY_LIST = []
22 | EMPTY_DICT = {}
23 | SKIPPED_NODES = {'script', 'style'}
24 | TITLE = 'title'
25 | TEXT = '#text'
26 |
27 |
28 | # ============================================================================
29 | def extract_text(node, metadata=None):
30 | node_name = node.get('nodeName', '').lower()
31 | if node_name not in SKIPPED_NODES:
32 | children = node.get('children', EMPTY_LIST)
33 | if node_name == TEXT:
34 | text_value = node.get('nodeValue', '').strip()
35 | if text_value:
36 | yield text_value
37 | elif node_name == TITLE:
38 | title = ' '.join(
39 | chain.from_iterable(extract_text(child, None) for child in children)
40 | )
41 | if metadata is not None:
42 | metadata['title'] = title
43 | else:
44 | yield title
45 | else:
46 | if children:
47 | for text_value in chain.from_iterable(
48 | extract_text(child, metadata) for child in children
49 | ):
50 | yield text_value
51 | content_doc = node.get('contentDocument')
52 | if content_doc:
53 | for text_value in extract_text(content_doc, None):
54 | yield text_value
55 |
56 |
57 | # ============================================================================
58 | class CrawlProxyApp(FrontEndApp):
59 | def __init__(self, config_file=None, custom_config=None):
60 | self.colls_dir = os.path.join(os.environ.get('VOLUME_DIR', '.'), 'collections')
61 |
62 | # ensure collections dir exists for auto-index
63 | os.makedirs(self.colls_dir, exist_ok=True)
64 |
65 | super(CrawlProxyApp, self).__init__(
66 | config_file='./config.yaml', custom_config=custom_config
67 | )
68 |
69 | self.redis = redis.StrictRedis.from_url(
70 | os.environ['REDIS_URL'], decode_responses=True
71 | )
72 |
73 | self.collections_checked = set()
74 |
75 | self.custom_record_path = (
76 | self.recorder_path + '&put_record={rec_type}&url={url}'
77 | )
78 |
79 | self.solr_ingester = FullTextIngester()
80 |
81 | def ensure_coll_exists(self, coll):
82 | if coll == 'live':
83 | return
84 |
85 | if coll in self.collections_checked:
86 | return
87 |
88 | try:
89 | m = CollectionsManager(coll, colls_dir=self.colls_dir, must_exist=False)
90 | m.add_collection()
91 | except FileExistsError:
92 | pass
93 |
94 | self.collections_checked.add(coll)
95 |
96 | def proxy_route_request(self, url, environ):
97 | try:
98 | key = 'up:' + environ['REMOTE_ADDR']
99 | timestamp, coll, mode, cache = self.redis.hmget(
100 | key, 'timestamp', 'coll', 'mode', 'cache'
101 | )
102 |
103 | # environ['pywb_redis_key'] = key
104 | environ['pywb_proxy_default_timestamp'] = timestamp or timestamp_now()
105 | environ['pywb_cache'] = cache
106 |
107 | if coll:
108 | self.ensure_coll_exists(coll)
109 |
110 | if mode == 'replay' or coll == 'live':
111 | proxy_prefix = '/' + coll + '/bn_/'
112 | else:
113 | proxy_prefix = '/' + coll + '/record/bn_/'
114 | else:
115 | proxy_prefix = self.proxy_prefix
116 |
117 | except Exception as e:
118 | traceback.print_exc()
119 |
120 | return proxy_prefix + url
121 |
122 | def _init_routes(self):
123 | super(CrawlProxyApp, self)._init_routes()
124 | self.url_map.add(
125 | Rule(
126 | '/api/screenshot/', endpoint=self.put_screenshot, methods=['PUT']
127 | )
128 | )
129 |
130 | self.url_map.add(
131 | Rule('/api/dom/', endpoint=self.put_raw_dom, methods=['PUT'])
132 | )
133 |
134 | self.url_map.add(
135 | Rule('/api/search/', endpoint=self.page_search, methods=['GET'])
136 | )
137 |
138 | self.url_map.add(
139 | Rule('//search', endpoint=self.serve_orig_coll_page, methods=['GET'])
140 | )
141 |
142 | def serve_orig_coll_page(self, environ, coll='$root'):
143 | return super(CrawlProxyApp, self).serve_coll_page(environ, coll)
144 |
145 | def serve_coll_page(self, environ, coll='$root'):
146 | if not self.is_valid_coll(coll):
147 | self.raise_not_found(environ, 'No handler for "/{0}"'.format(coll))
148 |
149 | self.setup_paths(environ, coll)
150 |
151 | view = BaseInsertView(self.rewriterapp.jinja_env, 'fullsearch.html')
152 |
153 | wb_prefix = environ.get('SCRIPT_NAME', '')
154 | if wb_prefix:
155 | wb_prefix += '/'
156 |
157 | content = view.render_to_string(environ, wb_prefix=wb_prefix, coll=coll)
158 |
159 | return WbResponse.text_response(
160 | content, content_type='text/html; charset="utf-8"'
161 | )
162 |
163 | def page_search(self, environ, coll):
164 | params = dict(parse_qsl(environ.get('QUERY_STRING')))
165 |
166 | result = self.solr_ingester.query_solr(coll, params)
167 |
168 | return WbResponse.json_response(result)
169 |
170 | def put_screenshot(self, environ, coll):
171 | data = environ['wsgi.input'].read()
172 | params = dict(parse_qsl(environ.get('QUERY_STRING')))
173 |
174 | return self.put_record(
175 | environ, coll, 'urn:screenshot:{url}', 'resource', params, data
176 | )
177 |
178 | def put_raw_dom(self, environ, coll):
179 | text = environ['wsgi.input'].read()
180 | params = dict(parse_qsl(environ.get('QUERY_STRING')))
181 |
182 | res = self.put_record(environ, coll, 'urn:dom:{url}', 'metadata', params, text)
183 |
184 | self.solr_ingester.ingest(coll, text, params)
185 | return res
186 |
187 | def put_record(self, environ, coll, target_uri_format, rec_type, params, data):
188 | self.ensure_coll_exists(coll)
189 |
190 | headers = {'Content-Type': environ.get('CONTENT_TYPE', 'text/plain')}
191 |
192 | url = params.get('url')
193 |
194 | if not url:
195 | return WbResponse.json_response({'error': 'no url'})
196 |
197 | timestamp = params.get('timestamp')
198 | if timestamp:
199 | headers['WARC-Date'] = timestamp_to_iso_date(timestamp)
200 |
201 | target_uri = target_uri_format.format(url=url)
202 | put_url = self.custom_record_path.format(
203 | url=target_uri, coll=coll, rec_type=rec_type
204 | )
205 | res = requests.put(put_url, headers=headers, data=data)
206 |
207 | res = res.json()
208 |
209 | return WbResponse.json_response(res)
210 |
211 | def serve_content(self, environ, *args, **kwargs):
212 | res = super(CrawlProxyApp, self).serve_content(environ, *args, **kwargs)
213 |
214 | if (
215 | environ.get('pywb_cache') == 'always'
216 | and res.status_headers.statusline.startswith('200')
217 | and environ.get('HTTP_REFERER')
218 | ):
219 |
220 | res.status_headers.headers.append(
221 | ('Cache-Control', 'public, max-age=31536000, immutable')
222 | )
223 |
224 | return res
225 |
226 |
227 | # =============================================================================
228 | class FullTextIngester:
229 | def __init__(self):
230 | self.solr_api = 'http://solr:8983/solr/browsertrix/update/json/docs?commit=true'
231 | self.solr_update_api = 'http://solr:8983/solr/browsertrix/update?commit=true'
232 | self.solr_select_api = 'http://solr:8983/solr/browsertrix/select'
233 |
234 | self.page_query = '?q=title_t:*&fq=coll_s:{coll}&fl=title_t,url_s,timestamp_ss,has_screenshot_b&rows={rows}&start={start}&sort=timestamp_ss+{sort}'
235 | self.text_query = '?q={q}&fq={fq}&fl=id,title_t,url_s,timestamp_ss,has_screenshot_b&hl=true&hl.fl=content_t&hl.snippets=3&rows={rows}&start={start}'
236 |
237 | def update_if_dupe(self, digest, coll, url, timestamp, timestamp_dt):
238 | try:
239 | query = 'digest_s:"{0}" AND coll_s:{1} AND url_s:"{2}"'.format(
240 | digest, coll, url
241 | )
242 | resp = requests.get(self.solr_select_api, params={'q': query, 'fl': 'id'})
243 |
244 | resp = resp.json()
245 | resp = resp.get('response')
246 | if not resp:
247 | return False
248 |
249 | docs = resp.get('docs')
250 | if not docs:
251 | return False
252 |
253 | id_ = docs[0].get('id')
254 | if not id_:
255 | return False
256 |
257 | add_cmd = {
258 | 'add': {
259 | 'doc': {
260 | 'id': id_,
261 | 'timestamp_ss': {'add': timestamp},
262 | 'timestamp_dts': {'add': timestamp_dt},
263 | }
264 | }
265 | }
266 |
267 | resp = requests.post(self.solr_update_api, json=add_cmd)
268 | return True
269 |
270 | except Exception as e:
271 | print(e)
272 | return False
273 |
274 | def ingest(self, coll, text, params):
275 | parsed = json.loads(text)
276 | mdata = {}
277 | content = "\n".join(text for text in extract_text(parsed["root"], mdata))
278 | title = mdata.get('title')
279 | url = params.get('url')
280 | timestamp_ss = params.get('timestamp')
281 | timestamp_dts = timestamp_to_iso_date(timestamp_ss)
282 | has_screenshot_b = params.get('hasScreenshot') == '1'
283 |
284 | title = title or url
285 |
286 | digest = self.get_digest(content)
287 |
288 | if self.update_if_dupe(digest, coll, url, timestamp_ss, timestamp_dts):
289 | return
290 |
291 | data = {
292 | 'coll_s': coll,
293 | 'title_t': title,
294 | 'content_t': content,
295 | 'url_s': url,
296 | 'digest_s': digest,
297 | 'timestamp_ss': timestamp_ss,
298 | 'timestamp_dts': timestamp_dts,
299 | 'has_screenshot_b': has_screenshot_b,
300 | }
301 |
302 | result = requests.post(self.solr_api, json=data)
303 |
304 | def get_digest(self, text):
305 | m = hashlib.sha1()
306 | m.update(text.encode('utf-8'))
307 | return 'sha1:' + base64.b32encode(m.digest()).decode('utf-8')
308 |
309 | def query_solr(self, coll, params):
310 | search = params.get('search')
311 |
312 | start = int(params.get('start', 0))
313 |
314 | rows = int(params.get('limit', 10))
315 |
316 | sort = params.get('sort', 'asc')
317 |
318 | if not search:
319 | qurl = self.solr_select_api + self.page_query.format(
320 | coll=coll, start=start, rows=rows, sort=sort
321 | )
322 | res = requests.get(qurl)
323 |
324 | res = res.json()
325 | resp = res.get('response', {})
326 | docs = resp.get('docs')
327 |
328 | return {
329 | 'total': resp.get('numFound'),
330 | 'results': [
331 | {
332 | 'title': doc.get('title_t'),
333 | 'url': doc.get('url_s'),
334 | 'timestamp': doc.get('timestamp_ss'),
335 | 'has_screenshot': doc.get('has_screenshot_b'),
336 | }
337 | for doc in docs
338 | ],
339 | }
340 |
341 | else:
342 | query = 'content_t:"{q}" OR title_t:"{q}" OR url_s:"*{q}*"'.format(
343 | q=search, coll=coll
344 | )
345 | res = requests.get(
346 | self.solr_select_api
347 | + self.text_query.format(
348 | q=query, start=start, rows=rows, fq='coll_s:' + coll
349 | )
350 | )
351 |
352 | res = res.json()
353 | resp = res.get('response', {})
354 | docs = resp.get('docs')
355 | hl = res.get('highlighting', {})
356 |
357 | return {
358 | 'total': resp.get('numFound'),
359 | 'results': [
360 | {
361 | 'title': doc.get('title_t'),
362 | 'url': doc.get('url_s'),
363 | 'timestamp': doc.get('timestamp_ss'),
364 | 'has_screenshot': doc.get('has_screenshot_b'),
365 | 'matched': hl.get(doc.get('id'), {}).get('content_t'),
366 | }
367 | for doc in docs
368 | ],
369 | }
370 |
371 |
372 | # =============================================================================
373 | application = CrawlProxyApp()
374 |
--------------------------------------------------------------------------------
/pywb/run.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | mkdir -p collections
4 |
5 | uwsgi uwsgi.ini
6 |
7 |
--------------------------------------------------------------------------------
/pywb/static/browsertrix-logo.svg:
--------------------------------------------------------------------------------
1 |
2 |
--------------------------------------------------------------------------------
/pywb/templates/fullsearch.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | {% if metadata %} {{ metadata.title if metadata.title else coll }} {% else %} {{ coll }} {% endif %} Pages
6 |
7 |
67 |
195 |
196 |
197 |
198 |
199 |
202 |
211 |
212 |
213 |
214 |
215 |
216 |
217 |
218 |
219 |
220 |
8 |
9 | [](https://github.com/ambv/black) [](https://travis-ci.org/webrecorder/browsertrix)
10 |
11 | ## High Fidelity Browser-Based Crawling Automation
12 |
13 | Browsertrix is a brand new toolset from the Webrecorder project for automating browsers to perform complex scripted behaviors
14 | as well as crawl multiple pages. (The name was originally used for an older project with similar goals).
15 |
16 | Browsertrix is a system for orchestrating Docker-based Chrome browsers, crawling processes, behavior systems, web archiving capture and replay, and full-text search.
17 |
18 | It includes the following features:
19 | * Crawling via customizable YAML-based crawl spec
20 | * High-fidelity browser-based crawlers (controlled via [webrecorder/autobrowser](https://github.com/webrecorder/autobrowser))
21 | * Execution of complex, domain-specific in-page behaviors (provided by [webrecorder/behaviors](https://github.com/webrecorder/behaviors))
22 | * Capture or replay into designated [pywb](https://github.com/webrecorder/pywb) collections
23 | * Screenshot creation of each page (optional).
24 | * Text extraction for each page and full text search via Solr (optional).
25 | * Support for customized browser profiles to minimize capture of private information.
26 |
27 | ## Getting Started
28 |
29 | ### Installing Browsertrix
30 |
31 | Browsertrix is currently designed to run with Docker and Docker Compose.
32 | The Browsertrix CLI requires local Python 3.6+.
33 |
34 | To install, run:
35 |
36 | ```bash
37 | git clone https://github.com/webrecorder/browsertrix
38 | cd browsertrix
39 | python setup.py install
40 | ./install-browsers.sh
41 | docker-compose build
42 | docker-compose up -d
43 | ```
44 |
45 | The `install-browsers.sh` script installs additional Docker images necessary for dynamic browser creation.
46 | The script can be used to update the images as well.
47 |
48 | ### Installing Browsertrix CLI
49 |
50 | The Browsertrix CLI is installed by running `python setup.py install` and includes full functionality for running crawls and creating browser profiles.
51 |
52 | Once installed, browsertrix commands are available via the `browsertrix` command.
53 |
54 | ## Creating a Crawl
55 |
56 | To create a crawl, first a crawl spec should be defined in a yaml file.
57 | An example spec, [sample-crawls/example.yaml](sample-crawls/example.yaml) might look as follows:
58 |
59 | ```yaml
60 | crawls:
61 | - name: example
62 | crawl_type: all-links
63 | num_browsers: 1
64 |
65 | coll: example
66 | mode: record
67 |
68 | seed_urls:
69 | - https://www.iana.org/
70 | ```
71 |
72 | Then, simply run `browsertrix crawl create sample-crawls/example.yaml --watch`
73 |
74 | The `--watch` param will also result in the crawling browser opening in a new browser window via vnc connection.
75 |
76 | If started successfully, the output will be similar to:
77 | ```
78 | Crawl Created and Started: cf30281efc7a
79 | Status: running
80 | Opening Browser 1 of 1 (CKVEMACNI6YBUKLQI6UKKBLB) for crawl cf30281efc7a
81 | ```
82 |
83 | To view all running crawls, simply run `browsertrix crawl list` which should result in output similar to:
84 |
85 | ```
86 | CRAWL ID NAME STARTED DURATION STATUS CRAWL TYPE COLL MODE TO CRAWL PENDING SEEN BROWSERS TABS
87 | cf30281efc7a example 0:00:35 ago 0:00:10 running all-links example record 15 1 25 1 1
88 | ```
89 |
90 | To get more detailed info on the crawl, run `browsertrix crawl info --urls