├── .gitattributes ├── .github ├── ISSUE_TEMPLATE │ ├── 1-bug.yml │ ├── 2-feature-change.yml │ ├── 3-localization-request.yml │ ├── 4-docs-change.yml │ ├── 5-new-feature.yml │ ├── 6-task.yml │ └── config.yml ├── PULL_REQUEST_TEMPLATE │ └── ui_changes.md └── workflows │ ├── ansible-lint.yaml │ ├── backend-lint.yaml │ ├── deploy-dev.yaml │ ├── docs-publish.yaml │ ├── frontend-lint-test-build.yaml │ ├── k3d-ci.yaml │ ├── k3d-log-ci.yaml │ ├── k3d-nightly-ci.yaml │ ├── microk8s-ci.yaml │ ├── password-check.yaml │ ├── project-assign-issue.yml │ ├── publish-helm-chart.yaml │ ├── release.yaml │ └── weblate-reformat.yaml ├── .gitignore ├── .pre-commit-config.yaml ├── .vscode ├── extensions.json ├── settings.json └── snippets.code-snippets ├── LICENSE ├── NOTICE ├── README.md ├── ansible ├── .editorconfig ├── .tool-versions ├── Pipfile ├── Pipfile.lock ├── README.md ├── ansible.cfg ├── deploys │ └── .gitignore ├── do_setup.yml ├── install_k3s.yml ├── install_microk8s.yml ├── inventory │ ├── digital_ocean │ │ ├── group_vars │ │ │ └── main.yml │ │ └── hosts.ini │ ├── hosts.example │ ├── microk8s │ │ ├── group_vars │ │ │ └── example.yml │ │ └── hosts.ini │ └── sample-k3s │ │ ├── group_vars │ │ └── all.yml │ │ └── hosts.ini ├── lint-cfg.yml ├── playbooks │ └── do_teardown.yml └── roles │ ├── btrix │ ├── deploy │ │ └── tasks │ │ │ └── main.yml │ ├── install │ │ └── tasks │ │ │ └── main.yml │ ├── prereq │ │ └── tasks │ │ │ └── main.yml │ └── templates │ │ └── k8s-manifest.yaml.j2 │ ├── digital_ocean │ ├── setup │ │ ├── tasks │ │ │ └── main.yml │ │ └── templates │ │ │ └── do-values.template.yaml │ └── teardown │ │ └── tasks │ │ └── main.yml │ ├── download │ └── tasks │ │ └── main.yml │ ├── k3s │ ├── master │ │ ├── defaults │ │ │ └── main.yml │ │ ├── tasks │ │ │ └── main.yml │ │ └── templates │ │ │ └── k3s.service.j2 │ └── node │ │ ├── tasks │ │ └── main.yml │ │ └── templates │ │ └── k3s.service.j2 │ ├── microk8s │ ├── common │ │ ├── defaults │ │ │ └── main.yml │ │ ├── handlers │ │ │ └── main.yml │ │ ├── tasks │ │ │ └── main.yml │ │ └── templates │ │ │ └── btrix_values.j2 │ ├── debian │ │ └── tasks │ │ │ └── main.yml │ └── redhat │ │ └── tasks │ │ └── main.yml │ ├── prereq │ └── tasks │ │ └── main.yml │ └── reset │ └── tasks │ ├── main.yml │ └── umount_with_children.yml ├── assets └── browsertrix-lockup-color-dynamic.svg ├── backend ├── .pylintrc ├── Dockerfile ├── btrixcloud │ ├── __init__.py │ ├── auth.py │ ├── background_jobs.py │ ├── basecrawls.py │ ├── colls.py │ ├── crawlconfigs.py │ ├── crawlmanager.py │ ├── crawls.py │ ├── db.py │ ├── emailsender.py │ ├── invites.py │ ├── k8sapi.py │ ├── main.py │ ├── main_bg.py │ ├── main_migrations.py │ ├── main_op.py │ ├── migrations │ │ ├── __init__.py │ │ ├── migration_0001_archives_to_orgs.py │ │ ├── migration_0002_crawlconfig_crawlstats.py │ │ ├── migration_0003_mutable_crawl_configs.py │ │ ├── migration_0004_config_seeds.py │ │ ├── migration_0005_operator_scheduled_jobs.py │ │ ├── migration_0006_precompute_crawl_stats.py │ │ ├── migration_0007_colls_and_config_update.py │ │ ├── migration_0008_precompute_crawl_file_stats.py │ │ ├── migration_0009_crawl_types.py │ │ ├── migration_0010_collection_total_size.py │ │ ├── migration_0011_crawl_timeout_configmap.py │ │ ├── migration_0012_notes_to_description.py │ │ ├── migration_0013_crawl_name.py │ │ ├── migration_0014_to_collection_ids.py │ │ ├── migration_0015_org_storage_usage.py │ │ ├── migration_0016_operator_scheduled_jobs_v2.py │ │ ├── migration_0017_storage_by_type.py │ │ ├── migration_0018_usernames.py │ │ ├── migration_0019_org_slug.py │ │ ├── migration_0020_org_storage_refs.py │ │ ├── migration_0021_profile_filenames.py │ │ ├── migration_0022_partial_complete.py │ │ ├── migration_0023_available_extra_exec_mins.py │ │ ├── migration_0024_crawlerchannel.py │ │ ├── migration_0025_workflow_db_configmap_fixes.py │ │ ├── migration_0026_crawl_review_status.py │ │ ├── migration_0027_profile_modified.py │ │ ├── migration_0028_page_files_errors.py │ │ ├── migration_0029_remove_workflow_configmaps.py │ │ ├── migration_0030_user_invites_flatten.py │ │ ├── migration_0031_org_created.py │ │ ├── migration_0032_dupe_org_names.py │ │ ├── migration_0033_crawl_quota_states.py │ │ ├── migration_0034_drop_invalid_crc.py │ │ ├── migration_0035_fix_failed_logins.py │ │ ├── migration_0036_coll_visibility.py │ │ ├── migration_0037_upload_pages.py │ │ ├── migration_0038_org_last_crawl_finished.py │ │ ├── migration_0039_coll_slugs.py │ │ ├── migration_0040_archived_item_page_count.py │ │ ├── migration_0041_pages_snapshots.py │ │ ├── migration_0042_page_filenames.py │ │ ├── migration_0043_unset_file_expireat.py │ │ ├── migration_0044_coll_stats.py │ │ ├── migration_0045_crawl_counts.py │ │ ├── migration_0046_invalid_lang.py │ │ └── migration_0047_scale_to_browser_windows.py │ ├── models.py │ ├── operator │ │ ├── __init__.py │ │ ├── baseoperator.py │ │ ├── bgjobs.py │ │ ├── crawls.py │ │ ├── cronjobs.py │ │ ├── models.py │ │ └── profiles.py │ ├── ops.py │ ├── orgs.py │ ├── pages.py │ ├── pagination.py │ ├── profiles.py │ ├── storages.py │ ├── subs.py │ ├── uploads.py │ ├── users.py │ ├── utils.py │ ├── version.py │ └── webhooks.py ├── dev-requirements.txt ├── mypy.ini ├── requirements.txt ├── test-requirements.txt ├── test │ ├── __init__.py │ ├── conftest.py │ ├── data │ │ ├── example-2.wacz │ │ ├── example.wacz │ │ ├── org-export.json │ │ └── thumbnail.jpg │ ├── echo_server.py │ ├── test_api.py │ ├── test_collections.py │ ├── test_crawl_config_search_values.py │ ├── test_crawl_config_tags.py │ ├── test_crawlconfigs.py │ ├── test_filter_sort_results.py │ ├── test_login.py │ ├── test_org.py │ ├── test_org_subs.py │ ├── test_permissions.py │ ├── test_profiles.py │ ├── test_qa.py │ ├── test_run_crawl.py │ ├── test_stop_cancel_crawl.py │ ├── test_uploads.py │ ├── test_users.py │ ├── test_utils.py │ ├── test_webhooks.py │ ├── test_workflow_auto_add_to_collection.py │ ├── test_y_org_import_export.py │ ├── test_z_delete_org.py │ └── utils.py └── test_nightly │ ├── __init__.py │ ├── conftest.py │ ├── test_concurrent_crawl_limit.py │ ├── test_crawl_errors.py │ ├── test_crawl_logs.py │ ├── test_crawl_timeout.py │ ├── test_crawlconfig_crawl_stats.py │ ├── test_delete_crawls.py │ ├── test_execution_minutes_quota.py │ ├── test_free_register.py │ ├── test_invite_expiration.py │ ├── test_max_crawl_size_limit.py │ ├── test_org_deletion.py │ ├── test_storage_quota.py │ ├── test_upload_replicas.py │ ├── test_z_background_jobs.py │ └── utils.py ├── btrix ├── chart ├── .helmignore ├── Chart.lock ├── Chart.yaml ├── README.md ├── admin │ └── logging │ │ ├── .gitignore │ │ ├── Chart.yaml │ │ ├── README.md │ │ ├── scripts │ │ ├── eck_install.sh │ │ ├── eck_uninstall.sh │ │ ├── kibana_export.ndjson │ │ ├── kibana_exports.sh │ │ └── kibana_imports.sh │ │ ├── templates │ │ ├── elasticsearch.yaml │ │ ├── fluentd.yaml │ │ ├── ingress.yaml │ │ ├── kibana.yaml │ │ └── logging.yaml │ │ └── values.yaml ├── app-templates │ ├── background_job.yaml │ ├── crawl_configmap.yaml │ ├── crawl_cron_job.yaml │ ├── crawl_job.yaml │ ├── crawler.yaml │ ├── profile_job.yaml │ ├── profilebrowser.yaml │ ├── qa_configmap.yaml │ ├── redis.yaml │ ├── replica_deletion_cron_job.yaml │ └── replica_job.yaml ├── btrix-crds │ ├── Chart.yaml │ └── templates │ │ ├── crawlerjob.yaml │ │ └── profilejob.yaml ├── charts │ ├── btrix-admin-logging-0.1.0.tgz │ ├── btrix-crds-0.1.1.tgz │ ├── btrix-proxies-0.1.0.tgz │ └── metacontroller-helm-4.11.11.tgz ├── email-templates │ ├── failed_bg_job │ ├── invite │ ├── password_reset │ ├── sub_cancel │ └── validate ├── examples │ ├── k3s-hosted.yaml │ ├── local-config.yaml │ ├── local-logging.yaml │ └── microk8s-hosted.yaml ├── proxies │ ├── Chart.yaml │ ├── templates │ │ └── proxies.yaml │ └── values.yaml ├── templates │ ├── backend.yaml │ ├── configmap.yaml │ ├── frontend.yaml │ ├── ingress.yaml │ ├── minio.yaml │ ├── mongo.yaml │ ├── namespaces.yaml │ ├── networkpolicies.yaml │ ├── operators.yaml │ ├── priorities.yaml │ ├── role.yaml │ ├── secrets.yaml │ ├── service.yaml │ └── signer.yaml ├── test │ ├── microk8s-ci.yaml │ ├── test-nightly-addons.yaml │ └── test.yaml └── values.yaml ├── configs └── signing.sample.yaml ├── frontend ├── .dockerignore ├── .editorconfig ├── .eslintrc.js ├── .gitignore ├── .husky │ └── pre-commit ├── .prettierignore ├── .storybook │ ├── main.ts │ └── preview.ts ├── .vscode ├── .yarnrc ├── 00-browsertrix-nginx-init.sh ├── Dockerfile ├── README.md ├── config │ ├── define.js │ ├── tailwind │ │ └── plugins │ │ │ ├── attributes.js │ │ │ ├── contain.js │ │ │ ├── content-visibility.js │ │ │ └── parts.js │ └── webpack │ │ └── shoelace.js ├── custom-elements-manifest.config.mjs ├── docs │ ├── .gitignore │ ├── copy-api-docs.sh │ ├── docs │ │ ├── CNAME │ │ ├── assets │ │ │ ├── brand │ │ │ │ ├── browsertrix-icon-white.svg │ │ │ │ └── favicon.svg │ │ │ └── fonts │ │ │ │ ├── Inter-Italic.var.woff2 │ │ │ │ ├── Inter.var.woff2 │ │ │ │ └── Recursive_VF_1.084.woff2 │ │ ├── deploy │ │ │ ├── admin │ │ │ │ ├── org-import-export.md │ │ │ │ └── upgrade-notes.md │ │ │ ├── ansible │ │ │ │ ├── digitalocean.md │ │ │ │ ├── k3s.md │ │ │ │ └── microk8s.md │ │ │ ├── customization.md │ │ │ ├── index.md │ │ │ ├── local.md │ │ │ ├── proxies.md │ │ │ └── remote.md │ │ ├── develop │ │ │ ├── docs.md │ │ │ ├── frontend-dev.md │ │ │ ├── index.md │ │ │ ├── local-dev-setup.md │ │ │ ├── localization.md │ │ │ └── ui │ │ │ │ ├── components.md │ │ │ │ ├── design-action-menus.md │ │ │ │ ├── design-status-icons.md │ │ │ │ └── storybook.md │ │ ├── helm-repo │ │ │ └── .gitignore │ │ ├── index.md │ │ ├── js │ │ │ ├── embed.js │ │ │ └── insertversion.js │ │ ├── overrides │ │ │ ├── .icons │ │ │ │ ├── bootstrap │ │ │ │ │ ├── asterisk.svg │ │ │ │ │ ├── bug-fill.svg │ │ │ │ │ ├── chat-left-text-fill.svg │ │ │ │ │ ├── check-circle-fill.svg │ │ │ │ │ ├── dash-square-fill.svg │ │ │ │ │ ├── exclamation-circle-fill.svg │ │ │ │ │ ├── exclamation-diamond-fill.svg │ │ │ │ │ ├── exclamation-square-fill.svg │ │ │ │ │ ├── exclamation-triangle-fill.svg │ │ │ │ │ ├── exclamation-triangle.svg │ │ │ │ │ ├── eye.svg │ │ │ │ │ ├── file-earmark-text-fill.svg │ │ │ │ │ ├── github.svg │ │ │ │ │ ├── globe.svg │ │ │ │ │ ├── globe2.svg │ │ │ │ │ ├── hourglass-split.svg │ │ │ │ │ ├── info-circle-fill.svg │ │ │ │ │ ├── mastodon.svg │ │ │ │ │ ├── mortarboard-fill.svg │ │ │ │ │ ├── motherboard-fill.svg │ │ │ │ │ ├── pencil-fill.svg │ │ │ │ │ ├── pencil.svg │ │ │ │ │ ├── question-circle-fill.svg │ │ │ │ │ ├── quote.svg │ │ │ │ │ ├── slash-circle-fill.svg │ │ │ │ │ ├── three-dots-vertical.svg │ │ │ │ │ ├── upload.svg │ │ │ │ │ ├── x-octagon-fill.svg │ │ │ │ │ └── youtube.svg │ │ │ │ └── btrix │ │ │ │ │ └── status-dot.svg │ │ │ ├── main.html │ │ │ └── partials │ │ │ │ ├── actions.html │ │ │ │ └── integrations │ │ │ │ └── analytics │ │ │ │ ├── custom.html │ │ │ │ └── plausible.html │ │ ├── stylesheets │ │ │ └── extra.css │ │ └── user-guide │ │ │ ├── archived-items.md │ │ │ ├── browser-profiles.md │ │ │ ├── collection.md │ │ │ ├── contribute.md │ │ │ ├── crawl-workflows.md │ │ │ ├── getting-started.md │ │ │ ├── index.md │ │ │ ├── join.md │ │ │ ├── org-members.md │ │ │ ├── org-settings.md │ │ │ ├── org.md │ │ │ ├── overview.md │ │ │ ├── presentation-sharing.md │ │ │ ├── public-collections-gallery.md │ │ │ ├── review.md │ │ │ ├── running-crawl.md │ │ │ ├── signup.md │ │ │ ├── user-settings.md │ │ │ └── workflow-setup.md │ └── mkdocs.yml ├── frontend.conf.template ├── index.d.ts ├── lib │ └── intl-durationformat.js ├── lit-localize.json ├── minio.conf ├── package.json ├── patches │ └── @shoelace-style+shoelace+2.5.2.patch ├── playwright.config.ts ├── postcss.config.js ├── prettier.config.js ├── public │ └── browsertrix-og.jpg ├── sample.env.local ├── scripts │ ├── get-resolved-playwright-version.js │ └── serve.js ├── src │ ├── __generated__ │ │ ├── locale-codes.ts │ │ └── locales │ │ │ ├── .keep │ │ │ ├── de.ts │ │ │ ├── es.ts │ │ │ ├── fr.ts │ │ │ └── pt.ts │ ├── __mocks__ │ │ ├── @formatjs │ │ │ └── intl-displaynames │ │ │ │ └── should-polyfill.js │ │ ├── _empty.js │ │ ├── api │ │ │ ├── archives │ │ │ │ └── [id] │ │ │ │ │ └── crawls.js │ │ │ ├── orgs │ │ │ │ └── [id].js │ │ │ └── settings.js │ │ └── shoelace.js │ ├── assets │ │ ├── brand │ │ │ └── browsertrix-lockup-color.svg │ │ ├── favicons │ │ │ ├── apple-touch-icon.png │ │ │ ├── docs-logo.svg │ │ │ ├── favicon.ico │ │ │ ├── favicon.svg │ │ │ ├── icon-192.png │ │ │ └── icon-512.png │ │ ├── fonts │ │ │ ├── Inter │ │ │ │ ├── Inter-italic.var.woff2 │ │ │ │ ├── Inter-roman.var.woff2 │ │ │ │ └── inter.css │ │ │ └── Recursive │ │ │ │ ├── Recursive_VF.woff2 │ │ │ │ └── recursive.css │ │ ├── icons │ │ │ ├── chevron-left.svg │ │ │ ├── chevron-right.svg │ │ │ ├── dot.svg │ │ │ ├── flask-fill.svg │ │ │ ├── microscope.svg │ │ │ ├── pencil-circle-dashed.svg │ │ │ └── replaywebpage.svg │ │ └── images │ │ │ ├── caret-down-fill.svg │ │ │ ├── caret-right-fill.svg │ │ │ ├── new-crawl-config_Seeded-Crawl.svg │ │ │ ├── new-crawl-config_URL-List.svg │ │ │ └── thumbnails │ │ │ ├── thumbnail-cyan.avif │ │ │ ├── thumbnail-green.avif │ │ │ ├── thumbnail-orange.avif │ │ │ └── thumbnail-yellow.avif │ ├── classes │ │ ├── BtrixElement.ts │ │ └── TailwindElement.ts │ ├── components │ │ ├── beta-badges.stylesheet.css │ │ ├── beta-badges.ts │ │ ├── detail-page-title.ts │ │ ├── document-title.ts │ │ ├── index.ts │ │ ├── not-found.ts │ │ ├── orgs-list.ts │ │ ├── screencast.ts │ │ ├── ui │ │ │ ├── README.md │ │ │ ├── alert.ts │ │ │ ├── badge.ts │ │ │ ├── button.ts │ │ │ ├── card.ts │ │ │ ├── code.ts │ │ │ ├── combobox.ts │ │ │ ├── config-details.ts │ │ │ ├── copy-button.ts │ │ │ ├── copy-field.ts │ │ │ ├── data-grid │ │ │ │ ├── cellDirective.ts │ │ │ │ ├── controllers │ │ │ │ │ ├── focus.ts │ │ │ │ │ └── rows.ts │ │ │ │ ├── data-grid-cell.ts │ │ │ │ ├── data-grid-row.ts │ │ │ │ ├── data-grid.stylesheet.css │ │ │ │ ├── data-grid.ts │ │ │ │ ├── events │ │ │ │ │ └── btrix-select-row.ts │ │ │ │ ├── index.ts │ │ │ │ ├── renderRows.ts │ │ │ │ └── types.ts │ │ │ ├── data-table.ts │ │ │ ├── desc-list.ts │ │ │ ├── details.ts │ │ │ ├── dialog.ts │ │ │ ├── field-error.ts │ │ │ ├── file-list.ts │ │ │ ├── format-date.ts │ │ │ ├── index.ts │ │ │ ├── inline-input.ts │ │ │ ├── language-select.ts │ │ │ ├── link.ts │ │ │ ├── markdown-editor.ts │ │ │ ├── markdown-viewer.ts │ │ │ ├── menu-item-link.ts │ │ │ ├── meter.ts │ │ │ ├── navigation │ │ │ │ ├── index.ts │ │ │ │ └── navigation-button.ts │ │ │ ├── numbered-list.ts │ │ │ ├── overflow-dropdown.ts │ │ │ ├── overflow-scroll.ts │ │ │ ├── pagination.ts │ │ │ ├── popover.ts │ │ │ ├── pw-strength-alert.ts │ │ │ ├── relative-duration.ts │ │ │ ├── search-combobox.ts │ │ │ ├── section-heading.ts │ │ │ ├── select-crawler-proxy.ts │ │ │ ├── select-crawler.ts │ │ │ ├── syntax-input.ts │ │ │ ├── tab-group │ │ │ │ ├── index.ts │ │ │ │ ├── tab-group.ts │ │ │ │ ├── tab-panel.ts │ │ │ │ └── tab.ts │ │ │ ├── tab-list.ts │ │ │ ├── table │ │ │ │ ├── index.ts │ │ │ │ ├── table-body.ts │ │ │ │ ├── table-cell.ts │ │ │ │ ├── table-footer.ts │ │ │ │ ├── table-head.ts │ │ │ │ ├── table-header-cell.ts │ │ │ │ ├── table-row.ts │ │ │ │ ├── table.stylesheet.css │ │ │ │ └── table.ts │ │ │ ├── tag-input.ts │ │ │ ├── tag.ts │ │ │ ├── time-input.ts │ │ │ ├── url-input.ts │ │ │ └── user-language-select.ts │ │ ├── utils │ │ │ ├── grouped-list.ts │ │ │ ├── index.ts │ │ │ └── observable.ts │ │ └── verified-badge.ts │ ├── context │ │ ├── org.ts │ │ └── view-state.ts │ ├── controllers │ │ ├── api.ts │ │ ├── clipboard.ts │ │ ├── formControl.ts │ │ ├── localize.ts │ │ ├── navigate.ts │ │ ├── notify.ts │ │ ├── observable.ts │ │ └── searchParams.ts │ ├── decorators │ │ ├── needLogin.test.ts │ │ └── needLogin.ts │ ├── events │ │ ├── btrix-change.ts │ │ ├── btrix-input.ts │ │ └── index.ts │ ├── features │ │ ├── accounts │ │ │ ├── index.ts │ │ │ ├── invite-form.ts │ │ │ └── sign-up-form.ts │ │ ├── admin │ │ │ ├── index.ts │ │ │ ├── stats.ts │ │ │ └── super-admin-banner.ts │ │ ├── archived-items │ │ │ ├── archived-item-list.ts │ │ │ ├── crawl-list.ts │ │ │ ├── crawl-log-table.ts │ │ │ ├── crawl-logs.ts │ │ │ ├── crawl-pending-exclusions.ts │ │ │ ├── crawl-queue.ts │ │ │ ├── crawl-status.ts │ │ │ ├── file-uploader.ts │ │ │ ├── index.ts │ │ │ ├── item-list-controls.ts │ │ │ └── item-metadata-editor.ts │ │ ├── browser-profiles │ │ │ ├── index.ts │ │ │ ├── new-browser-profile-dialog.ts │ │ │ ├── profile-browser.ts │ │ │ └── select-browser-profile.ts │ │ ├── collections │ │ │ ├── collection-create-dialog.ts │ │ │ ├── collection-edit-dialog.ts │ │ │ ├── collection-initial-view-dialog.ts │ │ │ ├── collection-items-dialog.ts │ │ │ ├── collection-snapshot-preview.ts │ │ │ ├── collection-thumbnail.ts │ │ │ ├── collection-workflow-list.ts │ │ │ ├── collections-add.ts │ │ │ ├── collections-grid-with-edit-dialog.ts │ │ │ ├── collections-grid.ts │ │ │ ├── edit-dialog │ │ │ │ ├── helpers │ │ │ │ │ ├── check-changed.ts │ │ │ │ │ ├── gather-state.ts │ │ │ │ │ ├── snapshots.ts │ │ │ │ │ └── submit-task.ts │ │ │ │ ├── presentation-section.ts │ │ │ │ └── sharing-section.ts │ │ │ ├── helpers │ │ │ │ └── share-link.ts │ │ │ ├── index.ts │ │ │ ├── select-collection-access.ts │ │ │ ├── select-collection-page.ts │ │ │ └── share-collection.ts │ │ ├── crawl-workflows │ │ │ ├── custom-behaviors-table-row.ts │ │ │ ├── custom-behaviors-table.ts │ │ │ ├── exclusion-editor.ts │ │ │ ├── index.ts │ │ │ ├── link-selector-table.ts │ │ │ ├── live-workflow-status.ts │ │ │ ├── new-workflow-dialog.ts │ │ │ ├── queue-exclusion-form.ts │ │ │ ├── queue-exclusion-table.ts │ │ │ ├── workflow-editor.ts │ │ │ └── workflow-list.ts │ │ ├── index.ts │ │ ├── org │ │ │ ├── index.ts │ │ │ ├── org-status-banner.ts │ │ │ └── usage-history-table.ts │ │ └── qa │ │ │ ├── index.ts │ │ │ ├── page-list │ │ │ ├── helpers │ │ │ │ ├── approval.ts │ │ │ │ ├── crawlCounts.ts │ │ │ │ ├── iconFor.ts │ │ │ │ ├── index.ts │ │ │ │ ├── issueCounts.ts │ │ │ │ └── severity.ts │ │ │ ├── index.ts │ │ │ ├── page-list.ts │ │ │ ├── test-data.ts │ │ │ └── ui │ │ │ │ ├── animate.ts │ │ │ │ ├── index.ts │ │ │ │ ├── page-details.ts │ │ │ │ ├── page-group.ts │ │ │ │ └── page.ts │ │ │ ├── page-qa-approval.ts │ │ │ ├── qa-run-dropdown.ts │ │ │ └── review-status.ts │ ├── global.ts │ ├── index.ejs │ ├── index.test.ts │ ├── index.ts │ ├── layouts │ │ ├── collections │ │ │ └── metadataColumn.ts │ │ ├── columns.ts │ │ ├── empty.ts │ │ ├── emptyMessage.ts │ │ ├── page.ts │ │ ├── pageError.ts │ │ ├── pageHeader.ts │ │ ├── pageSectionsWithNav.ts │ │ ├── panel.ts │ │ └── separator.ts │ ├── manifest.webmanifest │ ├── mixins │ │ └── FormControl.ts │ ├── pages │ │ ├── account-settings.ts │ │ ├── admin │ │ │ ├── admin.ts │ │ │ ├── index.ts │ │ │ └── users-invite.ts │ │ ├── collections │ │ │ ├── collection.ts │ │ │ └── index.ts │ │ ├── crawls.ts │ │ ├── index.ts │ │ ├── invite │ │ │ ├── accept.test.ts │ │ │ ├── accept.ts │ │ │ ├── join.test.ts │ │ │ ├── join.ts │ │ │ └── ui │ │ │ │ ├── inviteMessage.ts │ │ │ │ ├── org-form.test.ts │ │ │ │ └── org-form.ts │ │ ├── log-in.test.ts │ │ ├── log-in.ts │ │ ├── org │ │ │ ├── archived-item-detail │ │ │ │ ├── archived-item-detail.ts │ │ │ │ ├── index.ts │ │ │ │ └── ui │ │ │ │ │ └── qa.ts │ │ │ ├── archived-item-qa │ │ │ │ ├── archived-item-qa.stylesheet.css │ │ │ │ ├── archived-item-qa.ts │ │ │ │ ├── index.ts │ │ │ │ ├── types.ts │ │ │ │ └── ui │ │ │ │ │ ├── resources.ts │ │ │ │ │ ├── screenshots.ts │ │ │ │ │ ├── severityBadge.ts │ │ │ │ │ ├── spinner.ts │ │ │ │ │ └── text.ts │ │ │ ├── archived-items.ts │ │ │ ├── browser-profiles-detail.ts │ │ │ ├── browser-profiles-list.ts │ │ │ ├── browser-profiles-new.ts │ │ │ ├── collection-detail.ts │ │ │ ├── collections-list.ts │ │ │ ├── dashboard.ts │ │ │ ├── index.ts │ │ │ ├── settings │ │ │ │ ├── components │ │ │ │ │ ├── billing.ts │ │ │ │ │ ├── crawling-defaults.ts │ │ │ │ │ └── general.ts │ │ │ │ ├── settings.stylesheet.css │ │ │ │ └── settings.ts │ │ │ ├── types.ts │ │ │ ├── workflow-detail.ts │ │ │ ├── workflows-list.ts │ │ │ └── workflows-new.ts │ │ ├── orgs.ts │ │ ├── public │ │ │ ├── index.ts │ │ │ └── org.ts │ │ ├── reset-password.ts │ │ ├── sign-up.ts │ │ └── verify.ts │ ├── replayWebPage.d.ts │ ├── routes.ts │ ├── shoelace.ts │ ├── stories │ │ ├── Intro.mdx │ │ ├── components │ │ │ ├── Button.stories.ts │ │ │ ├── Button.ts │ │ │ ├── DataGrid.stories.ts │ │ │ ├── DataGrid.ts │ │ │ ├── DataTable.stories.ts │ │ │ ├── DataTable.ts │ │ │ ├── OverflowScroll.stories.ts │ │ │ ├── OverflowScroll.ts │ │ │ ├── Popover.stories.ts │ │ │ ├── Popover.ts │ │ │ ├── SyntaxInput.stories.ts │ │ │ ├── SyntaxInput.ts │ │ │ ├── Table.data.ts │ │ │ ├── Table.stories.ts │ │ │ ├── Table.ts │ │ │ └── decorators │ │ │ │ └── dataGridForm.ts │ │ ├── decorators │ │ │ └── orgDecorator.ts │ │ └── features │ │ │ ├── archived-items │ │ │ └── CrawlLogs.stories.ts │ │ │ ├── crawl-workflows │ │ │ ├── CustomBehaviorsTable.stories.ts │ │ │ ├── LinkSelectorTable.stories.ts │ │ │ ├── QueueExclusionForm.stories.ts │ │ │ └── QueueExclusionTable.stories.ts │ │ │ ├── excludeContainerProperties.ts │ │ │ └── org │ │ │ └── UsageHistoryTable.stories.ts │ ├── strings │ │ ├── collections │ │ │ ├── alerts.ts │ │ │ └── metadata.ts │ │ ├── crawl-workflows │ │ │ ├── errors.ts │ │ │ ├── infoText.ts │ │ │ ├── labels.ts │ │ │ ├── scopeType.ts │ │ │ └── section.ts │ │ ├── orgs │ │ │ └── alerts.ts │ │ ├── ui.ts │ │ ├── utils.ts │ │ └── validation.ts │ ├── styles.css │ ├── theme.stylesheet.css │ ├── theme.ts │ ├── trackEvents.ts │ ├── types │ │ ├── api.ts │ │ ├── auth.ts │ │ ├── billing.ts │ │ ├── browser.ts │ │ ├── collection.ts │ │ ├── crawlState.ts │ │ ├── crawler.ts │ │ ├── events.d.ts │ │ ├── lib │ │ │ └── intl.durationformat.d.ts │ │ ├── localization.ts │ │ ├── org.ts │ │ ├── qa.ts │ │ ├── user.ts │ │ ├── utils.ts │ │ └── workflow.ts │ └── utils │ │ ├── APIRouter.test.ts │ │ ├── APIRouter.ts │ │ ├── AuthService.test.ts │ │ ├── AuthService.ts │ │ ├── LiteElement.ts │ │ ├── PasswordService.ts │ │ ├── analytics.ts │ │ ├── api.ts │ │ ├── app.ts │ │ ├── crawler.ts │ │ ├── cron.ts │ │ ├── css.ts │ │ ├── events.ts │ │ ├── executionTimeFormatter.test.ts │ │ ├── executionTimeFormatter.ts │ │ ├── form.ts │ │ ├── localize.test.ts │ │ ├── localize.ts │ │ ├── notify.ts │ │ ├── orgs.ts │ │ ├── persist.ts │ │ ├── pluralize.ts │ │ ├── polyfills.ts │ │ ├── replay.ts │ │ ├── round-duration.test.ts │ │ ├── round-duration.ts │ │ ├── router.ts │ │ ├── slugify.ts │ │ ├── state.test.ts │ │ ├── state.ts │ │ ├── string.ts │ │ ├── tailwind.ts │ │ ├── timeoutCache.ts │ │ ├── user.ts │ │ ├── weakCache.test.ts │ │ ├── weakCache.ts │ │ └── workflow.ts ├── tailwind.config.js ├── tests │ ├── README.md │ └── login.spec.ts ├── tsconfig.eslint.json ├── tsconfig.json ├── vendor.webpack.config.js ├── web-test-runner.config.mjs ├── webpack.config.js ├── webpack.dev.js ├── webpack.prod.js ├── xliff │ ├── .keep │ ├── de.xlf │ ├── es.xlf │ ├── fr.xlf │ └── pt.xlf └── yarn.lock ├── pylintrc ├── scripts ├── build-backend.sh ├── build-frontend.sh ├── check_passwords.py ├── generate-helm-index.py ├── minikube-build-and-deploy.sh ├── minikube-reset.sh └── test │ └── test_check_passwords.py ├── test └── setup.sh ├── update-version.sh ├── version.txt └── yarn.lock /.gitattributes: -------------------------------------------------------------------------------- 1 | frontend/xliff/*.xlf linguist-generated 2 | frontend/src/__generated__/** linguist-generated 3 | frontend/yarn.lock linguist-generated 4 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/2-feature-change.yml: -------------------------------------------------------------------------------- 1 | name: Change Request 2 | description: Request new functionality or changes to an existing feature. 3 | title: "[Change]: " 4 | labels: ["enhancement", "idea"] 5 | type: "Feature" 6 | body: 7 | # Deployment type 8 | - type: dropdown 9 | id: deployment 10 | attributes: 11 | label: Browsertrix Host 12 | description: Are you self-hosting Browsertrix, or are you on a hosted plan? 13 | options: 14 | - Self-Hosted 15 | - Hosted by Webrecorder 16 | validations: 17 | required: true 18 | - type: textarea 19 | attributes: 20 | label: What change would you like to see? 21 | description: | 22 | A clear and concise description of the change to existing functionality. 23 | 24 | For general ideas on how to improve Browsertrix, or if you have questions about 25 | existing functionality, please check our [community forum](https://forum.webrecorder.net/) 26 | before submitting a request. 27 | placeholder: I would like to be able to ____________ so that I can ____________. 28 | validations: 29 | required: true 30 | # Additional details 31 | - type: textarea 32 | attributes: 33 | label: Additional details 34 | description: Any additional context that helps us understand this request. -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/3-localization-request.yml: -------------------------------------------------------------------------------- 1 | name: Localization Request 2 | description: Request a new language or translation. 3 | title: "[L10N]: " 4 | labels: ["localization"] 5 | type: "Task" 6 | body: 7 | - type: textarea 8 | attributes: 9 | label: Language 10 | description: Specify the language you'd like to add or translate. A list of currently supported languages can be found in our [Weblate project](https://hosted.weblate.org/engage/browsertrix/). 11 | validations: 12 | required: true 13 | - type: textarea 14 | attributes: 15 | label: Context 16 | description: Any background information that helps us understand the request. -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/4-docs-change.yml: -------------------------------------------------------------------------------- 1 | name: Documentation / User Guide Change 2 | description: Request an update to https://docs.browsertrix.com. 3 | title: "[Docs]: " 4 | labels: ["documentation"] 5 | type: "Task" 6 | body: 7 | - type: textarea 8 | attributes: 9 | label: Description 10 | description: A summary of the change. 11 | validations: 12 | required: true 13 | - type: textarea 14 | attributes: 15 | label: Context 16 | description: Additional context. -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/5-new-feature.yml: -------------------------------------------------------------------------------- 1 | name: (Webrecorder Team) Planned Feature 2 | description: For internal Webrecorder use only. 3 | title: "[Feature]: " 4 | labels: ["feature design", "ui/ux", "front end", "back end"] 5 | type: "Feature" 6 | projects: ["webrecorder/9"] 7 | body: 8 | - type: textarea 9 | attributes: 10 | label: Description 11 | description: A summary of the new or improved feature. 12 | validations: 13 | required: true 14 | - type: textarea 15 | attributes: 16 | label: Requirements 17 | description: A list of software requirements. 18 | validations: 19 | required: true 20 | - type: textarea 21 | attributes: 22 | label: Context 23 | description: Additional context. -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/6-task.yml: -------------------------------------------------------------------------------- 1 | name: (Webrecorder Team) Task 2 | description: For internal Webrecorder use only. 3 | title: "[Task]: " 4 | type: "Task" 5 | projects: ["webrecorder/9"] 6 | body: 7 | - type: textarea 8 | attributes: 9 | label: Description 10 | description: A summary of the task. 11 | validations: 12 | required: true 13 | - type: textarea 14 | attributes: 15 | label: Context 16 | description: Additional context. -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: false 2 | 3 | contact_links: 4 | - name: Ask a question 5 | url: https://forum.webrecorder.net/c/help/browsertrix/24 6 | about: Have a ("how do I...?") question? Not sure if your issue is reproducible? The best way to get help is on our community forum! 7 | - name: Report a replay issue 8 | about: Issues related to an archived item or collection not replaying properly should be reported in the ReplayWeb.page repo. 9 | url: https://github.com/webrecorder/replayweb.page/issues/new?&labels=replay+bug%2Cbug&projects=&template=replay-bug.yml&title=[Replay+Bug]%3A+ 10 | - name: Report a security vulnerability 11 | about: Please email security@webrecorder.org directly. We will follow up with you there! 12 | url: https://webrecorder.net/.well-known/security.txt 13 | - name: Read the docs 14 | url: https://docs.browsertrix.com 15 | about: Find solutions to common questions, such as how to install, develop, and deploy Browsertrix. 16 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE/ui_changes.md: -------------------------------------------------------------------------------- 1 | Resolves #issue_number 2 | 3 | ## Changes 4 | 5 | 6 | 7 | ## Manual testing 8 | 9 | 1. 10 | 11 | ## Screenshots 12 | 13 | | Page | Image/video | 14 | | ---- | ----------- | 15 | | | | 16 | 17 | 18 | -------------------------------------------------------------------------------- /.github/workflows/docs-publish.yaml: -------------------------------------------------------------------------------- 1 | name: Publish Docs 2 | on: 3 | workflow_dispatch: 4 | 5 | release: 6 | types: [published] 7 | 8 | permissions: 9 | contents: write 10 | 11 | jobs: 12 | deploy_docs: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v3 16 | - uses: actions/setup-python@v5 17 | with: 18 | python-version: 3.x 19 | - run: pip install mkdocs-material mkdocs-redirects requests pyyaml 20 | 21 | - name: Generate Helm Chart Index 22 | run: python ./scripts/generate-helm-index.py > ./frontend/docs/docs/helm-repo/index.yaml 23 | 24 | - name: Copy Docs Files 25 | run: frontend/docs/copy-api-docs.sh 26 | env: 27 | DOCS_SOURCE_URL: https://app.browsertrix.com 28 | ENABLE_ANALYTICS: true 29 | 30 | - name: Build Docs 31 | run: cd frontend/docs; mkdocs gh-deploy --force 32 | env: 33 | ENABLE_ANALYTICS: true 34 | -------------------------------------------------------------------------------- /.github/workflows/password-check.yaml: -------------------------------------------------------------------------------- 1 | # name: Password Check 2 | 3 | # on: 4 | # push: 5 | # paths: 6 | # - '*.yaml' 7 | # - '*.yml' 8 | # pull_request: 9 | # paths: 10 | # - '*.yaml' 11 | # - '*.yml' 12 | 13 | # jobs: 14 | # check: 15 | # runs-on: ubuntu-latest 16 | # steps: 17 | # - name: checkout 18 | # uses: actions/checkout@v3 19 | # with: 20 | # fetch-depth: 3 21 | 22 | # - name: Set up Python 23 | # uses: actions/setup-python@v4 24 | # with: 25 | # python-version: '3.10' 26 | 27 | # - name: Install dependencies 28 | # run: | 29 | # cd backend/ 30 | # python -m pip install --upgrade pip 31 | # pip install pyyaml 32 | 33 | # - name: Password Check 34 | # run: | 35 | # CHANGED_FILES=$(git diff --name-only HEAD^..HEAD) 36 | # echo $CHANGED_FILES 37 | # YML_FILES=$(echo "$CHANGED_FILES" | { grep ".yml$\|.yaml$" || true; }) 38 | # if [[ -n "$YML_FILES" ]]; then 39 | # python3 scripts/check_passwords.py $YML_FILES 40 | # fi 41 | -------------------------------------------------------------------------------- /.github/workflows/project-assign-issue.yml: -------------------------------------------------------------------------------- 1 | name: Update assigned issues in Webrecorder Projects 2 | 3 | on: 4 | issues: 5 | types: [assigned] 6 | 7 | env: 8 | todo: Todo 9 | done: Done! 10 | in_progress: Dev In Progress 11 | 12 | jobs: 13 | update-project-column: 14 | runs-on: ubuntu-latest 15 | if: github.event_name == 'issues' && github.event.action == 'assigned' 16 | steps: 17 | - name: Move issue to ${{ env.todo }} 18 | uses: leonsteinhaeuser/project-beta-automations@v2.1.0 19 | with: 20 | gh_token: ${{ secrets.GHPROJECT_TOKEN }} 21 | organization: webrecorder 22 | project_id: 9 23 | resource_node_id: ${{ github.event.issue.node_id }} 24 | status_value: ${{ env.todo }} # Target status 25 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | **/*.pyc 2 | **/node_modules/ 3 | **/config.env 4 | **/config.yaml 5 | **/signing.yaml 6 | .DS_Store 7 | # digital ocean custom values 8 | private.yml 9 | # microk8s playbook hosts 10 | hosts 11 | chart/local.yaml 12 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/psf/black 3 | rev: 24.1.1 4 | hooks: 5 | - id: black 6 | args: ["backend/btrixcloud/"] 7 | - repo: local 8 | hooks: 9 | - id: pylint 10 | name: pylint 11 | entry: cd backend && pylint 12 | language: system 13 | types: [python] 14 | args: ["btrixcloud/"] 15 | - repo: local 16 | hooks: 17 | - id: mypy 18 | name: mypy 19 | entry: cd backend && mypy 20 | language: python 21 | args: ["btrixcloud/"] 22 | - repo: local 23 | hooks: 24 | - id: husky-run-pre-commit 25 | name: husky 26 | language: system 27 | entry: frontend/.husky/pre-commit 28 | pass_filenames: false 29 | # - repo: local 30 | # hooks: 31 | # - id: password-check 32 | # name: password-check 33 | # language: system 34 | # types: [yaml] 35 | # entry: python3 scripts/check_passwords.py 36 | -------------------------------------------------------------------------------- /.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": [ 3 | "dbaeumer.vscode-eslint", 4 | "esbenp.prettier-vscode", 5 | "runem.lit-plugin", 6 | "bradlc.vscode-tailwindcss", 7 | "redhat.vscode-yaml", 8 | "streetsidesoftware.code-spell-checker", 9 | "ms-python.black-formatter", 10 | "ms-python.pylint", 11 | "unifiedjs.vscode-mdx" 12 | ] 13 | } 14 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Browsertrix 2 | Copyright (C) 2022 Webrecorder Software 3 | 4 | Released under the GNU Affero General Public License. 5 | See LICENSE for details. 6 | -------------------------------------------------------------------------------- /ansible/.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | # Defaults for all editor files 4 | [*] 5 | insert_final_newline = true 6 | indent_style = space 7 | indent_size = 4 8 | trim_trailing_whitespace = true 9 | 10 | # YAML is fussy about indenting and charset 11 | [*.yml] 12 | indent_style = space 13 | indent_size = 2 14 | continuation_indent_size = unset 15 | charset = utf-8 16 | 17 | # Markdown is fussy about indenting 18 | [*.md] 19 | indent_style = space 20 | indent_size = 4 21 | 22 | # Jinja2 template files 23 | [*.j2] 24 | end_of_line = lf 25 | -------------------------------------------------------------------------------- /ansible/.tool-versions: -------------------------------------------------------------------------------- 1 | python 3.11.2 2 | -------------------------------------------------------------------------------- /ansible/Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | url = "https://pypi.org/simple" 3 | verify_ssl = true 4 | name = "pypi" 5 | 6 | [packages] 7 | ansible = "*" 8 | molecule = {extras = ["docker"], version = "*"} 9 | certifi = "*" 10 | ansible-lint = "*" 11 | wcmatch = "*" # workaround ubuntu shipping a broken wcmatch package 12 | yamllint = "*" 13 | ansible-core = "*" 14 | docker = "*" 15 | boto3 = "*" 16 | jmespath = "*" 17 | mkdocs-material = "*" 18 | 19 | [dev-packages] 20 | 21 | [requires] 22 | python_version = "3.12" 23 | -------------------------------------------------------------------------------- /ansible/README.md: -------------------------------------------------------------------------------- 1 | ### Install 2 | 3 | Most current instructions to install will always be [here](https://docs.browsertrix.com/deploy/remote/) 4 | -------------------------------------------------------------------------------- /ansible/ansible.cfg: -------------------------------------------------------------------------------- 1 | [defaults] 2 | host_key_checking = False 3 | inventory = ./inventory 4 | -------------------------------------------------------------------------------- /ansible/deploys/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | -------------------------------------------------------------------------------- /ansible/do_setup.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: deploy browsertrix on digital ocean 3 | hosts: localhost 4 | connection: local 5 | gather_facts: false 6 | vars_files: 7 | - inventory/digital_ocean/group_vars/main.yml 8 | roles: 9 | - role: digital_ocean/setup 10 | - role: btrix/deploy 11 | -------------------------------------------------------------------------------- /ansible/install_k3s.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # Can be skipped if k3s is installed, this installs k3s 4 | - hosts: k3s_cluster 5 | gather_facts: yes 6 | connection: local # Comment if deploying to remote host 7 | become: yes 8 | roles: 9 | - role: prereq 10 | - role: download 11 | 12 | # Can be skipped if k3s is installed, this configures the master k3s node 13 | - hosts: controller 14 | connection: local # Comment if deploying to remote host 15 | become: yes 16 | roles: 17 | - role: k3s/master 18 | 19 | # Uncomment for multi-node deployment 20 | # - hosts: node 21 | # roles: 22 | # - role: k3s/node 23 | 24 | # Ansible controller to install browsertrix 25 | - hosts: 127.0.0.1 26 | connection: local 27 | become: yes # Can be removed if not using the btrix/prereq role 28 | roles: 29 | - role: btrix/prereq # Only required if you wish to install & configure Helm / Kubectl 30 | - role: btrix/install 31 | - role: btrix/deploy 32 | -------------------------------------------------------------------------------- /ansible/install_microk8s.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: all 3 | gather_facts: yes 4 | become: yes 5 | roles: 6 | - microk8s/debian # Change to redhat if using a redhat distro 7 | - microk8s/common 8 | - btrix/deploy 9 | 10 | handlers: 11 | - name: Reboot System 12 | ansible.builtin.reboot: 13 | when: 14 | - skip_handlers | default("false") == "false" 15 | 16 | - name: microk8s ready 17 | ansible.builtin.command: 18 | cmd: microk8s.status --wait-ready 19 | changed_when: false 20 | when: 21 | - skip_handlers | default("false") == "false" 22 | -------------------------------------------------------------------------------- /ansible/inventory/digital_ocean/group_vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | project_name: "default" 3 | 4 | main_node_size: "s-4vcpu-8gb" 5 | crawl_node_size: "c-4" 6 | droplet_region: "sfo3" 7 | 8 | node_pools: 9 | - name=main-app;size={{ main_node_size }};label=nodeType=main;min-nodes=1;max-nodes=2;count=1 10 | - name=crawling;size={{ crawl_node_size }};label=nodeType=crawling;taint=nodeType=crawling:NoSchedule;auto-scale=true;min-nodes=1;max-nodes=3;count=1 11 | 12 | enable_admin_addons: false 13 | 14 | admin_node_pool: 15 | name: admin-app 16 | size: s-4vcpu-8gb 17 | label: nodeType=admin 18 | count: 1 19 | 20 | db_name: "{{ project_name }}" 21 | k8s_name: "{{ project_name }}" 22 | 23 | bucket_name: "{{ project_name }}" 24 | bucket_path: "crawls" 25 | 26 | registry_name: "{{ project_name }}" 27 | 28 | domain: "browsertrix.cloud" 29 | subdomain: "{{ project_name }}" 30 | 31 | 32 | configure_kubectl: false 33 | use_do_registry: false 34 | image_tag: "latest" 35 | 36 | enable_signing: true 37 | signing_host: "signing" 38 | 39 | superuser_email: "dev@webrecorder.net" 40 | superuser_password: "PassW0rd!" 41 | 42 | org_name: "{{ project_name }}" 43 | 44 | registration_enabled: false 45 | 46 | cert_email: "{{ superuser_email }}" 47 | 48 | smtp_port: "" 49 | smtp_host: "" 50 | sender_email: "" 51 | reply_to_email: "" 52 | sender_password: "" 53 | -------------------------------------------------------------------------------- /ansible/inventory/digital_ocean/hosts.ini: -------------------------------------------------------------------------------- 1 | 127.0.0.1 2 | -------------------------------------------------------------------------------- /ansible/inventory/hosts.example: -------------------------------------------------------------------------------- 1 | 3.2.1.4 # ip address of your remote endpoint 2 | -------------------------------------------------------------------------------- /ansible/inventory/microk8s/group_vars/example.yml: -------------------------------------------------------------------------------- 1 | --- 2 | all: 3 | children: 4 | "deploy microk8s": 5 | vars: 6 | browsertrix_mongo_password: "CHANGE ME" 7 | browsertrix_superuser_password: "CHANGE ME" 8 | 9 | host_ip: "{{ hostvars[inventory_hostname]['ansible_default_ipv4']['address'] }}" 10 | domain_name: "{{inventory_hostname }}" 11 | signing_subdomain: "{{ signing_subdomain }}" 12 | signing_authtoken: "{{ 99999999 }}" 13 | enable_signing: true 14 | your_user: sysadmin 15 | ansible_user: sysadmin 16 | cert_email: infra@example.org 17 | # crawler_session_size_limit_bytes: 50000000000 18 | 19 | crawler_extra_args: "--screenshot view,thumbnail,fullPage" 20 | 21 | microk8s_dns_servers: 22 | - 1.1.1.1 23 | - 1.0.0.1 24 | -------------------------------------------------------------------------------- /ansible/inventory/microk8s/hosts.ini: -------------------------------------------------------------------------------- 1 | [microk8s] 2 | 127.0.0.1 3 | # Add more hosts here if you want to deploy to multiple nodes 4 | -------------------------------------------------------------------------------- /ansible/inventory/sample-k3s/group_vars/all.yml: -------------------------------------------------------------------------------- 1 | --- 2 | k3s_version: v1.22.3+k3s1 3 | ansible_user: debian 4 | systemd_dir: /etc/systemd/system 5 | controller_ip: "{{ hostvars[groups['controller'][0]]['ansible_host'] | default(groups['controller'][0]) }}" 6 | extra_server_args: "--disable traefik" 7 | extra_agent_args: "" 8 | project_name: browsertrix-cloud 9 | domain: my-domain.example.com 10 | email: test@example.com 11 | -------------------------------------------------------------------------------- /ansible/inventory/sample-k3s/hosts.ini: -------------------------------------------------------------------------------- 1 | [controller] 2 | # Set to the IP address of the k3s host node 3 | 127.0.0.1 4 | 5 | # Uncomment for multi-node deployment 6 | # [node] 7 | # 192.168.1.2 8 | 9 | [k3s_cluster:children] 10 | controller 11 | # Uncomment for multi-node deployment 12 | # node 13 | -------------------------------------------------------------------------------- /ansible/roles/btrix/deploy/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: deploy btrix 3 | ansible.builtin.command: helm upgrade --install -f {{ browsertrix_cloud_home | default('..') }}/chart/values.yaml -f {{ browsertrix_cloud_home | default('..') }}/chart/{{ project_name }}-values.yaml btrix {{ browsertrix_cloud_home | default('..') }}/chart/ 4 | register: helm_result 5 | changed_when: helm_result.rc == 0 6 | environment: 7 | KUBECONFIG: "/home/{{ ansible_user }}/.kube/config" 8 | tags: helm_upgrade 9 | -------------------------------------------------------------------------------- /ansible/roles/btrix/prereq/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Gather installed helm version, if there is any 3 | ansible.builtin.shell: helm version 4 | register: helm_result 5 | failed_when: helm_result.rc != 0 and helm_result.rc != 127 6 | # Since this is a reporting task, it should never change 7 | # as well run and register a result in any case 8 | changed_when: false 9 | check_mode: false 10 | 11 | - name: Install Helm 12 | ansible.builtin.shell: | 13 | curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 14 | chmod +700 get_helm.sh 15 | ./get_helm.sh 16 | when: helm_result.rc != 0 17 | 18 | - name: Install kubectl 19 | ansible.builtin.shell: | 20 | curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" 21 | install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl 22 | 23 | - name: Install jq 24 | ansible.builtin.package: 25 | name: jq 26 | state: present 27 | -------------------------------------------------------------------------------- /ansible/roles/btrix/templates/k8s-manifest.yaml.j2: -------------------------------------------------------------------------------- 1 | ingress_class: "nginx" 2 | 3 | mongo_auth: 4 | username: root 5 | password: example 6 | 7 | ingress: 8 | host: "{{ domain }}" 9 | cert_email: "{{ email }}" 10 | scheme: "https" 11 | tls: true 12 | -------------------------------------------------------------------------------- /ansible/roles/digital_ocean/teardown/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | create_backup: true 3 | delete_space: false 4 | 5 | -------------------------------------------------------------------------------- /ansible/roles/k3s/master/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | k3s_server_location: /var/lib/rancher/k3s 3 | -------------------------------------------------------------------------------- /ansible/roles/k3s/master/templates/k3s.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Lightweight Kubernetes 3 | Documentation=https://k3s.io 4 | After=network-online.target 5 | 6 | [Service] 7 | Type=notify 8 | ExecStartPre=-/sbin/modprobe br_netfilter 9 | ExecStartPre=-/sbin/modprobe overlay 10 | ExecStart=/usr/local/bin/k3s server --data-dir {{ k3s_server_location }} {{ extra_server_args | default("") }} 11 | KillMode=process 12 | Delegate=yes 13 | # Having non-zero Limit*s causes performance problems due to accounting overhead 14 | # in the kernel. We recommend using cgroups to do container-local accounting. 15 | LimitNOFILE=1048576 16 | LimitNPROC=infinity 17 | LimitCORE=infinity 18 | TasksMax=infinity 19 | TimeoutStartSec=0 20 | Restart=always 21 | RestartSec=5s 22 | 23 | [Install] 24 | WantedBy=multi-user.target 25 | -------------------------------------------------------------------------------- /ansible/roles/k3s/node/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Copy K3s service file 4 | template: 5 | src: "k3s.service.j2" 6 | dest: "{{ systemd_dir }}/k3s-node.service" 7 | owner: root 8 | group: root 9 | mode: 0755 10 | 11 | - name: Enable and check K3s service 12 | systemd: 13 | name: k3s-node 14 | daemon_reload: yes 15 | state: restarted 16 | enabled: yes 17 | -------------------------------------------------------------------------------- /ansible/roles/k3s/node/templates/k3s.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Lightweight Kubernetes 3 | Documentation=https://k3s.io 4 | After=network-online.target 5 | 6 | [Service] 7 | Type=notify 8 | ExecStartPre=-/sbin/modprobe br_netfilter 9 | ExecStartPre=-/sbin/modprobe overlay 10 | ExecStart=/usr/local/bin/k3s agent --server https://{{ controller_ip }}:6443 --token {{ hostvars[groups['controller'][0]]['token'] }} {{ extra_agent_args | default("") }} 11 | KillMode=process 12 | Delegate=yes 13 | # Having non-zero Limit*s causes performance problems due to accounting overhead 14 | # in the kernel. We recommend using cgroups to do container-local accounting. 15 | LimitNOFILE=1048576 16 | LimitNPROC=infinity 17 | LimitCORE=infinity 18 | TasksMax=infinity 19 | TimeoutStartSec=0 20 | Restart=always 21 | RestartSec=5s 22 | 23 | [Install] 24 | WantedBy=multi-user.target 25 | -------------------------------------------------------------------------------- /ansible/roles/microk8s/common/handlers/main.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webrecorder/browsertrix/54d29aec0585c358126861c03c2212596c5d2c3f/ansible/roles/microk8s/common/handlers/main.yml -------------------------------------------------------------------------------- /ansible/roles/microk8s/common/templates/btrix_values.j2: -------------------------------------------------------------------------------- 1 | default_org: {{ browsertrix_default_org | default('My Organization') }} 2 | 3 | crawler_extra_args: "{{ crawler_extra_args | default('') }}" 4 | 5 | superuser: 6 | email: {{ browsertrix_superuser_email | default('admin@example.com') }} 7 | password: {{ browsertrix_superuser_password | default('PASSW0RD!') }} 8 | 9 | mongo_auth: 10 | username: {{ browsertrix_mongo_username | default('root') }} 11 | password: {{ browsertrix_mongo_password | default('PASSWORD!') }} 12 | 13 | ingress: 14 | host: "{{ domain }}" 15 | cert_email: "{{ cert_email }}" 16 | tls: true 17 | 18 | # optional second-host for signing archives 19 | {% if signing_domain %} 20 | signer: 21 | enabled: true 22 | host: {{ signing_domain }} 23 | cert_email: {{ cert_email }} 24 | image_pull_policy: "IfNotPresent" 25 | auth_token: {{ signing_authtoken }} 26 | {% endif %} 27 | 28 | # required for microk8s 29 | ingress_class: "public" 30 | 31 | -------------------------------------------------------------------------------- /ansible/roles/microk8s/debian/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # =========================================== 3 | # Install microk8s 4 | - name: microk8s | ensure dependencies are installed (Debian) 5 | ansible.builtin.apt: 6 | name: 7 | - snapd 8 | - fuse 9 | - udev 10 | - git 11 | - acl 12 | state: present 13 | update_cache: true 14 | cache_valid_time: "{{ microk8s_cache_valid_time }}" 15 | when: 16 | - ansible_os_family == "Debian" 17 | tags: 18 | - microk8s 19 | - microk8s.dependencies 20 | - microk8s.dependencies.apt 21 | 22 | - name: microk8s | start and enable services (Debian) 23 | ansible.builtin.service: 24 | name: "{{ microk8s_service }}" 25 | state: started 26 | enabled: true 27 | loop: 28 | - udev 29 | loop_control: 30 | loop_var: microk8s_service 31 | label: "{{ microk8s_service }}" 32 | when: 33 | - ansible_os_family == "Debian" 34 | tags: 35 | - microk8s 36 | - microk8s.dependencies 37 | - microk8s.dependencies.services 38 | -------------------------------------------------------------------------------- /ansible/roles/reset/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Disable services 3 | systemd: 4 | name: "{{ item }}" 5 | state: stopped 6 | enabled: no 7 | failed_when: false 8 | with_items: 9 | - k3s 10 | - k3s-node 11 | 12 | - name: pkill -9 -f "k3s/data/[^/]+/bin/containerd-shim-runc" 13 | register: pkill_containerd_shim_runc 14 | command: pkill -9 -f "k3s/data/[^/]+/bin/containerd-shim-runc" 15 | changed_when: "pkill_containerd_shim_runc.rc == 0" 16 | failed_when: false 17 | 18 | - name: Umount k3s filesystems 19 | include_tasks: umount_with_children.yml 20 | with_items: 21 | - /run/k3s 22 | - /var/lib/kubelet 23 | - /run/netns 24 | - /var/lib/rancher/k3s 25 | loop_control: 26 | loop_var: mounted_fs 27 | 28 | - name: Remove service files, binaries and data 29 | file: 30 | name: "{{ item }}" 31 | state: absent 32 | with_items: 33 | - /usr/local/bin/k3s 34 | - "{{ systemd_dir }}/k3s.service" 35 | - "{{ systemd_dir }}/k3s-node.service" 36 | - /etc/rancher/k3s 37 | - /var/lib/kubelet 38 | - /var/lib/rancher/k3s 39 | 40 | - name: daemon_reload 41 | systemd: 42 | daemon_reload: yes 43 | -------------------------------------------------------------------------------- /ansible/roles/reset/tasks/umount_with_children.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Get the list of mounted filesystems 3 | shell: set -o pipefail && cat /proc/mounts | awk '{ print $2}' | grep -E "^{{ mounted_fs }}" 4 | register: get_mounted_filesystems 5 | args: 6 | executable: /bin/bash 7 | failed_when: false 8 | changed_when: get_mounted_filesystems.stdout | length > 0 9 | check_mode: false 10 | 11 | - name: Umount filesystem 12 | mount: 13 | path: "{{ item }}" 14 | state: unmounted 15 | with_items: 16 | "{{ get_mounted_filesystems.stdout_lines | reverse | list }}" 17 | -------------------------------------------------------------------------------- /backend/.pylintrc: -------------------------------------------------------------------------------- 1 | [MAIN] 2 | extension-pkg-whitelist=pydantic 3 | ignore-paths=btrixcloud/migrations/migration_0028_page_files_errors.py 4 | disable=too-many-positional-arguments,invalid-name 5 | -------------------------------------------------------------------------------- /backend/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM docker.io/library/python:3.12-slim 2 | 3 | RUN apt-get update \ 4 | && apt-get install -y --no-install-recommends git \ 5 | && apt-get purge -y --auto-remove \ 6 | && rm -rf /var/lib/apt/lists/* 7 | 8 | WORKDIR /app 9 | 10 | ADD requirements.txt /app 11 | 12 | RUN pip install -r requirements.txt 13 | 14 | ADD btrixcloud/ /app/btrixcloud/ 15 | 16 | EXPOSE 8000 17 | -------------------------------------------------------------------------------- /backend/btrixcloud/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webrecorder/browsertrix/54d29aec0585c358126861c03c2212596c5d2c3f/backend/btrixcloud/__init__.py -------------------------------------------------------------------------------- /backend/btrixcloud/migrations/migration_0002_crawlconfig_crawlstats.py: -------------------------------------------------------------------------------- 1 | """ 2 | Migration 0002 - Dropping CrawlConfig crawl stats 3 | """ 4 | 5 | from btrixcloud.migrations import BaseMigration 6 | 7 | 8 | MIGRATION_VERSION = "0002" 9 | 10 | 11 | class Migration(BaseMigration): 12 | """Migration class.""" 13 | 14 | # pylint: disable=unused-argument 15 | def __init__(self, mdb, **kwargs): 16 | super().__init__(mdb, migration_version=MIGRATION_VERSION) 17 | 18 | async def migrate_up(self): 19 | """Perform migration up. 20 | 21 | Drop crawl statistics fields from crawl_config collection documents 22 | as these are now generated dynamically from a join as needed in API 23 | endpoints. 24 | """ 25 | crawl_configs = self.mdb["crawl_configs"] 26 | await crawl_configs.update_many({}, {"$unset": {"crawlCount": 1}}) 27 | await crawl_configs.update_many({}, {"$unset": {"lastCrawlId": 1}}) 28 | await crawl_configs.update_many({}, {"$unset": {"lastCrawlTime": 1}}) 29 | await crawl_configs.update_many({}, {"$unset": {"lastCrawlState": 1}}) 30 | -------------------------------------------------------------------------------- /backend/btrixcloud/migrations/migration_0006_precompute_crawl_stats.py: -------------------------------------------------------------------------------- 1 | """ 2 | Migration 0006 - Precomputing workflow crawl stats 3 | """ 4 | 5 | from btrixcloud.crawlconfigs import stats_recompute_all 6 | from btrixcloud.migrations import BaseMigration 7 | 8 | 9 | MIGRATION_VERSION = "0006" 10 | 11 | 12 | class Migration(BaseMigration): 13 | """Migration class.""" 14 | 15 | # pylint: disable=unused-argument 16 | def __init__(self, mdb, **kwargs): 17 | super().__init__(mdb, migration_version=MIGRATION_VERSION) 18 | 19 | async def migrate_up(self): 20 | """Perform migration up. 21 | 22 | Add data on workflow crawl statistics that was previously dynamically 23 | computed when needed to the database. 24 | """ 25 | # pylint: disable=duplicate-code 26 | crawl_configs = self.mdb["crawl_configs"] 27 | crawls = self.mdb["crawls"] 28 | 29 | async for config in crawl_configs.find({"inactive": {"$ne": True}}): 30 | config_id = config["_id"] 31 | try: 32 | await stats_recompute_all(crawl_configs, crawls, config_id) 33 | # pylint: disable=broad-exception-caught 34 | except Exception as err: 35 | print(f"Unable to update workflow {config_id}: {err}", flush=True) 36 | -------------------------------------------------------------------------------- /backend/btrixcloud/migrations/migration_0008_precompute_crawl_file_stats.py: -------------------------------------------------------------------------------- 1 | """ 2 | Migration 0008 - Precomputing crawl file stats 3 | """ 4 | 5 | from btrixcloud.crawls import recompute_crawl_file_count_and_size 6 | from btrixcloud.migrations import BaseMigration 7 | 8 | 9 | MIGRATION_VERSION = "0008" 10 | 11 | 12 | class Migration(BaseMigration): 13 | """Migration class.""" 14 | 15 | # pylint: disable=unused-argument 16 | def __init__(self, mdb, **kwargs): 17 | super().__init__(mdb, migration_version=MIGRATION_VERSION) 18 | 19 | async def migrate_up(self): 20 | """Perform migration up. 21 | 22 | Add data on crawl file count and size to database that was previously 23 | dynamically generated in the API endpoints. 24 | """ 25 | # pylint: disable=duplicate-code 26 | crawls = self.mdb["crawls"] 27 | 28 | async for crawl in crawls.find({}): 29 | crawl_id = crawl["_id"] 30 | try: 31 | await recompute_crawl_file_count_and_size(crawls, crawl_id) 32 | # pylint: disable=broad-exception-caught 33 | except Exception as err: 34 | print(f"Unable to update crawl {crawl_id}: {err}", flush=True) 35 | -------------------------------------------------------------------------------- /backend/btrixcloud/migrations/migration_0009_crawl_types.py: -------------------------------------------------------------------------------- 1 | """ 2 | Migration 0009 - Crawl types 3 | """ 4 | 5 | from btrixcloud.migrations import BaseMigration 6 | 7 | 8 | MIGRATION_VERSION = "0009" 9 | 10 | 11 | class Migration(BaseMigration): 12 | """Migration class.""" 13 | 14 | # pylint: disable=unused-argument 15 | def __init__(self, mdb, **kwargs): 16 | super().__init__(mdb, migration_version=MIGRATION_VERSION) 17 | 18 | async def migrate_up(self): 19 | """Perform migration up. 20 | 21 | Add type "crawl" to all existing crawls that don't already have a type 22 | """ 23 | # pylint: disable=duplicate-code 24 | crawls = self.mdb["crawls"] 25 | try: 26 | await crawls.update_many( 27 | {"type": {"$eq": None}}, {"$set": {"type": "crawl"}} 28 | ) 29 | # pylint: disable=broad-exception-caught 30 | except Exception as err: 31 | print(f"Error adding type 'crawl' to existing crawls: {err}", flush=True) 32 | -------------------------------------------------------------------------------- /backend/btrixcloud/migrations/migration_0010_collection_total_size.py: -------------------------------------------------------------------------------- 1 | """ 2 | Migration 0010 - Precomputing collection total size 3 | """ 4 | 5 | from btrixcloud.colls import CollectionOps 6 | from btrixcloud.migrations import BaseMigration 7 | 8 | 9 | MIGRATION_VERSION = "0010" 10 | 11 | 12 | class Migration(BaseMigration): 13 | """Migration class.""" 14 | 15 | # pylint: disable=unused-argument 16 | def __init__(self, mdb, **kwargs): 17 | super().__init__(mdb, migration_version=MIGRATION_VERSION) 18 | 19 | async def migrate_up(self): 20 | """Perform migration up. 21 | 22 | Recompute collection data to include totalSize. 23 | """ 24 | # pylint: disable=duplicate-code 25 | coll_ops = CollectionOps(self.mdb, None, None, None) 26 | 27 | async for coll in coll_ops.collections.find({}): 28 | coll_id = coll["_id"] 29 | try: 30 | await coll_ops.update_collection_counts_and_tags(coll_id) 31 | # pylint: disable=broad-exception-caught 32 | except Exception as err: 33 | print(f"Unable to update collection {coll_id}: {err}", flush=True) 34 | -------------------------------------------------------------------------------- /backend/btrixcloud/migrations/migration_0012_notes_to_description.py: -------------------------------------------------------------------------------- 1 | """ 2 | Migration 0012 - Notes to description 3 | """ 4 | 5 | from btrixcloud.migrations import BaseMigration 6 | 7 | 8 | MIGRATION_VERSION = "0012" 9 | 10 | 11 | class Migration(BaseMigration): 12 | """Migration class.""" 13 | 14 | # pylint: disable=unused-argument 15 | def __init__(self, mdb, **kwargs): 16 | super().__init__(mdb, migration_version=MIGRATION_VERSION) 17 | 18 | async def migrate_up(self): 19 | """Perform migration up. 20 | 21 | Rename crawl notes field to description. 22 | """ 23 | # pylint: disable=duplicate-code 24 | crawls = self.mdb["crawls"] 25 | try: 26 | await crawls.update_many({}, {"$rename": {"notes": "description"}}) 27 | # pylint: disable=broad-exception-caught 28 | except Exception as err: 29 | print(f"Error renaming crawl notes to description: {err}", flush=True) 30 | -------------------------------------------------------------------------------- /backend/btrixcloud/migrations/migration_0014_to_collection_ids.py: -------------------------------------------------------------------------------- 1 | """ 2 | Migration 0014 - collections to collectionIDs 3 | """ 4 | 5 | from btrixcloud.migrations import BaseMigration 6 | 7 | 8 | MIGRATION_VERSION = "0014" 9 | 10 | 11 | class Migration(BaseMigration): 12 | """Migration class.""" 13 | 14 | # pylint: disable=unused-argument 15 | def __init__(self, mdb, **kwargs): 16 | super().__init__(mdb, migration_version=MIGRATION_VERSION) 17 | 18 | async def migrate_up(self): 19 | """Perform migration up. 20 | 21 | Rename crawl 'collections' field to 'collectionIds' 22 | """ 23 | # pylint: disable=duplicate-code 24 | crawls = self.mdb["crawls"] 25 | try: 26 | await crawls.update_many({}, {"$rename": {"collections": "collectionIds"}}) 27 | # pylint: disable=broad-exception-caught 28 | except Exception as err: 29 | print( 30 | f"Error renaming crawl 'collections' to 'collectionIds': {err}", 31 | flush=True, 32 | ) 33 | -------------------------------------------------------------------------------- /backend/btrixcloud/migrations/migration_0019_org_slug.py: -------------------------------------------------------------------------------- 1 | """ 2 | Migration 0019 - Organization slug 3 | """ 4 | 5 | from btrixcloud.migrations import BaseMigration 6 | from btrixcloud.utils import slug_from_name 7 | 8 | 9 | MIGRATION_VERSION = "0019" 10 | 11 | 12 | class Migration(BaseMigration): 13 | """Migration class.""" 14 | 15 | # pylint: disable=unused-argument 16 | def __init__(self, mdb, **kwargs): 17 | super().__init__(mdb, migration_version=MIGRATION_VERSION) 18 | 19 | async def migrate_up(self): 20 | """Perform migration up. 21 | 22 | Add slug to all existing orgs. 23 | """ 24 | # pylint: disable=duplicate-code 25 | mdb_orgs = self.mdb["organizations"] 26 | async for org in mdb_orgs.find({"slug": {"$eq": None}}): 27 | oid = org["_id"] 28 | slug = slug_from_name(org["name"]) 29 | try: 30 | await mdb_orgs.find_one_and_update( 31 | {"_id": oid}, {"$set": {"slug": slug}} 32 | ) 33 | # pylint: disable=broad-exception-caught 34 | except Exception as err: 35 | print(f"Error adding slug to org {oid}: {err}", flush=True) 36 | -------------------------------------------------------------------------------- /backend/btrixcloud/migrations/migration_0025_workflow_db_configmap_fixes.py: -------------------------------------------------------------------------------- 1 | """ 2 | Migration 0025 -- fix workflow database and configmap issues. 3 | """ 4 | 5 | from btrixcloud.migrations import BaseMigration 6 | 7 | 8 | MIGRATION_VERSION = "0025" 9 | 10 | 11 | class Migration(BaseMigration): 12 | """Migration class.""" 13 | 14 | # pylint: disable=unused-argument 15 | def __init__(self, mdb, **kwargs): 16 | super().__init__(mdb, migration_version=MIGRATION_VERSION) 17 | 18 | async def migrate_up(self): 19 | """Perform migration up. 20 | 21 | Set crawlTimeout to 0 in any workflows where it is not set, and 22 | update configmap for each workflow to include crawlerChannel. 23 | """ 24 | mdb_crawl_configs = self.mdb["crawl_configs"] 25 | try: 26 | await mdb_crawl_configs.update_many( 27 | {"crawlTimeout": None}, 28 | {"$set": {"crawlTimeout": 0}}, 29 | ) 30 | # pylint: disable=broad-except 31 | except Exception: 32 | print( 33 | "Error updating null crawlconfig crawlTimeouts to 0", 34 | flush=True, 35 | ) 36 | -------------------------------------------------------------------------------- /backend/btrixcloud/migrations/migration_0027_profile_modified.py: -------------------------------------------------------------------------------- 1 | """ 2 | Migration 0027 - Profile modified date fallback 3 | """ 4 | 5 | from btrixcloud.migrations import BaseMigration 6 | 7 | 8 | MIGRATION_VERSION = "0027" 9 | 10 | 11 | class Migration(BaseMigration): 12 | """Migration class.""" 13 | 14 | # pylint: disable=unused-argument 15 | def __init__(self, mdb, **kwargs): 16 | super().__init__(mdb, migration_version=MIGRATION_VERSION) 17 | 18 | async def migrate_up(self): 19 | """Perform migration up. 20 | 21 | If profile doesn't have modified date, set to created 22 | """ 23 | # pylint: disable=duplicate-code 24 | profiles = self.mdb["profiles"] 25 | try: 26 | await profiles.update_many( 27 | {"modified": None}, [{"$set": {"modified": "$created"}}] 28 | ) 29 | # pylint: disable=broad-exception-caught 30 | except Exception as err: 31 | print( 32 | f"Error adding modified date to profiles: {err}", 33 | flush=True, 34 | ) 35 | -------------------------------------------------------------------------------- /backend/btrixcloud/migrations/migration_0029_remove_workflow_configmaps.py: -------------------------------------------------------------------------------- 1 | """ 2 | Migration 0028 - Page files and errors 3 | """ 4 | 5 | from btrixcloud.migrations import BaseMigration 6 | from btrixcloud.crawlmanager import CrawlManager 7 | 8 | 9 | MIGRATION_VERSION = "0029" 10 | 11 | 12 | class Migration(BaseMigration): 13 | """Migration class.""" 14 | 15 | # pylint: disable=unused-argument 16 | def __init__(self, mdb, **kwargs): 17 | super().__init__(mdb, migration_version=MIGRATION_VERSION) 18 | 19 | async def migrate_up(self): 20 | """delete all workflow-scoped configmaps""" 21 | crawl_manager = CrawlManager() 22 | await crawl_manager.core_api.delete_collection_namespaced_config_map( 23 | namespace=crawl_manager.namespace, label_selector="btrix.crawlconfig" 24 | ) 25 | -------------------------------------------------------------------------------- /backend/btrixcloud/migrations/migration_0034_drop_invalid_crc.py: -------------------------------------------------------------------------------- 1 | """ 2 | Migration 0034 -- remove crc32 from CrawlFile 3 | """ 4 | 5 | from btrixcloud.migrations import BaseMigration 6 | 7 | 8 | MIGRATION_VERSION = "0034" 9 | 10 | 11 | class Migration(BaseMigration): 12 | """Migration class.""" 13 | 14 | # pylint: disable=unused-argument 15 | def __init__(self, mdb, **kwargs): 16 | super().__init__(mdb, migration_version=MIGRATION_VERSION) 17 | 18 | async def migrate_up(self): 19 | """Perform migration up. 20 | 21 | Remove crc32 field from all crawl files 22 | """ 23 | crawls_db = self.mdb["crawls"] 24 | 25 | try: 26 | res = await crawls_db.update_many( 27 | {"files.crc32": {"$exists": 1}}, 28 | {"$unset": {"files.$[].crc32": 1}}, 29 | ) 30 | updated = res.modified_count 31 | print(f"{updated} crawls migrated to remove crc32 from files", flush=True) 32 | # pylint: disable=broad-exception-caught 33 | except Exception as err: 34 | print( 35 | f"Error migrating crawl files to remove crc32: {err}", 36 | flush=True, 37 | ) 38 | -------------------------------------------------------------------------------- /backend/btrixcloud/migrations/migration_0035_fix_failed_logins.py: -------------------------------------------------------------------------------- 1 | """ 2 | Migration 0035 -- fix model for failed logins 3 | """ 4 | 5 | from btrixcloud.migrations import BaseMigration 6 | 7 | 8 | MIGRATION_VERSION = "0035" 9 | 10 | 11 | class Migration(BaseMigration): 12 | """Migration class.""" 13 | 14 | # pylint: disable=unused-argument 15 | def __init__(self, mdb, **kwargs): 16 | super().__init__(mdb, migration_version=MIGRATION_VERSION) 17 | 18 | async def migrate_up(self): 19 | """Perform migration up. 20 | 21 | Set created from attempted.attempted 22 | """ 23 | failed_logins = self.mdb["logins"] 24 | 25 | try: 26 | res = await failed_logins.update_many( 27 | {"attempted.attempted": {"$exists": 1}}, 28 | [{"$set": {"attempted": "$attempted.attempted"}}], 29 | ) 30 | updated = res.modified_count 31 | print(f"{updated} failed logins fixed", flush=True) 32 | # pylint: disable=broad-exception-caught 33 | except Exception as err: 34 | print( 35 | f"Error fixing failed logins: {err}", 36 | flush=True, 37 | ) 38 | -------------------------------------------------------------------------------- /backend/btrixcloud/operator/__init__.py: -------------------------------------------------------------------------------- 1 | """operators module""" 2 | 3 | from .profiles import ProfileOperator 4 | from .bgjobs import BgJobOperator 5 | from .cronjobs import CronJobOperator 6 | from .crawls import CrawlOperator 7 | from .baseoperator import K8sOpAPI 8 | 9 | operator_classes = [ProfileOperator, BgJobOperator, CronJobOperator, CrawlOperator] 10 | 11 | 12 | # ============================================================================ 13 | def init_operator_api(app, *args): 14 | """registers webhook handlers for metacontroller""" 15 | 16 | k8s = K8sOpAPI() 17 | 18 | operators = [] 19 | for cls in operator_classes: 20 | oper = cls(k8s, *args) 21 | oper.init_routes(app) 22 | operators.append(oper) 23 | 24 | @app.get("/healthz", include_in_schema=False) 25 | async def healthz(): 26 | return {} 27 | 28 | return k8s 29 | -------------------------------------------------------------------------------- /backend/btrixcloud/pagination.py: -------------------------------------------------------------------------------- 1 | """API pagination""" 2 | 3 | from typing import Any, List, Optional 4 | 5 | 6 | DEFAULT_PAGE_SIZE = 1_000 7 | 8 | 9 | # ============================================================================ 10 | def paginated_format( 11 | items: Optional[List[Any]], 12 | total: int, 13 | page: int = 1, 14 | page_size: int = DEFAULT_PAGE_SIZE, 15 | ): 16 | """Return items in paged format.""" 17 | return {"items": items, "total": total, "page": page, "pageSize": page_size} 18 | -------------------------------------------------------------------------------- /backend/btrixcloud/version.py: -------------------------------------------------------------------------------- 1 | """current version""" 2 | 3 | __version__ = "1.17.0-beta.0" 4 | -------------------------------------------------------------------------------- /backend/dev-requirements.txt: -------------------------------------------------------------------------------- 1 | black 2 | pylint 3 | mypy==1.10.1 4 | -------------------------------------------------------------------------------- /backend/mypy.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | plugins = pydantic.mypy 3 | ignore_missing_imports = True 4 | disable_error_code = var-annotated 5 | -------------------------------------------------------------------------------- /backend/requirements.txt: -------------------------------------------------------------------------------- 1 | gunicorn 2 | uvicorn[standard] 3 | fastapi==0.103.2 4 | motor 5 | passlib 6 | PyJWT==2.8.0 7 | pydantic==2.8.2 8 | email-validator 9 | loguru 10 | aiofiles 11 | kubernetes-asyncio==29.0.0 12 | kubernetes 13 | aiobotocore 14 | requests 15 | redis>=5.0.0 16 | pyyaml 17 | jinja2 18 | humanize 19 | python-multipart 20 | pathvalidate 21 | https://github.com/ikreymer/stream-zip/archive/refs/heads/crc32-optional.zip 22 | backoff>=2.2.1 23 | python-slugify>=8.0.1 24 | types_aiobotocore_s3 25 | types-redis 26 | types-python-slugify 27 | types-pyYAML 28 | remotezip 29 | json-stream 30 | aiostream 31 | iso639-lang>=2.6.0 32 | -------------------------------------------------------------------------------- /backend/test-requirements.txt: -------------------------------------------------------------------------------- 1 | -r requirements.txt 2 | 3 | pytest 4 | requests 5 | boto3 6 | -------------------------------------------------------------------------------- /backend/test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webrecorder/browsertrix/54d29aec0585c358126861c03c2212596c5d2c3f/backend/test/__init__.py -------------------------------------------------------------------------------- /backend/test/data/example-2.wacz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webrecorder/browsertrix/54d29aec0585c358126861c03c2212596c5d2c3f/backend/test/data/example-2.wacz -------------------------------------------------------------------------------- /backend/test/data/example.wacz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webrecorder/browsertrix/54d29aec0585c358126861c03c2212596c5d2c3f/backend/test/data/example.wacz -------------------------------------------------------------------------------- /backend/test/data/thumbnail.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webrecorder/browsertrix/54d29aec0585c358126861c03c2212596c5d2c3f/backend/test/data/thumbnail.jpg -------------------------------------------------------------------------------- /backend/test/echo_server.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | A web server to record POST requests and return them on a GET request 4 | """ 5 | from http.server import HTTPServer, BaseHTTPRequestHandler 6 | import json 7 | 8 | BIND_HOST = "0.0.0.0" 9 | PORT = 18080 10 | 11 | post_bodies = [] 12 | 13 | 14 | class EchoServerHTTPRequestHandler(BaseHTTPRequestHandler): 15 | def do_GET(self): 16 | self.send_response(200) 17 | self.end_headers() 18 | self.wfile.write(json.dumps({"post_bodies": post_bodies}).encode("utf-8")) 19 | 20 | def do_POST(self): 21 | content_length = int(self.headers.get("content-length", 0)) 22 | body = self.rfile.read(content_length) 23 | self.send_response(200) 24 | if self.path.endswith("/portalUrl"): 25 | self.send_header("Content-Type", "application/json") 26 | self.end_headers() 27 | self.wfile.write( 28 | json.dumps({"portalUrl": "https://portal.example.com/path/"}).encode( 29 | "utf-8" 30 | ) 31 | ) 32 | else: 33 | self.end_headers() 34 | 35 | post_bodies.append(json.loads(body.decode("utf-8").replace("'", '"'))) 36 | 37 | 38 | httpd = HTTPServer((BIND_HOST, PORT), EchoServerHTTPRequestHandler) 39 | httpd.serve_forever() 40 | -------------------------------------------------------------------------------- /backend/test/test_login.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | from .conftest import API_PREFIX, ADMIN_USERNAME, ADMIN_PW 4 | 5 | 6 | def test_login_invalid(): 7 | password = "invalid" 8 | r = requests.post( 9 | f"{API_PREFIX}/auth/jwt/login", 10 | data={ 11 | "username": ADMIN_USERNAME, 12 | "password": password, 13 | "grant_type": "password", 14 | }, 15 | ) 16 | data = r.json() 17 | 18 | assert r.status_code == 400 19 | assert data["detail"] == "login_bad_credentials" 20 | 21 | 22 | def test_login(): 23 | r = requests.post( 24 | f"{API_PREFIX}/auth/jwt/login", 25 | data={ 26 | "username": ADMIN_USERNAME, 27 | "password": ADMIN_PW, 28 | "grant_type": "password", 29 | }, 30 | ) 31 | data = r.json() 32 | 33 | assert r.status_code == 200, data["detail"] 34 | assert data["token_type"] == "bearer" 35 | assert data["access_token"] 36 | access_token = data["access_token"] 37 | -------------------------------------------------------------------------------- /backend/test/test_utils.py: -------------------------------------------------------------------------------- 1 | """utils tests""" 2 | 3 | import pytest 4 | 5 | from btrixcloud.utils import slug_from_name 6 | 7 | 8 | @pytest.mark.parametrize( 9 | "name,expected_slug", 10 | [ 11 | ("Default org", "default-org"), 12 | ("User's org", "users-org"), 13 | ("User's @ org", "users-org"), 14 | ("Org with åccénted charactêrs", "org-with-accented-characters"), 15 | ("Org with åccénted! charactêrs@!", "org-with-accented-characters"), 16 | ("cATs! 🐈🐈‍⬛", "cats"), 17 | ], 18 | ) 19 | def test_slug_from_name(name: str, expected_slug: str): 20 | assert slug_from_name(name) == expected_slug 21 | -------------------------------------------------------------------------------- /backend/test/utils.py: -------------------------------------------------------------------------------- 1 | """Test utilities.""" 2 | 3 | 4 | def read_in_chunks(fh, blocksize=1024): 5 | """Lazy function (generator) to read a file piece by piece. 6 | Default chunk size: 1k.""" 7 | while True: 8 | data = fh.read(blocksize) 9 | if not data: 10 | break 11 | yield data 12 | -------------------------------------------------------------------------------- /backend/test_nightly/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webrecorder/browsertrix/54d29aec0585c358126861c03c2212596c5d2c3f/backend/test_nightly/__init__.py -------------------------------------------------------------------------------- /backend/test_nightly/test_crawl_errors.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | from .conftest import API_PREFIX 4 | 5 | 6 | def test_get_crawl_errors(admin_auth_headers, default_org_id, error_crawl_id): 7 | r = requests.get( 8 | f"{API_PREFIX}/orgs/{default_org_id}/crawls/{error_crawl_id}/errors", 9 | headers=admin_auth_headers, 10 | ) 11 | assert r.status_code == 200 12 | data = r.json() 13 | assert data["total"] > 0 14 | assert data["items"] 15 | -------------------------------------------------------------------------------- /backend/test_nightly/test_max_crawl_size_limit.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import time 3 | 4 | from .conftest import API_PREFIX 5 | 6 | 7 | def test_max_crawl_size(admin_auth_headers, default_org_id, max_crawl_size_crawl_id): 8 | # Verify that crawl has started 9 | r = requests.get( 10 | f"{API_PREFIX}/orgs/{default_org_id}/crawls/{max_crawl_size_crawl_id}/replay.json", 11 | headers=admin_auth_headers, 12 | ) 13 | assert r.status_code == 200 14 | data = r.json() 15 | assert data["state"] in ( 16 | "starting", 17 | "running", 18 | "generate-wacz", 19 | "uploading-wacz", 20 | "pending-wait", 21 | ) 22 | 23 | # Wait some time to let crawl start, hit max size limit, and gracefully stop 24 | time.sleep(240) 25 | 26 | # Verify crawl was stopped 27 | r = requests.get( 28 | f"{API_PREFIX}/orgs/{default_org_id}/crawls/{max_crawl_size_crawl_id}/replay.json", 29 | headers=admin_auth_headers, 30 | ) 31 | assert r.status_code == 200 32 | data = r.json() 33 | assert data["state"] == "complete" 34 | -------------------------------------------------------------------------------- /chart/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | frontend/ 25 | -------------------------------------------------------------------------------- /chart/Chart.lock: -------------------------------------------------------------------------------- 1 | dependencies: 2 | - name: btrix-admin-logging 3 | repository: file://./admin/logging 4 | version: 0.1.0 5 | - name: btrix-crds 6 | repository: file://./btrix-crds 7 | version: 0.1.1 8 | - name: metacontroller-helm 9 | repository: oci://ghcr.io/metacontroller 10 | version: 4.11.11 11 | - name: btrix-proxies 12 | repository: file://./proxies/ 13 | version: 0.1.0 14 | digest: sha256:2fd9472f857e9e3eacdcc616a3cffac5bb2951411cc2d34aea84253092225ecf 15 | generated: "2024-08-15T11:19:17.884682494+02:00" 16 | -------------------------------------------------------------------------------- /chart/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: browsertrix 3 | description: A chart for running the Webrecorder Browsertrix System 4 | type: application 5 | icon: https://webrecorder.net/assets/icon.png 6 | 7 | # Browsertrix and Chart Version 8 | version: v1.17.0-beta.0 9 | 10 | dependencies: 11 | - name: btrix-admin-logging 12 | version: 0.1.0 13 | condition: addons.admin.logging 14 | repository: file://./admin/logging 15 | - name: btrix-crds 16 | version: 0.1.1 17 | repository: file://./btrix-crds 18 | - name: metacontroller-helm 19 | version: 4.11.11 20 | repository: "oci://ghcr.io/metacontroller" 21 | - name: btrix-proxies 22 | version: 0.1.0 23 | condition: btrix-proxies.enabled 24 | repository: file://./proxies/ 25 | -------------------------------------------------------------------------------- /chart/README.md: -------------------------------------------------------------------------------- 1 | ## Update Helm dependencies 2 | 3 | * It needs to update Helm charts after changing its dependencies (e.g. logging) 4 | 5 | ``` 6 | $ helm dependency update . 7 | ``` 8 | 9 | ### Update metacontroller 10 | 11 | ``` 12 | #!/bin/bash 13 | 14 | # intall metacontroller 15 | git clone --depth=1 https://github.com/metacontroller/metacontroller.git 16 | cd metacontroller 17 | helm package deploy/helm/metacontroller --destination deploy/helm 18 | cd .. 19 | 20 | # update dependency 21 | helm dependency update 22 | ``` 23 | 24 | * Bump up the metacontroller version in `Chart.yaml` 25 | -------------------------------------------------------------------------------- /chart/admin/logging/.gitignore: -------------------------------------------------------------------------------- 1 | old 2 | _tmp_prod_ 3 | -------------------------------------------------------------------------------- /chart/admin/logging/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: btrix-admin-logging 3 | description: A chart for running the Webrecorder Browsertrix System - admin services 4 | type: application 5 | icon: https://webrecorder.net/assets/icon.png 6 | 7 | # This is the chart version. This version number should be incremented each time you make changes 8 | # to the chart and its templates, including the app version. 9 | # Versions are expected to follow Semantic Versioning (https://semver.org/) 10 | version: 0.1.0 11 | 12 | # This is the version number of the application being deployed. This version number should be 13 | # incremented each time you make changes to the application. Versions are not expected to 14 | # follow Semantic Versioning. They should reflect the version the application is using. 15 | appVersion: 0.1.0 16 | -------------------------------------------------------------------------------- /chart/admin/logging/scripts/eck_install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | kubectl create namespace btrix-admin 4 | kubectl create -f https://download.elastic.co/downloads/eck/2.5.0/crds.yaml 5 | kubectl apply -f https://download.elastic.co/downloads/eck/2.5.0/operator.yaml 6 | 7 | # kubectl label nodes docker-desktop nodeType=admin 8 | kubectl get nodes 9 | kubectl get nodes -o wide -o jsonpath='{.items[*].metadata.labels}' | jq . 10 | -------------------------------------------------------------------------------- /chart/admin/logging/scripts/eck_uninstall.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | kubectl delete -f https://download.elastic.co/downloads/eck/2.5.0/operator.yaml 4 | kubectl delete -f https://download.elastic.co/downloads/eck/2.5.0/crds.yaml 5 | kubectl delete namespace btrix-admin 6 | -------------------------------------------------------------------------------- /chart/admin/logging/scripts/kibana_imports.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ES_USER="elastic" 4 | ES_PASS=$(kubectl get secret btrixlog-es-elastic-user -n btrix-admin -o go-template='{{.data.elastic | base64decode}}') 5 | KIBANA_INGRESS="kibana-main" 6 | HOSTNAME=`kubectl get ingress -A | grep $KIBANA_INGRESS | awk '{print $4}'` 7 | KIBANA_URL="https://${HOSTNAME}/kibana" 8 | EXPORT_FN="./kibana_export.ndjson" 9 | 10 | echo "use $KIBANA_URL" 11 | 12 | curl -k --user $ES_USER:$ES_PASS -X POST \ 13 | "${KIBANA_URL}/api/saved_objects/_import" \ 14 | -H "kbn-xsrf: true" \ 15 | --form file=@$EXPORT_FN 16 | -------------------------------------------------------------------------------- /chart/admin/logging/templates/logging.yaml: -------------------------------------------------------------------------------- 1 | {{ if .Values.logging.enabled }} 2 | 3 | {{ if not .Values.logging.fileMode }} 4 | {{ include "es.install" . }} 5 | {{ include "kb.install" . }} 6 | {{ include "ingress.install" . }} 7 | {{ end }} 8 | 9 | {{ include "fluentd.install" . }} 10 | 11 | {{ end }} 12 | -------------------------------------------------------------------------------- /chart/admin/logging/values.yaml: -------------------------------------------------------------------------------- 1 | logging: 2 | namespace: btrix-admin 3 | enabled: true 4 | dedicatedNode: 5 | enabled: true 6 | nodeType: admin 7 | fileMode: true 8 | ingress: 9 | tls: false 10 | host: localhost 11 | path: /kibana 12 | elasticsearch: 13 | local: false 14 | cpu: 1 15 | mem: 4Gi 16 | opt: -Xms2g -Xmx2g 17 | volumeEnabled: false 18 | volumeSize: 1Gi 19 | kibana: 20 | local: false 21 | cpu: 1 22 | mem: 1Gi 23 | opt: --max-old-space-size=1024 24 | fluentd: 25 | logVar: /var/log 26 | logPathContainers: /var/lib/docker/containers 27 | cpu: 60m 28 | mem: 200Mi 29 | -------------------------------------------------------------------------------- /chart/app-templates/crawl_configmap.yaml: -------------------------------------------------------------------------------- 1 | # ------- 2 | # CONFIGMAP 3 | # ------- 4 | apiVersion: v1 5 | kind: ConfigMap 6 | metadata: 7 | name: {{ name }} 8 | namespace: {{ namespace }} 9 | labels: 10 | crawl: {{ id }} 11 | role: crawler 12 | oid: {{ oid }} 13 | cid: {{ cid }} 14 | 15 | data: 16 | crawl-config.json: {{ config | tojson }} 17 | -------------------------------------------------------------------------------- /chart/app-templates/crawl_cron_job.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: batch/v1 2 | kind: CronJob 3 | metadata: 4 | name: "{{ id }}" 5 | labels: 6 | btrix.crawlconfig: "{{ cid }}" 7 | btrix.org: "{{ oid }}" 8 | {% if userid %} 9 | btrix.userid: "{{ userid }}" 10 | {% endif %} 11 | role: "cron-job" 12 | 13 | spec: 14 | concurrencyPolicy: Forbid 15 | successfulJobsHistoryLimit: 0 16 | failedJobsHistoryLimit: 2 17 | 18 | schedule: "{{ schedule }}" 19 | 20 | jobTemplate: 21 | metadata: 22 | labels: 23 | btrix.crawlconfig: "{{ cid }}" 24 | role: "scheduled-crawljob" 25 | 26 | spec: 27 | suspend: true 28 | template: 29 | spec: 30 | restartPolicy: Never 31 | containers: 32 | - name: noop 33 | image: "docker.io/tianon/true" 34 | imagePullPolicy: IfNotPresent 35 | -------------------------------------------------------------------------------- /chart/app-templates/crawl_job.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: btrix.cloud/v1 2 | kind: CrawlJob 3 | metadata: 4 | name: crawljob-{{ id }} 5 | labels: 6 | crawl: "{{ id }}" 7 | role: {{ "qa-job" if qa_source else "job" }} 8 | btrix.org: "{{ oid }}" 9 | btrix.user: "{{ userid }}" 10 | btrix.storage: "{{ storage_name }}" 11 | 12 | spec: 13 | selector: 14 | matchLabels: 15 | crawl: "{{ id }}" 16 | 17 | id: "{{ id }}" 18 | userid: "{{ userid }}" 19 | cid: "{{ cid }}" 20 | oid: "{{ oid }}" 21 | scale: {{ scale }} 22 | browserWindows: {{ browser_windows }} 23 | 24 | profile_filename: "{{ profile_filename }}" 25 | storage_filename: "{{ storage_filename }}" 26 | 27 | maxCrawlSize: {{ max_crawl_size if not qa_source else 0 }} 28 | timeout: {{ timeout if not qa_source else 0 }} 29 | qaSourceCrawlId: "{{ qa_source }}" 30 | 31 | manual: {{ manual }} 32 | crawlerChannel: "{{ crawler_channel }}" 33 | ttlSecondsAfterFinished: {{ 30 if not qa_source else 0 }} 34 | warcPrefix: "{{ warc_prefix }}" 35 | 36 | storageName: "{{ storage_name }}" 37 | 38 | proxyId: "{{ proxy_id }}" 39 | 40 | pausedAt: "{{ pausedAt }}" 41 | 42 | -------------------------------------------------------------------------------- /chart/app-templates/profile_job.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: btrix.cloud/v1 2 | kind: ProfileJob 3 | metadata: 4 | name: profilejob-{{ id }} 5 | labels: 6 | browser: "{{ id }}" 7 | role: "job" 8 | btrix.org: {{ oid }} 9 | btrix.user: {{ userid }} 10 | {%- if base_profile %} 11 | btrix.baseprofile: "{{ base_profile }}" 12 | {%- endif %} 13 | btrix.storage: "{{ storage_name }}" 14 | 15 | spec: 16 | selector: 17 | matchLabels: 18 | browser: "{{ id }}" 19 | 20 | id: "{{ id }}" 21 | userid: "{{ userid }}" 22 | oid: "{{ oid }}" 23 | 24 | storageName: "{{ storage_name }}" 25 | crawlerImage: "{{ crawler_image }}" 26 | imagePullPolicy: "{{ image_pull_policy }}" 27 | 28 | startUrl: "{{ url }}" 29 | profileFilename: "{{ profile_filename }}" 30 | vncPassword: "{{ vnc_password }}" 31 | 32 | proxyId: "{{ proxy_id }}" 33 | 34 | {% if expire_time %} 35 | expireTime: "{{ expire_time }}" 36 | {% endif %} 37 | -------------------------------------------------------------------------------- /chart/app-templates/qa_configmap.yaml: -------------------------------------------------------------------------------- 1 | # ------- 2 | # CONFIGMAP 3 | # ------- 4 | apiVersion: v1 5 | kind: ConfigMap 6 | metadata: 7 | name: {{ name }} 8 | namespace: {{ namespace }} 9 | labels: 10 | crawl: {{ id }} 11 | role: crawler 12 | 13 | data: 14 | qa-config.json: {{ qa_source_replay_json | tojson }} 15 | -------------------------------------------------------------------------------- /chart/btrix-crds/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: btrix-crds 3 | description: Webrecorder Browsertrix Custom CRDs 4 | type: application 5 | icon: https://webrecorder.net/assets/icon.png 6 | 7 | # This is the chart version. This version number should be incremented each time you make changes 8 | # to the chart and its templates, including the app version. 9 | # Versions are expected to follow Semantic Versioning (https://semver.org/) 10 | version: 0.1.1 11 | 12 | # This is the version number of the application being deployed. This version number should be 13 | # incremented each time you make changes to the application. Versions are not expected to 14 | # follow Semantic Versioning. They should reflect the version the application is using. 15 | appVersion: 0.1.1 16 | -------------------------------------------------------------------------------- /chart/btrix-crds/templates/profilejob.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | name: profilejobs.btrix.cloud 6 | spec: 7 | scope: Namespaced 8 | group: btrix.cloud 9 | names: 10 | kind: ProfileJob 11 | plural: profilejobs 12 | singular: profilejob 13 | shortNames: 14 | - pjs 15 | 16 | versions: 17 | - name: v1 18 | served: true 19 | storage: true 20 | subresources: 21 | status: {} 22 | 23 | schema: 24 | openAPIV3Schema: 25 | type: object 26 | properties: 27 | spec: 28 | type: object 29 | x-kubernetes-preserve-unknown-fields: true 30 | 31 | status: 32 | type: object 33 | x-kubernetes-preserve-unknown-fields: true 34 | 35 | additionalPrinterColumns: 36 | - name: Expire At 37 | type: string 38 | jsonPath: .spec.expireTime 39 | description: Time Browser will Expire 40 | 41 | - name: Start Url 42 | type: string 43 | jsonPath: .spec.startUrl 44 | description: Starting Url 45 | -------------------------------------------------------------------------------- /chart/charts/btrix-admin-logging-0.1.0.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webrecorder/browsertrix/54d29aec0585c358126861c03c2212596c5d2c3f/chart/charts/btrix-admin-logging-0.1.0.tgz -------------------------------------------------------------------------------- /chart/charts/btrix-crds-0.1.1.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webrecorder/browsertrix/54d29aec0585c358126861c03c2212596c5d2c3f/chart/charts/btrix-crds-0.1.1.tgz -------------------------------------------------------------------------------- /chart/charts/btrix-proxies-0.1.0.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webrecorder/browsertrix/54d29aec0585c358126861c03c2212596c5d2c3f/chart/charts/btrix-proxies-0.1.0.tgz -------------------------------------------------------------------------------- /chart/charts/metacontroller-helm-4.11.11.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webrecorder/browsertrix/54d29aec0585c358126861c03c2212596c5d2c3f/chart/charts/metacontroller-helm-4.11.11.tgz -------------------------------------------------------------------------------- /chart/email-templates/failed_bg_job: -------------------------------------------------------------------------------- 1 | Failed Background Job 2 | ~~~ 3 | Failed Background Job 4 | --------------------- 5 | {% if org %} 6 | Organization: {{ org.name }} ({{ job.oid }}) 7 | {% endif %} 8 | Job type: {{ job.type }} 9 | 10 | Job ID: {{ job.id }} 11 | Started: {{ job.started.isoformat(sep=" ", timespec="seconds") }}Z 12 | Finished: {{ finished.isoformat(sep=" ", timespec="seconds") }}Z 13 | 14 | {% if job.object_type %} 15 | Object type: {{ job.object_type }} 16 | {% endif %} 17 | 18 | {% if job.object_id %} 19 | Object ID: {{ job.object_id }} 20 | {% endif %} 21 | 22 | {% if job.file_path %} 23 | File path: {{ job.file_path }} 24 | {% endif %} 25 | 26 | {% if job.replica_storage %} 27 | Replica storage name: {{ job.replica_storage.name }} 28 | {% endif %} 29 | -------------------------------------------------------------------------------- /chart/email-templates/password_reset: -------------------------------------------------------------------------------- 1 | Password Reset 2 | ~~~ 3 | We received your password reset request. 4 | 5 | If you were locked out of your account, this request is sent automatically. 6 | 7 | If you did not attempt to log in and did not request this email, please let us know immediately at: 8 | {{ support_email }} 9 | 10 | Please click here: {{ origin }}/reset-password?token={{ token }} to create a new password. 11 | -------------------------------------------------------------------------------- /chart/email-templates/validate: -------------------------------------------------------------------------------- 1 | Welcome to Browsertrix, Verify your Registration. 2 | ~~~ 3 | Please verify your registration for Browsertrix for {{ receiver_email }} 4 | 5 | You can verify by clicking here: {{ origin }}/verify?token={{ token }} 6 | 7 | The verification token is: {{ token }} 8 | -------------------------------------------------------------------------------- /chart/examples/local-logging.yaml: -------------------------------------------------------------------------------- 1 | ingress: 2 | host: "myhostname" 3 | scheme: "https" 4 | tls: false 5 | 6 | ingress_class: "nginx" 7 | # for microk8s's ingress controller 8 | # ingress_class: "public" 9 | 10 | addons: 11 | admin: 12 | logging: true 13 | 14 | btrix-admin-logging: 15 | logging: 16 | enabled: true 17 | dedicatedNode: 18 | enabled: false 19 | ingress: 20 | tls: false 21 | class: "nginx" 22 | host: "myhostname" 23 | elasticsearch: 24 | local: true 25 | kibana: 26 | local: true 27 | -------------------------------------------------------------------------------- /chart/proxies/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: btrix-proxies 3 | description: A chart deploying the configmap and secrets required for using proxies with Browsertrix 4 | type: application 5 | icon: https://webrecorder.net/assets/icon.png 6 | 7 | # This is the chart version. This version number should be incremented each time you make changes 8 | # to the chart and its templates, including the app version. 9 | # Versions are expected to follow Semantic Versioning (https://semver.org/) 10 | version: 0.1.0 11 | 12 | # This is the version number of the application being deployed. This version number should be 13 | # incremented each time you make changes to the application. Versions are not expected to 14 | # follow Semantic Versioning. They should reflect the version the application is using. 15 | appVersion: 0.1.0 16 | -------------------------------------------------------------------------------- /chart/proxies/templates/proxies.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.proxies }} 2 | --- 3 | apiVersion: v1 4 | kind: Secret 5 | metadata: 6 | name: proxies 7 | namespace: {{ .Values.crawler_namespace | default "crawlers" }} 8 | type: Opaque 9 | stringData: 10 | {{- range .Values.proxies }} 11 | 12 | {{- if .ssh_private_key }} 13 | {{ .id }}-private-key: | 14 | {{ .ssh_private_key | indent 4 }} 15 | {{- end }} 16 | 17 | {{- if .ssh_host_public_key }} 18 | {{ .id }}-known-hosts: | 19 | {{ .ssh_host_public_key | indent 4 }} 20 | {{- end }} 21 | 22 | {{- end }} 23 | --- 24 | apiVersion: v1 25 | kind: Secret 26 | metadata: 27 | name: ops-proxy-configs 28 | namespace: {{ .Release.Namespace }} 29 | 30 | type: Opaque 31 | data: 32 | crawler_proxies_last_update: {{ now | unixEpoch | toString | b64enc | quote }} 33 | crawler_proxies.json: {{ .Values.proxies | toJson | b64enc | quote }} 34 | {{- end }} 35 | -------------------------------------------------------------------------------- /chart/proxies/values.yaml: -------------------------------------------------------------------------------- 1 | proxies: [] # see proxies description in main helm chart 2 | crawler_namespace: crawlers # namespace to deploy ssh keys to 3 | -------------------------------------------------------------------------------- /chart/templates/namespaces.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Namespace 4 | metadata: 5 | name: {{ .Values.crawler_namespace }} 6 | labels: 7 | release: {{ .Release.Name }} 8 | annotations: 9 | "helm.sh/resource-policy": keep 10 | 11 | 12 | -------------------------------------------------------------------------------- /chart/templates/priorities.yaml: -------------------------------------------------------------------------------- 1 | {{ $max_browser_windows := not (empty .Values.max_browser_windows) | ternary (int .Values.max_browser_windows) (mul (int .Values.max_crawl_scale) (int .Values.crawler_browser_instances) ) }} 2 | 3 | 4 | {{- range untilStep 0 $max_browser_windows 1 }} 5 | --- 6 | apiVersion: scheduling.k8s.io/v1 7 | kind: PriorityClass 8 | metadata: 9 | name: crawl-pri-{{ . }} 10 | value: {{ sub 0 . }} 11 | globalDefault: false 12 | description: "Priority for crawl instance #{{ . }}" 13 | 14 | {{- end }} 15 | 16 | {{- range untilStep 0 $max_browser_windows 1 }} 17 | --- 18 | apiVersion: scheduling.k8s.io/v1 19 | kind: PriorityClass 20 | metadata: 21 | name: qa-crawl-pri-{{ . }} 22 | value: {{ sub -2 . }} 23 | globalDefault: false 24 | description: "Priority for QA crawl instance #{{ . }}" 25 | 26 | {{- end }} 27 | 28 | # Lower Priority for Background Jobs 29 | --- 30 | apiVersion: scheduling.k8s.io/v1 31 | kind: PriorityClass 32 | metadata: 33 | name: bg-job 34 | value: -1000 35 | globalDefault: false 36 | description: "Priority for background jobs" 37 | 38 | 39 | -------------------------------------------------------------------------------- /chart/templates/service.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: crawler 6 | namespace: {{ .Values.crawler_namespace }} 7 | 8 | spec: 9 | clusterIP: None 10 | publishNotReadyAddresses: true 11 | selector: 12 | role: crawler 13 | 14 | ports: 15 | - protocol: TCP 16 | port: 9037 17 | name: screencast 18 | 19 | --- 20 | apiVersion: v1 21 | kind: Service 22 | metadata: 23 | name: redis 24 | namespace: {{ .Values.crawler_namespace }} 25 | 26 | spec: 27 | clusterIP: None 28 | publishNotReadyAddresses: true 29 | selector: 30 | role: redis 31 | 32 | ports: 33 | - protocol: TCP 34 | port: 6379 35 | name: redis 36 | 37 | --- 38 | apiVersion: v1 39 | kind: Service 40 | metadata: 41 | name: browser 42 | namespace: {{ .Values.crawler_namespace }} 43 | 44 | spec: 45 | clusterIP: None 46 | publishNotReadyAddresses: true 47 | selector: 48 | role: browser 49 | 50 | ports: 51 | - protocol: TCP 52 | port: 9223 53 | name: browser-api 54 | 55 | - protocol: TCP 56 | port: 9222 57 | name: browser-ws 58 | -------------------------------------------------------------------------------- /chart/test/microk8s-ci.yaml: -------------------------------------------------------------------------------- 1 | # microk8s overrides for ci 2 | # ------------------------- 3 | 4 | # use local images 5 | backend_image: "localhost:32000/webrecorder/browsertrix-backend:latest" 6 | frontend_image: "localhost:32000/webrecorder/browsertrix-frontend:latest" 7 | 8 | backend_pull_policy: "IfNotPresent" 9 | frontend_pull_policy: "IfNotPresent" 10 | 11 | # for testing only 12 | crawler_extra_cpu_per_browser: 300m 13 | 14 | crawler_extra_memory_per_browser: 256Mi 15 | -------------------------------------------------------------------------------- /chart/test/test-nightly-addons.yaml: -------------------------------------------------------------------------------- 1 | # additional settings for nightly tests 2 | 3 | invite_expire_seconds: 10 4 | 5 | max_pages_per_crawl: 300 6 | 7 | 8 | # enable to allow access to minio directly 9 | minio_local_access_port: 30090 10 | 11 | minio_local_bucket_name: &local_bucket_name "btrix-test-data" 12 | 13 | # for checking registration 14 | registration_enabled: "1" 15 | 16 | storages: 17 | - name: "default" 18 | type: "s3" 19 | access_key: "ADMIN" 20 | secret_key: "PASSW0RD" 21 | bucket_name: *local_bucket_name 22 | 23 | endpoint_url: "http://local-minio:9000/" 24 | is_default_primary: true 25 | access_endpoint_url: "/data/" 26 | 27 | - name: "replica-0" 28 | type: "s3" 29 | access_key: "ADMIN" 30 | secret_key: "PASSW0RD" 31 | bucket_name: "replica-0" 32 | 33 | endpoint_url: "http://local-minio:9000/" 34 | is_default_replica: true 35 | 36 | 37 | -------------------------------------------------------------------------------- /configs/signing.sample.yaml: -------------------------------------------------------------------------------- 1 | signing: 2 | domain: example.com # domain to retrieve a cert for (passed to ACME servers, required) 3 | email: test@example.com # email for acme auth (passed to ACME servers, required) 4 | port: 80 # local port for acme domain check (should be 80, change if running behind a proxy) 5 | 6 | output: /data # dir to store the keys and certs (for internal use) 7 | 8 | staging: False # generate staging certs 9 | 10 | # optional: set a 'cross-singing' CA and private key 11 | # this will be used along with ACME (Lets Encrypt) to sign the same CSR 12 | # csca_cert: 13 | # csca_private_key: 14 | 15 | # rfc3161 timestamp authority cert chain + timestamp urls 16 | # at least one required, if multiple, one is selected at random 17 | timestamping: 18 | # time server cert chain (cert + ca cert) 19 | # pkg:// url to load from python package data 20 | - certfile: pkg://authsign.trusted/ts-chain.pem 21 | url: http://freetsa.org/tsr # timeserver URL 22 | -------------------------------------------------------------------------------- /frontend/.dockerignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | src/stories 3 | -------------------------------------------------------------------------------- /frontend/.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | end_of_line = lf 5 | insert_final_newline = true 6 | indent_style = space 7 | indent_size = 2 8 | -------------------------------------------------------------------------------- /frontend/.gitignore: -------------------------------------------------------------------------------- 1 | ## editors 2 | /.idea 3 | 4 | ## system files 5 | .DS_Store 6 | 7 | ## npm 8 | /node_modules/ 9 | /npm-debug.log 10 | 11 | ## testing 12 | /coverage/ 13 | 14 | ## temp folders 15 | /.tmp/ 16 | /tmp/ 17 | 18 | # build 19 | /dist/ 20 | 21 | # dotenv 22 | .env 23 | .env.* 24 | 25 | storybook-static 26 | custom-elements.json 27 | /test-results/ 28 | /playwright-report/ 29 | /playwright/.cache/ 30 | 31 | *storybook.log 32 | -------------------------------------------------------------------------------- /frontend/.husky/pre-commit: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | . "$(dirname -- "$0")/_/husky.sh" 3 | 4 | # run hook only if frontend src changed 5 | if git diff --name-only --cached | grep --quiet 'frontend/src/'; 6 | then 7 | cd frontend 8 | npx lint-staged 9 | else 10 | echo "(no frontend/src changes - skipping pre-commit hook)" 11 | fi 12 | -------------------------------------------------------------------------------- /frontend/.prettierignore: -------------------------------------------------------------------------------- 1 | __generated__ 2 | __mocks__ 3 | assets 4 | dist 5 | docs 6 | -------------------------------------------------------------------------------- /frontend/.storybook/preview.ts: -------------------------------------------------------------------------------- 1 | import "@/global"; 2 | import "@/components/ui"; 3 | 4 | import { 5 | setCustomElementsManifest, 6 | type Preview, 7 | } from "@storybook/web-components"; 8 | 9 | import customElements from "@/__generated__/custom-elements.json"; 10 | 11 | // Automatically document component properties 12 | setCustomElementsManifest(customElements); 13 | 14 | const preview: Preview = { 15 | parameters: { 16 | actions: { argTypesRegex: "^on[A-Z].*" }, 17 | controls: { 18 | expanded: true, 19 | matchers: { 20 | color: /(background|color)$/i, 21 | date: /Date$/i, 22 | }, 23 | }, 24 | }, 25 | }; 26 | 27 | export default preview; 28 | -------------------------------------------------------------------------------- /frontend/.vscode: -------------------------------------------------------------------------------- 1 | ../.vscode -------------------------------------------------------------------------------- /frontend/.yarnrc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webrecorder/browsertrix/54d29aec0585c358126861c03c2212596c5d2c3f/frontend/.yarnrc -------------------------------------------------------------------------------- /frontend/README.md: -------------------------------------------------------------------------------- 1 | [Development guide](../docs/develop/frontend-dev.md) 2 | -------------------------------------------------------------------------------- /frontend/config/define.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Global constants to make available to build 3 | * 4 | * @TODO Consolidate webpack and web-test-runner esbuild configs 5 | */ 6 | const path = require("path"); 7 | 8 | const isDevServer = process.env.WEBPACK_SERVE; 9 | 10 | const dotEnvPath = path.resolve( 11 | process.cwd(), 12 | `.env${isDevServer ? `.local` : ""}`, 13 | ); 14 | require("dotenv").config({ 15 | path: dotEnvPath, 16 | }); 17 | 18 | const WEBSOCKET_HOST = 19 | isDevServer && process.env.API_BASE_URL 20 | ? new URL(process.env.API_BASE_URL).host 21 | : process.env.WEBSOCKET_HOST || ""; 22 | 23 | module.exports = { 24 | "window.process.env.WEBSOCKET_HOST": JSON.stringify(WEBSOCKET_HOST), 25 | }; 26 | -------------------------------------------------------------------------------- /frontend/config/tailwind/plugins/attributes.js: -------------------------------------------------------------------------------- 1 | const plugin = require("tailwindcss/plugin"); 2 | module.exports = plugin(function ({ matchVariant }) { 3 | matchVariant("attr", (value) => `&[${value}]`); 4 | matchVariant("group-attr", (value) => `:merge(.group)[${value}] &`); 5 | matchVariant("peer-attr", (value) => `:merge(.peer)[${value}] ~ &`); 6 | }); 7 | -------------------------------------------------------------------------------- /frontend/config/tailwind/plugins/contain.js: -------------------------------------------------------------------------------- 1 | const plugin = require("tailwindcss/plugin"); 2 | module.exports = plugin( 3 | ({ matchUtilities, theme }) => { 4 | matchUtilities( 5 | { 6 | contain: (value) => ({ 7 | contain: value, 8 | }), 9 | }, 10 | { values: theme("contain") }, 11 | ); 12 | }, 13 | { 14 | theme: { 15 | contain: { 16 | none: "none", 17 | strict: "strict", 18 | content: "content", 19 | size: "size", 20 | "inline-size": "inline-size", 21 | layout: "layout", 22 | style: "style", 23 | paint: "paint", 24 | }, 25 | }, 26 | }, 27 | ); 28 | -------------------------------------------------------------------------------- /frontend/config/tailwind/plugins/content-visibility.js: -------------------------------------------------------------------------------- 1 | const plugin = require("tailwindcss/plugin"); 2 | module.exports = plugin(function ({ addUtilities }) { 3 | addUtilities({ 4 | ".content-auto": { 5 | "content-visibility": "auto", 6 | }, 7 | ".content-hidden": { 8 | "content-visibility": "hidden", 9 | }, 10 | ".content-visible": { 11 | "content-visibility": "visible", 12 | }, 13 | }); 14 | }); 15 | -------------------------------------------------------------------------------- /frontend/config/tailwind/plugins/parts.js: -------------------------------------------------------------------------------- 1 | const plugin = require("tailwindcss/plugin"); 2 | module.exports = plugin(function ({ matchVariant }) { 3 | matchVariant("part", (value) => `&::part(${value})`); 4 | }); 5 | -------------------------------------------------------------------------------- /frontend/config/webpack/shoelace.js: -------------------------------------------------------------------------------- 1 | /* eslint-env node */ 2 | const path = require("path"); 3 | 4 | const shoelaceAssetsSrcPath = path.resolve( 5 | __dirname, 6 | "../..", 7 | "node_modules/@shoelace-style/shoelace/dist/assets", 8 | ); 9 | const shoelaceAssetsPublicPath = "shoelace/assets"; 10 | 11 | module.exports = { 12 | shoelaceAssetsSrcPath, 13 | shoelaceAssetsPublicPath, 14 | }; 15 | -------------------------------------------------------------------------------- /frontend/docs/.gitignore: -------------------------------------------------------------------------------- 1 | site 2 | -------------------------------------------------------------------------------- /frontend/docs/copy-api-docs.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | CURR=$(dirname "${BASH_SOURCE[0]}") 3 | 4 | TARGET=$CURR/docs/api/ 5 | mkdir $TARGET 6 | curl "$DOCS_SOURCE_URL/api/openapi.json" > $TARGET/openapi.json 7 | curl "$DOCS_SOURCE_URL/api/redoc" > $TARGET/index.html 8 | curl "$DOCS_SOURCE_URL/docs-logo.svg" > $TARGET/../docs-logo.svg 9 | 10 | if [ -n $ENABLE_ANALYTICS ]; then 11 | SCRIPT_1=' ' 12 | SCRIPT_2=' ' 13 | awk "1;//{ print \"$SCRIPT_1\"; print \"$SCRIPT_2\" }" $TARGET/index.html > $TARGET/index.html.new 14 | mv $TARGET/index.html.new $TARGET/index.html 15 | fi 16 | -------------------------------------------------------------------------------- /frontend/docs/docs/CNAME: -------------------------------------------------------------------------------- 1 | docs.browsertrix.com 2 | -------------------------------------------------------------------------------- /frontend/docs/docs/assets/fonts/Inter-Italic.var.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webrecorder/browsertrix/54d29aec0585c358126861c03c2212596c5d2c3f/frontend/docs/docs/assets/fonts/Inter-Italic.var.woff2 -------------------------------------------------------------------------------- /frontend/docs/docs/assets/fonts/Inter.var.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webrecorder/browsertrix/54d29aec0585c358126861c03c2212596c5d2c3f/frontend/docs/docs/assets/fonts/Inter.var.woff2 -------------------------------------------------------------------------------- /frontend/docs/docs/assets/fonts/Recursive_VF_1.084.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webrecorder/browsertrix/54d29aec0585c358126861c03c2212596c5d2c3f/frontend/docs/docs/assets/fonts/Recursive_VF_1.084.woff2 -------------------------------------------------------------------------------- /frontend/docs/docs/develop/ui/storybook.md: -------------------------------------------------------------------------------- 1 | # Using Storybook 2 | 3 | [Storybook](https://storybook.js.org/) is a tool for documenting and building UI components in isolation. Component documentation is organized into ["stories"](https://storybook.js.org/docs/writing-stories) that show a variety of possible rendered states of a UI component. 4 | 5 | Browsertrix component stories live in `frontend/src/stories`. Component attributes that are public properties (i.e. defined with Lit `@property({ type: Type })`) or documented in a TSDoc comment will automatically appear in stories through the [Custom Elements Manifest](https://custom-elements-manifest.open-wc.org/analyzer/getting-started/) file. 6 | 7 | To develop using Storybook, run: 8 | 9 | ```sh 10 | yarn storybook:watch 11 | ``` 12 | 13 | This will open Storybook in your default browser. Changes to Browsertrix components and stories wil automatically refresh the page. 14 | -------------------------------------------------------------------------------- /frontend/docs/docs/helm-repo/.gitignore: -------------------------------------------------------------------------------- 1 | !.gitignore 2 | -------------------------------------------------------------------------------- /frontend/docs/docs/js/embed.js: -------------------------------------------------------------------------------- 1 | if (window.self !== window.top) { 2 | // Within iframe--assume this is an iframe embedded in the Browsertrix app. 3 | const style = document.createElement("style"); 4 | 5 | // Decrease text size without decreasing element size and overall spacing 6 | style.innerText = `.md-typeset { font-size: 0.7rem; }`; 7 | 8 | window.document.body.appendChild(style); 9 | } 10 | -------------------------------------------------------------------------------- /frontend/docs/docs/js/insertversion.js: -------------------------------------------------------------------------------- 1 | const KEY = "/.__source"; 2 | let retries = 0; 3 | 4 | function loadVersion() { 5 | const value = self.sessionStorage.getItem(KEY); 6 | if (value) { 7 | parseVersion(value); 8 | } else if (retries++ < 10) { 9 | setTimeout(loadVersion, 500); 10 | } 11 | } 12 | 13 | function parseVersion(string) { 14 | const version = JSON.parse(string).version; 15 | if (!version) { 16 | return; 17 | } 18 | 19 | const elems = document.querySelectorAll("insert-version"); 20 | for (const elem of elems) { 21 | try { 22 | const code = elem.parentElement.nextElementSibling.querySelector("code"); 23 | code.childNodes.forEach((node) => { 24 | if (node.nodeType === Node.TEXT_NODE) { 25 | node.nodeValue = node.nodeValue.replaceAll("VERSION", version); 26 | } 27 | }); 28 | } catch (e) {} 29 | } 30 | } 31 | 32 | if (window.location.pathname.startsWith("/deploy/local")) { 33 | window.addEventListener("load", () => loadVersion()); 34 | } 35 | -------------------------------------------------------------------------------- /frontend/docs/docs/overrides/.icons/bootstrap/asterisk.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /frontend/docs/docs/overrides/.icons/bootstrap/bug-fill.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /frontend/docs/docs/overrides/.icons/bootstrap/chat-left-text-fill.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /frontend/docs/docs/overrides/.icons/bootstrap/check-circle-fill.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /frontend/docs/docs/overrides/.icons/bootstrap/dash-square-fill.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /frontend/docs/docs/overrides/.icons/bootstrap/exclamation-circle-fill.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /frontend/docs/docs/overrides/.icons/bootstrap/exclamation-diamond-fill.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /frontend/docs/docs/overrides/.icons/bootstrap/exclamation-square-fill.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /frontend/docs/docs/overrides/.icons/bootstrap/exclamation-triangle-fill.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /frontend/docs/docs/overrides/.icons/bootstrap/exclamation-triangle.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /frontend/docs/docs/overrides/.icons/bootstrap/eye.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /frontend/docs/docs/overrides/.icons/bootstrap/file-earmark-text-fill.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /frontend/docs/docs/overrides/.icons/bootstrap/github.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /frontend/docs/docs/overrides/.icons/bootstrap/hourglass-split.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /frontend/docs/docs/overrides/.icons/bootstrap/info-circle-fill.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /frontend/docs/docs/overrides/.icons/bootstrap/mastodon.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /frontend/docs/docs/overrides/.icons/bootstrap/mortarboard-fill.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /frontend/docs/docs/overrides/.icons/bootstrap/motherboard-fill.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /frontend/docs/docs/overrides/.icons/bootstrap/pencil-fill.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /frontend/docs/docs/overrides/.icons/bootstrap/pencil.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /frontend/docs/docs/overrides/.icons/bootstrap/question-circle-fill.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /frontend/docs/docs/overrides/.icons/bootstrap/quote.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /frontend/docs/docs/overrides/.icons/bootstrap/slash-circle-fill.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /frontend/docs/docs/overrides/.icons/bootstrap/three-dots-vertical.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /frontend/docs/docs/overrides/.icons/bootstrap/upload.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /frontend/docs/docs/overrides/.icons/bootstrap/x-octagon-fill.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /frontend/docs/docs/overrides/.icons/bootstrap/youtube.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /frontend/docs/docs/overrides/.icons/btrix/status-dot.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /frontend/docs/docs/overrides/main.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} {% block icons %} {% set icon_path = 2 | "overrides/.icons/bootstrap/" %} {{ super() }} {% endblock %} 3 | -------------------------------------------------------------------------------- /frontend/docs/docs/overrides/partials/integrations/analytics/custom.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /frontend/docs/docs/overrides/partials/integrations/analytics/plausible.html: -------------------------------------------------------------------------------- 1 | {% if config.extra.enable_analytics %} 2 | 3 | 4 | {% endif %} 5 | -------------------------------------------------------------------------------- /frontend/docs/docs/user-guide/contribute.md: -------------------------------------------------------------------------------- 1 | # Contribute 2 | 3 | We hope our user guide is a useful tool for you. Like Browsertrix itself, our user guide is open source. We greatly appreciate any feedback and open source contributions to our docs [on GitHub](https://github.com/webrecorder/browsertrix). 4 | 5 | Other ways to contribute: 6 | 7 | 1. Answer questions from the web archiving community on the [community help forum](https://forum.webrecorder.net/c/help/5). 8 | 2. [Let us know](mailto:docs-feedback@webrecorder.net) how we can improve our documentation. 9 | 3. If you encounter any bugs while using Browsertrix, please open a [GitHub issue](https://github.com/webrecorder/browsertrix/issues/new/choose) or [contact support](mailto:support@webrecorder.org). -------------------------------------------------------------------------------- /frontend/docs/docs/user-guide/join.md: -------------------------------------------------------------------------------- 1 | # Join an Existing Org 2 | 3 | If you've received an email inviting you to an org, you can join the org by accepting the invitation. 4 | 5 | If you don't have a Browsertrix account, you'll be prompted to [create a password and display name](./signup.md#configure-your-user-account). -------------------------------------------------------------------------------- /frontend/docs/docs/user-guide/org.md: -------------------------------------------------------------------------------- 1 | # Introduction to Orgs 2 | 3 | A Browsertrix org, or organization, is your workspace for web archiving. If you’re archiving collaboratively, an org workspace can be shared between team members. 4 | 5 | Every Browsertrix user belongs to one or more orgs. You'll create an org during sign-up if you sign up for Browsertrix through one of our hosted plans. 6 | 7 | Billing is managed per-org. Depending on your plan, your org may have monthly quotas for storage and minutes of crawl time. These quotas can be increased by upgrading your plan. 8 | 9 | You can change your org name, org URL, default workflow settings, and more from the [**Settings**](./org-settings.md) page. -------------------------------------------------------------------------------- /frontend/docs/docs/user-guide/user-settings.md: -------------------------------------------------------------------------------- 1 | # Change Your Name, Email, or Password 2 | 3 | ## Display Name 4 | 5 | Specify how you want your name to be shown to other org members. For example, your display name will be shown with the crawl workflows that you run. This display name will be used for all orgs that you're a part of. 6 | 7 | ## Email 8 | 9 | Update the email that you use to log in and receive Browsertrix emails from. 10 | 11 | ## Password 12 | 13 | Update the password that you use to login. For your security, we enforce strong passwords. For more information on we secure your account by enforcing strong passwords, see [zxcvbn](https://zxcvbn-ts.github.io/zxcvbn/guide/). 14 | -------------------------------------------------------------------------------- /frontend/index.d.ts: -------------------------------------------------------------------------------- 1 | declare module "*.avif"; 2 | declare module "*.svg"; 3 | declare module "*.webp"; 4 | declare module "*.css"; 5 | declare module "regex-colorize"; 6 | 7 | /** 8 | * Flattens to a normal string type, but preserves string literal suggestions 9 | */ 10 | type AnyString = string & {}; 11 | -------------------------------------------------------------------------------- /frontend/lib/intl-durationformat.js: -------------------------------------------------------------------------------- 1 | // Polyfill doesn't have an export--provide our own 2 | require("@formatjs/intl-durationformat/lib/polyfill"); 3 | 4 | module.exports = window.Intl.DurationFormat; 5 | -------------------------------------------------------------------------------- /frontend/lit-localize.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://raw.githubusercontent.com/lit/lit/main/packages/localize-tools/config.schema.json", 3 | "sourceLocale": "en", 4 | "targetLocales": ["de", "fr", "es", "pt"], 5 | "tsConfig": "tsconfig.json", 6 | "output": { 7 | "mode": "runtime", 8 | "outputDir": "src/__generated__/locales", 9 | "localeCodesModule": "src/__generated__/locale-codes.ts" 10 | }, 11 | "interchange": { 12 | "format": "xliff", 13 | "xliffDir": "xliff" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /frontend/minio.conf: -------------------------------------------------------------------------------- 1 | location $LOCAL_ACCESS_PATH { 2 | proxy_pass http://$LOCAL_MINIO_HOST/$LOCAL_BUCKET/; 3 | proxy_redirect off; 4 | proxy_buffering off; 5 | 6 | client_max_body_size 0; 7 | } 8 | 9 | -------------------------------------------------------------------------------- /frontend/postcss.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('postcss-load-config').Config} */ 2 | module.exports = { 3 | plugins: { 4 | "tailwindcss/nesting": {}, 5 | tailwindcss: {}, 6 | autoprefixer: {}, 7 | }, 8 | }; 9 | -------------------------------------------------------------------------------- /frontend/prettier.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import("@ianvs/prettier-plugin-sort-imports").PrettierConfig} */ 2 | module.exports = { 3 | plugins: [ 4 | "@ianvs/prettier-plugin-sort-imports", 5 | "@prettier/plugin-xml", 6 | "prettier-plugin-tailwindcss", 7 | ], 8 | tailwindFunctions: ["tw"], 9 | importOrder: [ 10 | "", 11 | "", 12 | "", 13 | "", 14 | // Parent directory items 15 | "^\\.\\.$", 16 | "^\\.\\.(/.+)$", 17 | "", 18 | // This directory items 19 | "^\\.(/.+)$", 20 | "", 21 | "^\\.$", 22 | "", 23 | "^@/(.*)$", 24 | "^~assets/(.*)", 25 | "", 26 | ], 27 | importOrderParserPlugins: ["typescript", "decorators-legacy"], 28 | overrides: [ 29 | { 30 | files: "**/*.xlf", 31 | options: { 32 | parser: "xml", 33 | proseWrap: "never", 34 | printWidth: Infinity, 35 | xmlSortAttributesByKey: false, 36 | xmlWhitespaceSensitivity: "preserve", 37 | xmlSelfClosingSpace: false, 38 | }, 39 | }, 40 | { 41 | files: "**/*.mdx", 42 | options: { 43 | proseWrap: "always", 44 | }, 45 | }, 46 | ], 47 | }; 48 | -------------------------------------------------------------------------------- /frontend/public/browsertrix-og.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webrecorder/browsertrix/54d29aec0585c358126861c03c2212596c5d2c3f/frontend/public/browsertrix-og.jpg -------------------------------------------------------------------------------- /frontend/sample.env.local: -------------------------------------------------------------------------------- 1 | API_BASE_URL= 2 | DOCS_URL=https://docs.browsertrix.com/ 3 | E2E_USER_EMAIL= 4 | E2E_USER_PASSWORD= 5 | GLITCHTIP_DSN= 6 | INJECT_EXTRA= 7 | -------------------------------------------------------------------------------- /frontend/scripts/get-resolved-playwright-version.js: -------------------------------------------------------------------------------- 1 | const fs = require("fs"); 2 | 3 | const lockfile = require("@yarnpkg/lockfile"); 4 | 5 | let file = fs.readFileSync("yarn.lock", "utf8"); 6 | let json = lockfile.parse(file); 7 | 8 | console.log( 9 | Object.entries(json.object).find(([pkg]) => 10 | pkg.startsWith("@playwright/test"), 11 | )[1].version, 12 | ); 13 | -------------------------------------------------------------------------------- /frontend/src/__generated__/locale-codes.ts: -------------------------------------------------------------------------------- 1 | // Do not modify this file by hand! 2 | // Re-generate this file by running lit-localize. 3 | 4 | /** 5 | * The locale code that templates in this source code are written in. 6 | */ 7 | export const sourceLocale = `en`; 8 | 9 | /** 10 | * The other locale codes that this application is localized into. Sorted 11 | * lexicographically. 12 | */ 13 | export const targetLocales = [ 14 | `de`, 15 | `es`, 16 | `fr`, 17 | `pt`, 18 | ] as const; 19 | 20 | /** 21 | * All valid project locale codes. Sorted lexicographically. 22 | */ 23 | export const allLocales = [ 24 | `de`, 25 | `en`, 26 | `es`, 27 | `fr`, 28 | `pt`, 29 | ] as const; 30 | -------------------------------------------------------------------------------- /frontend/src/__generated__/locales/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webrecorder/browsertrix/54d29aec0585c358126861c03c2212596c5d2c3f/frontend/src/__generated__/locales/.keep -------------------------------------------------------------------------------- /frontend/src/__mocks__/@formatjs/intl-displaynames/should-polyfill.js: -------------------------------------------------------------------------------- 1 | import { stub } from "sinon"; 2 | 3 | export const shouldPolyfill = stub(() => false); 4 | -------------------------------------------------------------------------------- /frontend/src/__mocks__/_empty.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Use to mock files in tests. 3 | * 4 | * Usage in web-test-runner.config.mjs: 5 | * importMap: { 6 | * imports: { 7 | * 'styles.css': '/src/__mocks__/_empty.js' 8 | * }, 9 | * }, 10 | */ 11 | export default ""; 12 | -------------------------------------------------------------------------------- /frontend/src/__mocks__/api/settings.js: -------------------------------------------------------------------------------- 1 | export default { 2 | registrationEnabled: false, 3 | jwtTokenLifetime: 0, 4 | defaultBehaviorTimeSeconds: 0, 5 | defaultPageLoadTimeSeconds: 0, 6 | maxPagesPerCrawl: 0, 7 | maxBrowserWindows: 0, 8 | billingEnabled: true, 9 | signUpUrl: "", 10 | salesEmail: "", 11 | supportEmail: "", 12 | }; 13 | -------------------------------------------------------------------------------- /frontend/src/__mocks__/shoelace.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Use to mock src/shoelace.ts 3 | */ 4 | -------------------------------------------------------------------------------- /frontend/src/assets/favicons/apple-touch-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webrecorder/browsertrix/54d29aec0585c358126861c03c2212596c5d2c3f/frontend/src/assets/favicons/apple-touch-icon.png -------------------------------------------------------------------------------- /frontend/src/assets/favicons/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webrecorder/browsertrix/54d29aec0585c358126861c03c2212596c5d2c3f/frontend/src/assets/favicons/favicon.ico -------------------------------------------------------------------------------- /frontend/src/assets/favicons/icon-192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webrecorder/browsertrix/54d29aec0585c358126861c03c2212596c5d2c3f/frontend/src/assets/favicons/icon-192.png -------------------------------------------------------------------------------- /frontend/src/assets/favicons/icon-512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webrecorder/browsertrix/54d29aec0585c358126861c03c2212596c5d2c3f/frontend/src/assets/favicons/icon-512.png -------------------------------------------------------------------------------- /frontend/src/assets/fonts/Inter/Inter-italic.var.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webrecorder/browsertrix/54d29aec0585c358126861c03c2212596c5d2c3f/frontend/src/assets/fonts/Inter/Inter-italic.var.woff2 -------------------------------------------------------------------------------- /frontend/src/assets/fonts/Inter/Inter-roman.var.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webrecorder/browsertrix/54d29aec0585c358126861c03c2212596c5d2c3f/frontend/src/assets/fonts/Inter/Inter-roman.var.woff2 -------------------------------------------------------------------------------- /frontend/src/assets/fonts/Inter/inter.css: -------------------------------------------------------------------------------- 1 | @font-face { 2 | font-family: 'Inter var'; 3 | font-weight: 100 900; 4 | font-display: swap; 5 | font-style: normal; 6 | font-named-instance: 'Regular'; 7 | src: url("Inter-roman.var.woff2?v=3.19") format("woff2"); 8 | } 9 | @font-face { 10 | font-family: 'Inter var'; 11 | font-weight: 100 900; 12 | font-display: swap; 13 | font-style: italic; 14 | font-named-instance: 'Italic'; 15 | src: url("Inter-italic.var.woff2?v=3.19") format("woff2"); 16 | } -------------------------------------------------------------------------------- /frontend/src/assets/fonts/Recursive/Recursive_VF.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webrecorder/browsertrix/54d29aec0585c358126861c03c2212596c5d2c3f/frontend/src/assets/fonts/Recursive/Recursive_VF.woff2 -------------------------------------------------------------------------------- /frontend/src/assets/fonts/Recursive/recursive.css: -------------------------------------------------------------------------------- 1 | /* See https://github.com/arrowtype/recursive/tree/main#opentype-features for font-feature-settings examples & more configuration info */ 2 | @font-face { 3 | font-family: 'Recursive var'; 4 | font-weight: 300 1000; 5 | font-display: swap; 6 | font-style: oblique 0deg 15deg; 7 | font-variation-settings: 'MONO' 1, 'CASL' 0, 'slnt' 0, 'CRSV' 0; 8 | src: url("./Recursive_VF.woff2?v=1.085") format("woff2"); 9 | font-feature-settings: "ss02", "ss08", "ss12"; 10 | } -------------------------------------------------------------------------------- /frontend/src/assets/icons/chevron-left.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /frontend/src/assets/icons/chevron-right.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /frontend/src/assets/icons/dot.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /frontend/src/assets/icons/flask-fill.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /frontend/src/assets/icons/microscope.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /frontend/src/assets/icons/replaywebpage.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /frontend/src/assets/images/caret-down-fill.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /frontend/src/assets/images/caret-right-fill.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /frontend/src/assets/images/thumbnails/thumbnail-cyan.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webrecorder/browsertrix/54d29aec0585c358126861c03c2212596c5d2c3f/frontend/src/assets/images/thumbnails/thumbnail-cyan.avif -------------------------------------------------------------------------------- /frontend/src/assets/images/thumbnails/thumbnail-green.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webrecorder/browsertrix/54d29aec0585c358126861c03c2212596c5d2c3f/frontend/src/assets/images/thumbnails/thumbnail-green.avif -------------------------------------------------------------------------------- /frontend/src/assets/images/thumbnails/thumbnail-orange.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webrecorder/browsertrix/54d29aec0585c358126861c03c2212596c5d2c3f/frontend/src/assets/images/thumbnails/thumbnail-orange.avif -------------------------------------------------------------------------------- /frontend/src/assets/images/thumbnails/thumbnail-yellow.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webrecorder/browsertrix/54d29aec0585c358126861c03c2212596c5d2c3f/frontend/src/assets/images/thumbnails/thumbnail-yellow.avif -------------------------------------------------------------------------------- /frontend/src/classes/BtrixElement.ts: -------------------------------------------------------------------------------- 1 | import { TailwindElement } from "./TailwindElement"; 2 | 3 | import { APIController } from "@/controllers/api"; 4 | import { LocalizeController } from "@/controllers/localize"; 5 | import { NavigateController } from "@/controllers/navigate"; 6 | import { NotifyController } from "@/controllers/notify"; 7 | import appState, { use } from "@/utils/state"; 8 | 9 | export class BtrixElement extends TailwindElement { 10 | /** Access and react to updates to shared state */ 11 | @use() 12 | appState = appState; 13 | 14 | readonly api = new APIController(this); 15 | readonly notify = new NotifyController(this); 16 | readonly navigate = new NavigateController(this); 17 | readonly localize = new LocalizeController(this); 18 | 19 | protected get authState() { 20 | return this.appState.auth; 21 | } 22 | 23 | protected get userInfo() { 24 | return this.appState.userInfo; 25 | } 26 | 27 | protected get userOrg() { 28 | return this.appState.userOrg; 29 | } 30 | 31 | protected get orgId() { 32 | return this.appState.orgId; 33 | } 34 | 35 | protected get orgSlugState() { 36 | return this.appState.orgSlug; 37 | } 38 | 39 | protected get org() { 40 | return this.appState.org; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /frontend/src/classes/TailwindElement.ts: -------------------------------------------------------------------------------- 1 | import { LitElement } from "lit"; 2 | 3 | import { theme } from "@/theme"; 4 | 5 | export class TailwindElement extends LitElement { 6 | connectedCallback(): void { 7 | super.connectedCallback(); 8 | // Insert the compiled Tailwind css into the shadow root. 9 | // This has the benefit of not requiring a whole copy of compiled Tailwind 10 | // for every TailwindElement, so we still get the benefits of atomic CSS. 11 | // And because Tailwind uses `@layer`[^1], the order of declarations ends up 12 | // correct, and you can use component styles with `static styles = ...`, 13 | // *and* you can use Tailwind functions and directives in those styles 14 | // thanks to `postcss-lit`. 15 | // 16 | // [^1]: (see https://tailwindcss.com/docs/adding-custom-styles#using-css-and-layer), 17 | if (this.shadowRoot) { 18 | this.shadowRoot.adoptedStyleSheets = [ 19 | ...this.shadowRoot.adoptedStyleSheets, 20 | theme, 21 | ]; 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /frontend/src/components/beta-badges.stylesheet.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; 4 | 5 | @layer components { 6 | :host { 7 | @apply contents !leading-[0]; 8 | } 9 | sl-tooltip { 10 | @apply [--max-width:theme(spacing.64)]; 11 | } 12 | sl-tooltip::part(body), 13 | sl-tooltip::part(base__arrow) { 14 | @apply bg-brand-green; 15 | } 16 | 17 | sl-tooltip::part(body) { 18 | @apply w-auto text-balance text-white; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /frontend/src/components/document-title.ts: -------------------------------------------------------------------------------- 1 | import { LitElement, type PropertyValues } from "lit"; 2 | import { customElement, property } from "lit/decorators.js"; 3 | 4 | /** 5 | * Updates user's browser title bar 6 | */ 7 | @customElement("btrix-document-title") 8 | export class DocumentTitle extends LitElement { 9 | @property({ type: String }) 10 | title = ""; 11 | 12 | disconnectedCallback(): void { 13 | // Reset back to default title 14 | document.title = "Browsertrix"; 15 | 16 | super.disconnectedCallback(); 17 | } 18 | 19 | willUpdate(changedProperties: PropertyValues) { 20 | if (changedProperties.has("title") && this.title) { 21 | document.title = this.title; 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /frontend/src/components/index.ts: -------------------------------------------------------------------------------- 1 | import "./ui"; 2 | import "./utils"; 3 | import "./document-title"; 4 | 5 | import("./orgs-list"); 6 | import("./not-found"); 7 | import("./screencast"); 8 | import("./beta-badges"); 9 | import("./detail-page-title"); 10 | import("./verified-badge"); 11 | -------------------------------------------------------------------------------- /frontend/src/components/ui/README.md: -------------------------------------------------------------------------------- 1 | # Reusable UI Components 2 | 3 | Components here should be reusable across Browsertrix (and perhaps eventually across other Webrecorder projects as well). They should generally not depend on auth, not be specific to a single page or view, and have well-defined (and typed) inputs and outputs. 4 | 5 | 6 | -------------------------------------------------------------------------------- /frontend/src/components/ui/card.ts: -------------------------------------------------------------------------------- 1 | import { html } from "lit"; 2 | import { customElement } from "lit/decorators.js"; 3 | 4 | import { TailwindElement } from "@/classes/TailwindElement"; 5 | 6 | @customElement("btrix-card") 7 | export class Card extends TailwindElement { 8 | render() { 9 | return html` 10 |
11 |
15 | 16 |
17 |
18 | 19 |
20 | 21 |
22 | `; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /frontend/src/components/ui/data-grid/cellDirective.ts: -------------------------------------------------------------------------------- 1 | import { Directive, type PartInfo } from "lit/directive.js"; 2 | 3 | import type { DataGridCell } from "./data-grid-cell"; 4 | import type { GridColumn } from "./types"; 5 | 6 | /** 7 | * Directive for replacing `renderCell` and `renderEditCell` 8 | * methods with custom render functions. 9 | */ 10 | export class CellDirective extends Directive { 11 | private readonly element?: DataGridCell; 12 | 13 | constructor(partInfo: PartInfo & { element?: DataGridCell }) { 14 | super(partInfo); 15 | this.element = partInfo.element; 16 | } 17 | 18 | render(col: GridColumn) { 19 | if (!this.element) return; 20 | 21 | if (col.renderCell) { 22 | this.element.renderCell = col.renderCell; 23 | } 24 | 25 | if (col.renderEditCell) { 26 | this.element.renderEditCell = col.renderEditCell; 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /frontend/src/components/ui/data-grid/events/btrix-select-row.ts: -------------------------------------------------------------------------------- 1 | import type { GridItem, GridRowId } from "../types"; 2 | 3 | export type BtrixSelectRowEvent = CustomEvent<{ 4 | id: GridRowId; 5 | item: T; 6 | }>; 7 | 8 | declare global { 9 | interface GlobalEventHandlersEventMap { 10 | "btrix-select-row": BtrixSelectRowEvent; 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /frontend/src/components/ui/data-grid/index.ts: -------------------------------------------------------------------------------- 1 | import "./data-grid"; 2 | import "./data-grid-cell"; 3 | import "./data-grid-row"; 4 | -------------------------------------------------------------------------------- /frontend/src/components/ui/data-grid/renderRows.ts: -------------------------------------------------------------------------------- 1 | import { type TemplateResult } from "lit"; 2 | import { repeat } from "lit/directives/repeat.js"; 3 | import type { EmptyObject } from "type-fest"; 4 | 5 | import type { GridItem, GridRowId, GridRows } from "./types"; 6 | 7 | export function renderRows( 8 | rows: GridRows, 9 | renderRow: ( 10 | { id, item }: { id: GridRowId; item: T | EmptyObject }, 11 | index: number, 12 | ) => TemplateResult, 13 | ) { 14 | return repeat( 15 | rows, 16 | ([id]) => id, 17 | ([id, item], i) => renderRow({ id, item }, i), 18 | ); 19 | } 20 | -------------------------------------------------------------------------------- /frontend/src/components/ui/field-error.ts: -------------------------------------------------------------------------------- 1 | import { localized } from "@lit/localize"; 2 | import { html } from "lit"; 3 | import { customElement, property } from "lit/decorators.js"; 4 | 5 | import { TailwindElement } from "@/classes/TailwindElement"; 6 | 7 | @customElement("btrix-field-error") 8 | @localized() 9 | export class FieldError extends TailwindElement { 10 | @property({ type: Boolean }) 11 | hidden = true; 12 | 13 | render() { 14 | return html`
19 | 20 |
`; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /frontend/src/components/ui/index.ts: -------------------------------------------------------------------------------- 1 | import "./alert"; 2 | import "./badge"; 3 | import "./button"; 4 | import "./card"; 5 | import "./data-table"; 6 | import "./desc-list"; 7 | import "./dialog"; 8 | import "./link"; 9 | import "./navigation"; 10 | import "./tab-group"; 11 | import "./tab-list"; 12 | import "./url-input"; 13 | 14 | import("./code"); 15 | import("./combobox"); 16 | import("./config-details"); 17 | import("./copy-button"); 18 | import("./copy-field"); 19 | import("./data-grid"); 20 | import("./details"); 21 | import("./file-list"); 22 | import("./format-date"); 23 | import("./inline-input"); 24 | import("./language-select"); 25 | import("./markdown-editor"); 26 | import("./markdown-viewer"); 27 | import("./menu-item-link"); 28 | import("./meter"); 29 | import("./numbered-list"); 30 | import("./overflow-dropdown"); 31 | import("./overflow-scroll"); 32 | import("./pagination"); 33 | import("./popover"); 34 | import("./pw-strength-alert"); 35 | import("./relative-duration"); 36 | import("./search-combobox"); 37 | import("./section-heading"); 38 | import("./select-crawler-proxy"); 39 | import("./select-crawler"); 40 | import("./syntax-input"); 41 | import("./table"); 42 | import("./tag-input"); 43 | import("./tag"); 44 | import("./time-input"); 45 | import("./user-language-select"); 46 | -------------------------------------------------------------------------------- /frontend/src/components/ui/inline-input.ts: -------------------------------------------------------------------------------- 1 | import SlInput from "@shoelace-style/shoelace/dist/components/input/input.js"; 2 | import { css } from "lit"; 3 | import { customElement, property } from "lit/decorators.js"; 4 | 5 | /** 6 | * Input to use inline with text. 7 | * 8 | * @attr value 9 | * @attr max 10 | * @attr min 11 | * @attr maxlength 12 | * @attr minlength 13 | */ 14 | @customElement("btrix-inline-input") 15 | export class InlineInput extends SlInput { 16 | @property({ type: String, reflect: true }) 17 | size: SlInput["size"] = "small"; 18 | 19 | @property({ type: String, reflect: true }) 20 | inputmode: SlInput["inputmode"] = "numeric"; 21 | 22 | @property({ type: String, reflect: true }) 23 | autocomplete: SlInput["autocomplete"] = "off"; 24 | 25 | static styles = [ 26 | SlInput.styles, 27 | css` 28 | :host { 29 | --sl-input-height-small: var(--sl-font-size-x-large); 30 | --sl-input-color: var(--sl-color-neutral-500); 31 | } 32 | 33 | .input--small .input__control { 34 | text-align: center; 35 | padding: 0 0.5ch; 36 | } 37 | `, 38 | ]; 39 | } 40 | -------------------------------------------------------------------------------- /frontend/src/components/ui/navigation/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./navigation-button"; 2 | -------------------------------------------------------------------------------- /frontend/src/components/ui/section-heading.ts: -------------------------------------------------------------------------------- 1 | import { css, html, LitElement } from "lit"; 2 | import { customElement } from "lit/decorators.js"; 3 | 4 | /** 5 | * Styled section heading 6 | * 7 | * Usage example: 8 | * ```ts 9 | * Text 10 | * ``` 11 | */ 12 | @customElement("btrix-section-heading") 13 | export class SectionHeading extends LitElement { 14 | // postcss-lit-disable-next-line 15 | static styles = css` 16 | .heading { 17 | display: flex; 18 | align-items: center; 19 | gap: 0.5rem; 20 | font-size: var(--sl-font-size-medium); 21 | color: var(--sl-color-neutral-500); 22 | min-height: 2rem; 23 | line-height: 1; 24 | border-bottom: 1px solid var(--sl-panel-border-color); 25 | margin-bottom: var(--margin); 26 | } 27 | `; 28 | 29 | render() { 30 | return html`
`; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /frontend/src/components/ui/tab-group/index.ts: -------------------------------------------------------------------------------- 1 | import "./tab"; 2 | import "./tab-group"; 3 | import "./tab-panel"; 4 | -------------------------------------------------------------------------------- /frontend/src/components/ui/tab-group/tab-panel.ts: -------------------------------------------------------------------------------- 1 | import { html } from "lit"; 2 | import { customElement, property } from "lit/decorators.js"; 3 | 4 | import { TailwindElement } from "@/classes/TailwindElement"; 5 | 6 | @customElement("btrix-tab-group-panel") 7 | export class TabGroupPanel extends TailwindElement { 8 | @property({ type: String }) 9 | name = ""; 10 | 11 | @property({ type: Boolean }) 12 | active = false; 13 | 14 | render() { 15 | return html` 16 |
23 | 24 |
25 | `; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /frontend/src/components/ui/table/index.ts: -------------------------------------------------------------------------------- 1 | import "./table"; 2 | import "./table-body"; 3 | import "./table-cell"; 4 | import "./table-footer"; 5 | import "./table-head"; 6 | import "./table-header-cell"; 7 | import "./table-row"; 8 | -------------------------------------------------------------------------------- /frontend/src/components/ui/table/table-body.ts: -------------------------------------------------------------------------------- 1 | import { css, html, LitElement } from "lit"; 2 | import { customElement, property } from "lit/decorators.js"; 3 | 4 | /** 5 | * @cssproperty --btrix-row-gap CSS value for `grid-row-gap 6 | */ 7 | @customElement("btrix-table-body") 8 | export class TableBody extends LitElement { 9 | static styles = css` 10 | :host { 11 | grid-column: 1 / -1; 12 | display: grid; 13 | grid-template-columns: subgrid; 14 | grid-row-gap: var(--btrix-row-gap, 0); 15 | color: var(--sl-color-neutral-900); 16 | } 17 | `; 18 | 19 | @property({ type: String, reflect: true, noAccessor: true }) 20 | role = "rowgroup"; 21 | 22 | render() { 23 | return html``; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /frontend/src/components/ui/table/table-footer.ts: -------------------------------------------------------------------------------- 1 | import { customElement } from "lit/decorators.js"; 2 | 3 | import { TableBody } from "./table-body"; 4 | 5 | @customElement("btrix-table-footer") 6 | export class TableFooter extends TableBody {} 7 | -------------------------------------------------------------------------------- /frontend/src/components/ui/table/table-head.ts: -------------------------------------------------------------------------------- 1 | import { css, html, LitElement } from "lit"; 2 | import { 3 | customElement, 4 | property, 5 | queryAssignedElements, 6 | } from "lit/decorators.js"; 7 | 8 | import { type TableHeaderCell } from "./table-header-cell"; 9 | 10 | /** 11 | * @csspart base 12 | */ 13 | @customElement("btrix-table-head") 14 | export class TableHead extends LitElement { 15 | static styles = css` 16 | :host { 17 | grid-column: 1 / -1; 18 | display: grid; 19 | grid-template-columns: subgrid; 20 | color: var(--sl-color-neutral-700); 21 | font-size: var(--sl-font-size-x-small); 22 | line-height: 1; 23 | } 24 | `; 25 | 26 | @property({ type: String, reflect: true, noAccessor: true }) 27 | role = "rowgroup"; 28 | 29 | @property({ type: Number, reflect: true, noAccessor: true }) 30 | colCount = 1; 31 | 32 | @queryAssignedElements({ 33 | selector: "btrix-table-header-cell", 34 | flatten: true, 35 | }) 36 | private readonly headerCells!: TableHeaderCell[]; 37 | 38 | render() { 39 | return html` 40 | 41 | `; 42 | } 43 | 44 | private onSlotChange() { 45 | this.colCount = this.headerCells.length || 1; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /frontend/src/components/ui/table/table-header-cell.ts: -------------------------------------------------------------------------------- 1 | import { html } from "lit"; 2 | import { customElement, property } from "lit/decorators.js"; 3 | 4 | import { TableCell } from "./table-cell"; 5 | 6 | import type { SortDirection as Direction } from "@/types/utils"; 7 | 8 | export type SortValues = "ascending" | "descending" | "none"; 9 | export const SortDirection = new Map([ 10 | [-1, "descending"], 11 | [1, "ascending"], 12 | ]); 13 | 14 | @customElement("btrix-table-header-cell") 15 | export class TableHeaderCell extends TableCell { 16 | @property({ type: String, reflect: true, noAccessor: true }) 17 | role = "columnheader"; 18 | 19 | @property({ type: String, reflect: true }) 20 | ariaSort: SortValues = "none"; 21 | 22 | render() { 23 | return html` `; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /frontend/src/components/ui/table/table-row.ts: -------------------------------------------------------------------------------- 1 | import { css, html } from "lit"; 2 | import { customElement, property } from "lit/decorators.js"; 3 | 4 | import { TailwindElement } from "@/classes/TailwindElement"; 5 | 6 | @customElement("btrix-table-row") 7 | export class TableRow extends TailwindElement { 8 | static styles = css` 9 | :host { 10 | grid-column: 1 / -1; 11 | display: grid; 12 | grid-template-columns: subgrid; 13 | position: relative; 14 | } 15 | `; 16 | 17 | @property({ type: String, reflect: true, noAccessor: true }) 18 | role = "row"; 19 | 20 | render() { 21 | return html``; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /frontend/src/components/ui/table/table.stylesheet.css: -------------------------------------------------------------------------------- 1 | btrix-table-cell[rowClickTarget] { 2 | display: grid; 3 | grid-template-columns: subgrid; 4 | white-space: nowrap; 5 | overflow: hidden; 6 | } 7 | 8 | btrix-table-cell .rowClickTarget { 9 | max-width: 100%; 10 | } 11 | 12 | btrix-table-cell sl-tooltip > *, 13 | btrix-table-cell btrix-popover > * { 14 | /* Place above .rowClickTarget::after overlay */ 15 | z-index: 1; 16 | } 17 | 18 | btrix-table-cell .rowClickTarget::after { 19 | content: ""; 20 | display: block; 21 | position: absolute; 22 | inset: 0; 23 | grid-column: clickable-start / clickable-end; 24 | } 25 | 26 | btrix-table-cell .rowClickTarget:focus-visible { 27 | outline: var(--sl-focus-ring); 28 | outline-offset: -0.25rem; 29 | border-radius: 0.5rem; 30 | } 31 | -------------------------------------------------------------------------------- /frontend/src/components/utils/index.ts: -------------------------------------------------------------------------------- 1 | import("./observable"); 2 | -------------------------------------------------------------------------------- /frontend/src/components/utils/observable.ts: -------------------------------------------------------------------------------- 1 | import { html, LitElement } from "lit"; 2 | import { customElement, property } from "lit/decorators.js"; 3 | 4 | import { ObservableController } from "@/controllers/observable"; 5 | 6 | /** 7 | * Observe element with Intersection Observer API. 8 | * 9 | * @example Usage: 10 | * ``` 11 | * 12 | * Observe me! 13 | * 14 | * ``` 15 | * 16 | * @fires btrix-intersect IntersectionEventDetail 17 | */ 18 | @customElement("btrix-observable") 19 | export class Observable extends LitElement { 20 | @property({ type: Object }) 21 | options?: IntersectionObserverInit; 22 | 23 | private observable?: ObservableController; 24 | 25 | connectedCallback(): void { 26 | super.connectedCallback(); 27 | this.observable = new ObservableController(this, this.options); 28 | } 29 | 30 | firstUpdated() { 31 | this.observable?.observe(this); 32 | } 33 | 34 | render() { 35 | return html``; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /frontend/src/components/verified-badge.ts: -------------------------------------------------------------------------------- 1 | import { localized, msg } from "@lit/localize"; 2 | import { html } from "lit"; 3 | import { customElement } from "lit/decorators.js"; 4 | 5 | import { BtrixElement } from "@/classes/BtrixElement"; 6 | 7 | @localized() 8 | @customElement("btrix-verified-badge") 9 | export class Component extends BtrixElement { 10 | render() { 11 | return html` 12 | 18 | ${msg( 20 | "Verified", 21 | )} 23 | 24 | `; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /frontend/src/context/org.ts: -------------------------------------------------------------------------------- 1 | import { createContext } from "@lit/context"; 2 | 3 | import type { ProxiesAPIResponse } from "@/types/crawler"; 4 | 5 | export type ProxiesContext = ProxiesAPIResponse | null; 6 | 7 | export const proxiesContext = createContext("proxies"); 8 | -------------------------------------------------------------------------------- /frontend/src/context/view-state.ts: -------------------------------------------------------------------------------- 1 | import { createContext } from "@lit/context"; 2 | 3 | import { type ViewState } from "@/utils/APIRouter"; 4 | 5 | export type ViewStateContext = ViewState | null; 6 | 7 | export const viewStateContext = createContext("viewState"); 8 | -------------------------------------------------------------------------------- /frontend/src/controllers/localize.ts: -------------------------------------------------------------------------------- 1 | import { LocalizeController as SlLocalizeController } from "@shoelace-style/localize"; 2 | import type { Options as PrettyMsOptions } from "pretty-ms"; 3 | 4 | import localize from "@/utils/localize"; 5 | import roundDuration from "@/utils/round-duration"; 6 | 7 | export class LocalizeController extends SlLocalizeController { 8 | /** 9 | * Custom number formatter 10 | */ 11 | readonly number = localize.number; 12 | 13 | /** 14 | * Custom date formatter that takes missing `Z` into account 15 | */ 16 | readonly date = localize.date; 17 | 18 | /** 19 | * Custom duration formatter 20 | */ 21 | readonly duration = localize.duration; 22 | 23 | readonly ordinal = localize.ordinal; 24 | 25 | readonly humanizeDuration = (value: number, options?: PrettyMsOptions) => { 26 | const duration = roundDuration(value, options); 27 | 28 | if (options?.colonNotation) 29 | return localize.duration(duration, { style: "digital" }); 30 | 31 | if (options?.verbose) return localize.duration(duration, { style: "long" }); 32 | 33 | return localize.duration(duration); 34 | }; 35 | 36 | readonly bytes = localize.bytes; 37 | } 38 | -------------------------------------------------------------------------------- /frontend/src/decorators/needLogin.test.ts: -------------------------------------------------------------------------------- 1 | import { expect } from "@esm-bundle/chai"; 2 | import { spy } from "sinon"; 3 | 4 | import type LiteElement from "../utils/LiteElement"; 5 | import appState, { AppStateService } from "../utils/state"; 6 | 7 | import needLogin from "./needLogin"; 8 | 9 | describe("needLogin", () => { 10 | beforeEach(() => { 11 | AppStateService.resetAll(); 12 | window.sessionStorage.clear(); 13 | }); 14 | 15 | it("dispatches the correct event on need log in", () => { 16 | const dispatchEventSpy = spy(); 17 | class LiteElementMock { 18 | dispatchEvent = dispatchEventSpy; 19 | } 20 | 21 | const Element = needLogin( 22 | // @ts-expect-error not stubbing full BtrixElement 23 | class TestElement extends LiteElementMock { 24 | appState = appState; 25 | } as unknown as { 26 | new (...args: unknown[]): LiteElement; 27 | }, 28 | ); 29 | 30 | const element = new Element(); 31 | element.update(new Map()); 32 | 33 | expect(dispatchEventSpy.getCall(0).firstArg.type).to.equal( 34 | "btrix-need-login", 35 | ); 36 | }); 37 | }); 38 | -------------------------------------------------------------------------------- /frontend/src/decorators/needLogin.ts: -------------------------------------------------------------------------------- 1 | import { type BtrixElement } from "@/classes/BtrixElement"; 2 | import AuthService from "@/utils/AuthService"; 3 | 4 | /** 5 | * Block rendering and dispatch event if user is not logged in. 6 | * When using with other class decorators, `@needLogin` should 7 | * be closest to the component (see usage example.) 8 | * 9 | * @example Usage: 10 | * ```ts 11 | * @customElement("my-component") 12 | * @needLogin 13 | * MyComponent extends LiteElement {} 14 | * ``` 15 | * 16 | * @fires btrix-need-login 17 | */ 18 | export default function needLogin< 19 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 20 | T extends { new (...args: any[]): BtrixElement }, 21 | >(constructor: T) { 22 | return class extends constructor { 23 | update(changedProperties: Map) { 24 | if (this.authState) { 25 | super.update(changedProperties); 26 | } else { 27 | this.dispatchEvent( 28 | AuthService.createNeedLoginEvent({ 29 | redirectUrl: `${window.location.pathname}${window.location.search}${window.location.hash}`, 30 | }), 31 | ); 32 | } 33 | } 34 | }; 35 | } 36 | -------------------------------------------------------------------------------- /frontend/src/events/btrix-change.ts: -------------------------------------------------------------------------------- 1 | export type BtrixChangeEvent = CustomEvent<{ value: T }>; 2 | 3 | declare global { 4 | interface GlobalEventHandlersEventMap { 5 | "btrix-change": BtrixChangeEvent; 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /frontend/src/events/btrix-input.ts: -------------------------------------------------------------------------------- 1 | export type BtrixInputEvent = CustomEvent<{ value: T }>; 2 | 3 | declare global { 4 | interface GlobalEventHandlersEventMap { 5 | "btrix-input": BtrixInputEvent; 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /frontend/src/events/index.ts: -------------------------------------------------------------------------------- 1 | import "./btrix-change"; 2 | import "./btrix-input"; 3 | -------------------------------------------------------------------------------- /frontend/src/features/accounts/index.ts: -------------------------------------------------------------------------------- 1 | import("./invite-form"); 2 | import("./sign-up-form"); 3 | -------------------------------------------------------------------------------- /frontend/src/features/admin/index.ts: -------------------------------------------------------------------------------- 1 | import "./stats"; 2 | import "./super-admin-banner"; 3 | -------------------------------------------------------------------------------- /frontend/src/features/archived-items/index.ts: -------------------------------------------------------------------------------- 1 | import("./archived-item-list"); 2 | import("./crawl-list"); 3 | import("./crawl-log-table"); 4 | import("./crawl-logs"); 5 | import("./item-metadata-editor"); 6 | import("./crawl-pending-exclusions"); 7 | import("./crawl-queue"); 8 | import("./crawl-status"); 9 | import("./file-uploader"); 10 | import("./item-list-controls"); 11 | -------------------------------------------------------------------------------- /frontend/src/features/browser-profiles/index.ts: -------------------------------------------------------------------------------- 1 | import("./new-browser-profile-dialog"); 2 | import("./profile-browser"); 3 | import("./select-browser-profile"); 4 | -------------------------------------------------------------------------------- /frontend/src/features/collections/edit-dialog/helpers/snapshots.ts: -------------------------------------------------------------------------------- 1 | import { type SnapshotItem } from "../../select-collection-page"; 2 | 3 | import { type CollectionThumbnailSource } from "@/types/collection"; 4 | 5 | export function sourceToSnapshot( 6 | source: CollectionThumbnailSource | null, 7 | ): SnapshotItem | null { 8 | if (source == null) return null; 9 | return { 10 | pageId: source.urlPageId, 11 | status: 200, 12 | ts: source.urlTs, 13 | url: source.url, 14 | }; 15 | } 16 | 17 | export function snapshotToSource( 18 | source: SnapshotItem | null, 19 | ): CollectionThumbnailSource | null { 20 | if (source == null) return null; 21 | return { 22 | urlPageId: source.pageId, 23 | urlTs: source.ts, 24 | url: source.url, 25 | }; 26 | } 27 | -------------------------------------------------------------------------------- /frontend/src/features/collections/helpers/share-link.ts: -------------------------------------------------------------------------------- 1 | import { RouteNamespace } from "@/routes"; 2 | import { CollectionAccess, type Collection } from "@/types/collection"; 3 | 4 | export function collectionShareLink( 5 | collection: 6 | | (Pick & Partial>) 7 | | undefined, 8 | privateSlug: string | null, 9 | publicSlug: string | null, 10 | ) { 11 | const baseUrl = `${window.location.protocol}//${window.location.hostname}${window.location.port ? `:${window.location.port}` : ""}`; 12 | if (collection) { 13 | return `${baseUrl}/${ 14 | collection.access === CollectionAccess.Private 15 | ? `${RouteNamespace.PrivateOrgs}/${privateSlug}/collections/view/${collection.id}` 16 | : `${RouteNamespace.PublicOrgs}/${publicSlug}/collections/${collection.slug}` 17 | }`; 18 | } 19 | return ""; 20 | } 21 | -------------------------------------------------------------------------------- /frontend/src/features/collections/index.ts: -------------------------------------------------------------------------------- 1 | import("./collections-add"); 2 | import("./collections-grid"); 3 | import("./collections-grid-with-edit-dialog"); 4 | import("./collection-items-dialog"); 5 | import("./collection-edit-dialog"); 6 | import("./collection-create-dialog"); 7 | import("./collection-initial-view-dialog"); 8 | import("./collection-workflow-list"); 9 | import("./select-collection-access"); 10 | import("./select-collection-page"); 11 | import("./share-collection"); 12 | import("./collection-thumbnail"); 13 | import("./edit-dialog/sharing-section"); 14 | -------------------------------------------------------------------------------- /frontend/src/features/crawl-workflows/index.ts: -------------------------------------------------------------------------------- 1 | import("./custom-behaviors-table"); 2 | import("./exclusion-editor"); 3 | import("./live-workflow-status"); 4 | import("./link-selector-table"); 5 | import("./new-workflow-dialog"); 6 | import("./queue-exclusion-form"); 7 | import("./queue-exclusion-table"); 8 | import("./workflow-editor"); 9 | import("./workflow-list"); 10 | -------------------------------------------------------------------------------- /frontend/src/features/index.ts: -------------------------------------------------------------------------------- 1 | import "./accounts"; 2 | import "./archived-items"; 3 | import "./browser-profiles"; 4 | import "./collections"; 5 | import "./crawl-workflows"; 6 | import "./org"; 7 | import "./qa"; 8 | 9 | import("./admin"); 10 | -------------------------------------------------------------------------------- /frontend/src/features/org/index.ts: -------------------------------------------------------------------------------- 1 | import("./org-status-banner"); 2 | import("./usage-history-table"); 3 | -------------------------------------------------------------------------------- /frontend/src/features/qa/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./page-list"; 2 | import("./page-qa-approval"); 3 | import("./qa-run-dropdown"); 4 | import("./review-status"); 5 | -------------------------------------------------------------------------------- /frontend/src/features/qa/page-list/helpers/approval.ts: -------------------------------------------------------------------------------- 1 | import { msg } from "@lit/localize"; 2 | 3 | import type { ArchivedItemPage } from "@/types/crawler"; 4 | import { cached } from "@/utils/weakCache"; 5 | 6 | export type ReviewStatus = "approved" | "rejected" | "commentOnly" | null; 7 | 8 | export const approvalFromPage = cached( 9 | (page: ArchivedItemPage): ReviewStatus => 10 | page.approved == null 11 | ? page.notes?.length 12 | ? "commentOnly" 13 | : null 14 | : page.approved 15 | ? "approved" 16 | : "rejected", 17 | ); 18 | 19 | export const labelFor = cached((status: ReviewStatus) => { 20 | switch (status) { 21 | // Approval 22 | case "approved": 23 | return msg("Approved"); 24 | case "rejected": 25 | return msg("Rejected"); 26 | case "commentOnly": 27 | return msg("Comments Only"); 28 | 29 | // No data 30 | default: 31 | return; 32 | } 33 | }); 34 | -------------------------------------------------------------------------------- /frontend/src/features/qa/page-list/helpers/crawlCounts.ts: -------------------------------------------------------------------------------- 1 | import { cached } from "@/utils/weakCache"; 2 | 3 | type Optional = T | undefined | null; 4 | 5 | export const crawlCounts = cached( 6 | (bad: Optional, good: Optional) => { 7 | if (bad == null || good == null) return null; 8 | return `${good}/${good + bad}`; 9 | }, 10 | ); 11 | -------------------------------------------------------------------------------- /frontend/src/features/qa/page-list/helpers/index.ts: -------------------------------------------------------------------------------- 1 | import type { ArchivedItemPage } from "@/types/crawler"; 2 | 3 | export const pageIsReviewed = (page: ArchivedItemPage) => 4 | page.approved != null || !!page.notes?.length; 5 | 6 | export * from "./crawlCounts"; 7 | export * from "./iconFor"; 8 | export * from "./issueCounts"; 9 | export * from "./severity"; 10 | export type * from "./severity"; 11 | -------------------------------------------------------------------------------- /frontend/src/features/qa/page-list/helpers/severity.ts: -------------------------------------------------------------------------------- 1 | import { tw } from "@/utils/tailwind"; 2 | import { cached } from "@/utils/weakCache"; 3 | 4 | export type Severity = "severe" | "moderate" | "good" | null; 5 | 6 | export const severityFromMatch = cached( 7 | (match: number | undefined | null): Severity => { 8 | if (match == null) return null; 9 | // TODO extract configs for match thresholds 10 | if (match < 0.5) return "severe"; 11 | if (match < 0.9) return "moderate"; 12 | return "good"; 13 | }, 14 | ); 15 | 16 | export const severityFromResourceCounts = cached( 17 | (bad: number | undefined, good: number | undefined): Severity => { 18 | if (bad == null || good == null) return null; 19 | // TODO extract configs for resource count thresholds 20 | const total = bad + good; 21 | if (bad > 10 || bad / total > 0.5) return "severe"; 22 | if (bad > 0) return "moderate"; 23 | return "good"; 24 | }, 25 | ); 26 | 27 | export const textColorFromSeverity = cached((severity: Severity) => { 28 | switch (severity) { 29 | case "good": 30 | return tw`text-green-600`; 31 | case "moderate": 32 | return tw`text-yellow-500`; 33 | case "severe": 34 | return tw`text-red-500`; 35 | default: 36 | return ""; 37 | } 38 | }); 39 | -------------------------------------------------------------------------------- /frontend/src/features/qa/page-list/index.ts: -------------------------------------------------------------------------------- 1 | import "./ui"; 2 | import "./page-list"; 3 | -------------------------------------------------------------------------------- /frontend/src/features/qa/page-list/ui/index.ts: -------------------------------------------------------------------------------- 1 | import "./page-group"; 2 | import "./page"; 3 | -------------------------------------------------------------------------------- /frontend/src/global.ts: -------------------------------------------------------------------------------- 1 | import "broadcastchannel-polyfill"; 2 | import "construct-style-sheets-polyfill"; 3 | import "./shoelace"; 4 | import "./assets/fonts/Inter/inter.css"; 5 | import "./assets/fonts/Recursive/recursive.css"; 6 | import "./styles.css"; 7 | 8 | import { theme } from "@/theme"; 9 | 10 | // Make theme CSS available in document 11 | document.adoptedStyleSheets = [theme]; 12 | -------------------------------------------------------------------------------- /frontend/src/layouts/empty.ts: -------------------------------------------------------------------------------- 1 | import { html } from "lit"; 2 | 3 | import { stringFor } from "@/strings/ui"; 4 | 5 | export const notSpecified = html` 6 | ${stringFor.notSpecified} 7 | `; 8 | 9 | export const none = html` 10 | ${stringFor.none} 11 | `; 12 | -------------------------------------------------------------------------------- /frontend/src/layouts/emptyMessage.ts: -------------------------------------------------------------------------------- 1 | import clsx from "clsx"; 2 | import { html, nothing, type TemplateResult } from "lit"; 3 | 4 | import { tw } from "@/utils/tailwind"; 5 | 6 | export function emptyMessage({ 7 | message, 8 | detail, 9 | actions, 10 | }: { 11 | message: TemplateResult | string; 12 | detail?: TemplateResult | string; 13 | actions?: TemplateResult; 14 | }) { 15 | return html` 16 |
17 |

25 | ${message} 26 |

27 | ${detail 28 | ? html`

31 | ${detail} 32 |

` 33 | : nothing} 34 | ${actions} 35 |
36 | `; 37 | } 38 | -------------------------------------------------------------------------------- /frontend/src/layouts/pageError.ts: -------------------------------------------------------------------------------- 1 | import { html, nothing, type TemplateResult } from "lit"; 2 | 3 | /** 4 | * Render a full page error, like 404 or 500 for primary resources. 5 | */ 6 | export function pageError({ 7 | heading, 8 | detail, 9 | primaryAction, 10 | secondaryAction, 11 | }: { 12 | heading: string | TemplateResult; 13 | detail: string | TemplateResult; 14 | primaryAction: TemplateResult; 15 | secondaryAction?: TemplateResult; 16 | }) { 17 | return html` 18 |
19 |

22 | ${heading} 23 |

24 |

${detail}

25 |
${primaryAction}
26 | ${secondaryAction 27 | ? html`

${secondaryAction}

` 28 | : nothing} 29 |
30 | `; 31 | } 32 | -------------------------------------------------------------------------------- /frontend/src/layouts/panel.ts: -------------------------------------------------------------------------------- 1 | import { html, type TemplateResult } from "lit"; 2 | import { ifDefined } from "lit/directives/if-defined.js"; 3 | 4 | import { pageHeading } from "./page"; 5 | 6 | export function panelHeader({ 7 | heading, 8 | actions, 9 | }: { 10 | heading: string | Parameters[0]; 11 | actions?: TemplateResult; 12 | }) { 13 | return html` 14 |
15 | ${typeof heading === "string" 16 | ? pageHeading({ content: heading }) 17 | : pageHeading(heading)} 18 | ${actions} 19 |
20 | `; 21 | } 22 | 23 | export function panelBody({ content }: { content: TemplateResult }) { 24 | return html`
${content}
`; 25 | } 26 | 27 | /** 28 | * @TODO Refactor components to use panel 29 | */ 30 | export function panel({ 31 | heading, 32 | actions, 33 | body, 34 | id, 35 | className, 36 | }: { 37 | body: TemplateResult; 38 | id?: string; 39 | className?: string; 40 | } & Parameters[0]) { 41 | return html`
42 | ${panelHeader({ heading, actions })} ${body} 43 |
`; 44 | } 45 | -------------------------------------------------------------------------------- /frontend/src/layouts/separator.ts: -------------------------------------------------------------------------------- 1 | import { html } from "lit"; 2 | 3 | /** 4 | * For separatoring text in the same line, e.g. for breadcrumbs or item details 5 | */ 6 | export function textSeparator() { 7 | return html`/ `; 12 | } 13 | -------------------------------------------------------------------------------- /frontend/src/manifest.webmanifest: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Browsertrix", 3 | "short_name": "Browsertrix", 4 | "display": "browser", 5 | "background_color": "#fff", 6 | "description": "Automated Browser-Based Crawling at Scale.", 7 | "icons": [ 8 | { "src": "/icon-192.png", "type": "image/png", "sizes": "192x192" }, 9 | { "src": "/icon-512.png", "type": "image/png", "sizes": "512x512" } 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /frontend/src/pages/admin/index.ts: -------------------------------------------------------------------------------- 1 | import "./admin"; 2 | import "./users-invite"; 3 | -------------------------------------------------------------------------------- /frontend/src/pages/collections/index.ts: -------------------------------------------------------------------------------- 1 | import "./collection"; 2 | -------------------------------------------------------------------------------- /frontend/src/pages/index.ts: -------------------------------------------------------------------------------- 1 | import "./org"; 2 | 3 | import(/* webpackChunkName: "sign-up" */ "./sign-up"); 4 | import(/* webpackChunkName: "log-in" */ "./log-in"); 5 | import(/* webpackChunkName: "crawls" */ "./crawls"); 6 | import(/* webpackChunkName: "join" */ "./invite/join"); 7 | import(/* webpackChunkName: "verify" */ "./verify"); 8 | import(/* webpackChunkName: "reset-password" */ "./reset-password"); 9 | import(/* webpackChunkName: "accept-invite" */ "./invite/accept"); 10 | import(/* webpackChunkName: "account-settings" */ "./account-settings"); 11 | import(/* webpackChunkName: "collections" */ "./collections"); 12 | import(/* webpackChunkName: "public" */ "./public"); 13 | -------------------------------------------------------------------------------- /frontend/src/pages/org/archived-item-detail/index.ts: -------------------------------------------------------------------------------- 1 | import "./archived-item-detail"; 2 | -------------------------------------------------------------------------------- /frontend/src/pages/org/archived-item-qa/archived-item-qa.stylesheet.css: -------------------------------------------------------------------------------- 1 | article > * { 2 | min-height: 0; 3 | } 4 | 5 | .qa-grid { 6 | grid-template: 7 | "header" 8 | "pageToolbar" 9 | "tabGroup" 10 | "pageList"; 11 | grid-template-columns: 100%; 12 | grid-template-rows: repeat(5, max-content); 13 | } 14 | 15 | /* Tailwind 'lg' responsive size */ 16 | @media only screen and (min-width: 1024px) { 17 | .qa-grid { 18 | grid-template: 19 | "header header" 20 | "pageToolbar pageList" 21 | "tabGroup pageList"; 22 | grid-template-columns: 1fr 35rem; 23 | grid-template-rows: repeat(2, min-content) 1fr; 24 | } 25 | } 26 | 27 | .grid--header { 28 | grid-area: header; 29 | } 30 | 31 | .grid--pageToolbar { 32 | grid-area: pageToolbar; 33 | } 34 | 35 | .grid--tabGroup { 36 | grid-area: tabGroup; 37 | } 38 | 39 | .grid--pageList { 40 | grid-area: pageList; 41 | } 42 | 43 | sl-image-comparer::part(divider) { 44 | --divider-width: 1rem; 45 | border-left: 1px solid var(--sl-panel-border-color); 46 | border-right: 1px solid var(--sl-panel-border-color); 47 | box-shadow: var(--sl-shadow-large); 48 | } 49 | 50 | sl-image-comparer::part(handle) { 51 | background-color: transparent; 52 | } 53 | -------------------------------------------------------------------------------- /frontend/src/pages/org/archived-item-qa/index.ts: -------------------------------------------------------------------------------- 1 | import "./archived-item-qa"; 2 | -------------------------------------------------------------------------------- /frontend/src/pages/org/archived-item-qa/types.ts: -------------------------------------------------------------------------------- 1 | export const TABS = ["screenshots", "text", "resources", "replay"] as const; 2 | export type QATab = (typeof TABS)[number]; 3 | 4 | export type GoodBad = { 5 | good: number; 6 | bad: number; 7 | }; 8 | 9 | export type BlobPayload = { blobUrl: string }; 10 | export type TextPayload = { text: string }; 11 | export type ReplayPayload = { replayUrl: string }; 12 | export type ResourcesPayload = { resources: { [key: string]: GoodBad } }; 13 | export type ReplayData = Partial< 14 | BlobPayload & TextPayload & ReplayPayload & ResourcesPayload 15 | > | null; 16 | -------------------------------------------------------------------------------- /frontend/src/pages/org/archived-item-qa/ui/severityBadge.ts: -------------------------------------------------------------------------------- 1 | import { html } from "lit"; 2 | 3 | import type { BadgeVariant } from "@/components/ui/badge"; 4 | // import { tw } from "@/utils/tailwind"; 5 | import { severityFromMatch } from "@/features/qa/page-list/helpers"; 6 | import { formatPercentage } from "@/features/qa/page-list/ui/page-details"; 7 | 8 | export function renderSeverityBadge(value?: number | null) { 9 | if (value === undefined || value === null) { 10 | return; 11 | } 12 | 13 | let variant: BadgeVariant = "neutral"; 14 | switch (severityFromMatch(value)) { 15 | case "severe": 16 | variant = "danger"; 17 | break; 18 | case "moderate": 19 | variant = "warning"; 20 | break; 21 | case "good": 22 | variant = "success"; 23 | break; 24 | default: 25 | break; 26 | } 27 | 28 | return html` 29 | ${formatPercentage(value, 0)}% 30 | `; 31 | } 32 | -------------------------------------------------------------------------------- /frontend/src/pages/org/archived-item-qa/ui/spinner.ts: -------------------------------------------------------------------------------- 1 | import clsx, { type ClassValue } from "clsx"; 2 | import { html } from "lit"; 3 | 4 | export function renderSpinner(className?: ClassValue) { 5 | return html`
11 | 12 |
`; 13 | } 14 | -------------------------------------------------------------------------------- /frontend/src/pages/org/settings/settings.stylesheet.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; 4 | 5 | @layer components { 6 | sl-radio.radio-card { 7 | @apply cursor-pointer rounded-md border border-neutral-300 p-2 transition-colors; 8 | &:hover { 9 | @apply border-neutral-400; 10 | } 11 | &[aria-checked="true"] { 12 | @apply border-primary bg-primary-50; 13 | } 14 | &::part(base) { 15 | @apply grid grid-cols-[auto_minmax(0,1fr)] gap-x-1; 16 | } 17 | &::part(control) { 18 | @apply col-start-1 col-end-2 row-start-1 row-end-2; 19 | } 20 | &::part(label) { 21 | @apply col-start-1 col-end-3 row-start-1 row-end-3 ml-0 grid flex-auto grid-cols-subgrid gap-y-2; 22 | } 23 | } 24 | sl-details.details-card { 25 | @apply col-span-2; 26 | &::part(header) { 27 | @apply p-2; 28 | } 29 | &::part(content) { 30 | @apply p-2 pt-0; 31 | } 32 | } 33 | sl-radio.radio-card[aria-checked="true"] sl-details.details-card { 34 | &::part(base) { 35 | @apply border-primary/50; 36 | } 37 | &::part(summary-icon) { 38 | @apply text-primary-700; 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /frontend/src/pages/org/types.ts: -------------------------------------------------------------------------------- 1 | export * from "@/types/crawler"; 2 | -------------------------------------------------------------------------------- /frontend/src/pages/public/index.ts: -------------------------------------------------------------------------------- 1 | import "./org"; 2 | -------------------------------------------------------------------------------- /frontend/src/replayWebPage.d.ts: -------------------------------------------------------------------------------- 1 | import type { Embed as ReplayWebPage } from "replaywebpage"; 2 | 3 | type RwpUrlChangeEvent = CustomEvent<{ 4 | type: "urlchange"; 5 | view: "pages" | "replay"; 6 | replayNotFoundError: boolean; 7 | title?: string; 8 | ts?: string; 9 | url?: string; 10 | }>; 11 | 12 | declare global { 13 | interface HTMLElementTagNameMap { 14 | "replay-web-page": ReplayWebPage; 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /frontend/src/stories/Intro.mdx: -------------------------------------------------------------------------------- 1 | import { Meta } from "@storybook/blocks"; 2 | 3 | 4 | 5 | # Introduction 6 | 7 | {/* TODO Consolidate with storybook.md in frontend/docs */} 8 | 9 | Browsertrix component stories live in `frontend/src/stories`. Component 10 | attributes that are public properties (i.e. defined with Lit 11 | `@property({ type: Type })`) or documented in a TSDoc comment will automatically 12 | appear in stories through the 13 | [Custom Elements Manifest](https://custom-elements-manifest.open-wc.org/analyzer/getting-started/) 14 | file. 15 | 16 | ## Troubleshooting 17 | 18 | **Component attributes aren't updating in Storybook** 19 | 20 | Ensure you're running Storybook with `yarn storybook:watch` instead of 21 | `yarn storybook`. The "watch" script automatically regenerates 22 | `custom-elements.json`, which is the source of element attributes documentation. 23 | 24 | ## Storybook Docs 25 | 26 | - [How to write stories](https://storybook.js.org/docs/writing-stories/?renderer=web-components) 27 | - [About autodocs](https://storybook.js.org/docs/writing-docs/autodocs/?renderer=web-components) 28 | - [Configure Storybook](https://storybook.js.org/docs/configure/?renderer=web-components) 29 | -------------------------------------------------------------------------------- /frontend/src/stories/components/Button.ts: -------------------------------------------------------------------------------- 1 | import { html } from "lit"; 2 | import { ifDefined } from "lit/directives/if-defined.js"; 3 | 4 | import type { Button } from "@/components/ui/button"; 5 | 6 | import "@/components/ui/button"; 7 | 8 | export type RenderProps = Button; 9 | 10 | export const renderButton = ({ 11 | variant, 12 | filled, 13 | label, 14 | raised, 15 | loading, 16 | href, 17 | }: Partial) => { 18 | return html` 19 | 27 | ${label} 28 | 29 | `; 30 | }; 31 | -------------------------------------------------------------------------------- /frontend/src/stories/components/DataTable.ts: -------------------------------------------------------------------------------- 1 | import { html } from "lit"; 2 | import { ifDefined } from "lit/directives/if-defined.js"; 3 | 4 | import type { DataTable } from "@/components/ui/data-table"; 5 | 6 | import "@/components/ui/table"; 7 | import "@/components/ui/data-table"; 8 | 9 | export type RenderProps = DataTable & { 10 | classes?: string; 11 | }; 12 | 13 | const columns = ["A", "B", "C"] satisfies RenderProps["columns"]; 14 | const rows = Array.from({ length: 5 }).map((_, i) => 15 | columns.map((col) => `${col}${i + 1}`), 16 | ) satisfies RenderProps["columns"]; 17 | 18 | export const defaultArgs = { 19 | columns, 20 | rows, 21 | } satisfies Pick; 22 | 23 | export const renderDataTable = ({ 24 | columns, 25 | rows, 26 | columnWidths, 27 | classes, 28 | }: Partial) => { 29 | return html` 30 | 36 | 37 | `; 38 | }; 39 | -------------------------------------------------------------------------------- /frontend/src/stories/components/OverflowScroll.stories.ts: -------------------------------------------------------------------------------- 1 | import type { Meta, StoryObj } from "@storybook/web-components"; 2 | 3 | import { renderOverflowScroll, type RenderProps } from "./OverflowScroll"; 4 | 5 | const meta = { 6 | title: "Components/Overflow Scroll", 7 | component: "btrix-overflow-scroll", 8 | tags: ["autodocs"], 9 | render: renderOverflowScroll, 10 | argTypes: { 11 | direction: { 12 | control: { type: "select" }, 13 | options: ["horizontal"] satisfies RenderProps["direction"][], 14 | }, 15 | scrim: { 16 | control: { type: "boolean" }, 17 | }, 18 | }, 19 | } satisfies Meta; 20 | 21 | export default meta; 22 | 23 | type Story = StoryObj; 24 | 25 | export const Default: Story = { 26 | args: { 27 | direction: "horizontal", 28 | scrim: true, 29 | }, 30 | }; 31 | -------------------------------------------------------------------------------- /frontend/src/stories/components/OverflowScroll.ts: -------------------------------------------------------------------------------- 1 | import clsx from "clsx"; 2 | import { html } from "lit"; 3 | import { ifDefined } from "lit/directives/if-defined.js"; 4 | 5 | import { defaultArgs, renderTable } from "./Table"; 6 | 7 | import "@/components/ui/overflow-scroll"; 8 | 9 | import type { OverflowScroll } from "@/components/ui/overflow-scroll"; 10 | import { tw } from "@/utils/tailwind"; 11 | 12 | export type RenderProps = OverflowScroll; 13 | 14 | export const renderOverflowScroll = ({ 15 | direction, 16 | scrim, 17 | }: Partial) => { 18 | return html` 19 |
22 | 26 | 27 | ${renderTable({ 28 | ...defaultArgs, 29 | classes: clsx( 30 | ...defaultArgs.classes, 31 | tw`w-[800px] rounded border bg-neutral-50 p-2 [--btrix-table-cell-padding:var(--sl-spacing-2x-small)]`, 32 | ), 33 | })} 34 | 35 |
36 | `; 37 | }; 38 | -------------------------------------------------------------------------------- /frontend/src/stories/components/Popover.ts: -------------------------------------------------------------------------------- 1 | import { html, nothing, type TemplateResult } from "lit"; 2 | import { ifDefined } from "lit/directives/if-defined.js"; 3 | 4 | import type { Popover } from "@/components/ui/popover"; 5 | 6 | import "@/components/ui/popover"; 7 | 8 | export type RenderProps = Popover & { 9 | anchor: TemplateResult; 10 | slottedContent: TemplateResult; 11 | }; 12 | 13 | export const renderComponent = ({ 14 | content, 15 | placement, 16 | open, 17 | anchor, 18 | slottedContent, 19 | }: Partial) => { 20 | return html` 21 | 27 | ${anchor} 28 | ${slottedContent 29 | ? html`
${slottedContent}
` 30 | : nothing} 31 | 32 | `; 33 | }; 34 | -------------------------------------------------------------------------------- /frontend/src/stories/components/SyntaxInput.ts: -------------------------------------------------------------------------------- 1 | import { html } from "lit"; 2 | import { ifDefined } from "lit/directives/if-defined.js"; 3 | 4 | import { Language } from "@/components/ui/code"; 5 | import type { SyntaxInput } from "@/components/ui/syntax-input"; 6 | 7 | import "@/components/ui/syntax-input"; 8 | 9 | export { Language }; 10 | 11 | export type RenderProps = Pick & { 12 | name?: string; 13 | }; 14 | 15 | export const defaultArgs = { 16 | value: "
Edit me
", 17 | language: Language.XML, 18 | placeholder: "Enter HTML", 19 | } satisfies Partial; 20 | 21 | export const renderComponent = (opts: Partial) => html` 22 | 31 | `; 32 | -------------------------------------------------------------------------------- /frontend/src/stories/features/crawl-workflows/QueueExclusionForm.stories.ts: -------------------------------------------------------------------------------- 1 | import type { Meta, StoryObj } from "@storybook/web-components"; 2 | import { html } from "lit"; 3 | 4 | import { argTypes } from "../excludeContainerProperties"; 5 | 6 | import type { QueueExclusionForm } from "@/features/crawl-workflows/queue-exclusion-form"; 7 | 8 | import "@/features/crawl-workflows/queue-exclusion-form"; 9 | 10 | const meta = { 11 | title: "Features/Queue Exclusion Form", 12 | component: "btrix-queue-exclusion-form", 13 | tags: ["autodocs"], 14 | render: (args) => html` 15 | 19 | `, 20 | argTypes: { 21 | ...argTypes, 22 | }, 23 | args: {}, 24 | } satisfies Meta; 25 | 26 | export default meta; 27 | type Story = StoryObj; 28 | 29 | export const Default: Story = { 30 | args: {}, 31 | }; 32 | 33 | export const Submitting: Story = { 34 | args: { 35 | isSubmitting: true, 36 | }, 37 | }; 38 | 39 | export const CustomErrorMessage: Story = { 40 | args: { 41 | fieldErrorMessage: "Please enter a valid exclusion value", 42 | }, 43 | }; 44 | -------------------------------------------------------------------------------- /frontend/src/stories/features/excludeContainerProperties.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Exclude `BtrixElement` and `StorybookOrg` properties from story controls 3 | */ 4 | import { StorybookOrg } from "../decorators/orgDecorator"; 5 | 6 | import { BtrixElement } from "@/classes/BtrixElement"; 7 | 8 | const controlOpts = { table: { disable: true } }; 9 | const argTypes: Record = { 10 | api: controlOpts, 11 | notify: controlOpts, 12 | navigate: controlOpts, 13 | localize: controlOpts, 14 | }; 15 | 16 | Object.getOwnPropertyNames(BtrixElement.prototype).forEach((prop) => { 17 | if (prop === "constructor") return; 18 | 19 | argTypes[prop] = controlOpts; 20 | }); 21 | 22 | Object.getOwnPropertyNames(StorybookOrg.prototype).forEach((prop) => { 23 | if (prop === "constructor") return; 24 | 25 | argTypes[prop] = controlOpts; 26 | }); 27 | 28 | export { argTypes }; 29 | -------------------------------------------------------------------------------- /frontend/src/strings/collections/alerts.ts: -------------------------------------------------------------------------------- 1 | import { msg } from "@lit/localize"; 2 | 3 | export const alerts = { 4 | orgNotPublicWarning: msg( 5 | "This doesn't have a public collections gallery enabled yet. To make this collection and basic org information public, update the org's gallery visibility setting.", 6 | ), 7 | }; 8 | -------------------------------------------------------------------------------- /frontend/src/strings/collections/metadata.ts: -------------------------------------------------------------------------------- 1 | import { msg } from "@lit/localize"; 2 | 3 | export const metadata = { 4 | dateLatest: msg("Collection Period"), 5 | uniquePageCount: msg("Unique Pages in Collection"), 6 | pageCount: msg("Total Pages Crawled"), 7 | totalSize: msg("Collection Size"), 8 | topPageHosts: msg("Top Page Hostnames"), 9 | }; 10 | -------------------------------------------------------------------------------- /frontend/src/strings/crawl-workflows/errors.ts: -------------------------------------------------------------------------------- 1 | // TODO Add all error codes 2 | // https://github.com/webrecorder/browsertrix/issues/2512 3 | export const errorFor = {}; 4 | -------------------------------------------------------------------------------- /frontend/src/strings/crawl-workflows/labels.ts: -------------------------------------------------------------------------------- 1 | import { msg } from "@lit/localize"; 2 | 3 | export const labelFor = { 4 | behaviors: msg("Behaviors"), 5 | customBehaviors: msg("Custom Behaviors"), 6 | autoscrollBehavior: msg("Autoscroll"), 7 | autoclickBehavior: msg("Autoclick"), 8 | pageLoadTimeoutSeconds: msg("Page Load Limit"), 9 | postLoadDelaySeconds: msg("Delay After Page Load"), 10 | behaviorTimeoutSeconds: "Behavior Limit", 11 | pageExtraDelaySeconds: msg("Delay Before Next Page"), 12 | selectLink: msg("Link Selectors"), 13 | clickSelector: msg("Click Selector"), 14 | }; 15 | -------------------------------------------------------------------------------- /frontend/src/strings/crawl-workflows/scopeType.ts: -------------------------------------------------------------------------------- 1 | import { msg } from "@lit/localize"; 2 | 3 | import type { WorkflowScopeType } from "@/types/workflow"; 4 | 5 | const scopeType: Record< 6 | (typeof WorkflowScopeType)[keyof typeof WorkflowScopeType], 7 | string 8 | > = { 9 | ["page-list"]: msg("List of Pages"), 10 | prefix: msg("Pages in Same Directory"), 11 | host: msg("Pages on Same Domain"), 12 | domain: msg("Pages on Same Domain + Subdomains"), 13 | "page-spa": msg("In-Page Links"), 14 | page: msg("Single Page"), 15 | custom: msg("Custom Page Prefix"), 16 | any: msg("Any Page"), 17 | }; 18 | 19 | export default scopeType; 20 | -------------------------------------------------------------------------------- /frontend/src/strings/crawl-workflows/section.ts: -------------------------------------------------------------------------------- 1 | import { msg } from "@lit/localize"; 2 | 3 | import { type SectionsEnum } from "@/utils/workflow"; 4 | 5 | const section: Record = { 6 | scope: msg("Scope"), 7 | limits: msg("Crawl Limits"), 8 | behaviors: msg("Page Behavior"), 9 | browserSettings: msg("Browser Settings"), 10 | scheduling: msg("Scheduling"), 11 | metadata: msg("Metadata"), 12 | }; 13 | 14 | export default section; 15 | -------------------------------------------------------------------------------- /frontend/src/strings/orgs/alerts.ts: -------------------------------------------------------------------------------- 1 | import { msg } from "@lit/localize"; 2 | 3 | export const alerts = { 4 | settingsUpdateSuccess: msg("Updated org settings."), 5 | settingsUpdateFailure: msg("Sorry, couldn't update org at this time."), 6 | }; 7 | -------------------------------------------------------------------------------- /frontend/src/strings/ui.ts: -------------------------------------------------------------------------------- 1 | import { msg } from "@lit/localize"; 2 | import { html, type TemplateResult } from "lit"; 3 | 4 | export const stringFor: Record = { 5 | noData: "--", 6 | notApplicable: msg("n/a"), 7 | notSpecified: msg("Not specified"), 8 | none: msg("None"), 9 | }; 10 | 11 | export const noData = stringFor.noData; 12 | export const notApplicable = stringFor.notApplicable; 13 | 14 | // TODO Refactor all generic confirmation messages to use utility 15 | export const deleteConfirmation = (name: string | TemplateResult) => 16 | msg(html` 17 | Are you sure you want to delete 18 | ${name}? 19 | `); 20 | -------------------------------------------------------------------------------- /frontend/src/strings/utils.ts: -------------------------------------------------------------------------------- 1 | import { msg, str } from "@lit/localize"; 2 | 3 | import { noData } from "@/strings/ui"; 4 | import localize from "@/utils/localize"; 5 | 6 | export const monthYearDateRange = ( 7 | startDate?: string | null, 8 | endDate?: string | null, 9 | ): string => { 10 | if (!startDate || !endDate) { 11 | return noData; 12 | } 13 | const format: Intl.DateTimeFormatOptions = { 14 | month: "long", 15 | year: "numeric", 16 | }; 17 | const startMonthYear = localize.date(startDate, format); 18 | const endMonthYear = localize.date(endDate, format); 19 | 20 | if (startMonthYear === endMonthYear) return endMonthYear; 21 | 22 | return msg(str`${startMonthYear} to ${endMonthYear}`, { 23 | desc: "Date range formatted to show full month name and year", 24 | }); 25 | }; 26 | -------------------------------------------------------------------------------- /frontend/src/strings/validation.ts: -------------------------------------------------------------------------------- 1 | import { msg } from "@lit/localize"; 2 | 3 | export const validationMessageFor: Partial< 4 | Record 5 | > = { 6 | valueMissing: msg("Please fill out this field."), 7 | }; 8 | -------------------------------------------------------------------------------- /frontend/src/styles.css: -------------------------------------------------------------------------------- 1 | /* Global styles */ 2 | .uppercase { 3 | letter-spacing: 0.06em; 4 | } 5 | 6 | /* Regex Colorizer no bg theme */ 7 | /* https://stevenlevithan.com/regex/colorizer/themes/nobg.css */ 8 | 9 | .regex { 10 | color: var(--sl-color-neutral-500); 11 | } 12 | /* metasequence */ 13 | .regex b { 14 | color: var(--sl-color-blue-500); 15 | } 16 | /* char class */ 17 | .regex i { 18 | color: var(--sl-color-lime-500); 19 | } 20 | /* char class: metasequence */ 21 | .regex i b { 22 | color: var(--sl-color-green-500); 23 | } 24 | /* char class: range-hyphen */ 25 | .regex i u { 26 | color: var(--sl-color-green-500); 27 | } 28 | /* group: depth 1 */ 29 | .regex b.g1 { 30 | color: var(--sl-color-orange-500); 31 | } 32 | /* group: depth 2 */ 33 | .regex b.g2 { 34 | color: var(--sl-color-amber-500); 35 | } 36 | /* group: depth 3 */ 37 | .regex b.g3 { 38 | color: var(--sl-color-rose-500); 39 | } 40 | /* group: depth 4 */ 41 | .regex b.g4 { 42 | color: var(--sl-color-teal-500); 43 | } 44 | /* group: depth 5 */ 45 | .regex b.g5 { 46 | color: var(--sl-color-fuchsia-500); 47 | } 48 | /* error */ 49 | .regex b.err { 50 | color: var(--sl-color-danger-500); 51 | } 52 | .regex b, 53 | .regex i, 54 | .regex u { 55 | font-weight: normal; 56 | font-style: normal; 57 | text-decoration: none; 58 | } 59 | -------------------------------------------------------------------------------- /frontend/src/theme.ts: -------------------------------------------------------------------------------- 1 | import themeCSS from "./theme.stylesheet.css"; 2 | 3 | // Create a new style sheet from the compiled theme CSS 4 | export const theme = new CSSStyleSheet(); 5 | theme.replaceSync(themeCSS as string); 6 | -------------------------------------------------------------------------------- /frontend/src/trackEvents.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * All available analytics tracking events 3 | */ 4 | 5 | export enum AnalyticsTrackEvent { 6 | /** 7 | * Generic 8 | */ 9 | PageView = "pageview", 10 | /** 11 | * Collections 12 | */ 13 | CopyShareCollectionLink = "Copy share collection link", 14 | DownloadPublicCollection = "Download public collection", 15 | /** 16 | * Workflows 17 | */ 18 | ExpandWorkflowFormSection = "Expand workflow form section", 19 | } 20 | -------------------------------------------------------------------------------- /frontend/src/types/auth.ts: -------------------------------------------------------------------------------- 1 | import { z } from "zod"; 2 | 3 | export const authSchema = z.object({ 4 | username: z.string(), 5 | headers: z.object({ 6 | Authorization: z.string(), 7 | }), 8 | /** Timestamp (milliseconds) when token expires */ 9 | tokenExpiresAt: z.number(), 10 | }); 11 | export type Auth = z.infer; 12 | -------------------------------------------------------------------------------- /frontend/src/types/billing.ts: -------------------------------------------------------------------------------- 1 | import { z } from "zod"; 2 | 3 | import { apiDateSchema } from "./api"; 4 | 5 | export enum SubscriptionStatus { 6 | Active = "active", 7 | Trialing = "trialing", 8 | TrialingCanceled = "trialing_canceled", 9 | PausedPaymentFailed = "paused_payment_failed", 10 | PaymentNeverMade = "payment_never_made", 11 | Cancelled = "cancelled", 12 | } 13 | export const subscriptionStatusSchema = z.nativeEnum(SubscriptionStatus); 14 | 15 | export const subscriptionSchema = z.object({ 16 | status: subscriptionStatusSchema, 17 | planId: z.string(), 18 | readOnlyOnCancel: z.boolean(), 19 | futureCancelDate: apiDateSchema.nullable(), 20 | subId: z.string(), 21 | }); 22 | export type Subscription = z.infer; 23 | 24 | export const billingPortalSchema = z.object({ 25 | portalUrl: z.string().url(), 26 | }); 27 | export type BillingPortal = z.infer; 28 | -------------------------------------------------------------------------------- /frontend/src/types/browser.ts: -------------------------------------------------------------------------------- 1 | export type Browser = { 2 | browserid: string; 3 | }; 4 | -------------------------------------------------------------------------------- /frontend/src/types/events.d.ts: -------------------------------------------------------------------------------- 1 | import { type APIEventMap } from "@/controllers/api"; 2 | import { type CopiedEventMap } from "@/controllers/clipboard"; 3 | import { type NavigateEventMap } from "@/controllers/navigate"; 4 | import { type NotifyEventMap } from "@/controllers/notify"; 5 | import { type UserGuideEventMap } from "@/index"; 6 | import { type AuthEventMap } from "@/utils/AuthService"; 7 | 8 | import "@/events"; 9 | 10 | /** 11 | * Declare custom events here so that typescript can find them. 12 | * Custom event names should be prefixed with `btrix-`. 13 | */ 14 | declare global { 15 | interface GlobalEventHandlersEventMap 16 | extends NavigateEventMap, 17 | NotifyEventMap, 18 | AuthEventMap, 19 | APIEventMap, 20 | UserGuideEventMap, 21 | CopiedEventMap {} 22 | } 23 | -------------------------------------------------------------------------------- /frontend/src/types/localization.ts: -------------------------------------------------------------------------------- 1 | import ISO6391, { type LanguageCode } from "iso-639-1"; 2 | import { z } from "zod"; 3 | 4 | import { allLocales } from "@/__generated__/locale-codes"; 5 | 6 | export { allLocales as translatedLocales }; 7 | 8 | export const translatedLocaleEnum = z.enum(allLocales); 9 | export type TranslatedLocaleEnum = z.infer; 10 | 11 | export const allLanguageCodes = ISO6391.getAllCodes(); 12 | export type AllLanguageCodes = readonly LanguageCode[]; 13 | 14 | export const languageCodeSchema = z.custom((val) => 15 | typeof val === "string" 16 | ? (allLanguageCodes as string[]).includes(val) 17 | : false, 18 | ); 19 | export type { LanguageCode }; 20 | -------------------------------------------------------------------------------- /frontend/src/types/qa.ts: -------------------------------------------------------------------------------- 1 | import type { ArchivedItemPage } from "@/types/crawler"; 2 | import type { CrawlState } from "@/types/crawlState"; 3 | 4 | export type QARun = { 5 | id: string; 6 | userName: string; 7 | started: string; // date 8 | finished: string; // date 9 | state: CrawlState; 10 | crawlExecSeconds: number; 11 | stats: { 12 | found: number; 13 | done: number; 14 | size: number; 15 | }; 16 | resources?: { crawlId: string; name: string; path: string }[]; 17 | stopping?: boolean | null; 18 | }; 19 | 20 | export type ArchivedItemQAPage = ArchivedItemPage & { 21 | qa: { 22 | screenshotMatch: number | null; 23 | textMatch: number | null; 24 | resourceCounts: { 25 | crawlGood: number; 26 | crawlBad: number; 27 | replayGood: number; 28 | replayBad: number; 29 | } | null; 30 | }; 31 | }; 32 | -------------------------------------------------------------------------------- /frontend/src/types/utils.ts: -------------------------------------------------------------------------------- 1 | import type { DebouncedFunc } from "lodash"; 2 | 3 | /** 4 | * Gets the underlying function type of a debounced function. 5 | * Useful because lit-plugin doesn't recognize debounced function types as callable 6 | * 7 | * @example 8 | * } > 9 | */ 10 | export type UnderlyingFunction = T extends DebouncedFunc ? F : T; 11 | 12 | type Enumerate< 13 | N extends number, 14 | Acc extends number[] = [], 15 | > = Acc["length"] extends N 16 | ? Acc[number] 17 | : Enumerate; 18 | 19 | /** Number literal range from `F` to `T` (exclusive) */ 20 | export type Range = Exclude< 21 | Enumerate, 22 | Enumerate 23 | >; 24 | 25 | export enum SortDirection { 26 | Descending = -1, 27 | Ascending = 1, 28 | } 29 | -------------------------------------------------------------------------------- /frontend/src/types/workflow.ts: -------------------------------------------------------------------------------- 1 | import { ScopeType } from "@/types/crawler"; 2 | 3 | export enum NewWorkflowOnlyScopeType { 4 | PageList = "page-list", 5 | } 6 | 7 | export const WorkflowScopeType = { ...ScopeType, ...NewWorkflowOnlyScopeType }; 8 | -------------------------------------------------------------------------------- /frontend/src/utils/analytics.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Custom tracking for analytics. 3 | * 4 | * Any third-party analytics script will need to have been made 5 | * available through the `extra.js` injected by the server. 6 | */ 7 | 8 | import { AnalyticsTrackEvent } from "../trackEvents"; 9 | 10 | export type AnalyticsTrackProps = { 11 | org_slug?: string | null; 12 | logged_in?: boolean; 13 | collection_slug?: string; 14 | section?: string; 15 | }; 16 | 17 | declare global { 18 | interface Window { 19 | btrixEvent?: ( 20 | event: string, 21 | extra?: { props?: AnalyticsTrackProps }, 22 | ) => void; 23 | } 24 | } 25 | 26 | export function track( 27 | event: `${AnalyticsTrackEvent}`, 28 | props?: AnalyticsTrackProps, 29 | ) { 30 | if (!window.btrixEvent) { 31 | return; 32 | } 33 | 34 | try { 35 | window.btrixEvent(event, { props }); 36 | } catch (err) { 37 | console.debug(err); 38 | } 39 | } 40 | 41 | export function pageView(props?: AnalyticsTrackProps) { 42 | track(AnalyticsTrackEvent.PageView, props); 43 | } 44 | -------------------------------------------------------------------------------- /frontend/src/utils/events.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Ignore events bubbling from children. Common use case would be 3 | * ignoring @sl-hide events from tooltips within a dialog. 4 | * 5 | * @example Usage: 6 | * ``` 7 | * console.log("Only current target!"))} 9 | * > 10 | * 11 | * ``` 12 | */ 13 | export function makeCurrentTargetHandler(t: EventTarget) { 14 | const currentTargetHandler: ( 15 | handler: (event: T) => void, 16 | ) => (event: T) => void = (handler) => (e) => { 17 | if (e.target === e.currentTarget) { 18 | handler.bind(t)(e); 19 | } 20 | }; 21 | 22 | return currentTargetHandler; 23 | } 24 | 25 | /** 26 | * Stop propgation shorthand. 27 | * 28 | * @example Usage: 29 | * ``` 30 | * 33 | * 34 | * ``` 35 | */ 36 | export function stopProp(e: Event) { 37 | e.stopPropagation(); 38 | } 39 | -------------------------------------------------------------------------------- /frontend/src/utils/orgs.ts: -------------------------------------------------------------------------------- 1 | import { AccessCode, type OrgData } from "@/types/org"; 2 | 3 | export * from "@/types/org"; 4 | 5 | export function isOwner(accessCode?: AccessCode): boolean { 6 | if (!accessCode) return false; 7 | 8 | return accessCode === AccessCode.owner; 9 | } 10 | 11 | export function isAdmin(accessCode?: AccessCode): boolean { 12 | if (!accessCode) return false; 13 | 14 | return accessCode >= AccessCode.owner; 15 | } 16 | 17 | export function isCrawler(accessCode?: AccessCode): boolean { 18 | if (!accessCode) return false; 19 | 20 | return accessCode >= AccessCode.crawler; 21 | } 22 | 23 | export function isArchivingDisabled( 24 | org?: OrgData | null, 25 | checkExecMinutesQuota = false, 26 | ): boolean { 27 | return Boolean( 28 | !org || 29 | org.readOnly || 30 | org.storageQuotaReached || 31 | (checkExecMinutesQuota ? org.execMinutesQuotaReached : false), 32 | ); 33 | } 34 | -------------------------------------------------------------------------------- /frontend/src/utils/polyfills.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-explicit-any, @typescript-eslint/no-unnecessary-condition */ 2 | // cSpell:disable 3 | 4 | /** 5 | * Object.entriesFrom() polyfill 6 | * @author Chris Ferdinandi 7 | * @license MIT 8 | */ 9 | 10 | if (!Object.fromEntries) { 11 | Object.fromEntries = function ( 12 | entries: Iterable, 13 | ): { [k: string]: T } { 14 | if (!entries?.[Symbol.iterator]) { 15 | throw new Error( 16 | "Object.fromEntries() requires a single iterable argument", 17 | ); 18 | } 19 | const obj: { [k: PropertyKey]: T } = {}; 20 | for (const [key, value] of entries) { 21 | obj[key] = value; 22 | } 23 | return obj; 24 | }; 25 | } 26 | -------------------------------------------------------------------------------- /frontend/src/utils/replay.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Format timestamp returned from API into string format 3 | * accepted by `` `ts`. 4 | */ 5 | export function formatRwpTimestamp(ts?: string | null): string | undefined { 6 | if (!ts) return; 7 | return ts.split(".")[0].replace(/\D/g, ""); 8 | } 9 | -------------------------------------------------------------------------------- /frontend/src/utils/router.ts: -------------------------------------------------------------------------------- 1 | import { ROUTES } from "@/routes"; 2 | import APIRouter from "@/utils/APIRouter"; 3 | import { cached } from "@/utils/weakCache"; 4 | 5 | const router = new APIRouter(ROUTES); 6 | 7 | export const urlForName = cached(router.urlForName); 8 | 9 | export default router; 10 | -------------------------------------------------------------------------------- /frontend/src/utils/slugify.ts: -------------------------------------------------------------------------------- 1 | import slugify from "slugify"; 2 | 3 | import localize from "./localize"; 4 | 5 | export default function slugifyStrict(value: string) { 6 | return slugify(value, { 7 | strict: true, 8 | lower: true, 9 | locale: localize.activeLanguage, 10 | }); 11 | } 12 | -------------------------------------------------------------------------------- /frontend/src/utils/string.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Escape string to use as regex 3 | * From https://github.com/tc39/proposal-regex-escaping/blob/main/polyfill.js#L3 4 | */ 5 | export function regexEscape(s: unknown) { 6 | return String(s).replace(/[\\^$*+?.()|[\]{}]/g, "\\$&"); 7 | } 8 | 9 | export function regexUnescape(s: unknown) { 10 | return String(s).replace(/(\\|\/\.\*)/g, ""); 11 | } 12 | -------------------------------------------------------------------------------- /frontend/src/utils/tailwind.ts: -------------------------------------------------------------------------------- 1 | export const tw = (strings: TemplateStringsArray, ...values: string[]) => 2 | String.raw({ raw: strings }, ...values); 3 | -------------------------------------------------------------------------------- /frontend/src/utils/timeoutCache.ts: -------------------------------------------------------------------------------- 1 | import { type Cache } from "./weakCache"; 2 | 3 | export function timeoutCache(seconds: number) { 4 | return class implements Cache { 5 | private readonly cache: { [key: string]: V } = Object.create(null); 6 | set(key: K | string, value: V) { 7 | if (typeof key !== "string") { 8 | key = JSON.stringify(key); 9 | } 10 | this.cache[key] = value; 11 | setTimeout(() => { 12 | try { 13 | delete this.cache[key]; 14 | } catch (_) { 15 | /* empty */ 16 | console.debug("missing key", key); 17 | } 18 | }, seconds * 1000); 19 | return this; 20 | } 21 | get(key: K | string) { 22 | if (typeof key !== "string") { 23 | key = JSON.stringify(key); 24 | } 25 | try { 26 | return this.cache[key]; 27 | } catch (_) { 28 | console.debug("missing key", key); 29 | /* empty */ 30 | } 31 | } 32 | has(key: K | string) { 33 | if (typeof key !== "string") { 34 | key = JSON.stringify(key); 35 | } 36 | return Object.prototype.hasOwnProperty.call(this.cache, key); 37 | } 38 | }; 39 | } 40 | -------------------------------------------------------------------------------- /frontend/src/utils/user.ts: -------------------------------------------------------------------------------- 1 | import type { APIUser } from "@/index"; 2 | import { type UserInfo } from "@/types/user"; 3 | 4 | export function formatAPIUser(userData: APIUser): UserInfo { 5 | return { 6 | id: userData.id, 7 | email: userData.email, 8 | name: userData.name, 9 | isVerified: userData.is_verified, 10 | isSuperAdmin: userData.is_superuser, 11 | orgs: userData.orgs, 12 | }; 13 | } 14 | -------------------------------------------------------------------------------- /frontend/tests/login.spec.ts: -------------------------------------------------------------------------------- 1 | import { test } from "@playwright/test"; 2 | import { chromium } from "playwright"; 3 | 4 | test("test", async ({ baseURL }) => { 5 | const browser = await chromium.launch({ headless: true }); 6 | const context = await browser.newContext(); 7 | const page = await context.newPage(); 8 | 9 | try { 10 | await page.goto(baseURL!); 11 | await page.waitForLoadState("load"); 12 | await page.waitForSelector('input[name="username"]'); 13 | await page.click('input[name="username"]'); 14 | 15 | const e2eEmail = process.env.E2E_USER_EMAIL; 16 | if (!e2eEmail) { 17 | throw new Error( 18 | "E2E_USER_EMAIL environment variable is not defined or null.", 19 | ); 20 | } 21 | await page.fill('input[name="username"]', e2eEmail); 22 | await page.click('input[name="password"]'); 23 | const e2ePassword = process.env.E2E_USER_PASSWORD; 24 | if (!e2ePassword) { 25 | throw new Error( 26 | "E2E_USER_PASSWORD environment variable is not defined or null.", 27 | ); 28 | } 29 | await page.fill('input[name="password"]', e2ePassword); 30 | await page.click('sl-button:has-text("Log In")'); 31 | } finally { 32 | await browser.close(); 33 | } 34 | }); 35 | -------------------------------------------------------------------------------- /frontend/tsconfig.eslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "types": ["@types/node"], 4 | "noEmit": true 5 | }, 6 | "extends": "./tsconfig.json", 7 | "include": ["**/*.ts", "**/*.js", "*.mjs"] 8 | } 9 | -------------------------------------------------------------------------------- /frontend/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "outDir": "./dist/", 4 | "module": "esnext", 5 | "target": "es6", 6 | "moduleResolution": "bundler", 7 | "allowJs": true, 8 | "strict": true, 9 | "noImplicitAny": true, 10 | "allowSyntheticDefaultImports": true, 11 | "experimentalDecorators": true, 12 | "importHelpers": true, 13 | "sourceMap": true, 14 | "inlineSources": true, 15 | "skipLibCheck": true, 16 | "esModuleInterop": true, 17 | "useDefineForClassFields": false, 18 | "plugins": [ 19 | { 20 | "name": "ts-lit-plugin", 21 | "strict": true, 22 | "rules": { 23 | "no-missing-import": "off", 24 | }, 25 | "maxNodeModuleImportDepth": -1, 26 | }, 27 | ], 28 | "incremental": true, 29 | "paths": { 30 | "@/*": ["./src/*"], 31 | "~assets/*": ["./src/assets/*"], 32 | }, 33 | "lib": ["DOM", "DOM.Iterable", "ES2021.WeakRef"], 34 | }, 35 | "include": ["**/*.ts"], 36 | "exclude": ["node_modules"], 37 | } 38 | -------------------------------------------------------------------------------- /frontend/vendor.webpack.config.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Separate vendor modules to speed up development rebuild 3 | */ 4 | const path = require("path"); 5 | 6 | const webpack = require("webpack"); 7 | 8 | module.exports = { 9 | entry: { 10 | lit: ["lit", "@lit/localize"], 11 | }, 12 | output: { 13 | path: path.join(__dirname, "dist/vendor"), 14 | filename: "dll.[name].js", 15 | library: "[name]_[fullhash]", 16 | }, 17 | plugins: [ 18 | new webpack.DllPlugin({ 19 | path: path.join(__dirname, "dist/vendor", "[name]-manifest.json"), 20 | name: "[name]_[fullhash]", 21 | }), 22 | ], 23 | }; 24 | -------------------------------------------------------------------------------- /frontend/webpack.prod.js: -------------------------------------------------------------------------------- 1 | // @ts-check 2 | 3 | const ESLintPlugin = require("eslint-webpack-plugin"); 4 | const TerserPlugin = require("terser-webpack-plugin"); 5 | const { merge } = require("webpack-merge"); 6 | 7 | const baseConfigs = require("./webpack.config.js"); 8 | const [main, vnc] = baseConfigs; 9 | 10 | module.exports = [ 11 | merge(main, { 12 | mode: "production", 13 | devtool: "source-map", 14 | 15 | // TODO figure out minifying lit templates 16 | optimization: { 17 | runtimeChunk: "single", 18 | splitChunks: { 19 | // Split both async and non-async chunks (only async by default) 20 | chunks: "all", 21 | }, 22 | minimize: true, 23 | minimizer: [ 24 | new TerserPlugin({ 25 | terserOptions: { 26 | compress: { 27 | drop_console: ["log", "info"], 28 | }, 29 | }, 30 | }), 31 | ], 32 | }, 33 | plugins: [ 34 | new ESLintPlugin({ 35 | failOnWarning: true, 36 | extensions: ["ts", "js"], 37 | }), 38 | ], 39 | }), 40 | { 41 | ...vnc, 42 | mode: "production", 43 | }, 44 | ]; 45 | -------------------------------------------------------------------------------- /frontend/xliff/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webrecorder/browsertrix/54d29aec0585c358126861c03c2212596c5d2c3f/frontend/xliff/.keep -------------------------------------------------------------------------------- /pylintrc: -------------------------------------------------------------------------------- 1 | [MESSAGE CONTROL] 2 | disable=duplicate-code 3 | -------------------------------------------------------------------------------- /scripts/build-backend.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | CURR=$(dirname "${BASH_SOURCE[0]}") 3 | 4 | docker build -t ${REGISTRY}webrecorder/browsertrix-backend:latest $CURR/../backend/ 5 | 6 | if [ -n "$REGISTRY" ]; then 7 | docker push ${REGISTRY}webrecorder/browsertrix-backend 8 | fi 9 | -------------------------------------------------------------------------------- /scripts/build-frontend.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | CURR=$(dirname "${BASH_SOURCE[0]}") 3 | 4 | DOCKER_BUILDKIT=1 docker build --build-arg GIT_COMMIT_HASH="$(git rev-parse --short HEAD)" --build-arg GIT_BRANCH_NAME="$(git rev-parse --abbrev-ref HEAD)" --build-arg --load -t ${REGISTRY}webrecorder/browsertrix-frontend:latest $CURR/../frontend/ 5 | 6 | if [ -n "$REGISTRY" ]; then 7 | docker push ${REGISTRY}webrecorder/browsertrix-frontend 8 | fi 9 | -------------------------------------------------------------------------------- /scripts/minikube-build-and-deploy.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | CURR=$(dirname "${BASH_SOURCE[0]}") 3 | 4 | eval $(minikube docker-env) 5 | for img in backend frontend; 6 | do 7 | sh "${CURR}/build-${img}.sh" 8 | done 9 | 10 | echo "Deploying helm chart..." 11 | helm upgrade --wait --install -f ./chart/values.yaml -f ./chart/local.yaml btrix ./chart/ 12 | 13 | until kubectl port-forward service/browsertrix-cloud-frontend 8000:80; do 14 | echo "Unable to forward service/browsertrix-cloud-frontend. Retrying.." >&2 15 | sleep 1 16 | done 17 | -------------------------------------------------------------------------------- /scripts/minikube-reset.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | if [ "$(minikube status | grep -o Running | wc -l)" -lt 3 ]; then 3 | echo "Error: Less than 3 components are running in Minikube" 4 | exit 1 5 | fi 6 | 7 | if kubectl config get-contexts | grep -q minikube; then 8 | kubectl config set-context minikube 9 | # ~~~ DANGER ZONE ~~~ 10 | echo "Uninstalling helm deployment and deleting pvcs" 11 | helm uninstall btrix 12 | minikube kubectl delete pvc minio-storage-pvc 13 | minikube kubectl delete pvc data-db-local-mongo-0 14 | fi 15 | -------------------------------------------------------------------------------- /update-version.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | version=`cat version.txt` 4 | jq ".version=\"$version\"" ./frontend/package.json > ./tmp-package.json 5 | mv ./tmp-package.json ./frontend/package.json 6 | 7 | echo '"""current version"""' > ./backend/btrixcloud/version.py 8 | echo "" >> ./backend/btrixcloud/version.py 9 | echo "__version__ = \"$version\"" >> ./backend/btrixcloud/version.py 10 | 11 | sed -E -i "" "s/^version:.*$/version: v$version/" chart/Chart.yaml 12 | 13 | sed -E -i "" "s/\/browsertrix-backend:[[:alnum:].-]+/\/browsertrix-backend:$version/" chart/values.yaml 14 | sed -E -i "" "s/\/browsertrix-frontend:[[:alnum:].-]+/\/browsertrix-frontend:$version/" chart/values.yaml 15 | -------------------------------------------------------------------------------- /version.txt: -------------------------------------------------------------------------------- 1 | 1.17.0-beta.0 2 | -------------------------------------------------------------------------------- /yarn.lock: -------------------------------------------------------------------------------- 1 | # THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. 2 | # yarn lockfile v1 3 | 4 | 5 | --------------------------------------------------------------------------------