├── .gitignore ├── COPYING ├── Howto.md ├── Readme.md ├── crawler ├── .dockerignore ├── .gitignore ├── LICENSE ├── build.sh ├── chrome_extensions │ └── webrtc-blocker │ │ ├── DOCUMENTATION.md │ │ ├── LICENSE │ │ ├── README.md │ │ ├── html │ │ └── options.html │ │ ├── img │ │ └── icon128.png │ │ ├── js │ │ ├── background.js │ │ └── options.js │ │ └── manifest.json ├── deploy_all.js ├── doc │ ├── Iliou_Towards_a_-framework_for_detecting_advanced_Web_bots.pdf │ ├── WebCrawling.pdf │ ├── bad-bot-report-2019.pdf │ ├── crawling_survey.pdf │ ├── gauravpandey44_docker-swarm.pdf │ ├── todo.md │ └── vastel-madweb20.pdf ├── docker-compose.yml ├── env │ └── skeleton_crawler.env ├── package-lock.json ├── package.json ├── push.sh ├── serverless.yml ├── src │ ├── browser_worker.ts │ ├── config.ts │ ├── crawl_queue.ts │ ├── enqueue.ts │ ├── handler.ts │ ├── helpers.ts │ ├── http_worker.ts │ ├── index.ts │ ├── metadata.ts │ ├── persistant_handler.ts │ ├── proxy.ts │ ├── proxy_server.ts │ ├── server │ │ ├── Readme.md │ │ ├── app.ts │ │ ├── main.controller.ts │ │ ├── middleware │ │ │ └── auth.ts │ │ ├── server.ts │ │ └── worker.service.ts │ ├── termination.ts │ └── worker.ts ├── test │ ├── Testing.md │ ├── clean_state_tests.ts │ ├── detection_tests.ts │ ├── direct_test.js │ ├── fingerprint │ │ ├── fingerprint2.js │ │ ├── index.html │ │ └── index2.html │ ├── fingerprint_tests.ts │ ├── integration_tests.ts │ ├── lambda_test.js │ ├── more_proxy_tests.ts │ ├── proxy_tests.ts │ ├── remote_tests.ts │ ├── screen.ts │ ├── test_server.ts │ ├── test_utils.ts │ ├── webrtc │ │ └── index2.html │ └── worker_test.ts ├── test_evasion_antoinevastel.png ├── test_evasion_intoli.png ├── test_evasion_sannysoft.png ├── tsconfig.json └── worker_images │ ├── with_alpine │ ├── .dockerignore │ ├── Dockerfile │ └── package.json │ └── with_node_slim │ ├── .dockerignore │ ├── Dockerfile │ ├── pptr_test.js │ └── start_xvfb_and_run_cmd.sh ├── docs ├── Proxy.md ├── diagram │ ├── arch_diagram.png │ ├── arch_diagram2.png │ ├── crawling infra architecture (2).drawio │ ├── crawling infra architecture (6).drawio │ └── crawling infra architecture.drawio └── images │ ├── ami_instance.png │ ├── crawl_task_config.png │ └── ec2_setup.png ├── lib ├── .gitignore ├── LICENSE ├── index.ts ├── misc │ ├── helpers.ts │ ├── http.ts │ ├── logger.ts │ ├── shell.ts │ └── stats.ts ├── package-lock.json ├── package.json ├── storage │ └── storage.ts ├── test │ └── test.js ├── tsconfig.json └── types │ ├── common.ts │ ├── proxy.ts │ ├── queue.ts │ └── workermeta.ts ├── master ├── .dockerignore ├── .gitignore ├── Dockerfile ├── LICENSE ├── certs │ ├── server.cert │ └── server.key ├── ctrl.js ├── deploy │ ├── build.sh │ ├── deploy.sh │ ├── dev.sh │ ├── env │ │ └── deploy.env │ ├── exclude.txt │ ├── login.sh │ ├── prod.sh │ └── test.sh ├── doc │ ├── Crawling Backends.md │ ├── Deploy.md │ ├── DeployNotes.md │ ├── Frontend.md │ ├── Proxy.md │ ├── QueueArch.md │ └── Swarm.md ├── docker-compose.dev.yml ├── docker-compose.prod.yml ├── docker-compose.yml ├── env │ ├── skeleton_development.env │ └── skeleton_production.env ├── frontend │ ├── .dockerignore │ ├── .gitignore │ ├── Dockerfile │ ├── README.md │ ├── launch.sh │ ├── package.json │ ├── public │ │ ├── favicon.ico │ │ ├── index.html │ │ ├── logo192.png │ │ ├── logo512.png │ │ ├── manifest.json │ │ └── robots.txt │ ├── src │ │ ├── App.css │ │ ├── App.test.tsx │ │ ├── App.tsx │ │ ├── common │ │ │ └── api.ts │ │ ├── components │ │ │ ├── config.tsx │ │ │ ├── items.tsx │ │ │ ├── logs.tsx │ │ │ ├── machines.tsx │ │ │ ├── proxies.tsx │ │ │ ├── task_create.tsx │ │ │ ├── task_detail.tsx │ │ │ ├── task_edit.tsx │ │ │ ├── tasks.tsx │ │ │ ├── test.tsx │ │ │ └── workers.tsx │ │ ├── index.css │ │ ├── index.tsx │ │ ├── logo.svg │ │ ├── react-app-env.d.ts │ │ ├── serviceWorker.ts │ │ └── setupTests.ts │ └── tsconfig.json ├── launch_frontend_interface.sh ├── nodemon.json ├── other │ ├── domain_count.js │ └── dump_collection.sh ├── package.json ├── queue.json ├── scheduler │ ├── Dockerfile │ ├── Readme.md │ ├── config.ts │ ├── crawl_worker │ │ └── docker-compose.yml │ ├── daemon.ts │ ├── run.ts │ ├── runner.ts │ ├── scheduler.conf.json │ └── swarm_worker_allocator.ts ├── src │ ├── app.ts │ ├── constants │ │ └── crawlTaskApi.constants.ts │ ├── db │ │ └── db.ts │ ├── fixtures │ │ └── proxies.ts │ ├── main.controller.ts │ ├── middleware │ │ └── auth.ts │ ├── models │ │ ├── config.ts │ │ ├── crawltask.model.js │ │ ├── crawltask.model.ts │ │ ├── machine.model.ts │ │ ├── proxy.model.ts │ │ ├── queue.model.ts │ │ └── workermeta.model.ts │ ├── public │ │ ├── fingerprint.html │ │ ├── scripts │ │ │ └── index.js │ │ └── stylesheets │ │ │ └── style.css │ ├── server.ts │ ├── services │ │ ├── api.service.ts │ │ ├── crawltask.service.ts │ │ ├── helpers.ts │ │ ├── infra.service.ts │ │ ├── lib.ts │ │ ├── proxy.service.ts │ │ ├── queue.service.ts │ │ ├── stats.service.ts │ │ └── test.service.ts │ └── swagger.json ├── tsconfig.json ├── tsconfig.prod.json └── wait-for.sh └── scripts ├── create_buckets.sh └── delete_buckets.sh /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/.gitignore -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/COPYING -------------------------------------------------------------------------------- /Howto.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/Howto.md -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/Readme.md -------------------------------------------------------------------------------- /crawler/.dockerignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/.dockerignore -------------------------------------------------------------------------------- /crawler/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/.gitignore -------------------------------------------------------------------------------- /crawler/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/LICENSE -------------------------------------------------------------------------------- /crawler/build.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/build.sh -------------------------------------------------------------------------------- /crawler/chrome_extensions/webrtc-blocker/DOCUMENTATION.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/chrome_extensions/webrtc-blocker/DOCUMENTATION.md -------------------------------------------------------------------------------- /crawler/chrome_extensions/webrtc-blocker/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/chrome_extensions/webrtc-blocker/LICENSE -------------------------------------------------------------------------------- /crawler/chrome_extensions/webrtc-blocker/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/chrome_extensions/webrtc-blocker/README.md -------------------------------------------------------------------------------- /crawler/chrome_extensions/webrtc-blocker/html/options.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/chrome_extensions/webrtc-blocker/html/options.html -------------------------------------------------------------------------------- /crawler/chrome_extensions/webrtc-blocker/img/icon128.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/chrome_extensions/webrtc-blocker/img/icon128.png -------------------------------------------------------------------------------- /crawler/chrome_extensions/webrtc-blocker/js/background.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/chrome_extensions/webrtc-blocker/js/background.js -------------------------------------------------------------------------------- /crawler/chrome_extensions/webrtc-blocker/js/options.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/chrome_extensions/webrtc-blocker/js/options.js -------------------------------------------------------------------------------- /crawler/chrome_extensions/webrtc-blocker/manifest.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/chrome_extensions/webrtc-blocker/manifest.json -------------------------------------------------------------------------------- /crawler/deploy_all.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/deploy_all.js -------------------------------------------------------------------------------- /crawler/doc/Iliou_Towards_a_-framework_for_detecting_advanced_Web_bots.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/doc/Iliou_Towards_a_-framework_for_detecting_advanced_Web_bots.pdf -------------------------------------------------------------------------------- /crawler/doc/WebCrawling.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/doc/WebCrawling.pdf -------------------------------------------------------------------------------- /crawler/doc/bad-bot-report-2019.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/doc/bad-bot-report-2019.pdf -------------------------------------------------------------------------------- /crawler/doc/crawling_survey.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/doc/crawling_survey.pdf -------------------------------------------------------------------------------- /crawler/doc/gauravpandey44_docker-swarm.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/doc/gauravpandey44_docker-swarm.pdf -------------------------------------------------------------------------------- /crawler/doc/todo.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/doc/todo.md -------------------------------------------------------------------------------- /crawler/doc/vastel-madweb20.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/doc/vastel-madweb20.pdf -------------------------------------------------------------------------------- /crawler/docker-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/docker-compose.yml -------------------------------------------------------------------------------- /crawler/env/skeleton_crawler.env: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/env/skeleton_crawler.env -------------------------------------------------------------------------------- /crawler/package-lock.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/package-lock.json -------------------------------------------------------------------------------- /crawler/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/package.json -------------------------------------------------------------------------------- /crawler/push.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/push.sh -------------------------------------------------------------------------------- /crawler/serverless.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/serverless.yml -------------------------------------------------------------------------------- /crawler/src/browser_worker.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/src/browser_worker.ts -------------------------------------------------------------------------------- /crawler/src/config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/src/config.ts -------------------------------------------------------------------------------- /crawler/src/crawl_queue.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/src/crawl_queue.ts -------------------------------------------------------------------------------- /crawler/src/enqueue.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/src/enqueue.ts -------------------------------------------------------------------------------- /crawler/src/handler.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/src/handler.ts -------------------------------------------------------------------------------- /crawler/src/helpers.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/src/helpers.ts -------------------------------------------------------------------------------- /crawler/src/http_worker.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/src/http_worker.ts -------------------------------------------------------------------------------- /crawler/src/index.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/src/index.ts -------------------------------------------------------------------------------- /crawler/src/metadata.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/src/metadata.ts -------------------------------------------------------------------------------- /crawler/src/persistant_handler.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/src/persistant_handler.ts -------------------------------------------------------------------------------- /crawler/src/proxy.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/src/proxy.ts -------------------------------------------------------------------------------- /crawler/src/proxy_server.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/src/proxy_server.ts -------------------------------------------------------------------------------- /crawler/src/server/Readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/src/server/Readme.md -------------------------------------------------------------------------------- /crawler/src/server/app.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/src/server/app.ts -------------------------------------------------------------------------------- /crawler/src/server/main.controller.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/src/server/main.controller.ts -------------------------------------------------------------------------------- /crawler/src/server/middleware/auth.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/src/server/middleware/auth.ts -------------------------------------------------------------------------------- /crawler/src/server/server.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/src/server/server.ts -------------------------------------------------------------------------------- /crawler/src/server/worker.service.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/src/server/worker.service.ts -------------------------------------------------------------------------------- /crawler/src/termination.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/src/termination.ts -------------------------------------------------------------------------------- /crawler/src/worker.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/src/worker.ts -------------------------------------------------------------------------------- /crawler/test/Testing.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/test/Testing.md -------------------------------------------------------------------------------- /crawler/test/clean_state_tests.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/test/clean_state_tests.ts -------------------------------------------------------------------------------- /crawler/test/detection_tests.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/test/detection_tests.ts -------------------------------------------------------------------------------- /crawler/test/direct_test.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/test/direct_test.js -------------------------------------------------------------------------------- /crawler/test/fingerprint/fingerprint2.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/test/fingerprint/fingerprint2.js -------------------------------------------------------------------------------- /crawler/test/fingerprint/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/test/fingerprint/index.html -------------------------------------------------------------------------------- /crawler/test/fingerprint/index2.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/test/fingerprint/index2.html -------------------------------------------------------------------------------- /crawler/test/fingerprint_tests.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/test/fingerprint_tests.ts -------------------------------------------------------------------------------- /crawler/test/integration_tests.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/test/integration_tests.ts -------------------------------------------------------------------------------- /crawler/test/lambda_test.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/test/lambda_test.js -------------------------------------------------------------------------------- /crawler/test/more_proxy_tests.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/test/more_proxy_tests.ts -------------------------------------------------------------------------------- /crawler/test/proxy_tests.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/test/proxy_tests.ts -------------------------------------------------------------------------------- /crawler/test/remote_tests.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/test/remote_tests.ts -------------------------------------------------------------------------------- /crawler/test/screen.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/test/screen.ts -------------------------------------------------------------------------------- /crawler/test/test_server.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/test/test_server.ts -------------------------------------------------------------------------------- /crawler/test/test_utils.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/test/test_utils.ts -------------------------------------------------------------------------------- /crawler/test/webrtc/index2.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/test/webrtc/index2.html -------------------------------------------------------------------------------- /crawler/test/worker_test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/test/worker_test.ts -------------------------------------------------------------------------------- /crawler/test_evasion_antoinevastel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/test_evasion_antoinevastel.png -------------------------------------------------------------------------------- /crawler/test_evasion_intoli.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/test_evasion_intoli.png -------------------------------------------------------------------------------- /crawler/test_evasion_sannysoft.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/test_evasion_sannysoft.png -------------------------------------------------------------------------------- /crawler/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/tsconfig.json -------------------------------------------------------------------------------- /crawler/worker_images/with_alpine/.dockerignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /crawler/worker_images/with_alpine/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/worker_images/with_alpine/Dockerfile -------------------------------------------------------------------------------- /crawler/worker_images/with_alpine/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/worker_images/with_alpine/package.json -------------------------------------------------------------------------------- /crawler/worker_images/with_node_slim/.dockerignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/worker_images/with_node_slim/.dockerignore -------------------------------------------------------------------------------- /crawler/worker_images/with_node_slim/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/worker_images/with_node_slim/Dockerfile -------------------------------------------------------------------------------- /crawler/worker_images/with_node_slim/pptr_test.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/worker_images/with_node_slim/pptr_test.js -------------------------------------------------------------------------------- /crawler/worker_images/with_node_slim/start_xvfb_and_run_cmd.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/crawler/worker_images/with_node_slim/start_xvfb_and_run_cmd.sh -------------------------------------------------------------------------------- /docs/Proxy.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/docs/Proxy.md -------------------------------------------------------------------------------- /docs/diagram/arch_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/docs/diagram/arch_diagram.png -------------------------------------------------------------------------------- /docs/diagram/arch_diagram2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/docs/diagram/arch_diagram2.png -------------------------------------------------------------------------------- /docs/diagram/crawling infra architecture (2).drawio: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/docs/diagram/crawling infra architecture (2).drawio -------------------------------------------------------------------------------- /docs/diagram/crawling infra architecture (6).drawio: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/docs/diagram/crawling infra architecture (6).drawio -------------------------------------------------------------------------------- /docs/diagram/crawling infra architecture.drawio: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/docs/diagram/crawling infra architecture.drawio -------------------------------------------------------------------------------- /docs/images/ami_instance.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/docs/images/ami_instance.png -------------------------------------------------------------------------------- /docs/images/crawl_task_config.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/docs/images/crawl_task_config.png -------------------------------------------------------------------------------- /docs/images/ec2_setup.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/docs/images/ec2_setup.png -------------------------------------------------------------------------------- /lib/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | .idea/ -------------------------------------------------------------------------------- /lib/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/lib/LICENSE -------------------------------------------------------------------------------- /lib/index.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/lib/index.ts -------------------------------------------------------------------------------- /lib/misc/helpers.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/lib/misc/helpers.ts -------------------------------------------------------------------------------- /lib/misc/http.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/lib/misc/http.ts -------------------------------------------------------------------------------- /lib/misc/logger.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/lib/misc/logger.ts -------------------------------------------------------------------------------- /lib/misc/shell.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/lib/misc/shell.ts -------------------------------------------------------------------------------- /lib/misc/stats.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/lib/misc/stats.ts -------------------------------------------------------------------------------- /lib/package-lock.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/lib/package-lock.json -------------------------------------------------------------------------------- /lib/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/lib/package.json -------------------------------------------------------------------------------- /lib/storage/storage.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/lib/storage/storage.ts -------------------------------------------------------------------------------- /lib/test/test.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/lib/test/test.js -------------------------------------------------------------------------------- /lib/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/lib/tsconfig.json -------------------------------------------------------------------------------- /lib/types/common.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/lib/types/common.ts -------------------------------------------------------------------------------- /lib/types/proxy.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/lib/types/proxy.ts -------------------------------------------------------------------------------- /lib/types/queue.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/lib/types/queue.ts -------------------------------------------------------------------------------- /lib/types/workermeta.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/lib/types/workermeta.ts -------------------------------------------------------------------------------- /master/.dockerignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/.dockerignore -------------------------------------------------------------------------------- /master/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/.gitignore -------------------------------------------------------------------------------- /master/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/Dockerfile -------------------------------------------------------------------------------- /master/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/LICENSE -------------------------------------------------------------------------------- /master/certs/server.cert: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/certs/server.cert -------------------------------------------------------------------------------- /master/certs/server.key: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/certs/server.key -------------------------------------------------------------------------------- /master/ctrl.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/ctrl.js -------------------------------------------------------------------------------- /master/deploy/build.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/deploy/build.sh -------------------------------------------------------------------------------- /master/deploy/deploy.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/deploy/deploy.sh -------------------------------------------------------------------------------- /master/deploy/dev.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/deploy/dev.sh -------------------------------------------------------------------------------- /master/deploy/env/deploy.env: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/deploy/env/deploy.env -------------------------------------------------------------------------------- /master/deploy/exclude.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/deploy/exclude.txt -------------------------------------------------------------------------------- /master/deploy/login.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/deploy/login.sh -------------------------------------------------------------------------------- /master/deploy/prod.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/deploy/prod.sh -------------------------------------------------------------------------------- /master/deploy/test.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/deploy/test.sh -------------------------------------------------------------------------------- /master/doc/Crawling Backends.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/doc/Crawling Backends.md -------------------------------------------------------------------------------- /master/doc/Deploy.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/doc/Deploy.md -------------------------------------------------------------------------------- /master/doc/DeployNotes.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/doc/DeployNotes.md -------------------------------------------------------------------------------- /master/doc/Frontend.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/doc/Frontend.md -------------------------------------------------------------------------------- /master/doc/Proxy.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/doc/Proxy.md -------------------------------------------------------------------------------- /master/doc/QueueArch.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/doc/QueueArch.md -------------------------------------------------------------------------------- /master/doc/Swarm.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/doc/Swarm.md -------------------------------------------------------------------------------- /master/docker-compose.dev.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/docker-compose.dev.yml -------------------------------------------------------------------------------- /master/docker-compose.prod.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/docker-compose.prod.yml -------------------------------------------------------------------------------- /master/docker-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/docker-compose.yml -------------------------------------------------------------------------------- /master/env/skeleton_development.env: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/env/skeleton_development.env -------------------------------------------------------------------------------- /master/env/skeleton_production.env: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/env/skeleton_production.env -------------------------------------------------------------------------------- /master/frontend/.dockerignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/frontend/.dockerignore -------------------------------------------------------------------------------- /master/frontend/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/frontend/.gitignore -------------------------------------------------------------------------------- /master/frontend/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/frontend/Dockerfile -------------------------------------------------------------------------------- /master/frontend/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/frontend/README.md -------------------------------------------------------------------------------- /master/frontend/launch.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/frontend/launch.sh -------------------------------------------------------------------------------- /master/frontend/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/frontend/package.json -------------------------------------------------------------------------------- /master/frontend/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/frontend/public/favicon.ico -------------------------------------------------------------------------------- /master/frontend/public/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/frontend/public/index.html -------------------------------------------------------------------------------- /master/frontend/public/logo192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/frontend/public/logo192.png -------------------------------------------------------------------------------- /master/frontend/public/logo512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/frontend/public/logo512.png -------------------------------------------------------------------------------- /master/frontend/public/manifest.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/frontend/public/manifest.json -------------------------------------------------------------------------------- /master/frontend/public/robots.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/frontend/public/robots.txt -------------------------------------------------------------------------------- /master/frontend/src/App.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/frontend/src/App.css -------------------------------------------------------------------------------- /master/frontend/src/App.test.tsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/frontend/src/App.test.tsx -------------------------------------------------------------------------------- /master/frontend/src/App.tsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/frontend/src/App.tsx -------------------------------------------------------------------------------- /master/frontend/src/common/api.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/frontend/src/common/api.ts -------------------------------------------------------------------------------- /master/frontend/src/components/config.tsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/frontend/src/components/config.tsx -------------------------------------------------------------------------------- /master/frontend/src/components/items.tsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/frontend/src/components/items.tsx -------------------------------------------------------------------------------- /master/frontend/src/components/logs.tsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/frontend/src/components/logs.tsx -------------------------------------------------------------------------------- /master/frontend/src/components/machines.tsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/frontend/src/components/machines.tsx -------------------------------------------------------------------------------- /master/frontend/src/components/proxies.tsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/frontend/src/components/proxies.tsx -------------------------------------------------------------------------------- /master/frontend/src/components/task_create.tsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/frontend/src/components/task_create.tsx -------------------------------------------------------------------------------- /master/frontend/src/components/task_detail.tsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/frontend/src/components/task_detail.tsx -------------------------------------------------------------------------------- /master/frontend/src/components/task_edit.tsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/frontend/src/components/task_edit.tsx -------------------------------------------------------------------------------- /master/frontend/src/components/tasks.tsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/frontend/src/components/tasks.tsx -------------------------------------------------------------------------------- /master/frontend/src/components/test.tsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/frontend/src/components/test.tsx -------------------------------------------------------------------------------- /master/frontend/src/components/workers.tsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/frontend/src/components/workers.tsx -------------------------------------------------------------------------------- /master/frontend/src/index.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/frontend/src/index.css -------------------------------------------------------------------------------- /master/frontend/src/index.tsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/frontend/src/index.tsx -------------------------------------------------------------------------------- /master/frontend/src/logo.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/frontend/src/logo.svg -------------------------------------------------------------------------------- /master/frontend/src/react-app-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | -------------------------------------------------------------------------------- /master/frontend/src/serviceWorker.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/frontend/src/serviceWorker.ts -------------------------------------------------------------------------------- /master/frontend/src/setupTests.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/frontend/src/setupTests.ts -------------------------------------------------------------------------------- /master/frontend/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/frontend/tsconfig.json -------------------------------------------------------------------------------- /master/launch_frontend_interface.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/launch_frontend_interface.sh -------------------------------------------------------------------------------- /master/nodemon.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/nodemon.json -------------------------------------------------------------------------------- /master/other/domain_count.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/other/domain_count.js -------------------------------------------------------------------------------- /master/other/dump_collection.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/other/dump_collection.sh -------------------------------------------------------------------------------- /master/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/package.json -------------------------------------------------------------------------------- /master/queue.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/queue.json -------------------------------------------------------------------------------- /master/scheduler/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/scheduler/Dockerfile -------------------------------------------------------------------------------- /master/scheduler/Readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/scheduler/Readme.md -------------------------------------------------------------------------------- /master/scheduler/config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/scheduler/config.ts -------------------------------------------------------------------------------- /master/scheduler/crawl_worker/docker-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/scheduler/crawl_worker/docker-compose.yml -------------------------------------------------------------------------------- /master/scheduler/daemon.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/scheduler/daemon.ts -------------------------------------------------------------------------------- /master/scheduler/run.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/scheduler/run.ts -------------------------------------------------------------------------------- /master/scheduler/runner.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/scheduler/runner.ts -------------------------------------------------------------------------------- /master/scheduler/scheduler.conf.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/scheduler/scheduler.conf.json -------------------------------------------------------------------------------- /master/scheduler/swarm_worker_allocator.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/scheduler/swarm_worker_allocator.ts -------------------------------------------------------------------------------- /master/src/app.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/src/app.ts -------------------------------------------------------------------------------- /master/src/constants/crawlTaskApi.constants.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/src/constants/crawlTaskApi.constants.ts -------------------------------------------------------------------------------- /master/src/db/db.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/src/db/db.ts -------------------------------------------------------------------------------- /master/src/fixtures/proxies.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/src/fixtures/proxies.ts -------------------------------------------------------------------------------- /master/src/main.controller.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/src/main.controller.ts -------------------------------------------------------------------------------- /master/src/middleware/auth.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/src/middleware/auth.ts -------------------------------------------------------------------------------- /master/src/models/config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/src/models/config.ts -------------------------------------------------------------------------------- /master/src/models/crawltask.model.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/src/models/crawltask.model.js -------------------------------------------------------------------------------- /master/src/models/crawltask.model.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/src/models/crawltask.model.ts -------------------------------------------------------------------------------- /master/src/models/machine.model.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/src/models/machine.model.ts -------------------------------------------------------------------------------- /master/src/models/proxy.model.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/src/models/proxy.model.ts -------------------------------------------------------------------------------- /master/src/models/queue.model.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/src/models/queue.model.ts -------------------------------------------------------------------------------- /master/src/models/workermeta.model.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/src/models/workermeta.model.ts -------------------------------------------------------------------------------- /master/src/public/fingerprint.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/src/public/fingerprint.html -------------------------------------------------------------------------------- /master/src/public/scripts/index.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/src/public/scripts/index.js -------------------------------------------------------------------------------- /master/src/public/stylesheets/style.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/src/public/stylesheets/style.css -------------------------------------------------------------------------------- /master/src/server.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/src/server.ts -------------------------------------------------------------------------------- /master/src/services/api.service.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/src/services/api.service.ts -------------------------------------------------------------------------------- /master/src/services/crawltask.service.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/src/services/crawltask.service.ts -------------------------------------------------------------------------------- /master/src/services/helpers.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/src/services/helpers.ts -------------------------------------------------------------------------------- /master/src/services/infra.service.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/src/services/infra.service.ts -------------------------------------------------------------------------------- /master/src/services/lib.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/src/services/lib.ts -------------------------------------------------------------------------------- /master/src/services/proxy.service.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/src/services/proxy.service.ts -------------------------------------------------------------------------------- /master/src/services/queue.service.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/src/services/queue.service.ts -------------------------------------------------------------------------------- /master/src/services/stats.service.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/src/services/stats.service.ts -------------------------------------------------------------------------------- /master/src/services/test.service.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/src/services/test.service.ts -------------------------------------------------------------------------------- /master/src/swagger.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/src/swagger.json -------------------------------------------------------------------------------- /master/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/tsconfig.json -------------------------------------------------------------------------------- /master/tsconfig.prod.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/tsconfig.prod.json -------------------------------------------------------------------------------- /master/wait-for.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/master/wait-for.sh -------------------------------------------------------------------------------- /scripts/create_buckets.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/scripts/create_buckets.sh -------------------------------------------------------------------------------- /scripts/delete_buckets.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NikolaiT/Crawling-Infrastructure/HEAD/scripts/delete_buckets.sh --------------------------------------------------------------------------------