├── .gitignore ├── .gitlab-ci.yml ├── CHANGELOG.md ├── LICENSE ├── README.md ├── docs ├── Makefile ├── make.bat └── source │ ├── Makefile │ ├── conf.py │ ├── dispatching.rst │ ├── index.rst │ ├── make.bat │ ├── modules.rst │ └── scraperx.rst ├── examples ├── README.md ├── config.yaml ├── dispatch_cookie_selenium.py ├── gen_cookie_requests.py ├── json_source.py ├── minimal.py ├── multiple_extractors.py ├── multiple_sources.py ├── page_dispatch.py ├── page_trigger_download.py └── qa_results.py ├── scraperx ├── __init__.py ├── arguments.py ├── config.py ├── dispatch.py ├── download.py ├── exceptions.py ├── extract.py ├── parsers.py ├── proxies.py ├── run_cli.py ├── save_to.py ├── scraper.py ├── test.py ├── trigger.py ├── user_agent.py ├── utils.py └── write.py ├── setup.py ├── tests ├── __init__.py ├── files │ ├── example.json │ └── windows_1252_encoded.json ├── test_parsers.py └── test_utils.py └── tox.ini /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/.gitignore -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/.gitlab-ci.yml -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/CHANGELOG.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/README.md -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/docs/Makefile -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/docs/make.bat -------------------------------------------------------------------------------- /docs/source/Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/docs/source/Makefile -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/docs/source/conf.py -------------------------------------------------------------------------------- /docs/source/dispatching.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/docs/source/dispatching.rst -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/docs/source/index.rst -------------------------------------------------------------------------------- /docs/source/make.bat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/docs/source/make.bat -------------------------------------------------------------------------------- /docs/source/modules.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/docs/source/modules.rst -------------------------------------------------------------------------------- /docs/source/scraperx.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/docs/source/scraperx.rst -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/examples/README.md -------------------------------------------------------------------------------- /examples/config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/examples/config.yaml -------------------------------------------------------------------------------- /examples/dispatch_cookie_selenium.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/examples/dispatch_cookie_selenium.py -------------------------------------------------------------------------------- /examples/gen_cookie_requests.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/examples/gen_cookie_requests.py -------------------------------------------------------------------------------- /examples/json_source.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/examples/json_source.py -------------------------------------------------------------------------------- /examples/minimal.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/examples/minimal.py -------------------------------------------------------------------------------- /examples/multiple_extractors.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/examples/multiple_extractors.py -------------------------------------------------------------------------------- /examples/multiple_sources.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/examples/multiple_sources.py -------------------------------------------------------------------------------- /examples/page_dispatch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/examples/page_dispatch.py -------------------------------------------------------------------------------- /examples/page_trigger_download.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/examples/page_trigger_download.py -------------------------------------------------------------------------------- /examples/qa_results.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/examples/qa_results.py -------------------------------------------------------------------------------- /scraperx/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/scraperx/__init__.py -------------------------------------------------------------------------------- /scraperx/arguments.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/scraperx/arguments.py -------------------------------------------------------------------------------- /scraperx/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/scraperx/config.py -------------------------------------------------------------------------------- /scraperx/dispatch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/scraperx/dispatch.py -------------------------------------------------------------------------------- /scraperx/download.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/scraperx/download.py -------------------------------------------------------------------------------- /scraperx/exceptions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/scraperx/exceptions.py -------------------------------------------------------------------------------- /scraperx/extract.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/scraperx/extract.py -------------------------------------------------------------------------------- /scraperx/parsers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/scraperx/parsers.py -------------------------------------------------------------------------------- /scraperx/proxies.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/scraperx/proxies.py -------------------------------------------------------------------------------- /scraperx/run_cli.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/scraperx/run_cli.py -------------------------------------------------------------------------------- /scraperx/save_to.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/scraperx/save_to.py -------------------------------------------------------------------------------- /scraperx/scraper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/scraperx/scraper.py -------------------------------------------------------------------------------- /scraperx/test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/scraperx/test.py -------------------------------------------------------------------------------- /scraperx/trigger.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/scraperx/trigger.py -------------------------------------------------------------------------------- /scraperx/user_agent.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/scraperx/user_agent.py -------------------------------------------------------------------------------- /scraperx/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/scraperx/utils.py -------------------------------------------------------------------------------- /scraperx/write.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/scraperx/write.py -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/setup.py -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/files/example.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/tests/files/example.json -------------------------------------------------------------------------------- /tests/files/windows_1252_encoded.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/tests/files/windows_1252_encoded.json -------------------------------------------------------------------------------- /tests/test_parsers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/tests/test_parsers.py -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/tests/test_utils.py -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtream1101/scraperx/HEAD/tox.ini --------------------------------------------------------------------------------