├── .github └── workflows │ ├── main.yml │ └── publish.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .readthedocs.yml ├── CHANGES.rst ├── Dockerfile ├── LICENSE ├── README.rst ├── artwork ├── logo.gif └── logo.png ├── docs ├── api.rst ├── changes.rst ├── conf.py ├── index.rst └── requirements.txt ├── pyproject.toml ├── scrapyrt ├── __init__.py ├── cmdline.py ├── conf │ ├── __init__.py │ ├── default_settings.py │ └── spider_settings.py ├── core.py ├── log.py ├── resources.py └── utils.py ├── tests ├── __init__.py ├── sample_data │ ├── spider-start-project │ │ ├── project │ │ │ ├── __init__.py │ │ │ ├── settings.py │ │ │ └── spiders │ │ │ │ ├── __init__.py │ │ │ │ ├── new.py │ │ │ │ ├── old.py │ │ │ │ └── universal.py │ │ └── scrapy.cfg │ └── testproject │ │ ├── scrapy.cfg │ │ └── testproject │ │ ├── __init__.py │ │ ├── items.py │ │ ├── settings.py │ │ ├── spider_templates │ │ └── testspider_startrequests.py.jinja │ │ └── spiders │ │ ├── __init__.py │ │ └── testspider.py ├── servers.py ├── spiders.py ├── test_cmdline.py ├── test_crawl_manager.py ├── test_crawler_runner.py ├── test_deprecations.py ├── test_log.py ├── test_log_observer.py ├── test_resource_crawl.py ├── test_resource_realtimeapi.py ├── test_resource_root.py ├── test_resource_serviceresource.py ├── test_settings │ ├── __init__.py │ ├── default_settings.py │ └── settings.py ├── test_spider_start.py ├── test_utils.py ├── testsite │ ├── app.py │ ├── index.html │ ├── page1.html │ ├── page2.html │ └── page3.html └── utils.py └── tox.ini /.github/workflows/main.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/.github/workflows/main.yml -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/.github/workflows/publish.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/.gitignore -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/.pre-commit-config.yaml -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/.readthedocs.yml -------------------------------------------------------------------------------- /CHANGES.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/CHANGES.rst -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/Dockerfile -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/LICENSE -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/README.rst -------------------------------------------------------------------------------- /artwork/logo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/artwork/logo.gif -------------------------------------------------------------------------------- /artwork/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/artwork/logo.png -------------------------------------------------------------------------------- /docs/api.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/docs/api.rst -------------------------------------------------------------------------------- /docs/changes.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CHANGES.rst 2 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/docs/conf.py -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/docs/index.rst -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/docs/requirements.txt -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/pyproject.toml -------------------------------------------------------------------------------- /scrapyrt/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /scrapyrt/cmdline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/scrapyrt/cmdline.py -------------------------------------------------------------------------------- /scrapyrt/conf/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/scrapyrt/conf/__init__.py -------------------------------------------------------------------------------- /scrapyrt/conf/default_settings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/scrapyrt/conf/default_settings.py -------------------------------------------------------------------------------- /scrapyrt/conf/spider_settings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/scrapyrt/conf/spider_settings.py -------------------------------------------------------------------------------- /scrapyrt/core.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/scrapyrt/core.py -------------------------------------------------------------------------------- /scrapyrt/log.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/scrapyrt/log.py -------------------------------------------------------------------------------- /scrapyrt/resources.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/scrapyrt/resources.py -------------------------------------------------------------------------------- /scrapyrt/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/scrapyrt/utils.py -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/tests/__init__.py -------------------------------------------------------------------------------- /tests/sample_data/spider-start-project/project/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/sample_data/spider-start-project/project/settings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/tests/sample_data/spider-start-project/project/settings.py -------------------------------------------------------------------------------- /tests/sample_data/spider-start-project/project/spiders/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/sample_data/spider-start-project/project/spiders/new.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/tests/sample_data/spider-start-project/project/spiders/new.py -------------------------------------------------------------------------------- /tests/sample_data/spider-start-project/project/spiders/old.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/tests/sample_data/spider-start-project/project/spiders/old.py -------------------------------------------------------------------------------- /tests/sample_data/spider-start-project/project/spiders/universal.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/tests/sample_data/spider-start-project/project/spiders/universal.py -------------------------------------------------------------------------------- /tests/sample_data/spider-start-project/scrapy.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/tests/sample_data/spider-start-project/scrapy.cfg -------------------------------------------------------------------------------- /tests/sample_data/testproject/scrapy.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/tests/sample_data/testproject/scrapy.cfg -------------------------------------------------------------------------------- /tests/sample_data/testproject/testproject/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/sample_data/testproject/testproject/items.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/tests/sample_data/testproject/testproject/items.py -------------------------------------------------------------------------------- /tests/sample_data/testproject/testproject/settings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/tests/sample_data/testproject/testproject/settings.py -------------------------------------------------------------------------------- /tests/sample_data/testproject/testproject/spider_templates/testspider_startrequests.py.jinja: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/tests/sample_data/testproject/testproject/spider_templates/testspider_startrequests.py.jinja -------------------------------------------------------------------------------- /tests/sample_data/testproject/testproject/spiders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/tests/sample_data/testproject/testproject/spiders/__init__.py -------------------------------------------------------------------------------- /tests/sample_data/testproject/testproject/spiders/testspider.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/tests/sample_data/testproject/testproject/spiders/testspider.py -------------------------------------------------------------------------------- /tests/servers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/tests/servers.py -------------------------------------------------------------------------------- /tests/spiders.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/tests/spiders.py -------------------------------------------------------------------------------- /tests/test_cmdline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/tests/test_cmdline.py -------------------------------------------------------------------------------- /tests/test_crawl_manager.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/tests/test_crawl_manager.py -------------------------------------------------------------------------------- /tests/test_crawler_runner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/tests/test_crawler_runner.py -------------------------------------------------------------------------------- /tests/test_deprecations.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/tests/test_deprecations.py -------------------------------------------------------------------------------- /tests/test_log.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/tests/test_log.py -------------------------------------------------------------------------------- /tests/test_log_observer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/tests/test_log_observer.py -------------------------------------------------------------------------------- /tests/test_resource_crawl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/tests/test_resource_crawl.py -------------------------------------------------------------------------------- /tests/test_resource_realtimeapi.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/tests/test_resource_realtimeapi.py -------------------------------------------------------------------------------- /tests/test_resource_root.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/tests/test_resource_root.py -------------------------------------------------------------------------------- /tests/test_resource_serviceresource.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/tests/test_resource_serviceresource.py -------------------------------------------------------------------------------- /tests/test_settings/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/tests/test_settings/__init__.py -------------------------------------------------------------------------------- /tests/test_settings/default_settings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/tests/test_settings/default_settings.py -------------------------------------------------------------------------------- /tests/test_settings/settings.py: -------------------------------------------------------------------------------- 1 | A = "B" 2 | -------------------------------------------------------------------------------- /tests/test_spider_start.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/tests/test_spider_start.py -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/tests/test_utils.py -------------------------------------------------------------------------------- /tests/testsite/app.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/tests/testsite/app.py -------------------------------------------------------------------------------- /tests/testsite/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/tests/testsite/index.html -------------------------------------------------------------------------------- /tests/testsite/page1.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/tests/testsite/page1.html -------------------------------------------------------------------------------- /tests/testsite/page2.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/tests/testsite/page2.html -------------------------------------------------------------------------------- /tests/testsite/page3.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/tests/testsite/page3.html -------------------------------------------------------------------------------- /tests/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/tests/utils.py -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapyrt/HEAD/tox.ini --------------------------------------------------------------------------------