├── .bandit.yml ├── .bumpversion.cfg ├── .cookiecutterrc ├── .coveragerc ├── .dockerignore ├── .editorconfig ├── .flake8 ├── .gitattributes ├── .github ├── ISSUE_TEMPLATE.md ├── PULL_REQUEST_TEMPLATE.md └── workflows │ ├── builds.yml │ ├── checks.yml │ ├── docs.yml │ └── tests.yml ├── .gitignore ├── .isort.cfg ├── .pre-commit-config.yaml ├── .python-version ├── .readthedocs.yml ├── .travis.yml ├── AUTHORS.rst ├── CONTRIBUTING.rst ├── Dockerfile ├── HISTORY.rst ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.rst ├── TODO.rst ├── VERSION ├── coverage.xml ├── docker-compose.yaml ├── docs ├── Makefile ├── authors.rst ├── conf.py ├── contributing.rst ├── history.rst ├── index.rst ├── installation.rst ├── make.bat ├── modules.rst ├── readme.rst ├── requirements.txt └── scrapy_redis.rst ├── example-project ├── Dockerfile ├── README.rst ├── docker-compose.yml ├── example │ ├── __init__.py │ ├── items.py │ ├── pipelines.py │ ├── settings.py │ └── spiders │ │ ├── __init__.py │ │ ├── dmoz.py │ │ ├── mycrawler_redis.py │ │ └── myspider_redis.py ├── process_items.py ├── requirements.txt └── scrapy.cfg ├── pylintrc ├── pytest.ini ├── requirements-tests.txt ├── requirements.txt ├── setup.cfg ├── setup.py ├── src └── scrapy_redis │ ├── __init__.py │ ├── connection.py │ ├── defaults.py │ ├── dupefilter.py │ ├── picklecompat.py │ ├── pipelines.py │ ├── queue.py │ ├── scheduler.py │ ├── spiders.py │ ├── stats.py │ └── utils.py ├── tests ├── test_connection.py ├── test_dupefilter.py ├── test_package_import.py ├── test_picklecompat.py ├── test_queue.py ├── test_scrapy_redis.py ├── test_spiders.py └── test_utils.py └── tox.ini /.bandit.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/.bandit.yml -------------------------------------------------------------------------------- /.bumpversion.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/.bumpversion.cfg -------------------------------------------------------------------------------- /.cookiecutterrc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/.cookiecutterrc -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/.coveragerc -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/.dockerignore -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/.editorconfig -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/.flake8 -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/.gitattributes -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/.github/ISSUE_TEMPLATE.md -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/.github/PULL_REQUEST_TEMPLATE.md -------------------------------------------------------------------------------- /.github/workflows/builds.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/.github/workflows/builds.yml -------------------------------------------------------------------------------- /.github/workflows/checks.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/.github/workflows/checks.yml -------------------------------------------------------------------------------- /.github/workflows/docs.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/.github/workflows/docs.yml -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/.github/workflows/tests.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/.gitignore -------------------------------------------------------------------------------- /.isort.cfg: -------------------------------------------------------------------------------- 1 | [settings] 2 | profile = black 3 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/.pre-commit-config.yaml -------------------------------------------------------------------------------- /.python-version: -------------------------------------------------------------------------------- 1 | 3.10.13 2 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/.readthedocs.yml -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/.travis.yml -------------------------------------------------------------------------------- /AUTHORS.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/AUTHORS.rst -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/CONTRIBUTING.rst -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/Dockerfile -------------------------------------------------------------------------------- /HISTORY.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/HISTORY.rst -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/LICENSE -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/MANIFEST.in -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/Makefile -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/README.rst -------------------------------------------------------------------------------- /TODO.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/TODO.rst -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 0.9.1 2 | -------------------------------------------------------------------------------- /coverage.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/coverage.xml -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/docker-compose.yaml -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/docs/Makefile -------------------------------------------------------------------------------- /docs/authors.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../AUTHORS.rst 2 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/docs/conf.py -------------------------------------------------------------------------------- /docs/contributing.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CONTRIBUTING.rst 2 | -------------------------------------------------------------------------------- /docs/history.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../HISTORY.rst 2 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/docs/index.rst -------------------------------------------------------------------------------- /docs/installation.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/docs/installation.rst -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/docs/make.bat -------------------------------------------------------------------------------- /docs/modules.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/docs/modules.rst -------------------------------------------------------------------------------- /docs/readme.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../README.rst 2 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/docs/requirements.txt -------------------------------------------------------------------------------- /docs/scrapy_redis.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/docs/scrapy_redis.rst -------------------------------------------------------------------------------- /example-project/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/example-project/Dockerfile -------------------------------------------------------------------------------- /example-project/README.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/example-project/README.rst -------------------------------------------------------------------------------- /example-project/docker-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/example-project/docker-compose.yml -------------------------------------------------------------------------------- /example-project/example/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /example-project/example/items.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/example-project/example/items.py -------------------------------------------------------------------------------- /example-project/example/pipelines.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/example-project/example/pipelines.py -------------------------------------------------------------------------------- /example-project/example/settings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/example-project/example/settings.py -------------------------------------------------------------------------------- /example-project/example/spiders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/example-project/example/spiders/__init__.py -------------------------------------------------------------------------------- /example-project/example/spiders/dmoz.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/example-project/example/spiders/dmoz.py -------------------------------------------------------------------------------- /example-project/example/spiders/mycrawler_redis.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/example-project/example/spiders/mycrawler_redis.py -------------------------------------------------------------------------------- /example-project/example/spiders/myspider_redis.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/example-project/example/spiders/myspider_redis.py -------------------------------------------------------------------------------- /example-project/process_items.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/example-project/process_items.py -------------------------------------------------------------------------------- /example-project/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/example-project/requirements.txt -------------------------------------------------------------------------------- /example-project/scrapy.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/example-project/scrapy.cfg -------------------------------------------------------------------------------- /pylintrc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/pylintrc -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/pytest.ini -------------------------------------------------------------------------------- /requirements-tests.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/requirements-tests.txt -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | scrapy>=2.6.0 2 | redis>=4.2 3 | six>=1.15 4 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/setup.cfg -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/setup.py -------------------------------------------------------------------------------- /src/scrapy_redis/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/src/scrapy_redis/__init__.py -------------------------------------------------------------------------------- /src/scrapy_redis/connection.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/src/scrapy_redis/connection.py -------------------------------------------------------------------------------- /src/scrapy_redis/defaults.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/src/scrapy_redis/defaults.py -------------------------------------------------------------------------------- /src/scrapy_redis/dupefilter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/src/scrapy_redis/dupefilter.py -------------------------------------------------------------------------------- /src/scrapy_redis/picklecompat.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/src/scrapy_redis/picklecompat.py -------------------------------------------------------------------------------- /src/scrapy_redis/pipelines.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/src/scrapy_redis/pipelines.py -------------------------------------------------------------------------------- /src/scrapy_redis/queue.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/src/scrapy_redis/queue.py -------------------------------------------------------------------------------- /src/scrapy_redis/scheduler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/src/scrapy_redis/scheduler.py -------------------------------------------------------------------------------- /src/scrapy_redis/spiders.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/src/scrapy_redis/spiders.py -------------------------------------------------------------------------------- /src/scrapy_redis/stats.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/src/scrapy_redis/stats.py -------------------------------------------------------------------------------- /src/scrapy_redis/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/src/scrapy_redis/utils.py -------------------------------------------------------------------------------- /tests/test_connection.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/tests/test_connection.py -------------------------------------------------------------------------------- /tests/test_dupefilter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/tests/test_dupefilter.py -------------------------------------------------------------------------------- /tests/test_package_import.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/tests/test_package_import.py -------------------------------------------------------------------------------- /tests/test_picklecompat.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/tests/test_picklecompat.py -------------------------------------------------------------------------------- /tests/test_queue.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/tests/test_queue.py -------------------------------------------------------------------------------- /tests/test_scrapy_redis.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/tests/test_scrapy_redis.py -------------------------------------------------------------------------------- /tests/test_spiders.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/tests/test_spiders.py -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/tests/test_utils.py -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmax/scrapy-redis/HEAD/tox.ini --------------------------------------------------------------------------------