├── .bumpversion.cfg ├── .codecov.yml ├── .github └── workflows │ ├── publish.yml │ └── test.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .readthedocs.yml ├── CHANGES.rst ├── LICENSE ├── README.rst ├── docs ├── Makefile ├── _ext │ └── __init__.py ├── changes.rst ├── conf.py ├── customization │ ├── index.rst │ ├── pages.rst │ └── spiders.rst ├── features │ └── search.rst ├── index.rst ├── make.bat ├── reference │ ├── api.rst │ ├── reqmeta.rst │ └── settings.rst ├── requirements.txt ├── setup.rst └── templates │ ├── article.rst │ ├── e-commerce.rst │ ├── google-search.rst │ ├── index.rst │ └── job-posting.rst ├── pyproject.toml ├── requirements-dev.txt ├── setup.cfg ├── setup.py ├── tests ├── __init__.py ├── conftest.py ├── incremental │ ├── test_collection_fp_manager.py │ ├── test_incremental_manager.py │ └── test_middleware.py ├── mockserver.py ├── pages │ ├── __init__.py │ ├── test_article_navigation_heuristics.py │ └── test_product_navigation_heuristics.py ├── test_addon.py ├── test_article.py ├── test_base.py ├── test_ecommerce.py ├── test_feeds.py ├── test_heuristics.py ├── test_job_posting.py ├── test_middlewares.py ├── test_params.py ├── test_params_location_param.py ├── test_search.py ├── test_serp.py ├── test_utils.py └── utils.py ├── tox.ini ├── utils ├── google-gl-updater │ ├── requirements.in │ ├── requirements.txt │ ├── template.py │ └── update.py └── google-hl-updater │ ├── requirements.in │ ├── requirements.txt │ ├── template.py │ └── update.py └── zyte_spider_templates ├── __init__.py ├── _addon.py ├── _geolocations.py ├── _incremental ├── __init__.py ├── manager.py └── middleware.py ├── _lang_codes.py ├── documentation.py ├── feeds.py ├── heuristics.py ├── middlewares.py ├── page_objects ├── __init__.py └── product_navigation_heuristics.py ├── pages ├── __init__.py ├── article_heuristics.py ├── product_navigation_heuristics.py └── search_request_template.py ├── params.py ├── spiders ├── __init__.py ├── _google_domains.py ├── _google_gl.py ├── _google_hl.py ├── article.py ├── base.py ├── ecommerce.py ├── job_posting.py └── serp.py └── utils.py /.bumpversion.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/.bumpversion.cfg -------------------------------------------------------------------------------- /.codecov.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/.codecov.yml -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/.github/workflows/publish.yml -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/.github/workflows/test.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/.gitignore -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/.pre-commit-config.yaml -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/.readthedocs.yml -------------------------------------------------------------------------------- /CHANGES.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/CHANGES.rst -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/LICENSE -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/README.rst -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/docs/Makefile -------------------------------------------------------------------------------- /docs/_ext/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/docs/_ext/__init__.py -------------------------------------------------------------------------------- /docs/changes.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CHANGES.rst 2 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/docs/conf.py -------------------------------------------------------------------------------- /docs/customization/index.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/docs/customization/index.rst -------------------------------------------------------------------------------- /docs/customization/pages.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/docs/customization/pages.rst -------------------------------------------------------------------------------- /docs/customization/spiders.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/docs/customization/spiders.rst -------------------------------------------------------------------------------- /docs/features/search.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/docs/features/search.rst -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/docs/index.rst -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/docs/make.bat -------------------------------------------------------------------------------- /docs/reference/api.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/docs/reference/api.rst -------------------------------------------------------------------------------- /docs/reference/reqmeta.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/docs/reference/reqmeta.rst -------------------------------------------------------------------------------- /docs/reference/settings.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/docs/reference/settings.rst -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/docs/requirements.txt -------------------------------------------------------------------------------- /docs/setup.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/docs/setup.rst -------------------------------------------------------------------------------- /docs/templates/article.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/docs/templates/article.rst -------------------------------------------------------------------------------- /docs/templates/e-commerce.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/docs/templates/e-commerce.rst -------------------------------------------------------------------------------- /docs/templates/google-search.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/docs/templates/google-search.rst -------------------------------------------------------------------------------- /docs/templates/index.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/docs/templates/index.rst -------------------------------------------------------------------------------- /docs/templates/job-posting.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/docs/templates/job-posting.rst -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/pyproject.toml -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | pre-commit 2 | pytest 3 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/setup.cfg -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/setup.py -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/tests/__init__.py -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/tests/conftest.py -------------------------------------------------------------------------------- /tests/incremental/test_collection_fp_manager.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/tests/incremental/test_collection_fp_manager.py -------------------------------------------------------------------------------- /tests/incremental/test_incremental_manager.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/tests/incremental/test_incremental_manager.py -------------------------------------------------------------------------------- /tests/incremental/test_middleware.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/tests/incremental/test_middleware.py -------------------------------------------------------------------------------- /tests/mockserver.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/tests/mockserver.py -------------------------------------------------------------------------------- /tests/pages/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/pages/test_article_navigation_heuristics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/tests/pages/test_article_navigation_heuristics.py -------------------------------------------------------------------------------- /tests/pages/test_product_navigation_heuristics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/tests/pages/test_product_navigation_heuristics.py -------------------------------------------------------------------------------- /tests/test_addon.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/tests/test_addon.py -------------------------------------------------------------------------------- /tests/test_article.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/tests/test_article.py -------------------------------------------------------------------------------- /tests/test_base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/tests/test_base.py -------------------------------------------------------------------------------- /tests/test_ecommerce.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/tests/test_ecommerce.py -------------------------------------------------------------------------------- /tests/test_feeds.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/tests/test_feeds.py -------------------------------------------------------------------------------- /tests/test_heuristics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/tests/test_heuristics.py -------------------------------------------------------------------------------- /tests/test_job_posting.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/tests/test_job_posting.py -------------------------------------------------------------------------------- /tests/test_middlewares.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/tests/test_middlewares.py -------------------------------------------------------------------------------- /tests/test_params.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/tests/test_params.py -------------------------------------------------------------------------------- /tests/test_params_location_param.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/tests/test_params_location_param.py -------------------------------------------------------------------------------- /tests/test_search.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/tests/test_search.py -------------------------------------------------------------------------------- /tests/test_serp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/tests/test_serp.py -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/tests/test_utils.py -------------------------------------------------------------------------------- /tests/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/tests/utils.py -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/tox.ini -------------------------------------------------------------------------------- /utils/google-gl-updater/requirements.in: -------------------------------------------------------------------------------- 1 | jinja2 2 | parsel 3 | requests 4 | -------------------------------------------------------------------------------- /utils/google-gl-updater/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/utils/google-gl-updater/requirements.txt -------------------------------------------------------------------------------- /utils/google-gl-updater/template.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/utils/google-gl-updater/template.py -------------------------------------------------------------------------------- /utils/google-gl-updater/update.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/utils/google-gl-updater/update.py -------------------------------------------------------------------------------- /utils/google-hl-updater/requirements.in: -------------------------------------------------------------------------------- 1 | jinja2 2 | parsel 3 | requests 4 | -------------------------------------------------------------------------------- /utils/google-hl-updater/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/utils/google-hl-updater/requirements.txt -------------------------------------------------------------------------------- /utils/google-hl-updater/template.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/utils/google-hl-updater/template.py -------------------------------------------------------------------------------- /utils/google-hl-updater/update.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/utils/google-hl-updater/update.py -------------------------------------------------------------------------------- /zyte_spider_templates/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/zyte_spider_templates/__init__.py -------------------------------------------------------------------------------- /zyte_spider_templates/_addon.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/zyte_spider_templates/_addon.py -------------------------------------------------------------------------------- /zyte_spider_templates/_geolocations.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/zyte_spider_templates/_geolocations.py -------------------------------------------------------------------------------- /zyte_spider_templates/_incremental/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /zyte_spider_templates/_incremental/manager.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/zyte_spider_templates/_incremental/manager.py -------------------------------------------------------------------------------- /zyte_spider_templates/_incremental/middleware.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/zyte_spider_templates/_incremental/middleware.py -------------------------------------------------------------------------------- /zyte_spider_templates/_lang_codes.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/zyte_spider_templates/_lang_codes.py -------------------------------------------------------------------------------- /zyte_spider_templates/documentation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/zyte_spider_templates/documentation.py -------------------------------------------------------------------------------- /zyte_spider_templates/feeds.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/zyte_spider_templates/feeds.py -------------------------------------------------------------------------------- /zyte_spider_templates/heuristics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/zyte_spider_templates/heuristics.py -------------------------------------------------------------------------------- /zyte_spider_templates/middlewares.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/zyte_spider_templates/middlewares.py -------------------------------------------------------------------------------- /zyte_spider_templates/page_objects/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/zyte_spider_templates/page_objects/__init__.py -------------------------------------------------------------------------------- /zyte_spider_templates/page_objects/product_navigation_heuristics.py: -------------------------------------------------------------------------------- 1 | from ..pages import HeuristicsProductNavigationPage 2 | -------------------------------------------------------------------------------- /zyte_spider_templates/pages/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/zyte_spider_templates/pages/__init__.py -------------------------------------------------------------------------------- /zyte_spider_templates/pages/article_heuristics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/zyte_spider_templates/pages/article_heuristics.py -------------------------------------------------------------------------------- /zyte_spider_templates/pages/product_navigation_heuristics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/zyte_spider_templates/pages/product_navigation_heuristics.py -------------------------------------------------------------------------------- /zyte_spider_templates/pages/search_request_template.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/zyte_spider_templates/pages/search_request_template.py -------------------------------------------------------------------------------- /zyte_spider_templates/params.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/zyte_spider_templates/params.py -------------------------------------------------------------------------------- /zyte_spider_templates/spiders/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /zyte_spider_templates/spiders/_google_domains.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/zyte_spider_templates/spiders/_google_domains.py -------------------------------------------------------------------------------- /zyte_spider_templates/spiders/_google_gl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/zyte_spider_templates/spiders/_google_gl.py -------------------------------------------------------------------------------- /zyte_spider_templates/spiders/_google_hl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/zyte_spider_templates/spiders/_google_hl.py -------------------------------------------------------------------------------- /zyte_spider_templates/spiders/article.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/zyte_spider_templates/spiders/article.py -------------------------------------------------------------------------------- /zyte_spider_templates/spiders/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/zyte_spider_templates/spiders/base.py -------------------------------------------------------------------------------- /zyte_spider_templates/spiders/ecommerce.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/zyte_spider_templates/spiders/ecommerce.py -------------------------------------------------------------------------------- /zyte_spider_templates/spiders/job_posting.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/zyte_spider_templates/spiders/job_posting.py -------------------------------------------------------------------------------- /zyte_spider_templates/spiders/serp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/zyte_spider_templates/spiders/serp.py -------------------------------------------------------------------------------- /zyte_spider_templates/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zytedata/zyte-spider-templates/HEAD/zyte_spider_templates/utils.py --------------------------------------------------------------------------------