├── .github └── workflows │ ├── ci.yml │ └── publish_to_pypi.yml ├── .gitignore ├── LICENSE ├── README.md ├── requirements.txt ├── scrapy_fieldstats ├── __init__.py └── fieldstats.py ├── setup.cfg ├── setup.py ├── tests ├── __init__.py └── test_scrapy_fieldstats.py └── tox.ini /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | build: 11 | 12 | runs-on: ubuntu-latest 13 | strategy: 14 | matrix: 15 | python-version: [3.8] 16 | 17 | steps: 18 | - uses: actions/checkout@v2 19 | - name: Set up Python ${{ matrix.python-version }} 20 | uses: actions/setup-python@v1 21 | with: 22 | python-version: ${{ matrix.python-version }} 23 | - name: Install dependencies 24 | run: | 25 | pip install -r requirements.txt 26 | - name: Lint with flake8 27 | run: | 28 | pip install flake8 29 | flake8 . 30 | - name: Test with pytest 31 | run: | 32 | pip install pytest 33 | pytest 34 | -------------------------------------------------------------------------------- /.github/workflows/publish_to_pypi.yml: -------------------------------------------------------------------------------- 1 | name: Upload Package to PyPI 2 | 3 | on: 4 | release: 5 | types: [created] 6 | 7 | jobs: 8 | deploy: 9 | 10 | runs-on: ubuntu-latest 11 | 12 | steps: 13 | - uses: actions/checkout@v2 14 | - name: Set up Python 15 | uses: actions/setup-python@v1 16 | with: 17 | python-version: '3.x' 18 | - name: Install dependencies 19 | run: | 20 | python -m pip install --upgrade pip 21 | pip install setuptools wheel twine 22 | - name: Build and publish 23 | env: 24 | TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} 25 | TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} 26 | run: | 27 | python setup.py sdist bdist_wheel 28 | twine upload dist/* 29 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | .static_storage/ 56 | .media/ 57 | local_settings.py 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | 106 | # editor 107 | .vscode 108 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | MIT License 3 | 4 | Copyright (c) 2017, Valdir Stumm Junior 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | 8 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 9 | 10 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 11 | 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Scrapy FieldStats 2 | ================= 3 | 4 | ![](https://github.com/stummjr/scrapy-fieldstats/workflows/CI/badge.svg) 5 | [![Downloads](https://pepy.tech/badge/scrapy-fieldstats)](https://pepy.tech/project/scrapy-fieldstats) 6 | 7 | A Scrapy extension that generates a summary of fields coverage from your scraped data. 8 | 9 | 10 | ## What? 11 | Upon finishing a job, Scrapy prints some useful stats about that job, such as: number of requests, responses, scraped items, etc. 12 | 13 | However, it's often useful to have an overview of the field coverage in such scraped items. Let's say you want to know the percentage of items missing the `price` field. That's when this extension comes into play! 14 | 15 | Check out an example: 16 | 17 | ```bash 18 | $ scrapy crawl example 19 | 2017-10-12 11:10:10 [scrapy.utils.log] INFO: Scrapy 1.4.0 started (bot: examplebot) 20 | ... 21 | 2017-10-12 11:10:20 [scrapy_fieldstats.fieldstats] INFO: Field stats: 22 | { 23 | 'author': { 24 |        'name': '100.0%', 25 |        'age':  '52.0%' 26 | }, 27 |    'image': '97.0%', 28 | 'title': '100.0%', 29 | 'price': '92.0%', 30 | 'stars': '47.5%' 31 | } 32 | 2017-10-12 11:10:20 [scrapy.statscollectors] INFO: Dumping Scrapy stats: 33 | ... 34 | ``` 35 | 36 | ## Installation 37 | First, pip install this package: 38 | 39 | ```bash 40 | $ pip install scrapy-fieldstats 41 | ``` 42 | 43 | ## Usage 44 | Enable the extension in your project's `settings.py` file, by adding the following lines: 45 | 46 | ```python 47 | EXTENSIONS = { 48 | 'scrapy_fieldstats.fieldstats.FieldStatsExtension': 10, 49 | } 50 | FIELDSTATS_ENABLED = True 51 | ``` 52 | That's all! Now run your job and have a look at the field stats. 53 | 54 | 55 | ## Settings 56 | The settings below can be defined as any other Scrapy settings, as described on [Scrapy docs](https://doc.scrapy.org/en/latest/topics/settings.html#populating-the-settings). 57 | 58 | * `FIELDSTATS_ENABLED`: to enable/disable the extension. 59 | * `FIELDSTATS_COUNTS_ONLY`: when `True`, the extension will output absolute counts, instead of percentages. 60 | * `FIELDSTATS_SKIP_NONE`: when `True`, `None` values won't be counted as existing values for fields. 61 | * `FIELDSTATS_ADD_TO_STATS`: when `True`, the extension will add the field coverage report to the job stats. 62 | 63 | 64 | ## Contributing 65 | If you spot a bug, or want to propose a new feature please create an issue in this project's 66 | [issue tracker](https://github.com/stummjr/scrapy-fieldstats/issues). 67 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | scrapy 2 | -------------------------------------------------------------------------------- /scrapy_fieldstats/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """Top-level package for Scrapy FieldStats.""" 4 | 5 | __author__ = """Valdir Stumm Junior""" 6 | __email__ = 'stummjr@gmail.com' 7 | __version__ = '0.2.0' 8 | -------------------------------------------------------------------------------- /scrapy_fieldstats/fieldstats.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | import logging 3 | import pprint 4 | 5 | from scrapy import signals 6 | from scrapy.exceptions import NotConfigured 7 | 8 | logger = logging.getLogger(__name__) 9 | 10 | 11 | class FieldStatsExtension(object): 12 | """ When enabled, the FieldStats extension logs the percentage of 13 | items coverage for a crawl. 14 | """ 15 | def __init__(self, show_counts=False, skip_none=True, add_to_stats=True): 16 | self.item_count = 0 17 | self.field_counts = {} 18 | self.show_counts = show_counts 19 | self.skip_none = skip_none 20 | self.add_to_stats = add_to_stats 21 | 22 | @classmethod 23 | def from_crawler(cls, crawler): 24 | if not crawler.settings.getbool('FIELDSTATS_ENABLED'): 25 | raise NotConfigured 26 | 27 | show_counts = crawler.settings.getbool('FIELDSTATS_COUNTS_ONLY', False) 28 | skip_none = crawler.settings.getbool('FIELDSTATS_SKIP_NONE', True) 29 | add_to_stats = crawler.settings.getbool('FIELDSTATS_ADD_TO_STATS', True) 30 | ext = cls(show_counts, skip_none, add_to_stats) 31 | crawler.signals.connect(ext.item_scraped, signal=signals.item_scraped) 32 | crawler.signals.connect(ext.spider_closed, signal=signals.spider_closed) 33 | return ext 34 | 35 | def item_scraped(self, item, spider): 36 | self.compute_item(item) 37 | 38 | def spider_closed(self, spider): 39 | if self.show_counts: 40 | report_data = self.field_counts 41 | else: 42 | report_data = self.build_fields_summary() 43 | 44 | message = 'Field stats:\n{}'.format(pprint.pformat(report_data)) 45 | logger.info(message) 46 | if self.add_to_stats: 47 | spider.crawler.stats.set_value('fields_coverage', report_data) 48 | 49 | def compute_item(self, item): 50 | self.item_count += 1 51 | self.count_item_fields(item) 52 | 53 | def count_item_fields(self, item, current_node=None): 54 | if current_node is None: 55 | current_node = self.field_counts 56 | 57 | for name, value in item.items(): 58 | if isinstance(value, dict): 59 | # recurse into nested items 60 | if name not in current_node: 61 | current_node[name] = {} 62 | self.count_item_fields(value, current_node=current_node[name]) 63 | continue 64 | 65 | if self.skip_none and value is None: 66 | continue 67 | 68 | if name not in current_node: 69 | current_node[name] = 0 70 | current_node[name] += 1 71 | 72 | def build_fields_summary(self, field_counts=None, fields_summary=None): 73 | if field_counts is None: 74 | field_counts = self.field_counts 75 | fields_summary = {} 76 | 77 | for name, value in field_counts.items(): 78 | if isinstance(value, dict): 79 | fields_summary[name] = {} 80 | self.build_fields_summary(field_counts[name], fields_summary[name]) 81 | else: 82 | field_percentage = int(value) * 100 / self.item_count 83 | fields_summary[name] = "{:.1f}%".format(field_percentage) 84 | 85 | return fields_summary 86 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 0.2.0 3 | commit = True 4 | tag = True 5 | 6 | [bumpversion:file:setup.py] 7 | search = version='{current_version}' 8 | replace = version='{new_version}' 9 | 10 | [bumpversion:file:scrapy_fieldstats/__init__.py] 11 | search = __version__ = '{current_version}' 12 | replace = __version__ = '{new_version}' 13 | 14 | [bdist_wheel] 15 | universal = 1 16 | 17 | [flake8] 18 | exclude = docs 19 | 20 | [aliases] 21 | test = pytest 22 | 23 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | from setuptools import setup, find_packages 4 | 5 | requirements = [] 6 | 7 | setup_requirements = [ 8 | 'pytest-runner', 9 | ] 10 | 11 | test_requirements = [ 12 | 'pytest', 13 | ] 14 | 15 | setup( 16 | name='scrapy-fieldstats', 17 | version='0.2.0', 18 | description="A Scrapy extension to generate a summary of fields coverage from your scraped data.", 19 | author="Valdir Stumm Junior", 20 | author_email='stummjr@gmail.com', 21 | url='https://github.com/stummjr/scrapy-fieldstats', 22 | packages=find_packages(include=['scrapy_fieldstats']), 23 | include_package_data=True, 24 | install_requires=requirements, 25 | license="MIT license", 26 | zip_safe=False, 27 | keywords='scrapy fields stats', 28 | classifiers=[ 29 | 'Development Status :: 2 - Pre-Alpha', 30 | 'Intended Audience :: Developers', 31 | 'License :: OSI Approved :: MIT License', 32 | 'Natural Language :: English', 33 | 'Programming Language :: Python :: 3.5', 34 | 'Programming Language :: Python :: 3.6', 35 | 'Programming Language :: Python :: 3.7', 36 | 'Programming Language :: Python :: 3.8', 37 | ], 38 | test_suite='tests', 39 | tests_require=test_requirements, 40 | setup_requires=setup_requirements, 41 | ) 42 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """Unit test package for scrapy_fieldstats.""" 4 | -------------------------------------------------------------------------------- /tests/test_scrapy_fieldstats.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from scrapy_fieldstats.fieldstats import FieldStatsExtension 3 | 4 | 5 | def extract_fake_items_and_compute_stats(fake_items, show_counts=False, skip_none=True): 6 | ext = FieldStatsExtension(skip_none=skip_none) 7 | for item in fake_items: 8 | ext.compute_item(item) 9 | 10 | if show_counts: 11 | return ext.field_counts 12 | return ext.build_fields_summary() 13 | 14 | 15 | def test_single_item(): 16 | fake_items = [{"field1": "value1"}] 17 | 18 | field_stats = extract_fake_items_and_compute_stats(fake_items) 19 | assert len(field_stats) == 1 20 | assert field_stats['field1'] == '100.0%' 21 | 22 | field_stats = extract_fake_items_and_compute_stats(fake_items, show_counts=True) 23 | assert len(field_stats) == 1 24 | assert field_stats['field1'] == 1 25 | 26 | 27 | def test_single_item_many_fields(): 28 | fake_items = [ 29 | { 30 | "field1": "value1", 31 | "field2": "value2", 32 | } 33 | ] 34 | 35 | field_stats = extract_fake_items_and_compute_stats(fake_items) 36 | assert len(field_stats) == 2 37 | assert field_stats['field1'] == '100.0%' 38 | assert field_stats['field2'] == '100.0%' 39 | 40 | field_stats = extract_fake_items_and_compute_stats(fake_items, show_counts=True) 41 | assert len(field_stats) == 2 42 | assert field_stats['field1'] == 1 43 | assert field_stats['field2'] == 1 44 | 45 | 46 | def test_many_items(): 47 | fake_items = [{"field1": "value1"}, {"field1": "value1"}] 48 | 49 | field_stats = extract_fake_items_and_compute_stats(fake_items) 50 | assert len(field_stats) == 1 51 | assert field_stats['field1'] == '100.0%' 52 | 53 | field_stats = extract_fake_items_and_compute_stats(fake_items, show_counts=True) 54 | assert len(field_stats) == 1 55 | assert field_stats['field1'] == 2 56 | 57 | 58 | def test_many_items_many_fields(): 59 | fake_items = [ 60 | { 61 | "field1": "value1", 62 | "field2": "value2", 63 | }, 64 | { 65 | "field1": "value1", 66 | "field2": "value2", 67 | } 68 | ] 69 | 70 | field_stats = extract_fake_items_and_compute_stats(fake_items) 71 | assert len(field_stats) == 2 72 | assert field_stats['field1'] == '100.0%' 73 | assert field_stats['field2'] == '100.0%' 74 | 75 | field_stats = extract_fake_items_and_compute_stats(fake_items, show_counts=True) 76 | assert len(field_stats) == 2 77 | assert field_stats['field1'] == 2 78 | assert field_stats['field2'] == 2 79 | 80 | 81 | def test_many_items_many_fields_missing_field(): 82 | fake_items = [ 83 | { 84 | "field1": "value1", 85 | }, 86 | { 87 | "field1": "value1", 88 | "field2": "value2", 89 | } 90 | ] 91 | 92 | field_stats = extract_fake_items_and_compute_stats(fake_items) 93 | assert len(field_stats) == 2 94 | assert field_stats['field1'] == '100.0%' 95 | assert field_stats['field2'] == '50.0%' 96 | 97 | field_stats = extract_fake_items_and_compute_stats(fake_items, show_counts=True) 98 | assert len(field_stats) == 2 99 | assert field_stats['field1'] == 2 100 | assert field_stats['field2'] == 1 101 | 102 | 103 | def test_many_items_many_fields_empty_field(): 104 | fake_items = [ 105 | { 106 | "field1": "value1", 107 | "field2": "", 108 | }, 109 | { 110 | "field1": "value1", 111 | "field2": "value2", 112 | } 113 | ] 114 | 115 | field_stats = extract_fake_items_and_compute_stats(fake_items) 116 | assert len(field_stats) == 2 117 | assert field_stats['field1'] == '100.0%' 118 | assert field_stats['field2'] == '100.0%' 119 | 120 | field_stats = extract_fake_items_and_compute_stats(fake_items, show_counts=True) 121 | assert len(field_stats) == 2 122 | assert field_stats['field1'] == 2 123 | assert field_stats['field2'] == 2 124 | 125 | 126 | def test_none_values(): 127 | fake_items = [ 128 | {"field1": None, "field2": ""}, 129 | {"field1": "value1", "field2": "value2"}, 130 | ] 131 | field_stats = extract_fake_items_and_compute_stats(fake_items, show_counts=True) 132 | assert field_stats['field1'] == 1 133 | assert field_stats['field2'] == 2 134 | 135 | field_stats = extract_fake_items_and_compute_stats(fake_items, show_counts=True, skip_none=False) 136 | assert field_stats['field1'] == 2 137 | assert field_stats['field2'] == 2 138 | 139 | 140 | def test_nested_items(): 141 | fake_items = [ 142 | { 143 | "field1": "value1", 144 | "field2": { 145 | "field2.1": "value2.1", 146 | "field2.2": "value2.2", 147 | "field2.3": { 148 | "field2.3.1": "value2.3.1", 149 | "field2.3.2": "value2.3.2", 150 | }, 151 | } 152 | }, 153 | { 154 | "field1": "value1", 155 | "field2": { 156 | "field2.1": "value2.1", 157 | "field2.3": { 158 | "field2.3.1": "value2.3.1", 159 | "field2.3.2": "", 160 | }, 161 | "field2.4": "value2.2", 162 | } 163 | } 164 | ] 165 | 166 | field_stats = extract_fake_items_and_compute_stats(fake_items) 167 | assert field_stats['field1'] == '100.0%' 168 | assert field_stats['field2']['field2.1'] == '100.0%' 169 | assert field_stats['field2']['field2.2'] == '50.0%' 170 | assert field_stats['field2']['field2.2'] == '50.0%' 171 | assert field_stats['field2']['field2.3']['field2.3.2'] == '100.0%' 172 | 173 | field_stats = extract_fake_items_and_compute_stats(fake_items, show_counts=True) 174 | assert field_stats['field1'] == 2 175 | assert field_stats['field2']['field2.1'] == 2 176 | assert field_stats['field2']['field2.2'] == 1 177 | assert field_stats['field2']['field2.2'] == 1 178 | assert field_stats['field2']['field2.3']['field2.3.2'] == 2 179 | 180 | 181 | def test_items_false_values(): 182 | fake_items = [ 183 | { 184 | "field1": False, 185 | "field2": 0, 186 | }, 187 | { 188 | "field1": True, 189 | "field2": 1, 190 | } 191 | ] 192 | 193 | field_stats = extract_fake_items_and_compute_stats(fake_items) 194 | assert field_stats['field1'] == '100.0%' 195 | assert field_stats['field2'] == '100.0%' 196 | 197 | field_stats = extract_fake_items_and_compute_stats(fake_items, show_counts=True) 198 | assert field_stats['field1'] == 2 199 | assert field_stats['field2'] == 2 200 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py38 3 | 4 | [testenv] 5 | deps = 6 | pytest 7 | scrapy 8 | commands = 9 | pytest 10 | 11 | [flake8] 12 | max-line-length = 120 13 | --------------------------------------------------------------------------------