├── .bumpversion.cfg ├── .gitignore ├── .travis.yml ├── README.rst ├── requirements.txt ├── scrapylib ├── __init__.py ├── constraints │ ├── __init__.py │ └── pipeline.py ├── crawlera.py ├── deltafetch.py ├── guid.py ├── hcf.py ├── hubproxy.py ├── links.py ├── magicfields.py ├── pipelines.py ├── processors │ ├── __init__.py │ └── date.py ├── proxy.py ├── querycleaner.py ├── redisqueue.py ├── spidertrace.py └── splitvariants.py ├── setup.py ├── tests ├── __init__.py ├── test_constraints.py ├── test_crawlera.py ├── test_deltafetch.py ├── test_hcf.py ├── test_hubproxy.py ├── test_links.py ├── test_magicfields.py ├── test_processors.py ├── test_querycleaner.py └── test_splitvariants.py └── tox.ini /.bumpversion.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapylib/HEAD/.bumpversion.cfg -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapylib/HEAD/.gitignore -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapylib/HEAD/.travis.yml -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapylib/HEAD/README.rst -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapylib/HEAD/requirements.txt -------------------------------------------------------------------------------- /scrapylib/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /scrapylib/constraints/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapylib/HEAD/scrapylib/constraints/__init__.py -------------------------------------------------------------------------------- /scrapylib/constraints/pipeline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapylib/HEAD/scrapylib/constraints/pipeline.py -------------------------------------------------------------------------------- /scrapylib/crawlera.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapylib/HEAD/scrapylib/crawlera.py -------------------------------------------------------------------------------- /scrapylib/deltafetch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapylib/HEAD/scrapylib/deltafetch.py -------------------------------------------------------------------------------- /scrapylib/guid.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapylib/HEAD/scrapylib/guid.py -------------------------------------------------------------------------------- /scrapylib/hcf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapylib/HEAD/scrapylib/hcf.py -------------------------------------------------------------------------------- /scrapylib/hubproxy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapylib/HEAD/scrapylib/hubproxy.py -------------------------------------------------------------------------------- /scrapylib/links.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapylib/HEAD/scrapylib/links.py -------------------------------------------------------------------------------- /scrapylib/magicfields.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapylib/HEAD/scrapylib/magicfields.py -------------------------------------------------------------------------------- /scrapylib/pipelines.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapylib/HEAD/scrapylib/pipelines.py -------------------------------------------------------------------------------- /scrapylib/processors/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapylib/HEAD/scrapylib/processors/__init__.py -------------------------------------------------------------------------------- /scrapylib/processors/date.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapylib/HEAD/scrapylib/processors/date.py -------------------------------------------------------------------------------- /scrapylib/proxy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapylib/HEAD/scrapylib/proxy.py -------------------------------------------------------------------------------- /scrapylib/querycleaner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapylib/HEAD/scrapylib/querycleaner.py -------------------------------------------------------------------------------- /scrapylib/redisqueue.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapylib/HEAD/scrapylib/redisqueue.py -------------------------------------------------------------------------------- /scrapylib/spidertrace.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapylib/HEAD/scrapylib/spidertrace.py -------------------------------------------------------------------------------- /scrapylib/splitvariants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapylib/HEAD/scrapylib/splitvariants.py -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapylib/HEAD/setup.py -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/test_constraints.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapylib/HEAD/tests/test_constraints.py -------------------------------------------------------------------------------- /tests/test_crawlera.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapylib/HEAD/tests/test_crawlera.py -------------------------------------------------------------------------------- /tests/test_deltafetch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapylib/HEAD/tests/test_deltafetch.py -------------------------------------------------------------------------------- /tests/test_hcf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapylib/HEAD/tests/test_hcf.py -------------------------------------------------------------------------------- /tests/test_hubproxy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapylib/HEAD/tests/test_hubproxy.py -------------------------------------------------------------------------------- /tests/test_links.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapylib/HEAD/tests/test_links.py -------------------------------------------------------------------------------- /tests/test_magicfields.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapylib/HEAD/tests/test_magicfields.py -------------------------------------------------------------------------------- /tests/test_processors.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapylib/HEAD/tests/test_processors.py -------------------------------------------------------------------------------- /tests/test_querycleaner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapylib/HEAD/tests/test_querycleaner.py -------------------------------------------------------------------------------- /tests/test_splitvariants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapylib/HEAD/tests/test_splitvariants.py -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/scrapylib/HEAD/tox.ini --------------------------------------------------------------------------------