├── .gitignore ├── MANIFEST.in ├── README.md ├── benchmark ├── __init__.py ├── bench_filter_queue.py ├── bench_mongo.py ├── bench_rabbitmq.py ├── bench_request_queue.py ├── bench_response.py └── bench_utils.py ├── bin ├── yascrapy_cache ├── yascrapy_producer └── yascrapy_worker ├── conf ├── common.json ├── core.json ├── migrate_rabbitmq.json ├── migrate_ssdb.json ├── proxy.json └── rotate_log.conf ├── docker ├── Dockerfile ├── bin │ ├── rabbitmq-start │ └── startall └── bloomd.conf ├── docs ├── Makefile ├── _static │ └── yascrapy.png ├── api_guide.rst ├── conf.py ├── getting_started.md ├── index.rst ├── installation.md ├── requirements.txt └── source │ ├── _static │ └── yascrapy.png │ ├── api_guide.rst │ ├── conf.py │ ├── getting_started.md │ ├── index.rst │ └── installation.md ├── sample_spider ├── __init__.py ├── items.py ├── sample_producer.py ├── sample_worker.py └── settings.py ├── setup.py ├── tools ├── delete_spider.py ├── migrate_rabbitmq.py ├── migrate_ssdb.py ├── proxyadmin.py ├── proxyconfig.py ├── rabbitmq_to_ssdb.py ├── rabbitmqadmin.py ├── responseadmin.py └── ssdbadmin.py └── yascrapy ├── __init__.py ├── base.py ├── bloomd.py ├── config.py ├── filter_queue.py ├── libs ├── __init__.py └── pybloomd.py ├── plugins ├── __init__.py ├── handle_error.py ├── handle_proxy.py └── mongo.py ├── rabbitmq.py ├── request_queue.py ├── response_queue.py ├── ssdb.py ├── tests ├── 404_err.html ├── test.html ├── test_config.py ├── test_error_handler.py ├── test_mongo.py ├── test_proxy_handler.py ├── test_response.py └── zh_404_err.html └── utils.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/.gitignore -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/README.md -------------------------------------------------------------------------------- /benchmark/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /benchmark/bench_filter_queue.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/benchmark/bench_filter_queue.py -------------------------------------------------------------------------------- /benchmark/bench_mongo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/benchmark/bench_mongo.py -------------------------------------------------------------------------------- /benchmark/bench_rabbitmq.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/benchmark/bench_rabbitmq.py -------------------------------------------------------------------------------- /benchmark/bench_request_queue.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/benchmark/bench_request_queue.py -------------------------------------------------------------------------------- /benchmark/bench_response.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/benchmark/bench_response.py -------------------------------------------------------------------------------- /benchmark/bench_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/benchmark/bench_utils.py -------------------------------------------------------------------------------- /bin/yascrapy_cache: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/bin/yascrapy_cache -------------------------------------------------------------------------------- /bin/yascrapy_producer: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/bin/yascrapy_producer -------------------------------------------------------------------------------- /bin/yascrapy_worker: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/bin/yascrapy_worker -------------------------------------------------------------------------------- /conf/common.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/conf/common.json -------------------------------------------------------------------------------- /conf/core.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/conf/core.json -------------------------------------------------------------------------------- /conf/migrate_rabbitmq.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/conf/migrate_rabbitmq.json -------------------------------------------------------------------------------- /conf/migrate_ssdb.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/conf/migrate_ssdb.json -------------------------------------------------------------------------------- /conf/proxy.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/conf/proxy.json -------------------------------------------------------------------------------- /conf/rotate_log.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/conf/rotate_log.conf -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/docker/Dockerfile -------------------------------------------------------------------------------- /docker/bin/rabbitmq-start: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/docker/bin/rabbitmq-start -------------------------------------------------------------------------------- /docker/bin/startall: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/docker/bin/startall -------------------------------------------------------------------------------- /docker/bloomd.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/docker/bloomd.conf -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/docs/Makefile -------------------------------------------------------------------------------- /docs/_static/yascrapy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/docs/_static/yascrapy.png -------------------------------------------------------------------------------- /docs/api_guide.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/docs/api_guide.rst -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/docs/conf.py -------------------------------------------------------------------------------- /docs/getting_started.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/docs/getting_started.md -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/docs/index.rst -------------------------------------------------------------------------------- /docs/installation.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/docs/installation.md -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/docs/requirements.txt -------------------------------------------------------------------------------- /docs/source/_static/yascrapy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/docs/source/_static/yascrapy.png -------------------------------------------------------------------------------- /docs/source/api_guide.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/docs/source/api_guide.rst -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/docs/source/conf.py -------------------------------------------------------------------------------- /docs/source/getting_started.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/docs/source/getting_started.md -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/docs/source/index.rst -------------------------------------------------------------------------------- /docs/source/installation.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/docs/source/installation.md -------------------------------------------------------------------------------- /sample_spider/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/sample_spider/__init__.py -------------------------------------------------------------------------------- /sample_spider/items.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/sample_spider/items.py -------------------------------------------------------------------------------- /sample_spider/sample_producer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/sample_spider/sample_producer.py -------------------------------------------------------------------------------- /sample_spider/sample_worker.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/sample_spider/sample_worker.py -------------------------------------------------------------------------------- /sample_spider/settings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/sample_spider/settings.py -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/setup.py -------------------------------------------------------------------------------- /tools/delete_spider.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/tools/delete_spider.py -------------------------------------------------------------------------------- /tools/migrate_rabbitmq.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/tools/migrate_rabbitmq.py -------------------------------------------------------------------------------- /tools/migrate_ssdb.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/tools/migrate_ssdb.py -------------------------------------------------------------------------------- /tools/proxyadmin.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/tools/proxyadmin.py -------------------------------------------------------------------------------- /tools/proxyconfig.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/tools/proxyconfig.py -------------------------------------------------------------------------------- /tools/rabbitmq_to_ssdb.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/tools/rabbitmq_to_ssdb.py -------------------------------------------------------------------------------- /tools/rabbitmqadmin.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/tools/rabbitmqadmin.py -------------------------------------------------------------------------------- /tools/responseadmin.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/tools/responseadmin.py -------------------------------------------------------------------------------- /tools/ssdbadmin.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/tools/ssdbadmin.py -------------------------------------------------------------------------------- /yascrapy/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /yascrapy/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/yascrapy/base.py -------------------------------------------------------------------------------- /yascrapy/bloomd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/yascrapy/bloomd.py -------------------------------------------------------------------------------- /yascrapy/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/yascrapy/config.py -------------------------------------------------------------------------------- /yascrapy/filter_queue.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/yascrapy/filter_queue.py -------------------------------------------------------------------------------- /yascrapy/libs/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /yascrapy/libs/pybloomd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/yascrapy/libs/pybloomd.py -------------------------------------------------------------------------------- /yascrapy/plugins/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /yascrapy/plugins/handle_error.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/yascrapy/plugins/handle_error.py -------------------------------------------------------------------------------- /yascrapy/plugins/handle_proxy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/yascrapy/plugins/handle_proxy.py -------------------------------------------------------------------------------- /yascrapy/plugins/mongo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/yascrapy/plugins/mongo.py -------------------------------------------------------------------------------- /yascrapy/rabbitmq.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/yascrapy/rabbitmq.py -------------------------------------------------------------------------------- /yascrapy/request_queue.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/yascrapy/request_queue.py -------------------------------------------------------------------------------- /yascrapy/response_queue.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/yascrapy/response_queue.py -------------------------------------------------------------------------------- /yascrapy/ssdb.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/yascrapy/ssdb.py -------------------------------------------------------------------------------- /yascrapy/tests/404_err.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/yascrapy/tests/404_err.html -------------------------------------------------------------------------------- /yascrapy/tests/test.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/yascrapy/tests/test.html -------------------------------------------------------------------------------- /yascrapy/tests/test_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/yascrapy/tests/test_config.py -------------------------------------------------------------------------------- /yascrapy/tests/test_error_handler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/yascrapy/tests/test_error_handler.py -------------------------------------------------------------------------------- /yascrapy/tests/test_mongo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/yascrapy/tests/test_mongo.py -------------------------------------------------------------------------------- /yascrapy/tests/test_proxy_handler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/yascrapy/tests/test_proxy_handler.py -------------------------------------------------------------------------------- /yascrapy/tests/test_response.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/yascrapy/tests/test_response.py -------------------------------------------------------------------------------- /yascrapy/tests/zh_404_err.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/yascrapy/tests/zh_404_err.html -------------------------------------------------------------------------------- /yascrapy/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianxunio/yascrapy/HEAD/yascrapy/utils.py --------------------------------------------------------------------------------