├── .gitignore ├── Dockerfile ├── README.md ├── apps ├── __init__.py ├── asynch │ ├── __init__.py │ ├── base.py │ └── tasks │ │ ├── __init__.py │ │ └── task_eastmoney │ │ ├── __init__.py │ │ ├── __main__.py │ │ ├── crawler.py │ │ ├── middleware.py │ │ └── pipeline.py └── periodic │ ├── __init__.py │ ├── base.py │ └── tasks │ ├── __init__.py │ ├── task_cookie │ ├── __init__.py │ └── __main__.py │ └── task_proxy │ ├── __init__.py │ ├── __main__.py │ ├── upstream.py │ └── validator.py ├── bin └── .gitkeep ├── common ├── __init__.py ├── exceptions.py ├── plugins │ ├── __init__.py │ ├── human │ │ ├── __init__.py │ │ ├── notify.py │ │ ├── slider.py │ │ └── verification.py │ └── storage │ │ ├── __init__.py │ │ ├── filestorage.py │ │ ├── mongostorage.py │ │ └── sqlitestorage.py ├── settings.py ├── singleton.py ├── sqlitedao.py └── timetrans.py ├── config ├── conf.d │ └── deadpool ├── config.yaml ├── jobs.d │ ├── async.d │ │ └── task_eastmoney.yaml │ └── periodic.d │ │ ├── task_cookie.yaml │ │ └── task_proxy.yaml └── jobs.yaml ├── contrib ├── __init__.py ├── elastic │ ├── __init__.py │ ├── base.py │ └── indices │ │ ├── __init__.py │ │ └── rlogs.py ├── mysql │ ├── __init__.py │ ├── base.py │ └── tables │ │ ├── __init__.py │ │ ├── base.py │ │ ├── cookie.py │ │ └── proxy.py └── redis │ ├── __init__.py │ └── base.py ├── data ├── custom │ └── pkuseg_user_dict.txt └── stopwords │ ├── baidu_stopwords.txt │ ├── cn_stopwords.txt │ ├── hit_stopwords.txt │ └── scu_stopwords.txt ├── deadpool ├── __init__.py └── celery.py ├── debug.log ├── doc ├── en │ ├── develop.md │ └── install.md ├── img │ ├── beat-start.png │ ├── eg-crawler.jpg │ ├── eg-middleware.jpg │ ├── eg-pipeline.jpg │ ├── main-start.jpg │ └── task-start.jpg ├── index.md └── zh │ ├── develop.md │ └── install.md ├── driver ├── chromedriver_linux64.zip ├── chromedriver_mac64.zip └── chromedriver_win32.zip ├── requirements.txt ├── result └── task_eastmoney.db3 ├── scripts ├── deadpool-beat.service └── deadpool.service ├── test ├── __init__.py ├── test_browser.py ├── test_hanlp.py ├── test_segments.py ├── test_settings.py ├── test_sqlitedao.py └── test_stopwords.py └── utils ├── __init__.py ├── driver.py ├── loader.py └── network.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/.gitignore -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/Dockerfile -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/README.md -------------------------------------------------------------------------------- /apps/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/apps/__init__.py -------------------------------------------------------------------------------- /apps/asynch/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/apps/asynch/__init__.py -------------------------------------------------------------------------------- /apps/asynch/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/apps/asynch/base.py -------------------------------------------------------------------------------- /apps/asynch/tasks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/apps/asynch/tasks/__init__.py -------------------------------------------------------------------------------- /apps/asynch/tasks/task_eastmoney/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/apps/asynch/tasks/task_eastmoney/__init__.py -------------------------------------------------------------------------------- /apps/asynch/tasks/task_eastmoney/__main__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/apps/asynch/tasks/task_eastmoney/__main__.py -------------------------------------------------------------------------------- /apps/asynch/tasks/task_eastmoney/crawler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/apps/asynch/tasks/task_eastmoney/crawler.py -------------------------------------------------------------------------------- /apps/asynch/tasks/task_eastmoney/middleware.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/apps/asynch/tasks/task_eastmoney/middleware.py -------------------------------------------------------------------------------- /apps/asynch/tasks/task_eastmoney/pipeline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/apps/asynch/tasks/task_eastmoney/pipeline.py -------------------------------------------------------------------------------- /apps/periodic/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/apps/periodic/__init__.py -------------------------------------------------------------------------------- /apps/periodic/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/apps/periodic/base.py -------------------------------------------------------------------------------- /apps/periodic/tasks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/apps/periodic/tasks/__init__.py -------------------------------------------------------------------------------- /apps/periodic/tasks/task_cookie/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/apps/periodic/tasks/task_cookie/__init__.py -------------------------------------------------------------------------------- /apps/periodic/tasks/task_cookie/__main__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/apps/periodic/tasks/task_cookie/__main__.py -------------------------------------------------------------------------------- /apps/periodic/tasks/task_proxy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/apps/periodic/tasks/task_proxy/__init__.py -------------------------------------------------------------------------------- /apps/periodic/tasks/task_proxy/__main__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/apps/periodic/tasks/task_proxy/__main__.py -------------------------------------------------------------------------------- /apps/periodic/tasks/task_proxy/upstream.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/apps/periodic/tasks/task_proxy/upstream.py -------------------------------------------------------------------------------- /apps/periodic/tasks/task_proxy/validator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/apps/periodic/tasks/task_proxy/validator.py -------------------------------------------------------------------------------- /bin/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /common/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/common/__init__.py -------------------------------------------------------------------------------- /common/exceptions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/common/exceptions.py -------------------------------------------------------------------------------- /common/plugins/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/common/plugins/__init__.py -------------------------------------------------------------------------------- /common/plugins/human/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/common/plugins/human/__init__.py -------------------------------------------------------------------------------- /common/plugins/human/notify.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/common/plugins/human/notify.py -------------------------------------------------------------------------------- /common/plugins/human/slider.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/common/plugins/human/slider.py -------------------------------------------------------------------------------- /common/plugins/human/verification.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/common/plugins/human/verification.py -------------------------------------------------------------------------------- /common/plugins/storage/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/common/plugins/storage/__init__.py -------------------------------------------------------------------------------- /common/plugins/storage/filestorage.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/common/plugins/storage/filestorage.py -------------------------------------------------------------------------------- /common/plugins/storage/mongostorage.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/common/plugins/storage/mongostorage.py -------------------------------------------------------------------------------- /common/plugins/storage/sqlitestorage.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /common/settings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/common/settings.py -------------------------------------------------------------------------------- /common/singleton.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/common/singleton.py -------------------------------------------------------------------------------- /common/sqlitedao.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/common/sqlitedao.py -------------------------------------------------------------------------------- /common/timetrans.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/common/timetrans.py -------------------------------------------------------------------------------- /config/conf.d/deadpool: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/config/conf.d/deadpool -------------------------------------------------------------------------------- /config/config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/config/config.yaml -------------------------------------------------------------------------------- /config/jobs.d/async.d/task_eastmoney.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/config/jobs.d/async.d/task_eastmoney.yaml -------------------------------------------------------------------------------- /config/jobs.d/periodic.d/task_cookie.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/config/jobs.d/periodic.d/task_cookie.yaml -------------------------------------------------------------------------------- /config/jobs.d/periodic.d/task_proxy.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/config/jobs.d/periodic.d/task_proxy.yaml -------------------------------------------------------------------------------- /config/jobs.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/config/jobs.yaml -------------------------------------------------------------------------------- /contrib/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/contrib/__init__.py -------------------------------------------------------------------------------- /contrib/elastic/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/contrib/elastic/__init__.py -------------------------------------------------------------------------------- /contrib/elastic/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/contrib/elastic/base.py -------------------------------------------------------------------------------- /contrib/elastic/indices/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/contrib/elastic/indices/__init__.py -------------------------------------------------------------------------------- /contrib/elastic/indices/rlogs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/contrib/elastic/indices/rlogs.py -------------------------------------------------------------------------------- /contrib/mysql/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/contrib/mysql/__init__.py -------------------------------------------------------------------------------- /contrib/mysql/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/contrib/mysql/base.py -------------------------------------------------------------------------------- /contrib/mysql/tables/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/contrib/mysql/tables/__init__.py -------------------------------------------------------------------------------- /contrib/mysql/tables/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/contrib/mysql/tables/base.py -------------------------------------------------------------------------------- /contrib/mysql/tables/cookie.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/contrib/mysql/tables/cookie.py -------------------------------------------------------------------------------- /contrib/mysql/tables/proxy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/contrib/mysql/tables/proxy.py -------------------------------------------------------------------------------- /contrib/redis/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/contrib/redis/__init__.py -------------------------------------------------------------------------------- /contrib/redis/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/contrib/redis/base.py -------------------------------------------------------------------------------- /data/custom/pkuseg_user_dict.txt: -------------------------------------------------------------------------------- 1 | OPEC+ 2 | 油市 3 | EIA 4 | RBOB -------------------------------------------------------------------------------- /data/stopwords/baidu_stopwords.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/data/stopwords/baidu_stopwords.txt -------------------------------------------------------------------------------- /data/stopwords/cn_stopwords.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/data/stopwords/cn_stopwords.txt -------------------------------------------------------------------------------- /data/stopwords/hit_stopwords.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/data/stopwords/hit_stopwords.txt -------------------------------------------------------------------------------- /data/stopwords/scu_stopwords.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/data/stopwords/scu_stopwords.txt -------------------------------------------------------------------------------- /deadpool/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/deadpool/__init__.py -------------------------------------------------------------------------------- /deadpool/celery.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/deadpool/celery.py -------------------------------------------------------------------------------- /debug.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/debug.log -------------------------------------------------------------------------------- /doc/en/develop.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /doc/en/install.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /doc/img/beat-start.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/doc/img/beat-start.png -------------------------------------------------------------------------------- /doc/img/eg-crawler.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/doc/img/eg-crawler.jpg -------------------------------------------------------------------------------- /doc/img/eg-middleware.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/doc/img/eg-middleware.jpg -------------------------------------------------------------------------------- /doc/img/eg-pipeline.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/doc/img/eg-pipeline.jpg -------------------------------------------------------------------------------- /doc/img/main-start.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/doc/img/main-start.jpg -------------------------------------------------------------------------------- /doc/img/task-start.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/doc/img/task-start.jpg -------------------------------------------------------------------------------- /doc/index.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /doc/zh/develop.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /doc/zh/install.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /driver/chromedriver_linux64.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/driver/chromedriver_linux64.zip -------------------------------------------------------------------------------- /driver/chromedriver_mac64.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/driver/chromedriver_mac64.zip -------------------------------------------------------------------------------- /driver/chromedriver_win32.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/driver/chromedriver_win32.zip -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/requirements.txt -------------------------------------------------------------------------------- /result/task_eastmoney.db3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/result/task_eastmoney.db3 -------------------------------------------------------------------------------- /scripts/deadpool-beat.service: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/scripts/deadpool-beat.service -------------------------------------------------------------------------------- /scripts/deadpool.service: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/scripts/deadpool.service -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/test/__init__.py -------------------------------------------------------------------------------- /test/test_browser.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/test/test_browser.py -------------------------------------------------------------------------------- /test/test_hanlp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/test/test_hanlp.py -------------------------------------------------------------------------------- /test/test_segments.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/test/test_segments.py -------------------------------------------------------------------------------- /test/test_settings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/test/test_settings.py -------------------------------------------------------------------------------- /test/test_sqlitedao.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/test/test_sqlitedao.py -------------------------------------------------------------------------------- /test/test_stopwords.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/test/test_stopwords.py -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/utils/__init__.py -------------------------------------------------------------------------------- /utils/driver.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/utils/driver.py -------------------------------------------------------------------------------- /utils/loader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/utils/loader.py -------------------------------------------------------------------------------- /utils/network.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ryuchen/DeadPool/HEAD/utils/network.py --------------------------------------------------------------------------------