├── mi ├── __init__.py ├── mi │ ├── __init__.py │ ├── tools │ │ ├── __init__.py │ │ ├── __init__.pyc │ │ ├── fuzzyMatching.pyc │ │ ├── gen_spiderFile_in_whiteList.pyc │ │ ├── gen_spiderFile_need_fuzzymatching.pyc │ │ └── gen_spiderFile_in_whiteList.py │ ├── pipelines │ │ ├── __init__.py │ │ ├── __init__.pyc │ │ ├── pipeline_mongo.pyc │ │ ├── pipeline_mysql.pyc │ │ └── pipeline_mongo.py │ ├── middlewares │ │ ├── __init__.py │ │ ├── __init__.pyc │ │ ├── middleware_monitor.pyc │ │ └── middleware_rotateUserAgent.pyc │ ├── scrapy_redis │ │ ├── __init__.py │ │ ├── queue.pyc │ │ ├── __init__.pyc │ │ ├── scheduler.pyc │ │ ├── connection.pyc │ │ ├── dupefilter.pyc │ │ ├── BloomfilterOnRedis.pyc │ │ ├── utils.py │ │ ├── connection.py │ │ ├── defaults.py │ │ ├── BloomfilterOnRedis.py │ │ ├── pipelines.py │ │ └── dupefilter.py │ ├── spiders_of_eCommerce │ │ ├── __init__.py │ │ ├── __init__.pyc │ │ ├── spider_jd.pyc │ │ ├── spider_gome.pyc │ │ ├── spider_amazon.pyc │ │ ├── spider_taobao.pyc │ │ ├── spider_tmall.pyc │ │ └── spider_dangdang.pyc │ ├── spiders_of_news_in_whiteList │ │ ├── __init__.py │ │ ├── __init__.pyc │ │ ├── spider_cctvcom.pyc │ │ ├── spider_huxiucom.pyc │ │ ├── spider_thepapercn.pyc │ │ ├── spider_cankaoxiaoxicom.pyc │ │ ├── spider_huxiucom.py │ │ ├── spider_thepapercn.py │ │ ├── spider_cctvcom.py │ │ └── spider_cankaoxiaoxicom.py │ ├── spiders_of_news_need_fuzzymatching │ │ ├── __init__.py │ │ ├── __init__.pyc │ │ ├── spider_sinacomcn.pyc │ │ ├── spider_mengzichengcn.pyc │ │ ├── spider_peoplecomcn.pyc │ │ ├── spider_chinadailycomcn.pyc │ │ ├── spider_sinacomcn.py │ │ ├── spider_peoplecomcn.py │ │ ├── spider_mengzichengcn.py │ │ └── spider_chinadailycomcn.py │ ├── items.pyc │ ├── __init__.pyc │ └── settings.pyc ├── settings.pyc ├── data_service.pyc ├── gen_settings.pyc ├── .idea │ ├── misc.xml │ ├── modules.xml │ └── mi.iml ├── scrapy.cfg ├── data_service.py └── settings.py ├── mi_manager ├── __init__.py ├── .idea │ ├── .name │ ├── misc.xml │ ├── modules.xml │ └── mi_manager.iml ├── daemon │ ├── __init__.py │ ├── model │ │ ├── __init__.py │ │ ├── task.pyc │ │ ├── mission.pyc │ │ ├── __init__.pyc │ │ ├── submission.pyc │ │ ├── submission.py │ │ ├── task.py │ │ └── mission.py │ ├── settings.pyc │ ├── data_service.pyc │ ├── gen_json_file.pyc │ ├── marathonHelper.pyc │ ├── jsons │ │ ├── mi.json │ │ ├── jd100jdcom.json │ │ ├── jd101jdcom.json │ │ ├── mis10jdcom.json │ │ ├── mis2jdcom.json │ │ ├── mis3jdcom.json │ │ ├── mis4jdcom.json │ │ ├── mis5jdcom.json │ │ ├── mis6jdcom.json │ │ ├── mis121jdcom.json │ │ ├── mis1huxiucom.json │ │ ├── mission1jdcom.json │ │ ├── mission1sinacomcn.json │ │ ├── mission1thepapercn.json │ │ ├── mission21huxiucom.json │ │ ├── mission21taobaocom.json │ │ ├── mission31gomecomcn.json │ │ ├── mogujiemogujiecom.json │ │ ├── mission1peoplecomcn.json │ │ ├── testmission16163com.json │ │ ├── mi20170630dangdangcom.json │ │ ├── mi20170630mengzichengcn.json │ │ ├── mission1cankaoxiaoxicom.json │ │ ├── mission21mengzichengcn.json │ │ ├── mission31chinabytecom.json │ │ ├── testmission16amazoncn.json │ │ ├── testmission16sinacomcn.json │ │ ├── mi20170630chinadailycomcn.json │ │ ├── filter_redis.json │ │ └── core_redis.json │ ├── mesosHelper.py │ ├── marathonHelper.py │ ├── gen_json_file.py │ └── settings.py ├── monitor │ ├── __init__.py │ ├── training │ │ ├── __init__.py │ │ └── 1498569115.pkl │ ├── spiderInitfiles_of_eCommerce │ │ ├── __init__.py │ │ ├── spiderInit_jd.com.py │ │ ├── spiderInit_gome.com.cn.py │ │ └── spiderInit_dangdang.com.py │ ├── static │ │ ├── const.js │ │ └── v2 │ │ │ ├── dist │ │ │ ├── img │ │ │ │ ├── user8-128x128.jpg │ │ │ │ ├── 1498442770678662.png │ │ │ │ └── 1498442770678662white.png │ │ │ └── css │ │ │ │ └── alt │ │ │ │ └── AdminLTE-fullcalendar.min.css │ │ │ ├── plugins │ │ │ ├── iCheck │ │ │ │ ├── flat │ │ │ │ │ ├── aero.png │ │ │ │ │ ├── blue.png │ │ │ │ │ ├── flat.png │ │ │ │ │ ├── grey.png │ │ │ │ │ ├── pink.png │ │ │ │ │ ├── red.png │ │ │ │ │ ├── aero@2x.png │ │ │ │ │ ├── blue@2x.png │ │ │ │ │ ├── flat@2x.png │ │ │ │ │ ├── green.png │ │ │ │ │ ├── grey@2x.png │ │ │ │ │ ├── orange.png │ │ │ │ │ ├── pink@2x.png │ │ │ │ │ ├── purple.png │ │ │ │ │ ├── red@2x.png │ │ │ │ │ ├── yellow.png │ │ │ │ │ ├── green@2x.png │ │ │ │ │ ├── orange@2x.png │ │ │ │ │ ├── purple@2x.png │ │ │ │ │ ├── yellow@2x.png │ │ │ │ │ ├── flat.css │ │ │ │ │ ├── red.css │ │ │ │ │ ├── aero.css │ │ │ │ │ ├── blue.css │ │ │ │ │ ├── grey.css │ │ │ │ │ ├── pink.css │ │ │ │ │ ├── green.css │ │ │ │ │ ├── orange.css │ │ │ │ │ ├── purple.css │ │ │ │ │ └── yellow.css │ │ │ │ ├── line │ │ │ │ │ ├── line.png │ │ │ │ │ └── line@2x.png │ │ │ │ ├── minimal │ │ │ │ │ ├── aero.png │ │ │ │ │ ├── blue.png │ │ │ │ │ ├── grey.png │ │ │ │ │ ├── pink.png │ │ │ │ │ ├── red.png │ │ │ │ │ ├── green.png │ │ │ │ │ ├── orange.png │ │ │ │ │ ├── purple.png │ │ │ │ │ ├── red@2x.png │ │ │ │ │ ├── yellow.png │ │ │ │ │ ├── aero@2x.png │ │ │ │ │ ├── blue@2x.png │ │ │ │ │ ├── green@2x.png │ │ │ │ │ ├── grey@2x.png │ │ │ │ │ ├── minimal.png │ │ │ │ │ ├── orange@2x.png │ │ │ │ │ ├── pink@2x.png │ │ │ │ │ ├── purple@2x.png │ │ │ │ │ ├── yellow@2x.png │ │ │ │ │ ├── minimal@2x.png │ │ │ │ │ ├── minimal.css │ │ │ │ │ ├── red.css │ │ │ │ │ ├── aero.css │ │ │ │ │ ├── blue.css │ │ │ │ │ ├── grey.css │ │ │ │ │ ├── pink.css │ │ │ │ │ └── green.css │ │ │ │ ├── square │ │ │ │ │ ├── aero.png │ │ │ │ │ ├── blue.png │ │ │ │ │ ├── green.png │ │ │ │ │ ├── grey.png │ │ │ │ │ ├── pink.png │ │ │ │ │ ├── red.png │ │ │ │ │ ├── aero@2x.png │ │ │ │ │ ├── blue@2x.png │ │ │ │ │ ├── grey@2x.png │ │ │ │ │ ├── orange.png │ │ │ │ │ ├── pink@2x.png │ │ │ │ │ ├── purple.png │ │ │ │ │ ├── red@2x.png │ │ │ │ │ ├── square.png │ │ │ │ │ ├── yellow.png │ │ │ │ │ ├── green@2x.png │ │ │ │ │ ├── orange@2x.png │ │ │ │ │ ├── purple@2x.png │ │ │ │ │ ├── square@2x.png │ │ │ │ │ ├── yellow@2x.png │ │ │ │ │ ├── square.css │ │ │ │ │ ├── red.css │ │ │ │ │ ├── aero.css │ │ │ │ │ ├── blue.css │ │ │ │ │ ├── grey.css │ │ │ │ │ ├── pink.css │ │ │ │ │ ├── green.css │ │ │ │ │ ├── orange.css │ │ │ │ │ └── purple.css │ │ │ │ ├── futurico │ │ │ │ │ ├── futurico.png │ │ │ │ │ ├── futurico@2x.png │ │ │ │ │ └── futurico.css │ │ │ │ ├── polaris │ │ │ │ │ ├── polaris.png │ │ │ │ │ ├── polaris@2x.png │ │ │ │ │ └── polaris.css │ │ │ │ └── all.css │ │ │ ├── datatables │ │ │ │ └── images │ │ │ │ │ ├── sort_asc.png │ │ │ │ │ ├── sort_both.png │ │ │ │ │ ├── sort_desc.png │ │ │ │ │ ├── sort_asc_disabled.png │ │ │ │ │ └── sort_desc_disabled.png │ │ │ └── datepicker │ │ │ │ └── locales │ │ │ │ ├── bootstrap-datepicker.kr.js │ │ │ │ ├── bootstrap-datepicker.gl.js │ │ │ │ ├── bootstrap-datepicker.ja.js │ │ │ │ ├── bootstrap-datepicker.hr.js │ │ │ │ ├── bootstrap-datepicker.zh-CN.js │ │ │ │ ├── bootstrap-datepicker.az.js │ │ │ │ ├── bootstrap-datepicker.cy.js │ │ │ │ ├── bootstrap-datepicker.he.js │ │ │ │ ├── bootstrap-datepicker.ms.js │ │ │ │ ├── bootstrap-datepicker.no.js │ │ │ │ ├── bootstrap-datepicker.ca.js │ │ │ │ ├── bootstrap-datepicker.el.js │ │ │ │ ├── bootstrap-datepicker.rs.js │ │ │ │ ├── bootstrap-datepicker.th.js │ │ │ │ ├── bootstrap-datepicker.bg.js │ │ │ │ ├── bootstrap-datepicker.es.js │ │ │ │ ├── bootstrap-datepicker.nl.js │ │ │ │ ├── bootstrap-datepicker.sl.js │ │ │ │ ├── bootstrap-datepicker.rs-latin.js │ │ │ │ ├── bootstrap-datepicker.nb.js │ │ │ │ ├── bootstrap-datepicker.sq.js │ │ │ │ ├── bootstrap-datepicker.ua.js │ │ │ │ ├── bootstrap-datepicker.id.js │ │ │ │ ├── bootstrap-datepicker.kk.js │ │ │ │ ├── bootstrap-datepicker.tr.js │ │ │ │ ├── bootstrap-datepicker.da.js │ │ │ │ ├── bootstrap-datepicker.pt-BR.js │ │ │ │ ├── bootstrap-datepicker.ru.js │ │ │ │ ├── bootstrap-datepicker.cs.js │ │ │ │ ├── bootstrap-datepicker.is.js │ │ │ │ ├── bootstrap-datepicker.sk.js │ │ │ │ ├── bootstrap-datepicker.sv.js │ │ │ │ ├── bootstrap-datepicker.fa.js │ │ │ │ ├── bootstrap-datepicker.hu.js │ │ │ │ ├── bootstrap-datepicker.ro.js │ │ │ │ ├── bootstrap-datepicker.zh-TW.js │ │ │ │ ├── bootstrap-datepicker.de.js │ │ │ │ ├── bootstrap-datepicker.fr.js │ │ │ │ ├── bootstrap-datepicker.mk.js │ │ │ │ ├── bootstrap-datepicker.vi.js │ │ │ │ ├── bootstrap-datepicker.it.js │ │ │ │ ├── bootstrap-datepicker.pt.js │ │ │ │ ├── bootstrap-datepicker.ar.js │ │ │ │ ├── bootstrap-datepicker.fi.js │ │ │ │ ├── bootstrap-datepicker.lv.js │ │ │ │ ├── bootstrap-datepicker.nl-BE.js │ │ │ │ ├── bootstrap-datepicker.lt.js │ │ │ │ ├── bootstrap-datepicker.pl.js │ │ │ │ ├── bootstrap-datepicker.sw.js │ │ │ │ ├── bootstrap-datepicker.ka.js │ │ │ │ └── bootstrap-datepicker.et.js │ │ │ ├── bootstrap │ │ │ ├── fonts │ │ │ │ ├── glyphicons-halflings-regular.eot │ │ │ │ ├── glyphicons-halflings-regular.ttf │ │ │ │ ├── glyphicons-halflings-regular.woff │ │ │ │ └── glyphicons-halflings-regular.woff2 │ │ │ └── js │ │ │ │ └── npm.js │ │ │ ├── mycommon.js │ │ │ └── page-target_urls.html │ ├── settings.pyc │ ├── classifier.pyc │ ├── data_service.pyc │ ├── gooseHelper.pyc │ ├── mongoHelper.pyc │ ├── monitor_init.pyc │ ├── mysqlHelper.pyc │ ├── mysql_init.pyc │ ├── url_extract_tools.pyc │ ├── upload │ │ └── swap.txt │ ├── url_extract_tools.py │ ├── monitor_init.py │ ├── redis_test_helper.py │ ├── mongoHelper.py │ ├── classifier.py │ └── settings.py ├── settings.pyc ├── gen_settings.pyc └── settings.py ├── ReadMe ├── er.png ├── bushutu.png ├── duibiao.png ├── jingxiangdabao.png ├── miliuchengtu.png ├── scrapyjiagou.png ├── dichengjiegoutu.png ├── scrapy_structure.png ├── mi_managermokuaitu.png └── mimanagerliuchengtu.jpg ├── .idea ├── misc.xml ├── vcs.xml ├── preferred-vcs.xml ├── modules.xml ├── version02.iml └── inspectionProfiles │ └── Project_Default.xml ├── .gitignore └── requirements.txt /mi/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mi/mi/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mi/mi/tools/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mi_manager/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mi/mi/pipelines/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mi/mi/middlewares/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mi/mi/scrapy_redis/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mi_manager/.idea/.name: -------------------------------------------------------------------------------- 1 | mi_master -------------------------------------------------------------------------------- /mi_manager/daemon/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mi_manager/monitor/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mi/mi/spiders_of_eCommerce/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mi_manager/daemon/model/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mi_manager/monitor/training/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mi/mi/spiders_of_news_in_whiteList/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mi/mi/spiders_of_news_need_fuzzymatching/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mi_manager/monitor/spiderInitfiles_of_eCommerce/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ReadMe/er.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/ReadMe/er.png -------------------------------------------------------------------------------- /mi/mi/items.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/mi/items.pyc -------------------------------------------------------------------------------- /mi/settings.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/settings.pyc -------------------------------------------------------------------------------- /mi_manager/monitor/static/const.js: -------------------------------------------------------------------------------- 1 | POST_URL_PREFIX = "http://127.0.0.1:5020" -------------------------------------------------------------------------------- /ReadMe/bushutu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/ReadMe/bushutu.png -------------------------------------------------------------------------------- /ReadMe/duibiao.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/ReadMe/duibiao.png -------------------------------------------------------------------------------- /mi/data_service.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/data_service.pyc -------------------------------------------------------------------------------- /mi/gen_settings.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/gen_settings.pyc -------------------------------------------------------------------------------- /mi/mi/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/mi/__init__.pyc -------------------------------------------------------------------------------- /mi/mi/settings.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/mi/settings.pyc -------------------------------------------------------------------------------- /ReadMe/jingxiangdabao.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/ReadMe/jingxiangdabao.png -------------------------------------------------------------------------------- /ReadMe/miliuchengtu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/ReadMe/miliuchengtu.png -------------------------------------------------------------------------------- /ReadMe/scrapyjiagou.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/ReadMe/scrapyjiagou.png -------------------------------------------------------------------------------- /mi/mi/tools/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/mi/tools/__init__.pyc -------------------------------------------------------------------------------- /mi_manager/settings.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/settings.pyc -------------------------------------------------------------------------------- /ReadMe/dichengjiegoutu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/ReadMe/dichengjiegoutu.png -------------------------------------------------------------------------------- /ReadMe/scrapy_structure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/ReadMe/scrapy_structure.png -------------------------------------------------------------------------------- /mi_manager/gen_settings.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/gen_settings.pyc -------------------------------------------------------------------------------- /ReadMe/mi_managermokuaitu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/ReadMe/mi_managermokuaitu.png -------------------------------------------------------------------------------- /ReadMe/mimanagerliuchengtu.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/ReadMe/mimanagerliuchengtu.jpg -------------------------------------------------------------------------------- /mi/mi/middlewares/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/mi/middlewares/__init__.pyc -------------------------------------------------------------------------------- /mi/mi/pipelines/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/mi/pipelines/__init__.pyc -------------------------------------------------------------------------------- /mi/mi/scrapy_redis/queue.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/mi/scrapy_redis/queue.pyc -------------------------------------------------------------------------------- /mi/mi/tools/fuzzyMatching.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/mi/tools/fuzzyMatching.pyc -------------------------------------------------------------------------------- /mi_manager/daemon/settings.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/daemon/settings.pyc -------------------------------------------------------------------------------- /mi/mi/scrapy_redis/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/mi/scrapy_redis/__init__.pyc -------------------------------------------------------------------------------- /mi/mi/scrapy_redis/scheduler.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/mi/scrapy_redis/scheduler.pyc -------------------------------------------------------------------------------- /mi_manager/daemon/model/task.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/daemon/model/task.pyc -------------------------------------------------------------------------------- /mi_manager/monitor/settings.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/settings.pyc -------------------------------------------------------------------------------- /mi/mi/pipelines/pipeline_mongo.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/mi/pipelines/pipeline_mongo.pyc -------------------------------------------------------------------------------- /mi/mi/pipelines/pipeline_mysql.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/mi/pipelines/pipeline_mysql.pyc -------------------------------------------------------------------------------- /mi/mi/scrapy_redis/connection.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/mi/scrapy_redis/connection.pyc -------------------------------------------------------------------------------- /mi/mi/scrapy_redis/dupefilter.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/mi/scrapy_redis/dupefilter.pyc -------------------------------------------------------------------------------- /mi_manager/daemon/data_service.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/daemon/data_service.pyc -------------------------------------------------------------------------------- /mi_manager/daemon/gen_json_file.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/daemon/gen_json_file.pyc -------------------------------------------------------------------------------- /mi_manager/daemon/model/mission.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/daemon/model/mission.pyc -------------------------------------------------------------------------------- /mi_manager/monitor/classifier.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/classifier.pyc -------------------------------------------------------------------------------- /mi_manager/monitor/data_service.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/data_service.pyc -------------------------------------------------------------------------------- /mi_manager/monitor/gooseHelper.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/gooseHelper.pyc -------------------------------------------------------------------------------- /mi_manager/monitor/mongoHelper.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/mongoHelper.pyc -------------------------------------------------------------------------------- /mi_manager/monitor/monitor_init.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/monitor_init.pyc -------------------------------------------------------------------------------- /mi_manager/monitor/mysqlHelper.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/mysqlHelper.pyc -------------------------------------------------------------------------------- /mi_manager/monitor/mysql_init.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/mysql_init.pyc -------------------------------------------------------------------------------- /mi_manager/daemon/marathonHelper.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/daemon/marathonHelper.pyc -------------------------------------------------------------------------------- /mi_manager/daemon/model/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/daemon/model/__init__.pyc -------------------------------------------------------------------------------- /mi/mi/middlewares/middleware_monitor.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/mi/middlewares/middleware_monitor.pyc -------------------------------------------------------------------------------- /mi/mi/spiders_of_eCommerce/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/mi/spiders_of_eCommerce/__init__.pyc -------------------------------------------------------------------------------- /mi/mi/spiders_of_eCommerce/spider_jd.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/mi/spiders_of_eCommerce/spider_jd.pyc -------------------------------------------------------------------------------- /mi_manager/daemon/model/submission.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/daemon/model/submission.pyc -------------------------------------------------------------------------------- /mi_manager/monitor/url_extract_tools.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/url_extract_tools.pyc -------------------------------------------------------------------------------- /mi/mi/scrapy_redis/BloomfilterOnRedis.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/mi/scrapy_redis/BloomfilterOnRedis.pyc -------------------------------------------------------------------------------- /mi/mi/spiders_of_eCommerce/spider_gome.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/mi/spiders_of_eCommerce/spider_gome.pyc -------------------------------------------------------------------------------- /mi_manager/monitor/training/1498569115.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/training/1498569115.pkl -------------------------------------------------------------------------------- /mi/mi/spiders_of_eCommerce/spider_amazon.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/mi/spiders_of_eCommerce/spider_amazon.pyc -------------------------------------------------------------------------------- /mi/mi/spiders_of_eCommerce/spider_taobao.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/mi/spiders_of_eCommerce/spider_taobao.pyc -------------------------------------------------------------------------------- /mi/mi/spiders_of_eCommerce/spider_tmall.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/mi/spiders_of_eCommerce/spider_tmall.pyc -------------------------------------------------------------------------------- /mi/mi/tools/gen_spiderFile_in_whiteList.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/mi/tools/gen_spiderFile_in_whiteList.pyc -------------------------------------------------------------------------------- /mi/mi/spiders_of_eCommerce/spider_dangdang.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/mi/spiders_of_eCommerce/spider_dangdang.pyc -------------------------------------------------------------------------------- /mi/mi/spiders_of_news_in_whiteList/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/mi/spiders_of_news_in_whiteList/__init__.pyc -------------------------------------------------------------------------------- /mi/mi/middlewares/middleware_rotateUserAgent.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/mi/middlewares/middleware_rotateUserAgent.pyc -------------------------------------------------------------------------------- /mi/mi/tools/gen_spiderFile_need_fuzzymatching.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/mi/tools/gen_spiderFile_need_fuzzymatching.pyc -------------------------------------------------------------------------------- /mi/mi/spiders_of_news_in_whiteList/spider_cctvcom.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/mi/spiders_of_news_in_whiteList/spider_cctvcom.pyc -------------------------------------------------------------------------------- /mi/mi/spiders_of_news_in_whiteList/spider_huxiucom.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/mi/spiders_of_news_in_whiteList/spider_huxiucom.pyc -------------------------------------------------------------------------------- /mi/mi/spiders_of_news_need_fuzzymatching/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/mi/spiders_of_news_need_fuzzymatching/__init__.pyc -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/dist/img/user8-128x128.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/dist/img/user8-128x128.jpg -------------------------------------------------------------------------------- /mi/mi/spiders_of_news_in_whiteList/spider_thepapercn.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/mi/spiders_of_news_in_whiteList/spider_thepapercn.pyc -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/flat/aero.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/flat/aero.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/flat/blue.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/flat/blue.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/flat/flat.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/flat/flat.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/flat/grey.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/flat/grey.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/flat/pink.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/flat/pink.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/flat/red.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/flat/red.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/line/line.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/line/line.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/dist/img/1498442770678662.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/dist/img/1498442770678662.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/flat/aero@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/flat/aero@2x.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/flat/blue@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/flat/blue@2x.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/flat/flat@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/flat/flat@2x.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/flat/green.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/flat/green.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/flat/grey@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/flat/grey@2x.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/flat/orange.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/flat/orange.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/flat/pink@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/flat/pink@2x.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/flat/purple.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/flat/purple.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/flat/red@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/flat/red@2x.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/flat/yellow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/flat/yellow.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/line/line@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/line/line@2x.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/minimal/aero.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/minimal/aero.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/minimal/blue.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/minimal/blue.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/minimal/grey.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/minimal/grey.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/minimal/pink.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/minimal/pink.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/minimal/red.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/minimal/red.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/square/aero.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/square/aero.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/square/blue.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/square/blue.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/square/green.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/square/green.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/square/grey.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/square/grey.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/square/pink.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/square/pink.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/square/red.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/square/red.png -------------------------------------------------------------------------------- /mi/mi/spiders_of_news_in_whiteList/spider_cankaoxiaoxicom.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/mi/spiders_of_news_in_whiteList/spider_cankaoxiaoxicom.pyc -------------------------------------------------------------------------------- /mi/mi/spiders_of_news_need_fuzzymatching/spider_sinacomcn.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/mi/spiders_of_news_need_fuzzymatching/spider_sinacomcn.pyc -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/flat/green@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/flat/green@2x.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/flat/orange@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/flat/orange@2x.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/flat/purple@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/flat/purple@2x.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/flat/yellow@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/flat/yellow@2x.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/minimal/green.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/minimal/green.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/minimal/orange.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/minimal/orange.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/minimal/purple.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/minimal/purple.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/minimal/red@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/minimal/red@2x.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/minimal/yellow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/minimal/yellow.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/square/aero@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/square/aero@2x.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/square/blue@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/square/blue@2x.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/square/grey@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/square/grey@2x.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/square/orange.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/square/orange.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/square/pink@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/square/pink@2x.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/square/purple.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/square/purple.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/square/red@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/square/red@2x.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/square/square.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/square/square.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/square/yellow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/square/yellow.png -------------------------------------------------------------------------------- /mi/mi/spiders_of_news_need_fuzzymatching/spider_mengzichengcn.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/mi/spiders_of_news_need_fuzzymatching/spider_mengzichengcn.pyc -------------------------------------------------------------------------------- /mi/mi/spiders_of_news_need_fuzzymatching/spider_peoplecomcn.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/mi/spiders_of_news_need_fuzzymatching/spider_peoplecomcn.pyc -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/dist/img/1498442770678662white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/dist/img/1498442770678662white.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/futurico/futurico.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/futurico/futurico.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/minimal/aero@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/minimal/aero@2x.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/minimal/blue@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/minimal/blue@2x.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/minimal/green@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/minimal/green@2x.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/minimal/grey@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/minimal/grey@2x.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/minimal/minimal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/minimal/minimal.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/minimal/orange@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/minimal/orange@2x.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/minimal/pink@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/minimal/pink@2x.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/minimal/purple@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/minimal/purple@2x.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/minimal/yellow@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/minimal/yellow@2x.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/polaris/polaris.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/polaris/polaris.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/square/green@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/square/green@2x.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/square/orange@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/square/orange@2x.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/square/purple@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/square/purple@2x.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/square/square@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/square/square@2x.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/square/yellow@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/square/yellow@2x.png -------------------------------------------------------------------------------- /mi/mi/spiders_of_news_need_fuzzymatching/spider_chinadailycomcn.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi/mi/spiders_of_news_need_fuzzymatching/spider_chinadailycomcn.pyc -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datatables/images/sort_asc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/datatables/images/sort_asc.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/minimal/minimal@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/minimal/minimal@2x.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/polaris/polaris@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/polaris/polaris@2x.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datatables/images/sort_both.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/datatables/images/sort_both.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datatables/images/sort_desc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/datatables/images/sort_desc.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/futurico/futurico@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/iCheck/futurico/futurico@2x.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/bootstrap/fonts/glyphicons-halflings-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/bootstrap/fonts/glyphicons-halflings-regular.eot -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/bootstrap/fonts/glyphicons-halflings-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/bootstrap/fonts/glyphicons-halflings-regular.ttf -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datatables/images/sort_asc_disabled.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/datatables/images/sort_asc_disabled.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datatables/images/sort_desc_disabled.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/plugins/datatables/images/sort_desc_disabled.png -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/bootstrap/fonts/glyphicons-halflings-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/bootstrap/fonts/glyphicons-halflings-regular.woff -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/bootstrap/fonts/glyphicons-halflings-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/724686158/mi/HEAD/mi_manager/monitor/static/v2/bootstrap/fonts/glyphicons-halflings-regular.woff2 -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /mi_manager/monitor/upload/swap.txt: -------------------------------------------------------------------------------- 1 | http://edition.cnn.com/2017/05/10/politics/white-hot-trump-comey/index.html 2 | http://en.chinaculture.org/news.html 3 | http://www.newyorker.com/ 4 | http://www.globaltimes.cn -------------------------------------------------------------------------------- /.idea/preferred-vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | ApexVCS 5 | 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by .ignore support plugin (hsz.mobi) 2 | .idea 3 | oldfile 4 | project.tar.gz 5 | mi.mi.settings.py 6 | mi_manager.daemon.settings.py 7 | mi_manager.monitor.settings.py 8 | 9 | *.py[cod] 10 | *$py.class -------------------------------------------------------------------------------- /mi/.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /mi/mi/scrapy_redis/utils.py: -------------------------------------------------------------------------------- 1 | import six 2 | 3 | 4 | def bytes_to_str(s, encoding='utf-8'): 5 | """Returns a str if a bytes object is given.""" 6 | if six.PY3 and isinstance(s, bytes): 7 | return s.decode(encoding) 8 | return s 9 | -------------------------------------------------------------------------------- /mi_manager/.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /mi/.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /mi_manager/monitor/url_extract_tools.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from tld import get_tld 3 | 4 | 5 | def extract_main_url(urls): 6 | res = None 7 | for i in urls: 8 | cur = get_tld(i, fail_silently=True) 9 | if cur: 10 | res = cur 11 | return res 12 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | scrapy 2 | redis 3 | pymongo 4 | bs4 5 | scrapy_redis 6 | requests 7 | lxml 8 | html5lib 9 | Flask 10 | twisted 11 | PyMySQL 12 | tld 13 | chardet 14 | jieba 15 | Pillow 16 | cssselect 17 | beautifulsoup 18 | nltk 19 | goose-extractor 20 | numpy 21 | scipy 22 | pandas 23 | scikit-learn==0.18.1 24 | xgboost -------------------------------------------------------------------------------- /mi/scrapy.cfg: -------------------------------------------------------------------------------- 1 | # Automatically created by: scrapy startproject 2 | # 3 | # For more information about the [deploy] section see: 4 | # https://scrapyd.readthedocs.org/en/latest/deploy.html 5 | 6 | [settings] 7 | default = mi.settings 8 | 9 | [deploy:mi_deploy] 10 | url = http://localhost:6800/ 11 | project = mi 12 | 13 | 14 | -------------------------------------------------------------------------------- /mi_manager/daemon/jsons/mi.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "mission1jdcommiv8", 3 | "cmd": null, 4 | "cpus": 1, 5 | "mem": 256, 6 | "disk": 256, 7 | "instances": 1, 8 | "container": { 9 | "docker": { 10 | "image": "mi:v8", 11 | "network": "HOST", 12 | "parameters": [] 13 | }, 14 | "type": "DOCKER", 15 | "volumes": [] 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /mi_manager/daemon/jsons/jd100jdcom.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "/spiders/jd100jdcom", 3 | "cmd": null, 4 | "cpus": 1, 5 | "mem": 256, 6 | "disk": 256, 7 | "instances": 1, 8 | "container": { 9 | "docker": { 10 | "image": "mi:v10", 11 | "network": "HOST", 12 | "parameters": [] 13 | }, 14 | "type": "DOCKER", 15 | "volumes": [] 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /mi_manager/daemon/jsons/jd101jdcom.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "/spiders/jd101jdcom", 3 | "cmd": null, 4 | "cpus": 1, 5 | "mem": 256, 6 | "disk": 256, 7 | "instances": 1, 8 | "container": { 9 | "docker": { 10 | "image": "mi:v10", 11 | "network": "HOST", 12 | "parameters": [] 13 | }, 14 | "type": "DOCKER", 15 | "volumes": [] 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /mi_manager/daemon/jsons/mis10jdcom.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "/spiders/mis10jdcom", 3 | "cmd": null, 4 | "cpus": 1, 5 | "mem": 256, 6 | "disk": 256, 7 | "instances": 1, 8 | "container": { 9 | "docker": { 10 | "image": "mi:v10", 11 | "network": "HOST", 12 | "parameters": [] 13 | }, 14 | "type": "DOCKER", 15 | "volumes": [] 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /mi_manager/daemon/jsons/mis2jdcom.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "/spiders/mis2jdcom", 3 | "cmd": null, 4 | "cpus": 1, 5 | "mem": 256, 6 | "disk": 256, 7 | "instances": 1, 8 | "container": { 9 | "docker": { 10 | "image": "mi:v10", 11 | "network": "HOST", 12 | "parameters": [] 13 | }, 14 | "type": "DOCKER", 15 | "volumes": [] 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /mi_manager/daemon/jsons/mis3jdcom.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "/spiders/mis3jdcom", 3 | "cmd": null, 4 | "cpus": 1, 5 | "mem": 256, 6 | "disk": 256, 7 | "instances": 1, 8 | "container": { 9 | "docker": { 10 | "image": "mi:v10", 11 | "network": "HOST", 12 | "parameters": [] 13 | }, 14 | "type": "DOCKER", 15 | "volumes": [] 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /mi_manager/daemon/jsons/mis4jdcom.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "/spiders/mis4jdcom", 3 | "cmd": null, 4 | "cpus": 1, 5 | "mem": 256, 6 | "disk": 256, 7 | "instances": 1, 8 | "container": { 9 | "docker": { 10 | "image": "mi:v10", 11 | "network": "HOST", 12 | "parameters": [] 13 | }, 14 | "type": "DOCKER", 15 | "volumes": [] 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /mi_manager/daemon/jsons/mis5jdcom.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "/spiders/mis5jdcom", 3 | "cmd": null, 4 | "cpus": 1, 5 | "mem": 256, 6 | "disk": 256, 7 | "instances": 1, 8 | "container": { 9 | "docker": { 10 | "image": "mi:v10", 11 | "network": "HOST", 12 | "parameters": [] 13 | }, 14 | "type": "DOCKER", 15 | "volumes": [] 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /mi_manager/daemon/jsons/mis6jdcom.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "/spiders/mis6jdcom", 3 | "cmd": null, 4 | "cpus": 1, 5 | "mem": 256, 6 | "disk": 256, 7 | "instances": 1, 8 | "container": { 9 | "docker": { 10 | "image": "mi:v10", 11 | "network": "HOST", 12 | "parameters": [] 13 | }, 14 | "type": "DOCKER", 15 | "volumes": [] 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /mi_manager/daemon/jsons/mis121jdcom.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "/spiders/mis121jdcom", 3 | "cmd": null, 4 | "cpus": 1, 5 | "mem": 256, 6 | "disk": 256, 7 | "instances": 1, 8 | "container": { 9 | "docker": { 10 | "image": "mi:v10", 11 | "network": "HOST", 12 | "parameters": [] 13 | }, 14 | "type": "DOCKER", 15 | "volumes": [] 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /mi_manager/daemon/jsons/mis1huxiucom.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "/spiders/mis1huxiucom", 3 | "cmd": null, 4 | "cpus": 1, 5 | "mem": 256, 6 | "disk": 256, 7 | "instances": 1, 8 | "container": { 9 | "docker": { 10 | "image": "mi:v10", 11 | "network": "HOST", 12 | "parameters": [] 13 | }, 14 | "type": "DOCKER", 15 | "volumes": [] 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /mi_manager/daemon/jsons/mission1jdcom.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "/spiders/mission1jdcom", 3 | "cmd": null, 4 | "cpus": 1, 5 | "mem": 256, 6 | "disk": 256, 7 | "instances": 1, 8 | "container": { 9 | "docker": { 10 | "image": "mi:v10", 11 | "network": "HOST", 12 | "parameters": [] 13 | }, 14 | "type": "DOCKER", 15 | "volumes": [] 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /mi_manager/daemon/jsons/mission1sinacomcn.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "/spiders/mission1sinacomcn", 3 | "cmd": null, 4 | "cpus": 1, 5 | "mem": 256, 6 | "disk": 256, 7 | "instances": 1, 8 | "container": { 9 | "docker": { 10 | "image": "mi:v10", 11 | "network": "HOST", 12 | "parameters": [] 13 | }, 14 | "type": "DOCKER", 15 | "volumes": [] 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /mi_manager/daemon/jsons/mission1thepapercn.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "/spiders/mission1thepapercn", 3 | "cmd": null, 4 | "cpus": 1, 5 | "mem": 256, 6 | "disk": 256, 7 | "instances": 1, 8 | "container": { 9 | "docker": { 10 | "image": "mi:v10", 11 | "network": "HOST", 12 | "parameters": [] 13 | }, 14 | "type": "DOCKER", 15 | "volumes": [] 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /mi_manager/daemon/jsons/mission21huxiucom.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "/spiders/mission21huxiucom", 3 | "cmd": null, 4 | "cpus": 1, 5 | "mem": 256, 6 | "disk": 256, 7 | "instances": 1, 8 | "container": { 9 | "docker": { 10 | "image": "mi:v10", 11 | "network": "HOST", 12 | "parameters": [] 13 | }, 14 | "type": "DOCKER", 15 | "volumes": [] 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /mi_manager/daemon/jsons/mission21taobaocom.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "/spiders/mission21taobaocom", 3 | "cmd": null, 4 | "cpus": 1, 5 | "mem": 256, 6 | "disk": 256, 7 | "instances": 1, 8 | "container": { 9 | "docker": { 10 | "image": "mi:v10", 11 | "network": "HOST", 12 | "parameters": [] 13 | }, 14 | "type": "DOCKER", 15 | "volumes": [] 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /mi_manager/daemon/jsons/mission31gomecomcn.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "/spiders/mission31gomecomcn", 3 | "cmd": null, 4 | "cpus": 1, 5 | "mem": 256, 6 | "disk": 256, 7 | "instances": 1, 8 | "container": { 9 | "docker": { 10 | "image": "mi:v10", 11 | "network": "HOST", 12 | "parameters": [] 13 | }, 14 | "type": "DOCKER", 15 | "volumes": [] 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /mi_manager/daemon/jsons/mogujiemogujiecom.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "/spiders/mogujiemogujiecom", 3 | "cmd": null, 4 | "cpus": 1, 5 | "mem": 256, 6 | "disk": 256, 7 | "instances": 1, 8 | "container": { 9 | "docker": { 10 | "image": "mi:v10", 11 | "network": "HOST", 12 | "parameters": [] 13 | }, 14 | "type": "DOCKER", 15 | "volumes": [] 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /mi_manager/daemon/jsons/mission1peoplecomcn.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "/spiders/mission1peoplecomcn", 3 | "cmd": null, 4 | "cpus": 1, 5 | "mem": 256, 6 | "disk": 256, 7 | "instances": 1, 8 | "container": { 9 | "docker": { 10 | "image": "mi:v10", 11 | "network": "HOST", 12 | "parameters": [] 13 | }, 14 | "type": "DOCKER", 15 | "volumes": [] 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /mi_manager/daemon/jsons/testmission16163com.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "/spiders/testmission16163com", 3 | "cmd": null, 4 | "cpus": 1, 5 | "mem": 256, 6 | "disk": 256, 7 | "instances": 1, 8 | "container": { 9 | "docker": { 10 | "image": "mi:v10", 11 | "network": "HOST", 12 | "parameters": [] 13 | }, 14 | "type": "DOCKER", 15 | "volumes": [] 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /mi_manager/daemon/jsons/mi20170630dangdangcom.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "/spiders/mi20170630dangdangcom", 3 | "cmd": null, 4 | "cpus": 1, 5 | "mem": 256, 6 | "disk": 256, 7 | "instances": 1, 8 | "container": { 9 | "docker": { 10 | "image": "mi:v10", 11 | "network": "HOST", 12 | "parameters": [] 13 | }, 14 | "type": "DOCKER", 15 | "volumes": [] 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /mi_manager/daemon/jsons/mi20170630mengzichengcn.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "/spiders/mi20170630mengzichengcn", 3 | "cmd": null, 4 | "cpus": 1, 5 | "mem": 256, 6 | "disk": 256, 7 | "instances": 1, 8 | "container": { 9 | "docker": { 10 | "image": "mi:v10", 11 | "network": "HOST", 12 | "parameters": [] 13 | }, 14 | "type": "DOCKER", 15 | "volumes": [] 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /mi_manager/daemon/jsons/mission1cankaoxiaoxicom.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "/spiders/mission1cankaoxiaoxicom", 3 | "cmd": null, 4 | "cpus": 1, 5 | "mem": 256, 6 | "disk": 256, 7 | "instances": 1, 8 | "container": { 9 | "docker": { 10 | "image": "mi:v10", 11 | "network": "HOST", 12 | "parameters": [] 13 | }, 14 | "type": "DOCKER", 15 | "volumes": [] 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /mi_manager/daemon/jsons/mission21mengzichengcn.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "/spiders/mission21mengzichengcn", 3 | "cmd": null, 4 | "cpus": 1, 5 | "mem": 256, 6 | "disk": 256, 7 | "instances": 1, 8 | "container": { 9 | "docker": { 10 | "image": "mi:v10", 11 | "network": "HOST", 12 | "parameters": [] 13 | }, 14 | "type": "DOCKER", 15 | "volumes": [] 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /mi_manager/daemon/jsons/mission31chinabytecom.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "/spiders/mission31chinabytecom", 3 | "cmd": null, 4 | "cpus": 1, 5 | "mem": 256, 6 | "disk": 256, 7 | "instances": 1, 8 | "container": { 9 | "docker": { 10 | "image": "mi:v10", 11 | "network": "HOST", 12 | "parameters": [] 13 | }, 14 | "type": "DOCKER", 15 | "volumes": [] 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /mi_manager/daemon/jsons/testmission16amazoncn.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "/spiders/testmission16amazoncn", 3 | "cmd": null, 4 | "cpus": 1, 5 | "mem": 256, 6 | "disk": 256, 7 | "instances": 1, 8 | "container": { 9 | "docker": { 10 | "image": "mi:v10", 11 | "network": "HOST", 12 | "parameters": [] 13 | }, 14 | "type": "DOCKER", 15 | "volumes": [] 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /mi_manager/daemon/jsons/testmission16sinacomcn.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "/spiders/testmission16sinacomcn", 3 | "cmd": null, 4 | "cpus": 1, 5 | "mem": 256, 6 | "disk": 256, 7 | "instances": 1, 8 | "container": { 9 | "docker": { 10 | "image": "mi:v10", 11 | "network": "HOST", 12 | "parameters": [] 13 | }, 14 | "type": "DOCKER", 15 | "volumes": [] 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /mi_manager/daemon/jsons/mi20170630chinadailycomcn.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "/spiders/mi20170630chinadailycomcn", 3 | "cmd": null, 4 | "cpus": 1, 5 | "mem": 256, 6 | "disk": 256, 7 | "instances": 1, 8 | "container": { 9 | "docker": { 10 | "image": "mi:v10", 11 | "network": "HOST", 12 | "parameters": [] 13 | }, 14 | "type": "DOCKER", 15 | "volumes": [] 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /mi_manager/.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /mi_manager/daemon/jsons/filter_redis.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "filterredis", 3 | "cmd": null, 4 | "cpus": 1, 5 | "mem": 8192, 6 | "disk": 8192, 7 | "instances": 1, 8 | "container": { 9 | "docker": { 10 | "image": "redis:3.2.8", 11 | "network": "HOST" 12 | }, 13 | "type": "DOCKER", 14 | "volumes": [] 15 | }, 16 | "portDefinitions": [ 17 | { 18 | "port": 0, 19 | "protocol": "tcp", 20 | "name": null, 21 | "labels": null 22 | } 23 | ], 24 | "env": {}, 25 | "labels": {}, 26 | "healthChecks": [] 27 | } -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/bootstrap/js/npm.js: -------------------------------------------------------------------------------- 1 | // This file is autogenerated via the `commonjs` Grunt task. You can require() this file in a CommonJS environment. 2 | require('../../js/transition.js') 3 | require('../../js/alert.js') 4 | require('../../js/button.js') 5 | require('../../js/carousel.js') 6 | require('../../js/collapse.js') 7 | require('../../js/dropdown.js') 8 | require('../../js/modal.js') 9 | require('../../js/tooltip.js') 10 | require('../../js/popover.js') 11 | require('../../js/scrollspy.js') 12 | require('../../js/tab.js') 13 | require('../../js/affix.js') -------------------------------------------------------------------------------- /mi_manager/daemon/model/submission.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | class Submisson(object): 3 | def __init__(self, name, detail, fathermission_name, resource_dic, weight): 4 | self.submission_name = name # 此值是子任务的唯一标志, 它是父任务名+爬虫名 5 | self.fathermission_name = fathermission_name 6 | self.resource_dic = resource_dic 7 | self.spider_name = detail['spider_name'] 8 | self.settings_name = detail['settings'] 9 | self.priority = float(detail['priority']) * float(weight) 10 | self.start_url = detail['start_url'] 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /mi_manager/daemon/mesosHelper.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import requests 3 | import settings 4 | import urllib2 5 | 6 | class MesosHelper(): 7 | def __init__(self): 8 | self.index = settings.MESOS_URL + '/#/' 9 | self.agent = settings.MESOS_URL + '/#/agents' 10 | 11 | 12 | def get_and_save_infos(self): 13 | req = urllib2.Request(self.index) 14 | res_data = urllib2.urlopen(req) 15 | res = res_data.read() 16 | print res 17 | 18 | 19 | if __name__ == '__main__': 20 | mesoshelper = MesosHelper() 21 | print mesoshelper.get_and_save_infos() -------------------------------------------------------------------------------- /mi_manager/daemon/model/task.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | class Task(object): 4 | 5 | def __init__(self, spider_name, resource_dic, settings_name, father_mission_name, start_url): 6 | self.spider_name = spider_name 7 | self.core_reids = resource_dic.get('core_reids') 8 | self.filter_redis = resource_dic.get('filter_redis') 9 | self.mongo = resource_dic.get('mongo') 10 | self.mysql = resource_dic.get('mysql') 11 | self.father_mission_name = father_mission_name 12 | self.settings_name = settings_name 13 | self.start_url = start_url -------------------------------------------------------------------------------- /mi/mi/scrapy_redis/connection.py: -------------------------------------------------------------------------------- 1 | import redis 2 | 3 | # Default values. 4 | FILTER_URL = None 5 | FILTER_HOST = 'localhost' 6 | FILTER_PORT = 6379 7 | FILTER_DB = 0 8 | 9 | 10 | def from_settings(settings): 11 | host = settings.get('FILTER_HOST', FILTER_HOST) 12 | port = settings.get('FILTER_PORT', FILTER_PORT) 13 | return redis.Redis(host=host, port=port) 14 | 15 | 16 | def from_settings_filter(settings): 17 | host = settings.get('FILTER_HOST', FILTER_HOST) 18 | port = settings.get('FILTER_PORT', FILTER_PORT) 19 | db = settings.get('FILTER_DB', FILTER_DB) 20 | return redis.Redis(host=host, port=port, db=db) -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.kr.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Korean translation for bootstrap-datepicker 3 | * Gu Youn 4 | */ 5 | ;(function($){ 6 | $.fn.datepicker.dates['kr'] = { 7 | days: ["일요일", "월요일", "화요일", "수요일", "목요일", "금요일", "토요일", "일요일"], 8 | daysShort: ["일", "월", "화", "수", "목", "금", "토", "일"], 9 | daysMin: ["일", "월", "화", "수", "목", "금", "토", "일"], 10 | months: ["1월", "2월", "3월", "4월", "5월", "6월", "7월", "8월", "9월", "10월", "11월", "12월"], 11 | monthsShort: ["1월", "2월", "3월", "4월", "5월", "6월", "7월", "8월", "9월", "10월", "11월", "12월"] 12 | }; 13 | }(jQuery)); 14 | -------------------------------------------------------------------------------- /mi_manager/monitor/monitor_init.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import redis 3 | import settings as prime_settings 4 | class MonitorInit(object): 5 | def start(self): 6 | print "尝试清空主监控器记录" 7 | try: 8 | # 连接数据库 9 | r = redis.Redis(prime_settings.REDIS_HOST, prime_settings.REDIS_PORT, db = prime_settings.MONITOR_DB) 10 | r.flushdb() 11 | # 清空monitor的四个队列 12 | for keyname in prime_settings.STATS_KEYS: 13 | r.delete(keyname) 14 | print "清空主监控器记录成功" 15 | except Exception: 16 | print "清空主监控器记录失败" 17 | finally: 18 | pass -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.gl.js: -------------------------------------------------------------------------------- 1 | ;(function($){ 2 | $.fn.datepicker.dates['gl'] = { 3 | days: ["Domingo", "Luns", "Martes", "Mércores", "Xoves", "Venres", "Sábado", "Domingo"], 4 | daysShort: ["Dom", "Lun", "Mar", "Mér", "Xov", "Ven", "Sáb", "Dom"], 5 | daysMin: ["Do", "Lu", "Ma", "Me", "Xo", "Ve", "Sa", "Do"], 6 | months: ["Xaneiro", "Febreiro", "Marzo", "Abril", "Maio", "Xuño", "Xullo", "Agosto", "Setembro", "Outubro", "Novembro", "Decembro"], 7 | monthsShort: ["Xan", "Feb", "Mar", "Abr", "Mai", "Xun", "Xul", "Ago", "Sep", "Out", "Nov", "Dec"], 8 | today: "Hoxe", 9 | clear: "Limpar" 10 | }; 11 | }(jQuery)); 12 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.ja.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Japanese translation for bootstrap-datepicker 3 | * Norio Suzuki 4 | */ 5 | ;(function($){ 6 | $.fn.datepicker.dates['ja'] = { 7 | days: ["日曜", "月曜", "火曜", "水曜", "木曜", "金曜", "土曜", "日曜"], 8 | daysShort: ["日", "月", "火", "水", "木", "金", "土", "日"], 9 | daysMin: ["日", "月", "火", "水", "木", "金", "土", "日"], 10 | months: ["1月", "2月", "3月", "4月", "5月", "6月", "7月", "8月", "9月", "10月", "11月", "12月"], 11 | monthsShort: ["1月", "2月", "3月", "4月", "5月", "6月", "7月", "8月", "9月", "10月", "11月", "12月"], 12 | today: "今日", 13 | format: "yyyy/mm/dd" 14 | }; 15 | }(jQuery)); 16 | -------------------------------------------------------------------------------- /mi_manager/monitor/redis_test_helper.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import redis 3 | import settings 4 | if __name__ == '__main__': 5 | r = redis.Redis(settings.REDIS_HOST, settings.REDIS_PORT, settings.SUBMISSION_DB) 6 | r.set('mission1_jd.com', '{"spider_name": "jd.com","father_mission_name": "mission1","settings": "默认设置1","priority": 5}') 7 | 8 | r = redis.Redis(settings.REDIS_HOST, settings.REDIS_PORT, settings.MISSION_DB) 9 | r.set('任务1', "{'submission_list': [{'name': 'mission1_jd.com', 'detail': {'spider_name': 'jd.com', 'settings': '默认设置1', 'priority': 2}}], 'weight': 0.8, 'start_time': 1498030984.74, 'resource_dic': {}, 'state': 'START', 'end_time': 1498031000.74}") 10 | -------------------------------------------------------------------------------- /mi_manager/daemon/jsons/core_redis.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "coreredis", 3 | "cmd": null, 4 | "cpus": 1, 5 | "mem": 2048, 6 | "disk": 2048, 7 | "instances": 1, 8 | "container": { 9 | "docker": { 10 | "image": "redis:3.2.8", 11 | "network": "HOST" 12 | }, 13 | "type": "DOCKER", 14 | "volumes": [ 15 | { 16 | "containerPath": "/data", 17 | "hostPath": "/redisdb/20170611", 18 | "mode": "RW" 19 | } 20 | ] 21 | }, 22 | "portDefinitions": [ 23 | { 24 | "port": 0, 25 | "protocol": "tcp", 26 | "name": null, 27 | "labels": null 28 | } 29 | ], 30 | "healthChecks": [], 31 | "labels": {}, 32 | "env": {} 33 | } -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.hr.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Croatian localisation 3 | */ 4 | ;(function($){ 5 | $.fn.datepicker.dates['hr'] = { 6 | days: ["Nedjelja", "Ponedjeljak", "Utorak", "Srijeda", "Četvrtak", "Petak", "Subota", "Nedjelja"], 7 | daysShort: ["Ned", "Pon", "Uto", "Sri", "Čet", "Pet", "Sub", "Ned"], 8 | daysMin: ["Ne", "Po", "Ut", "Sr", "Če", "Pe", "Su", "Ne"], 9 | months: ["Siječanj", "Veljača", "Ožujak", "Travanj", "Svibanj", "Lipanj", "Srpanj", "Kolovoz", "Rujan", "Listopad", "Studeni", "Prosinac"], 10 | monthsShort: ["Sij", "Velj", "Ožu", "Tra", "Svi", "Lip", "Srp", "Kol", "Ruj", "Lis", "Stu", "Pro"], 11 | today: "Danas" 12 | }; 13 | }(jQuery)); 14 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.zh-CN.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Simplified Chinese translation for bootstrap-datepicker 3 | * Yuan Cheung 4 | */ 5 | ;(function($){ 6 | $.fn.datepicker.dates['zh-CN'] = { 7 | days: ["星期日", "星期一", "星期二", "星期三", "星期四", "星期五", "星期六", "星期日"], 8 | daysShort: ["周日", "周一", "周二", "周三", "周四", "周五", "周六", "周日"], 9 | daysMin: ["日", "一", "二", "三", "四", "五", "六", "日"], 10 | months: ["一月", "二月", "三月", "四月", "五月", "六月", "七月", "八月", "九月", "十月", "十一月", "十二月"], 11 | monthsShort: ["一月", "二月", "三月", "四月", "五月", "六月", "七月", "八月", "九月", "十月", "十一月", "十二月"], 12 | today: "今日", 13 | format: "yyyy年mm月dd日", 14 | weekStart: 1 15 | }; 16 | }(jQuery)); 17 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.az.js: -------------------------------------------------------------------------------- 1 | // Azerbaijani 2 | ;(function($){ 3 | $.fn.datepicker.dates['az'] = { 4 | days: ["Bazar", "Bazar ertəsi", "Çərşənbə axşamı", "Çərşənbə", "Cümə axşamı", "Cümə", "Şənbə", "Bazar"], 5 | daysShort: ["B.", "B.e", "Ç.a", "Ç.", "C.a", "C.", "Ş.", "B."], 6 | daysMin: ["B.", "B.e", "Ç.a", "Ç.", "C.a", "C.", "Ş.", "B."], 7 | months: ["Yanvar", "Fevral", "Mart", "Aprel", "May", "İyun", "İyul", "Avqust", "Sentyabr", "Oktyabr", "Noyabr", "Dekabr"], 8 | monthsShort: ["Yan", "Fev", "Mar", "Apr", "May", "İyun", "İyul", "Avq", "Sen", "Okt", "Noy", "Dek"], 9 | today: "Bu gün", 10 | weekStart: 1 11 | }; 12 | }(jQuery)); 13 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.cy.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Welsh translation for bootstrap-datepicker 3 | * S. Morris 4 | */ 5 | ;(function($){ 6 | $.fn.datepicker.dates['cy'] = { 7 | days: ["Sul", "Llun", "Mawrth", "Mercher", "Iau", "Gwener", "Sadwrn", "Sul"], 8 | daysShort: ["Sul", "Llu", "Maw", "Mer", "Iau", "Gwe", "Sad", "Sul"], 9 | daysMin: ["Su", "Ll", "Ma", "Me", "Ia", "Gwe", "Sa", "Su"], 10 | months: ["Ionawr", "Chewfror", "Mawrth", "Ebrill", "Mai", "Mehefin", "Gorfennaf", "Awst", "Medi", "Hydref", "Tachwedd", "Rhagfyr"], 11 | monthsShort: ["Ion", "Chw", "Maw", "Ebr", "Mai", "Meh", "Gor", "Aws", "Med", "Hyd", "Tach", "Rha"], 12 | today: "Heddiw" 13 | }; 14 | }(jQuery)); 15 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.he.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Hebrew translation for bootstrap-datepicker 3 | * Sagie Maoz 4 | */ 5 | ;(function($){ 6 | $.fn.datepicker.dates['he'] = { 7 | days: ["ראשון", "שני", "שלישי", "רביעי", "חמישי", "שישי", "שבת", "ראשון"], 8 | daysShort: ["א", "ב", "ג", "ד", "ה", "ו", "ש", "א"], 9 | daysMin: ["א", "ב", "ג", "ד", "ה", "ו", "ש", "א"], 10 | months: ["ינואר", "פברואר", "מרץ", "אפריל", "מאי", "יוני", "יולי", "אוגוסט", "ספטמבר", "אוקטובר", "נובמבר", "דצמבר"], 11 | monthsShort: ["ינו", "פבר", "מרץ", "אפר", "מאי", "יונ", "יול", "אוג", "ספט", "אוק", "נוב", "דצמ"], 12 | today: "היום", 13 | rtl: true 14 | }; 15 | }(jQuery)); 16 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.ms.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Malay translation for bootstrap-datepicker 3 | * Ateman Faiz 4 | */ 5 | ;(function($){ 6 | $.fn.datepicker.dates['ms'] = { 7 | days: ["Ahad", "Isnin", "Selasa", "Rabu", "Khamis", "Jumaat", "Sabtu", "Ahad"], 8 | daysShort: ["Aha", "Isn", "Sel", "Rab", "Kha", "Jum", "Sab", "Aha"], 9 | daysMin: ["Ah", "Is", "Se", "Ra", "Kh", "Ju", "Sa", "Ah"], 10 | months: ["Januari", "Februari", "Mac", "April", "Mei", "Jun", "Julai", "Ogos", "September", "Oktober", "November", "Disember"], 11 | monthsShort: ["Jan", "Feb", "Mar", "Apr", "Mei", "Jun", "Jul", "Ogo", "Sep", "Okt", "Nov", "Dis"], 12 | today: "Hari Ini" 13 | }; 14 | }(jQuery)); 15 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.no.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Norwegian translation for bootstrap-datepicker 3 | **/ 4 | ;(function($){ 5 | $.fn.datepicker.dates['no'] = { 6 | days: ['Søndag','Mandag','Tirsdag','Onsdag','Torsdag','Fredag','Lørdag'], 7 | daysShort: ['Søn','Man','Tir','Ons','Tor','Fre','Lør'], 8 | daysMin: ['Sø','Ma','Ti','On','To','Fr','Lø'], 9 | months: ['Januar','Februar','Mars','April','Mai','Juni','Juli','August','September','Oktober','November','Desember'], 10 | monthsShort: ['Jan','Feb','Mar','Apr','Mai','Jun','Jul','Aug','Sep','Okt','Nov','Des'], 11 | today: 'I dag', 12 | clear: 'Nullstill', 13 | weekStart: 1, 14 | format: 'dd.mm.yyyy' 15 | }; 16 | }(jQuery)); 17 | -------------------------------------------------------------------------------- /mi/mi/scrapy_redis/defaults.py: -------------------------------------------------------------------------------- 1 | import redis 2 | 3 | 4 | # For standalone use. 5 | DUPEFILTER_KEY = 'dupefilter:%(timestamp)s' 6 | 7 | PIPELINE_KEY = '%(spider)s:items' 8 | 9 | REDIS_CLS = redis.StrictRedis 10 | REDIS_ENCODING = 'utf-8' 11 | # Sane connection defaults. 12 | REDIS_PARAMS = { 13 | 'socket_timeout': 30, 14 | 'socket_connect_timeout': 30, 15 | 'retry_on_timeout': True, 16 | 'encoding': REDIS_ENCODING, 17 | } 18 | 19 | SCHEDULER_QUEUE_KEY = '%(spider)s:requests' 20 | SCHEDULER_QUEUE_CLASS = 'scrapy_redis.queue.PriorityQueue' 21 | SCHEDULER_DUPEFILTER_KEY = '%(spider)s:dupefilter' 22 | SCHEDULER_DUPEFILTER_CLASS = 'scrapy_redis.dupefilter.RFPDupeFilter' 23 | 24 | START_URLS_KEY = '%(name)s:start_urls' 25 | START_URLS_AS_SET = False 26 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.ca.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Catalan translation for bootstrap-datepicker 3 | * J. Garcia 4 | */ 5 | ;(function($){ 6 | $.fn.datepicker.dates['ca'] = { 7 | days: ["Diumenge", "Dilluns", "Dimarts", "Dimecres", "Dijous", "Divendres", "Dissabte", "Diumenge"], 8 | daysShort: ["Diu", "Dil", "Dmt", "Dmc", "Dij", "Div", "Dis", "Diu"], 9 | daysMin: ["dg", "dl", "dt", "dc", "dj", "dv", "ds", "dg"], 10 | months: ["Gener", "Febrer", "Març", "Abril", "Maig", "Juny", "Juliol", "Agost", "Setembre", "Octubre", "Novembre", "Desembre"], 11 | monthsShort: ["Gen", "Feb", "Mar", "Abr", "Mai", "Jun", "Jul", "Ago", "Set", "Oct", "Nov", "Des"], 12 | today: "Avui" 13 | }; 14 | }(jQuery)); 15 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.el.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Greek translation for bootstrap-datepicker 3 | */ 4 | ;(function($){ 5 | $.fn.datepicker.dates['el'] = { 6 | days: ["Κυριακή", "Δευτέρα", "Τρίτη", "Τετάρτη", "Πέμπτη", "Παρασκευή", "Σάββατο", "Κυριακή"], 7 | daysShort: ["Κυρ", "Δευ", "Τρι", "Τετ", "Πεμ", "Παρ", "Σαβ", "Κυρ"], 8 | daysMin: ["Κυ", "Δε", "Τρ", "Τε", "Πε", "Πα", "Σα", "Κυ"], 9 | months: ["Ιανουάριος", "Φεβρουάριος", "Μάρτιος", "Απρίλιος", "Μάιος", "Ιούνιος", "Ιούλιος", "Αύγουστος", "Σεπτέμβριος", "Οκτώβριος", "Νοέμβριος", "Δεκέμβριος"], 10 | monthsShort: ["Ιαν", "Φεβ", "Μαρ", "Απρ", "Μάι", "Ιουν", "Ιουλ", "Αυγ", "Σεπ", "Οκτ", "Νοε", "Δεκ"], 11 | today: "Σήμερα" 12 | }; 13 | }(jQuery)); 14 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.rs.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Serbian cyrillic translation for bootstrap-datepicker 3 | * Bojan Milosavlević 4 | */ 5 | ;(function($){ 6 | $.fn.datepicker.dates['rs'] = { 7 | days: ["Недеља","Понедељак", "Уторак", "Среда", "Четвртак", "Петак", "Субота", "Недеља"], 8 | daysShort: ["Нед", "Пон", "Уто", "Сре", "Чет", "Пет", "Суб", "Нед"], 9 | daysMin: ["Н", "По", "У", "Ср", "Ч", "Пе", "Су", "Н"], 10 | months: ["Јануар", "Фебруар", "Март", "Април", "Мај", "Јун", "Јул", "Август", "Септембар", "Октобар", "Новембар", "Децембар"], 11 | monthsShort: ["Јан", "Феб", "Мар", "Апр", "Мај", "Јун", "Јул", "Авг", "Сеп", "Окт", "Нов", "Дец"], 12 | today: "Данас" 13 | }; 14 | }(jQuery)); 15 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.th.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Thai translation for bootstrap-datepicker 3 | * Suchau Jiraprapot 4 | */ 5 | ;(function($){ 6 | $.fn.datepicker.dates['th'] = { 7 | days: ["อาทิตย์", "จันทร์", "อังคาร", "พุธ", "พฤหัส", "ศุกร์", "เสาร์", "อาทิตย์"], 8 | daysShort: ["อา", "จ", "อ", "พ", "พฤ", "ศ", "ส", "อา"], 9 | daysMin: ["อา", "จ", "อ", "พ", "พฤ", "ศ", "ส", "อา"], 10 | months: ["มกราคม", "กุมภาพันธ์", "มีนาคม", "เมษายน", "พฤษภาคม", "มิถุนายน", "กรกฎาคม", "สิงหาคม", "กันยายน", "ตุลาคม", "พฤศจิกายน", "ธันวาคม"], 11 | monthsShort: ["ม.ค.", "ก.พ.", "มี.ค.", "เม.ย.", "พ.ค.", "มิ.ย.", "ก.ค.", "ส.ค.", "ก.ย.", "ต.ค.", "พ.ย.", "ธ.ค."], 12 | today: "วันนี้" 13 | }; 14 | }(jQuery)); 15 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.bg.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Bulgarian translation for bootstrap-datepicker 3 | * Apostol Apostolov 4 | */ 5 | ;(function($){ 6 | $.fn.datepicker.dates['bg'] = { 7 | days: ["Неделя", "Понеделник", "Вторник", "Сряда", "Четвъртък", "Петък", "Събота", "Неделя"], 8 | daysShort: ["Нед", "Пон", "Вто", "Сря", "Чет", "Пет", "Съб", "Нед"], 9 | daysMin: ["Н", "П", "В", "С", "Ч", "П", "С", "Н"], 10 | months: ["Януари", "Февруари", "Март", "Април", "Май", "Юни", "Юли", "Август", "Септември", "Октомври", "Ноември", "Декември"], 11 | monthsShort: ["Ян", "Фев", "Мар", "Апр", "Май", "Юни", "Юли", "Авг", "Сеп", "Окт", "Ное", "Дек"], 12 | today: "днес" 13 | }; 14 | }(jQuery)); 15 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.es.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Spanish translation for bootstrap-datepicker 3 | * Bruno Bonamin 4 | */ 5 | ;(function($){ 6 | $.fn.datepicker.dates['es'] = { 7 | days: ["Domingo", "Lunes", "Martes", "Miércoles", "Jueves", "Viernes", "Sábado", "Domingo"], 8 | daysShort: ["Dom", "Lun", "Mar", "Mié", "Jue", "Vie", "Sáb", "Dom"], 9 | daysMin: ["Do", "Lu", "Ma", "Mi", "Ju", "Vi", "Sa", "Do"], 10 | months: ["Enero", "Febrero", "Marzo", "Abril", "Mayo", "Junio", "Julio", "Agosto", "Septiembre", "Octubre", "Noviembre", "Diciembre"], 11 | monthsShort: ["Ene", "Feb", "Mar", "Abr", "May", "Jun", "Jul", "Ago", "Sep", "Oct", "Nov", "Dic"], 12 | today: "Hoy" 13 | }; 14 | }(jQuery)); 15 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.nl.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Dutch translation for bootstrap-datepicker 3 | * Reinier Goltstein 4 | */ 5 | ;(function($){ 6 | $.fn.datepicker.dates['nl'] = { 7 | days: ["Zondag", "Maandag", "Dinsdag", "Woensdag", "Donderdag", "Vrijdag", "Zaterdag", "Zondag"], 8 | daysShort: ["Zo", "Ma", "Di", "Wo", "Do", "Vr", "Za", "Zo"], 9 | daysMin: ["Zo", "Ma", "Di", "Wo", "Do", "Vr", "Za", "Zo"], 10 | months: ["Januari", "Februari", "Maart", "April", "Mei", "Juni", "Juli", "Augustus", "September", "Oktober", "November", "December"], 11 | monthsShort: ["Jan", "Feb", "Mrt", "Apr", "Mei", "Jun", "Jul", "Aug", "Sep", "Okt", "Nov", "Dec"], 12 | today: "Vandaag" 13 | }; 14 | }(jQuery)); 15 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.sl.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Slovene translation for bootstrap-datepicker 3 | * Gregor Rudolf 4 | */ 5 | ;(function($){ 6 | $.fn.datepicker.dates['sl'] = { 7 | days: ["Nedelja", "Ponedeljek", "Torek", "Sreda", "Četrtek", "Petek", "Sobota", "Nedelja"], 8 | daysShort: ["Ned", "Pon", "Tor", "Sre", "Čet", "Pet", "Sob", "Ned"], 9 | daysMin: ["Ne", "Po", "To", "Sr", "Če", "Pe", "So", "Ne"], 10 | months: ["Januar", "Februar", "Marec", "April", "Maj", "Junij", "Julij", "Avgust", "September", "Oktober", "November", "December"], 11 | monthsShort: ["Jan", "Feb", "Mar", "Apr", "Maj", "Jun", "Jul", "Avg", "Sep", "Okt", "Nov", "Dec"], 12 | today: "Danes" 13 | }; 14 | }(jQuery)); 15 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.rs-latin.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Serbian latin translation for bootstrap-datepicker 3 | * Bojan Milosavlević 4 | */ 5 | ;(function($){ 6 | $.fn.datepicker.dates['rs-latin'] = { 7 | days: ["Nedelja","Ponedeljak", "Utorak", "Sreda", "Četvrtak", "Petak", "Subota", "Nedelja"], 8 | daysShort: ["Ned", "Pon", "Uto", "Sre", "Čet", "Pet", "Sub", "Ned"], 9 | daysMin: ["N", "Po", "U", "Sr", "Č", "Pe", "Su", "N"], 10 | months: ["Januar", "Februar", "Mart", "April", "Maj", "Jun", "Jul", "Avgust", "Septembar", "Oktobar", "Novembar", "Decembar"], 11 | monthsShort: ["Jan", "Feb", "Mar", "Apr", "Maj", "Jun", "Jul", "Avg", "Sep", "Okt", "Nov", "Dec"], 12 | today: "Danas" 13 | }; 14 | }(jQuery)); 15 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.nb.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Norwegian (bokmål) translation for bootstrap-datepicker 3 | * Fredrik Sundmyhr 4 | */ 5 | ;(function($){ 6 | $.fn.datepicker.dates['nb'] = { 7 | days: ["Søndag", "Mandag", "Tirsdag", "Onsdag", "Torsdag", "Fredag", "Lørdag", "Søndag"], 8 | daysShort: ["Søn", "Man", "Tir", "Ons", "Tor", "Fre", "Lør", "Søn"], 9 | daysMin: ["Sø", "Ma", "Ti", "On", "To", "Fr", "Lø", "Sø"], 10 | months: ["Januar", "Februar", "Mars", "April", "Mai", "Juni", "Juli", "August", "September", "Oktober", "November", "Desember"], 11 | monthsShort: ["Jan", "Feb", "Mar", "Apr", "Mai", "Jun", "Jul", "Aug", "Sep", "Okt", "Nov", "Des"], 12 | today: "I Dag" 13 | }; 14 | }(jQuery)); 15 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.sq.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Albanian translation for bootstrap-datepicker 3 | * Tomor Pupovci 4 | */ 5 | ;(function($){ 6 | $.fn.datepicker.dates['sq'] = { 7 | days: ["E Diel", "E Hënë", "E martē", "E mërkurë", "E Enjte", "E Premte", "E Shtunë", "E Diel"], 8 | daysShort: ["Die", "Hën", "Mar", "Mër", "Enj", "Pre", "Shtu", "Die"], 9 | daysMin: ["Di", "Hë", "Ma", "Më", "En", "Pr", "Sht", "Di"], 10 | months: ["Janar", "Shkurt", "Mars", "Prill", "Maj", "Qershor", "Korrik", "Gusht", "Shtator", "Tetor", "Nëntor", "Dhjetor"], 11 | monthsShort: ["Jan", "Shk", "Mar", "Pri", "Maj", "Qer", "Korr", "Gu", "Sht", "Tet", "Nën", "Dhjet"], 12 | today: "Sot" 13 | }; 14 | }(jQuery)); 15 | 16 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.ua.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Ukrainian translation for bootstrap-datepicker 3 | * Igor Polynets 4 | */ 5 | ;(function($){ 6 | $.fn.datepicker.dates['ua'] = { 7 | days: ["Неділя", "Понеділок", "Вівторок", "Середа", "Четвер", "П'ятница", "Субота", "Неділя"], 8 | daysShort: ["Нед", "Пнд", "Втр", "Срд", "Чтв", "Птн", "Суб", "Нед"], 9 | daysMin: ["Нд", "Пн", "Вт", "Ср", "Чт", "Пт", "Сб", "Нд"], 10 | months: ["Cічень", "Лютий", "Березень", "Квітень", "Травень", "Червень", "Липень", "Серпень", "Вересень", "Жовтень", "Листопад", "Грудень"], 11 | monthsShort: ["Січ", "Лют", "Бер", "Кві", "Тра", "Чер", "Лип", "Сер", "Вер", "Жов", "Лис", "Гру"], 12 | today: "Сьогодні", 13 | weekStart: 1 14 | }; 15 | }(jQuery)); 16 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.id.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Bahasa translation for bootstrap-datepicker 3 | * Azwar Akbar 4 | */ 5 | ;(function($){ 6 | $.fn.datepicker.dates['id'] = { 7 | days: ["Minggu", "Senin", "Selasa", "Rabu", "Kamis", "Jumat", "Sabtu", "Minggu"], 8 | daysShort: ["Mgu", "Sen", "Sel", "Rab", "Kam", "Jum", "Sab", "Mgu"], 9 | daysMin: ["Mg", "Sn", "Sl", "Ra", "Ka", "Ju", "Sa", "Mg"], 10 | months: ["Januari", "Februari", "Maret", "April", "Mei", "Juni", "Juli", "Agustus", "September", "Oktober", "November", "Desember"], 11 | monthsShort: ["Jan", "Feb", "Mar", "Apr", "Mei", "Jun", "Jul", "Ags", "Sep", "Okt", "Nov", "Des"], 12 | today: "Hari Ini", 13 | clear: "Kosongkan" 14 | }; 15 | }(jQuery)); 16 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.kk.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Kazakh translation for bootstrap-datepicker 3 | * Yerzhan Tolekov 4 | */ 5 | ;(function($){ 6 | $.fn.datepicker.dates['kk'] = { 7 | days: ["Жексенбі", "Дүйсенбі", "Сейсенбі", "Сәрсенбі", "Бейсенбі", "Жұма", "Сенбі", "Жексенбі"], 8 | daysShort: ["Жек", "Дүй", "Сей", "Сәр", "Бей", "Жұм", "Сен", "Жек"], 9 | daysMin: ["Жк", "Дс", "Сс", "Ср", "Бс", "Жм", "Сн", "Жк"], 10 | months: ["Қаңтар", "Ақпан", "Наурыз", "Сәуір", "Мамыр", "Маусым", "Шілде", "Тамыз", "Қыркүйек", "Қазан", "Қараша", "Желтоқсан"], 11 | monthsShort: ["Қаң", "Ақп", "Нау", "Сәу", "Мамыр", "Мау", "Шлд", "Тмз", "Қыр", "Қзн", "Қар", "Жел"], 12 | today: "Бүгін", 13 | weekStart: 1 14 | }; 15 | }(jQuery)); 16 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.tr.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Turkish translation for bootstrap-datepicker 3 | * Serkan Algur 4 | */ 5 | ;(function($){ 6 | $.fn.datepicker.dates['tr'] = { 7 | days: ["Pazar", "Pazartesi", "Salı", "Çarşamba", "Perşembe", "Cuma", "Cumartesi", "Pazar"], 8 | daysShort: ["Pz", "Pzt", "Sal", "Çrş", "Prş", "Cu", "Cts", "Pz"], 9 | daysMin: ["Pz", "Pzt", "Sa", "Çr", "Pr", "Cu", "Ct", "Pz"], 10 | months: ["Ocak", "Şubat", "Mart", "Nisan", "Mayıs", "Haziran", "Temmuz", "Ağustos", "Eylül", "Ekim", "Kasım", "Aralık"], 11 | monthsShort: ["Oca", "Şub", "Mar", "Nis", "May", "Haz", "Tem", "Ağu", "Eyl", "Eki", "Kas", "Ara"], 12 | today: "Bugün", 13 | format: "dd.mm.yyyy" 14 | }; 15 | }(jQuery)); 16 | 17 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.da.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Danish translation for bootstrap-datepicker 3 | * Christian Pedersen 4 | */ 5 | ;(function($){ 6 | $.fn.datepicker.dates['da'] = { 7 | days: ["Søndag", "Mandag", "Tirsdag", "Onsdag", "Torsdag", "Fredag", "Lørdag", "Søndag"], 8 | daysShort: ["Søn", "Man", "Tir", "Ons", "Tor", "Fre", "Lør", "Søn"], 9 | daysMin: ["Sø", "Ma", "Ti", "On", "To", "Fr", "Lø", "Sø"], 10 | months: ["Januar", "Februar", "Marts", "April", "Maj", "Juni", "Juli", "August", "September", "Oktober", "November", "December"], 11 | monthsShort: ["Jan", "Feb", "Mar", "Apr", "Maj", "Jun", "Jul", "Aug", "Sep", "Okt", "Nov", "Dec"], 12 | today: "I Dag", 13 | clear: "Nulstil" 14 | }; 15 | }(jQuery)); 16 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.pt-BR.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Brazilian translation for bootstrap-datepicker 3 | * Cauan Cabral 4 | */ 5 | ;(function($){ 6 | $.fn.datepicker.dates['pt-BR'] = { 7 | days: ["Domingo", "Segunda", "Terça", "Quarta", "Quinta", "Sexta", "Sábado", "Domingo"], 8 | daysShort: ["Dom", "Seg", "Ter", "Qua", "Qui", "Sex", "Sáb", "Dom"], 9 | daysMin: ["Do", "Se", "Te", "Qu", "Qu", "Se", "Sa", "Do"], 10 | months: ["Janeiro", "Fevereiro", "Março", "Abril", "Maio", "Junho", "Julho", "Agosto", "Setembro", "Outubro", "Novembro", "Dezembro"], 11 | monthsShort: ["Jan", "Fev", "Mar", "Abr", "Mai", "Jun", "Jul", "Ago", "Set", "Out", "Nov", "Dez"], 12 | today: "Hoje", 13 | clear: "Limpar" 14 | }; 15 | }(jQuery)); 16 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.ru.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Russian translation for bootstrap-datepicker 3 | * Victor Taranenko 4 | */ 5 | ;(function($){ 6 | $.fn.datepicker.dates['ru'] = { 7 | days: ["Воскресенье", "Понедельник", "Вторник", "Среда", "Четверг", "Пятница", "Суббота", "Воскресенье"], 8 | daysShort: ["Вск", "Пнд", "Втр", "Срд", "Чтв", "Птн", "Суб", "Вск"], 9 | daysMin: ["Вс", "Пн", "Вт", "Ср", "Чт", "Пт", "Сб", "Вс"], 10 | months: ["Январь", "Февраль", "Март", "Апрель", "Май", "Июнь", "Июль", "Август", "Сентябрь", "Октябрь", "Ноябрь", "Декабрь"], 11 | monthsShort: ["Янв", "Фев", "Мар", "Апр", "Май", "Июн", "Июл", "Авг", "Сен", "Окт", "Ноя", "Дек"], 12 | today: "Сегодня", 13 | weekStart: 1 14 | }; 15 | }(jQuery)); 16 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.cs.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Czech translation for bootstrap-datepicker 3 | * Matěj Koubík 4 | * Fixes by Michal Remiš 5 | */ 6 | ;(function($){ 7 | $.fn.datepicker.dates['cs'] = { 8 | days: ["Neděle", "Pondělí", "Úterý", "Středa", "Čtvrtek", "Pátek", "Sobota", "Neděle"], 9 | daysShort: ["Ned", "Pon", "Úte", "Stř", "Čtv", "Pát", "Sob", "Ned"], 10 | daysMin: ["Ne", "Po", "Út", "St", "Čt", "Pá", "So", "Ne"], 11 | months: ["Leden", "Únor", "Březen", "Duben", "Květen", "Červen", "Červenec", "Srpen", "Září", "Říjen", "Listopad", "Prosinec"], 12 | monthsShort: ["Led", "Úno", "Bře", "Dub", "Kvě", "Čer", "Čnc", "Srp", "Zář", "Říj", "Lis", "Pro"], 13 | today: "Dnes" 14 | }; 15 | }(jQuery)); 16 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.is.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Icelandic translation for bootstrap-datepicker 3 | * Hinrik Örn Sigurðsson 4 | */ 5 | ;(function($){ 6 | $.fn.datepicker.dates['is'] = { 7 | days: ["Sunnudagur", "Mánudagur", "Þriðjudagur", "Miðvikudagur", "Fimmtudagur", "Föstudagur", "Laugardagur", "Sunnudagur"], 8 | daysShort: ["Sun", "Mán", "Þri", "Mið", "Fim", "Fös", "Lau", "Sun"], 9 | daysMin: ["Su", "Má", "Þr", "Mi", "Fi", "Fö", "La", "Su"], 10 | months: ["Janúar", "Febrúar", "Mars", "Apríl", "Maí", "Júní", "Júlí", "Ágúst", "September", "Október", "Nóvember", "Desember"], 11 | monthsShort: ["Jan", "Feb", "Mar", "Apr", "Maí", "Jún", "Júl", "Ágú", "Sep", "Okt", "Nóv", "Des"], 12 | today: "Í Dag" 13 | }; 14 | }(jQuery)); 15 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.sk.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Slovak translation for bootstrap-datepicker 3 | * Marek Lichtner 4 | * Fixes by Michal Remiš 5 | */ 6 | ;(function($){ 7 | $.fn.datepicker.dates["sk"] = { 8 | days: ["Nedeľa", "Pondelok", "Utorok", "Streda", "Štvrtok", "Piatok", "Sobota", "Nedeľa"], 9 | daysShort: ["Ned", "Pon", "Uto", "Str", "Štv", "Pia", "Sob", "Ned"], 10 | daysMin: ["Ne", "Po", "Ut", "St", "Št", "Pia", "So", "Ne"], 11 | months: ["Január", "Február", "Marec", "Apríl", "Máj", "Jún", "Júl", "August", "September", "Október", "November", "December"], 12 | monthsShort: ["Jan", "Feb", "Mar", "Apr", "Máj", "Jún", "Júl", "Aug", "Sep", "Okt", "Nov", "Dec"], 13 | today: "Dnes" 14 | }; 15 | }(jQuery)); 16 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.sv.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Swedish translation for bootstrap-datepicker 3 | * Patrik Ragnarsson 4 | */ 5 | ;(function($){ 6 | $.fn.datepicker.dates['sv'] = { 7 | days: ["Söndag", "Måndag", "Tisdag", "Onsdag", "Torsdag", "Fredag", "Lördag", "Söndag"], 8 | daysShort: ["Sön", "Mån", "Tis", "Ons", "Tor", "Fre", "Lör", "Sön"], 9 | daysMin: ["Sö", "Må", "Ti", "On", "To", "Fr", "Lö", "Sö"], 10 | months: ["Januari", "Februari", "Mars", "April", "Maj", "Juni", "Juli", "Augusti", "September", "Oktober", "November", "December"], 11 | monthsShort: ["Jan", "Feb", "Mar", "Apr", "Maj", "Jun", "Jul", "Aug", "Sep", "Okt", "Nov", "Dec"], 12 | today: "Idag", 13 | format: "yyyy-mm-dd", 14 | weekStart: 1 15 | }; 16 | }(jQuery)); 17 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.fa.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Persian translation for bootstrap-datepicker 3 | * Mostafa Rokooie 4 | */ 5 | ;(function($){ 6 | $.fn.datepicker.dates['fa'] = { 7 | days: ["یک‌شنبه", "دوشنبه", "سه‌شنبه", "چهارشنبه", "پنج‌شنبه", "جمعه", "شنبه", "یک‌شنبه"], 8 | daysShort: ["یک", "دو", "سه", "چهار", "پنج", "جمعه", "شنبه", "یک"], 9 | daysMin: ["ی", "د", "س", "چ", "پ", "ج", "ش", "ی"], 10 | months: ["ژانویه", "فوریه", "مارس", "آوریل", "مه", "ژوئن", "ژوئیه", "اوت", "سپتامبر", "اکتبر", "نوامبر", "دسامبر"], 11 | monthsShort: ["ژان", "فور", "مار", "آور", "مه", "ژون", "ژوی", "اوت", "سپت", "اکت", "نوا", "دسا"], 12 | today: "امروز", 13 | clear: "پاک کن", 14 | weekStart: 1, 15 | format: "yyyy/mm/dd" 16 | }; 17 | }(jQuery)); 18 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.hu.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Hungarian translation for bootstrap-datepicker 3 | * Sotus László 4 | */ 5 | ;(function($){ 6 | $.fn.datepicker.dates['hu'] = { 7 | days: ["Vasárnap", "Hétfő", "Kedd", "Szerda", "Csütörtök", "Péntek", "Szombat", "Vasárnap"], 8 | daysShort: ["Vas", "Hét", "Ked", "Sze", "Csü", "Pén", "Szo", "Vas"], 9 | daysMin: ["Va", "Hé", "Ke", "Sz", "Cs", "Pé", "Sz", "Va"], 10 | months: ["Január", "Február", "Március", "Április", "Május", "Június", "Július", "Augusztus", "Szeptember", "Október", "November", "December"], 11 | monthsShort: ["Jan", "Feb", "Már", "Ápr", "Máj", "Jún", "Júl", "Aug", "Sze", "Okt", "Nov", "Dec"], 12 | today: "Ma", 13 | weekStart: 1, 14 | format: "yyyy.mm.dd" 15 | }; 16 | }(jQuery)); 17 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.ro.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Romanian translation for bootstrap-datepicker 3 | * Cristian Vasile 4 | */ 5 | ;(function($){ 6 | $.fn.datepicker.dates['ro'] = { 7 | days: ["Duminică", "Luni", "Marţi", "Miercuri", "Joi", "Vineri", "Sâmbătă", "Duminică"], 8 | daysShort: ["Dum", "Lun", "Mar", "Mie", "Joi", "Vin", "Sâm", "Dum"], 9 | daysMin: ["Du", "Lu", "Ma", "Mi", "Jo", "Vi", "Sâ", "Du"], 10 | months: ["Ianuarie", "Februarie", "Martie", "Aprilie", "Mai", "Iunie", "Iulie", "August", "Septembrie", "Octombrie", "Noiembrie", "Decembrie"], 11 | monthsShort: ["Ian", "Feb", "Mar", "Apr", "Mai", "Iun", "Iul", "Aug", "Sep", "Oct", "Nov", "Dec"], 12 | today: "Astăzi", 13 | clear: "Șterge", 14 | weekStart: 1 15 | }; 16 | }(jQuery)); 17 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.zh-TW.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Traditional Chinese translation for bootstrap-datepicker 3 | * Rung-Sheng Jang 4 | * FrankWu Fix more appropriate use of Traditional Chinese habit 5 | */ 6 | ;(function($){ 7 | $.fn.datepicker.dates['zh-TW'] = { 8 | days: ["星期日", "星期一", "星期二", "星期三", "星期四", "星期五", "星期六", "星期日"], 9 | daysShort: ["週日", "週一", "週二", "週三", "週四", "週五", "週六", "週日"], 10 | daysMin: ["日", "一", "二", "三", "四", "五", "六", "日"], 11 | months: ["一月", "二月", "三月", "四月", "五月", "六月", "七月", "八月", "九月", "十月", "十一月", "十二月"], 12 | monthsShort: ["一月", "二月", "三月", "四月", "五月", "六月", "七月", "八月", "九月", "十月", "十一月", "十二月"], 13 | today: "今天", 14 | format: "yyyy年mm月dd日", 15 | weekStart: 1 16 | }; 17 | }(jQuery)); 18 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.de.js: -------------------------------------------------------------------------------- 1 | /** 2 | * German translation for bootstrap-datepicker 3 | * Sam Zurcher 4 | */ 5 | ;(function($){ 6 | $.fn.datepicker.dates['de'] = { 7 | days: ["Sonntag", "Montag", "Dienstag", "Mittwoch", "Donnerstag", "Freitag", "Samstag", "Sonntag"], 8 | daysShort: ["Son", "Mon", "Die", "Mit", "Don", "Fre", "Sam", "Son"], 9 | daysMin: ["So", "Mo", "Di", "Mi", "Do", "Fr", "Sa", "So"], 10 | months: ["Januar", "Februar", "März", "April", "Mai", "Juni", "Juli", "August", "September", "Oktober", "November", "Dezember"], 11 | monthsShort: ["Jan", "Feb", "Mär", "Apr", "Mai", "Jun", "Jul", "Aug", "Sep", "Okt", "Nov", "Dez"], 12 | today: "Heute", 13 | clear: "Löschen", 14 | weekStart: 1, 15 | format: "dd.mm.yyyy" 16 | }; 17 | }(jQuery)); 18 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.fr.js: -------------------------------------------------------------------------------- 1 | /** 2 | * French translation for bootstrap-datepicker 3 | * Nico Mollet 4 | */ 5 | ;(function($){ 6 | $.fn.datepicker.dates['fr'] = { 7 | days: ["Dimanche", "Lundi", "Mardi", "Mercredi", "Jeudi", "Vendredi", "Samedi", "Dimanche"], 8 | daysShort: ["Dim", "Lun", "Mar", "Mer", "Jeu", "Ven", "Sam", "Dim"], 9 | daysMin: ["D", "L", "Ma", "Me", "J", "V", "S", "D"], 10 | months: ["Janvier", "Février", "Mars", "Avril", "Mai", "Juin", "Juillet", "Août", "Septembre", "Octobre", "Novembre", "Décembre"], 11 | monthsShort: ["Jan", "Fév", "Mar", "Avr", "Mai", "Jui", "Jul", "Aou", "Sep", "Oct", "Nov", "Déc"], 12 | today: "Aujourd'hui", 13 | clear: "Effacer", 14 | weekStart: 1, 15 | format: "dd/mm/yyyy" 16 | }; 17 | }(jQuery)); 18 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.mk.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Macedonian translation for bootstrap-datepicker 3 | * Marko Aleksic 4 | */ 5 | ;(function($){ 6 | $.fn.datepicker.dates['mk'] = { 7 | days: ["Недела", "Понеделник", "Вторник", "Среда", "Четврток", "Петок", "Сабота", "Недела"], 8 | daysShort: ["Нед", "Пон", "Вто", "Сре", "Чет", "Пет", "Саб", "Нед"], 9 | daysMin: ["Не", "По", "Вт", "Ср", "Че", "Пе", "Са", "Не"], 10 | months: ["Јануари", "Февруари", "Март", "Април", "Мај", "Јуни", "Јули", "Август", "Септември", "Октомври", "Ноември", "Декември"], 11 | monthsShort: ["Јан", "Фев", "Мар", "Апр", "Мај", "Јун", "Јул", "Авг", "Сеп", "Окт", "Ное", "Дек"], 12 | today: "Денес", 13 | format: "dd.mm.yyyy" 14 | }; 15 | }(jQuery)); 16 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.vi.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Vietnamese translation for bootstrap-datepicker 3 | * An Vo 4 | */ 5 | ;(function($){ 6 | $.fn.datepicker.dates['vi'] = { 7 | days: ["Chủ nhật", "Thứ hai", "Thứ ba", "Thứ tư", "Thứ năm", "Thứ sáu", "Thứ bảy", "Chủ nhật"], 8 | daysShort: ["CN", "Thứ 2", "Thứ 3", "Thứ 4", "Thứ 5", "Thứ 6", "Thứ 7", "CN"], 9 | daysMin: ["CN", "T2", "T3", "T4", "T5", "T6", "T7", "CN"], 10 | months: ["Tháng 1", "Tháng 2", "Tháng 3", "Tháng 4", "Tháng 5", "Tháng 6", "Tháng 7", "Tháng 8", "Tháng 9", "Tháng 10", "Tháng 11", "Tháng 12"], 11 | monthsShort: ["Th1", "Th2", "Th3", "Th4", "Th5", "Th6", "Th7", "Th8", "Th9", "Th10", "Th11", "Th12"], 12 | today: "Hôm nay", 13 | clear: "Xóa", 14 | format: "dd/mm/yyyy" 15 | }; 16 | }(jQuery)); 17 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.it.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Italian translation for bootstrap-datepicker 3 | * Enrico Rubboli 4 | */ 5 | ;(function($){ 6 | $.fn.datepicker.dates['it'] = { 7 | days: ["Domenica", "Lunedì", "Martedì", "Mercoledì", "Giovedì", "Venerdì", "Sabato", "Domenica"], 8 | daysShort: ["Dom", "Lun", "Mar", "Mer", "Gio", "Ven", "Sab", "Dom"], 9 | daysMin: ["Do", "Lu", "Ma", "Me", "Gi", "Ve", "Sa", "Do"], 10 | months: ["Gennaio", "Febbraio", "Marzo", "Aprile", "Maggio", "Giugno", "Luglio", "Agosto", "Settembre", "Ottobre", "Novembre", "Dicembre"], 11 | monthsShort: ["Gen", "Feb", "Mar", "Apr", "Mag", "Giu", "Lug", "Ago", "Set", "Ott", "Nov", "Dic"], 12 | today: "Oggi", 13 | clear: "Cancella", 14 | weekStart: 1, 15 | format: "dd/mm/yyyy" 16 | }; 17 | }(jQuery)); 18 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.pt.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Portuguese translation for bootstrap-datepicker 3 | * Original code: Cauan Cabral 4 | * Tiago Melo 5 | */ 6 | ;(function($){ 7 | $.fn.datepicker.dates['pt'] = { 8 | days: ["Domingo", "Segunda", "Terça", "Quarta", "Quinta", "Sexta", "Sábado", "Domingo"], 9 | daysShort: ["Dom", "Seg", "Ter", "Qua", "Qui", "Sex", "Sáb", "Dom"], 10 | daysMin: ["Do", "Se", "Te", "Qu", "Qu", "Se", "Sa", "Do"], 11 | months: ["Janeiro", "Fevereiro", "Março", "Abril", "Maio", "Junho", "Julho", "Agosto", "Setembro", "Outubro", "Novembro", "Dezembro"], 12 | monthsShort: ["Jan", "Fev", "Mar", "Abr", "Mai", "Jun", "Jul", "Ago", "Set", "Out", "Nov", "Dez"], 13 | today: "Hoje", 14 | clear: "Limpar" 15 | }; 16 | }(jQuery)); 17 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.ar.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Arabic translation for bootstrap-datepicker 3 | * Mohammed Alshehri 4 | */ 5 | ;(function($){ 6 | $.fn.datepicker.dates['ar'] = { 7 | days: ["الأحد", "الاثنين", "الثلاثاء", "الأربعاء", "الخميس", "الجمعة", "السبت", "الأحد"], 8 | daysShort: ["أحد", "اثنين", "ثلاثاء", "أربعاء", "خميس", "جمعة", "سبت", "أحد"], 9 | daysMin: ["ح", "ن", "ث", "ع", "خ", "ج", "س", "ح"], 10 | months: ["يناير", "فبراير", "مارس", "أبريل", "مايو", "يونيو", "يوليو", "أغسطس", "سبتمبر", "أكتوبر", "نوفمبر", "ديسمبر"], 11 | monthsShort: ["يناير", "فبراير", "مارس", "أبريل", "مايو", "يونيو", "يوليو", "أغسطس", "سبتمبر", "أكتوبر", "نوفمبر", "ديسمبر"], 12 | today: "هذا اليوم", 13 | rtl: true 14 | }; 15 | }(jQuery)); 16 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.fi.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Finnish translation for bootstrap-datepicker 3 | * Jaakko Salonen 4 | */ 5 | ;(function($){ 6 | $.fn.datepicker.dates['fi'] = { 7 | days: ["sunnuntai", "maanantai", "tiistai", "keskiviikko", "torstai", "perjantai", "lauantai", "sunnuntai"], 8 | daysShort: ["sun", "maa", "tii", "kes", "tor", "per", "lau", "sun"], 9 | daysMin: ["su", "ma", "ti", "ke", "to", "pe", "la", "su"], 10 | months: ["tammikuu", "helmikuu", "maaliskuu", "huhtikuu", "toukokuu", "kesäkuu", "heinäkuu", "elokuu", "syyskuu", "lokakuu", "marraskuu", "joulukuu"], 11 | monthsShort: ["tam", "hel", "maa", "huh", "tou", "kes", "hei", "elo", "syy", "lok", "mar", "jou"], 12 | today: "tänään", 13 | weekStart: 1, 14 | format: "d.m.yyyy" 15 | }; 16 | }(jQuery)); 17 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.lv.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Latvian translation for bootstrap-datepicker 3 | * Artis Avotins 4 | */ 5 | 6 | ;(function($){ 7 | $.fn.datepicker.dates['lv'] = { 8 | days: ["Svētdiena", "Pirmdiena", "Otrdiena", "Trešdiena", "Ceturtdiena", "Piektdiena", "Sestdiena", "Svētdiena"], 9 | daysShort: ["Sv", "P", "O", "T", "C", "Pk", "S", "Sv"], 10 | daysMin: ["Sv", "Pr", "Ot", "Tr", "Ce", "Pk", "Se", "Sv"], 11 | months: ["Janvāris", "Februāris", "Marts", "Aprīlis", "Maijs", "Jūnijs", "Jūlijs", "Augusts", "Septembris", "Oktobris", "Novembris", "Decembris"], 12 | monthsShort: ["Jan", "Feb", "Mar", "Apr", "Mai", "Jūn", "Jūl", "Aug", "Sep", "Okt", "Nov", "Dec"], 13 | today: "Šodien", 14 | weekStart: 1 15 | }; 16 | }(jQuery)); 17 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.nl-BE.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Belgium-Dutch translation for bootstrap-datepicker 3 | * Julien Poulin 4 | */ 5 | ;(function($){ 6 | $.fn.datepicker.dates['nl-BE'] = { 7 | days: ["Zondag", "Maandag", "Dinsdag", "Woensdag", "Donderdag", "Vrijdag", "Zaterdag", "Zondag"], 8 | daysShort: ["Zo", "Ma", "Di", "Wo", "Do", "Vr", "Za", "Zo"], 9 | daysMin: ["Zo", "Ma", "Di", "Wo", "Do", "Vr", "Za", "Zo"], 10 | months: ["Januari", "Februari", "Maart", "April", "Mei", "Juni", "Juli", "Augustus", "September", "Oktober", "November", "December"], 11 | monthsShort: ["Jan", "Feb", "Mrt", "Apr", "Mei", "Jun", "Jul", "Aug", "Sep", "Okt", "Nov", "Dec"], 12 | today: "Vandaag", 13 | clear: "Leegmaken", 14 | weekStart: 1, 15 | format: "dd/mm/yyyy" 16 | }; 17 | }(jQuery)); 18 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.lt.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Lithuanian translation for bootstrap-datepicker 3 | * Šarūnas Gliebus 4 | */ 5 | 6 | ;(function($){ 7 | $.fn.datepicker.dates['lt'] = { 8 | days: ["Sekmadienis", "Pirmadienis", "Antradienis", "Trečiadienis", "Ketvirtadienis", "Penktadienis", "Šeštadienis", "Sekmadienis"], 9 | daysShort: ["S", "Pr", "A", "T", "K", "Pn", "Š", "S"], 10 | daysMin: ["Sk", "Pr", "An", "Tr", "Ke", "Pn", "Št", "Sk"], 11 | months: ["Sausis", "Vasaris", "Kovas", "Balandis", "Gegužė", "Birželis", "Liepa", "Rugpjūtis", "Rugsėjis", "Spalis", "Lapkritis", "Gruodis"], 12 | monthsShort: ["Sau", "Vas", "Kov", "Bal", "Geg", "Bir", "Lie", "Rugp", "Rugs", "Spa", "Lap", "Gru"], 13 | today: "Šiandien", 14 | weekStart: 1 15 | }; 16 | }(jQuery)); 17 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.pl.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Polish translation for bootstrap-datepicker 3 | * Robert 4 | */ 5 | ;(function($){ 6 | $.fn.datepicker.dates['pl'] = { 7 | days: ["Niedziela", "Poniedziałek", "Wtorek", "Środa", "Czwartek", "Piątek", "Sobota", "Niedziela"], 8 | daysShort: ["Nie", "Pn", "Wt", "Śr", "Czw", "Pt", "So", "Nie"], 9 | daysMin: ["N", "Pn", "Wt", "Śr", "Cz", "Pt", "So", "N"], 10 | months: ["Styczeń", "Luty", "Marzec", "Kwiecień", "Maj", "Czerwiec", "Lipiec", "Sierpień", "Wrzesień", "Październik", "Listopad", "Grudzień"], 11 | monthsShort: ["Sty", "Lu", "Mar", "Kw", "Maj", "Cze", "Lip", "Sie", "Wrz", "Pa", "Lis", "Gru"], 12 | today: "Dzisiaj", 13 | weekStart: 1 14 | }; 15 | }(jQuery)); 16 | -------------------------------------------------------------------------------- /mi/data_service.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # 调整电商类的start_url 4 | def get_start_url(spider_name, start_url): 5 | if spider_name == 'amazon.cn': 6 | start_url = 'https://www.amazon.cn/%E6%95%B0%E7%A0%81%E5%BD%B1%E9%9F%B3/dp/B014KP76G6/ref=sr_1_4?s=pc&ie=UTF8&qid=1493610822&sr=1-4&keywords=fire%E5%B9%B3%E6%9D%BF&th=1' 7 | elif spider_name == 'dangdang.com': 8 | start_url = 'http://www.dangdang.com' 9 | elif spider_name == 'gome.com.cn': 10 | start_url = 'http://www.gome.com.cn' 11 | elif spider_name == 'taobao.com': 12 | start_url = 'https://www.taobao.com' 13 | elif spider_name == 'tmall.com': 14 | start_url = 'https://www.taobao.com' 15 | elif spider_name == 'jd.com': 16 | if 'item' in start_url: 17 | pass 18 | else: 19 | start_url = 'https://item.jd.com/3312381.html' 20 | return start_url 21 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.sw.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Swahili translation for bootstrap-datepicker 3 | * Edwin Mugendi 4 | * Source: http://scriptsource.org/cms/scripts/page.php?item_id=entry_detail&uid=xnfaqyzcku 5 | */ 6 | ;(function($){ 7 | $.fn.datepicker.dates['sw'] = { 8 | days: ["Jumapili", "Jumatatu", "Jumanne", "Jumatano", "Alhamisi", "Ijumaa", "Jumamosi", "Jumapili"], 9 | daysShort: ["J2", "J3", "J4", "J5", "Alh", "Ij", "J1", "J2"], 10 | daysMin: ["2", "3", "4", "5", "A", "I", "1", "2"], 11 | months: ["Januari", "Februari", "Machi", "Aprili", "Mei", "Juni", "Julai", "Agosti", "Septemba", "Oktoba", "Novemba", "Desemba"], 12 | monthsShort: ["Jan", "Feb", "Mac", "Apr", "Mei", "Jun", "Jul", "Ago", "Sep", "Okt", "Nov", "Des"], 13 | today: "Leo" 14 | }; 15 | }(jQuery)); 16 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.ka.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Georgian translation for bootstrap-datepicker 3 | * Levan Melikishvili 4 | */ 5 | ;(function($){ 6 | $.fn.datepicker.dates['ka'] = { 7 | days: ["კვირა", "ორშაბათი", "სამშაბათი", "ოთხშაბათი", "ხუთშაბათი", "პარასკევი", "შაბათი", "კვირა"], 8 | daysShort: ["კვი", "ორშ", "სამ", "ოთხ", "ხუთ", "პარ", "შაბ", "კვი"], 9 | daysMin: ["კვ", "ორ", "სა", "ოთ", "ხუ", "პა", "შა", "კვ"], 10 | months: ["იანვარი", "თებერვალი", "მარტი", "აპრილი", "მაისი", "ივნისი", "ივლისი", "აგვისტო", "სექტემბერი", "ოქტომები", "ნოემბერი", "დეკემბერი"], 11 | monthsShort: ["იან", "თებ", "მარ", "აპრ", "მაი", "ივნ", "ივლ", "აგვ", "სექ", "ოქტ", "ნოე", "დეკ"], 12 | today: "დღეს", 13 | clear: "გასუფთავება", 14 | weekStart: 1, 15 | format: "dd.mm.yyyy" 16 | }; 17 | }(jQuery)); 18 | -------------------------------------------------------------------------------- /mi_manager/daemon/marathonHelper.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import requests 3 | class MarathonHelper(): 4 | def __init__(self): 5 | self.left = 6 6 | self.now = 0 7 | self.container_list = [] 8 | # 计算剩余负载能力 (待完善) 9 | def permit_container_number(self, need_cpu, need_mem, need_disk): 10 | return 1 11 | #return self.left - self.now 12 | 13 | def post_json_to_marathon(self, url, file, container): 14 | try: 15 | headers = {'Authorization': '(some auth code)', 'Content-Type': 'application/json'} 16 | response = requests.post(url, data=open(file, 'rb'), headers=headers) 17 | self.container_list.append(container) 18 | self.left = self.left - 1 19 | print '[Marathon]', '已收到指令,正在创建新容器' 20 | #print response.content 21 | except: 22 | pass 23 | #print '向marathon发post请求失败' 24 | 25 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/datepicker/locales/bootstrap-datepicker.et.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Estonian translation for bootstrap-datepicker 3 | * Ando Roots 4 | * Fixes by Illimar Tambek < 5 | */ 6 | ;(function($){ 7 | $.fn.datepicker.dates['et'] = { 8 | days: ["Pühapäev", "Esmaspäev", "Teisipäev", "Kolmapäev", "Neljapäev", "Reede", "Laupäev", "Pühapäev"], 9 | daysShort: ["Pühap", "Esmasp", "Teisip", "Kolmap", "Neljap", "Reede", "Laup", "Pühap"], 10 | daysMin: ["P", "E", "T", "K", "N", "R", "L", "P"], 11 | months: ["Jaanuar", "Veebruar", "Märts", "Aprill", "Mai", "Juuni", "Juuli", "August", "September", "Oktoober", "November", "Detsember"], 12 | monthsShort: ["Jaan", "Veebr", "Märts", "Apr", "Mai", "Juuni", "Juuli", "Aug", "Sept", "Okt", "Nov", "Dets"], 13 | today: "Täna", 14 | clear: "Tühjenda", 15 | weekStart: 1, 16 | format: "dd.mm.yyyy" 17 | }; 18 | }(jQuery)); 19 | -------------------------------------------------------------------------------- /mi_manager/daemon/gen_json_file.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import os 3 | import settings 4 | 5 | # 用于创建新容器的json的模板 6 | json_template = \ 7 | '''\ 8 | { 9 | "id": "/spiders/%s%s", 10 | "cmd": null, 11 | "cpus": %s, 12 | "mem": %s, 13 | "disk": %s, 14 | "instances": 1, 15 | "container": { 16 | "docker": { 17 | "image": "mi:%s", 18 | "network": "HOST", 19 | "parameters": [] 20 | }, 21 | "type": "DOCKER", 22 | "volumes": [] 23 | } 24 | } 25 | ''' 26 | # 传入爬虫子任务名 27 | def generate_json(mission_name, container_name, cpu, mem, disk): 28 | try: 29 | arr = (mission_name, container_name,cpu, mem, disk, settings.MI_VERSION) 30 | content = json_template % arr 31 | filename = os.getcwd() + settings.TEMP_PATH + '/jsons/'+ mission_name + container_name + '.json' 32 | with open(filename, 'w') as f: 33 | f.write(content) 34 | f.close() 35 | except: 36 | print 'fall' -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/mycommon.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by zhangzhengkun on 2017/6/13. 3 | */ 4 | 5 | String.prototype.format = function (args) { 6 | var result = this; 7 | if (arguments.length > 0) { 8 | if (arguments.length == 1 && typeof (args) == "object") { 9 | for (var key in args) { 10 | if (args[key] != undefined) { 11 | var reg = new RegExp("({" + key + "})", "g"); 12 | result = result.replace(reg, args[key]); 13 | } 14 | } 15 | } 16 | else { 17 | for (var i = 0; i < arguments.length; i++) { 18 | if (arguments[i] != undefined) { 19 | //var reg = new RegExp("({[" + i + "]})", "g");//这个在索引大于9时会有问题，谢谢何以笙箫的指出 20 | var reg = new RegExp("({)" + i + "(})", "g"); 21 | result = result.replace(reg, arguments[i]); 22 | } 23 | } 24 | } 25 | } 26 | return result; 27 | }; 28 | -------------------------------------------------------------------------------- /.idea/version02.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 19 | -------------------------------------------------------------------------------- /mi/mi/spiders_of_news_in_whiteList/spider_huxiucom.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from scrapy_redis.spiders import RedisCrawlSpider 3 | from mi.items import ArticleItem 4 | from scrapy.spiders import Rule 5 | from scrapy.linkextractors import LinkExtractor 6 | class Spider_huxiu(RedisCrawlSpider): 7 | name = 'huxiu.com' 8 | redis_key = 'huxiu.com:start_urls' 9 | allowed_domains = ['huxiu.com'] 10 | rules = [ 11 | Rule(LinkExtractor(allow=('https://www.huxiu.com/article/'),deny=()),callback='processArticle',follow=True) 12 | ] 13 | 14 | def processArticle(self,response): 15 | try: 16 | item = ArticleItem() 17 | title = response.xpath('''//h1/text()''').extract()[0] 18 | content = ''.join(response.xpath('''//div[@class='article-content-wrap']//p/text()''').extract()) 19 | item['articleTitle'] = title 20 | item['articleUrl'] = ''.join(response.url) 21 | item['articleContent'] = content 22 | yield item 23 | except: 24 | self.logger.info('item in article failed') 25 | -------------------------------------------------------------------------------- /mi/mi/spiders_of_news_in_whiteList/spider_thepapercn.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from scrapy_redis.spiders import RedisCrawlSpider 3 | from mi.items import ArticleItem 4 | from scrapy.spiders import Rule 5 | from scrapy.linkextractors import LinkExtractor 6 | class Spider_thepaper(RedisCrawlSpider): 7 | name = 'thepaper.cn' 8 | redis_key = 'thepaper.cn:start_urls' 9 | allowed_domains = ['thepaper.cn'] 10 | rules = [ 11 | Rule(LinkExtractor(allow=('http://www.thepaper.cn/newsDetail_forward_\d+'),deny=()),callback='processArticle',follow=True) 12 | ] 13 | 14 | def processArticle(self,response): 15 | try: 16 | item = ArticleItem() 17 | title = response.xpath('''//h1/text()''').extract()[0] 18 | content = ''.join(response.xpath('''//div[@class='news_txt']/text()''').extract()) 19 | item['articleTitle'] = title 20 | item['articleUrl'] = ''.join(response.url) 21 | item['articleContent'] = content 22 | yield item 23 | except: 24 | self.logger.info('item in article failed') 25 | -------------------------------------------------------------------------------- /mi_manager/monitor/spiderInitfiles_of_eCommerce/spiderInit_jd.com.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import redis 3 | import monitor.settings as prime_settings 4 | from monitor.mysqlHelper import MysqlHelper 5 | 6 | 7 | def init(): 8 | print "try insert eCommerceId......" 9 | try: 10 | db = MysqlHelper() 11 | sql = "insert into ECommerce(eCommerceName, eCommerceUrl) values('jd.com', 'https://item.jd.com');".encode(encoding='utf-8') 12 | db.insert(sql) 13 | print "insert eCommerceId success" 14 | except Exception: 15 | print "insert eCommerceId failed" 16 | 17 | print "pushing jd:start_url......" 18 | try: 19 | r = redis.Redis(prime_settings.REDIS_HOST, prime_settings.REDIS_PORT, db=prime_settings.FILTER_DB) 20 | r.delete("jd:start_urls") 21 | r.delete("jd:dupefilter" + "0") 22 | r.delete("jd:requests") 23 | r.lpush("jd:start_urls", 'https://item.jd.com/3312381.html') 24 | print "pushing jd:start_url success" 25 | except Exception: 26 | print "pushing jd:start_url failed" 27 | if __name__ == '__main__': 28 | init() 29 | -------------------------------------------------------------------------------- /mi/mi/spiders_of_news_in_whiteList/spider_cctvcom.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from scrapy_redis.spiders import RedisCrawlSpider 3 | from mi.items import ArticleItem 4 | from scrapy.spiders import Rule 5 | from scrapy.linkextractors import LinkExtractor 6 | class Spider_cctv(RedisCrawlSpider): 7 | name = 'cctv.com' 8 | redis_key = 'cctv.com:start_urls' 9 | allowed_domains = ['english.cctv.com'] 10 | rules = [ 11 | Rule(LinkExtractor(allow=('english.cctv.com/\d{4}/\d{2}/\d{2}'),deny=()),callback='processArticle',follow=True) 12 | ] 13 | 14 | def processArticle(self,response): 15 | try: 16 | item = ArticleItem() 17 | title = response.xpath('''//div[@id="jh604"]/div[@class="guojiA10232_ind03"]/h3/text()''').extract()[0] 18 | content = ''.join(response.xpath('''//div[@class='text']/p/text()''').extract()) 19 | item['articleTitle'] = title 20 | item['articleUrl'] = ''.join(response.url) 21 | item['articleContent'] = content 22 | yield item 23 | except: 24 | self.logger.info('item in article failed') 25 | -------------------------------------------------------------------------------- /mi_manager/monitor/spiderInitfiles_of_eCommerce/spiderInit_gome.com.cn.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import redis 3 | import monitor.settings as prime_settings 4 | from monitor.mysqlHelper import MysqlHelper 5 | 6 | def init(): 7 | print "try insert eCommerceId......" 8 | try: 9 | db = MysqlHelper() 10 | sql = "insert into ECommerce(eCommerceName, eCommerceUrl) values('gome.com.cn', 'http://www.gome.com.cn');".encode(encoding='utf-8') 11 | db.insert(sql) 12 | print "insert eCommerceId success" 13 | except Exception: 14 | print "insert eCommerceId failed" 15 | 16 | print "pushing guomei:start_url......" 17 | try: 18 | r = redis.Redis(prime_settings.REDIS_HOST, prime_settings.REDIS_PORT, db=prime_settings.FILTER_DB) 19 | r.delete("guomei:start_urls") 20 | r.delete("guomei:dupefilter" + "0") 21 | r.delete("guomei:requests") 22 | r.lpush("guomei:start_urls", 'http://www.gome.com.cn') 23 | print "pushing guomei:start_url success" 24 | except Exception: 25 | print "pushing guomei:start_url failed" 26 | if __name__ == '__main__': 27 | init() 28 | -------------------------------------------------------------------------------- /mi_manager/monitor/mongoHelper.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import redis 3 | import pymongo 4 | import settings as prime_settings 5 | 6 | class MongoHelper(): 7 | 8 | def connectMongo(self, host, port): 9 | conn = pymongo.MongoClient(host, port) 10 | return conn 11 | 12 | def connectDatabase(self, host, port, dbname): 13 | conn = self.connectMongo(host, port) 14 | db = conn[dbname] 15 | return db 16 | 17 | def discovery_and_connectDatabase_for_mission(self, mission_name): 18 | try: 19 | r = redis.Redis(prime_settings.REDIS_HOST, prime_settings.REDIS_PORT, prime_settings.MISSION_DB) 20 | mongo_dbname = eval(r.get(mission_name))['resource_dic']['mongo'] 21 | r = redis.Redis(prime_settings.REDIS_HOST, prime_settings.REDIS_PORT, prime_settings.RESOURCES_MONGO_DB) 22 | mysql_detail = r.get(mongo_dbname) 23 | dic = eval(mysql_detail) 24 | conn = self.connectMongo(dic['host'], int(dic['post'])) 25 | db = conn[mission_name] 26 | return db 27 | except: 28 | print '无法连接到任务[' + mission_name + ']所使用的Mongo数据库' 29 | -------------------------------------------------------------------------------- /mi_manager/monitor/spiderInitfiles_of_eCommerce/spiderInit_dangdang.com.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import redis 3 | import monitor.settings as prime_settings 4 | from monitor.mysqlHelper import MysqlHelper 5 | 6 | def init(): 7 | print "try insert eCommerceId......" 8 | try: 9 | db = MysqlHelper() 10 | sql = "insert into ECommerce(eCommerceName, eCommerceUrl) values('dangdang.com', 'http://www.dangdang.com');".encode(encoding='utf-8') 11 | db.insert(sql) 12 | print "insert eCommerceId success" 13 | except Exception: 14 | print "insert eCommerceId failed" 15 | 16 | print "pushing dangdang:start_url......" 17 | try: 18 | r = redis.Redis(prime_settings.REDIS_HOST, prime_settings.REDIS_PORT, db=prime_settings.FILTER_DB) 19 | r.delete("dangdang:start_urls") 20 | r.delete("dangdang:dupefilter" + "0") 21 | r.delete("dangdang:requests") 22 | r.lpush("dangdang:start_urls", 'http://www.dangdang.com') 23 | print "pushing dangdang:start_url success" 24 | except Exception: 25 | print "pushing dangdang:start_url failed" 26 | if __name__ == '__main__': 27 | init() 28 | -------------------------------------------------------------------------------- /mi/mi/spiders_of_news_in_whiteList/spider_cankaoxiaoxicom.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from scrapy_redis.spiders import RedisCrawlSpider 3 | from mi.items import ArticleItem 4 | from scrapy.spiders import Rule 5 | from scrapy.linkextractors import LinkExtractor 6 | class Spider_cankaoxiaoxi(RedisCrawlSpider): 7 | name = 'cankaoxiaoxi.com' 8 | redis_key = 'cankaoxiaoxi.com:start_urls' 9 | allowed_domains = ['cankaoxiaoxi.com'] 10 | rules = [ 11 | Rule(LinkExtractor(allow=('cankaoxiaoxi.com/\w+/\d{8}'),deny=()),callback='processArticle',follow=True) 12 | ] 13 | 14 | def processArticle(self,response): 15 | try: 16 | item = ArticleItem() 17 | title = response.xpath('''//div[@class="column"]//div[@class="bg-content"]/h1/text()''').extract()[0] 18 | content = ''.join(response.xpath('''//div[@class='inner']/div[@class="fs-small cont-detail det article-content ov"]/p/text()''').extract()) 19 | item['articleTitle'] = title 20 | item['articleUrl'] = ''.join(response.url) 21 | item['articleContent'] = content 22 | yield item 23 | except: 24 | self.logger.info('item in article failed') 25 | -------------------------------------------------------------------------------- /mi_manager/.idea/mi_manager.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 20 | -------------------------------------------------------------------------------- /.idea/inspectionProfiles/Project_Default.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 28 | -------------------------------------------------------------------------------- /mi/mi/scrapy_redis/BloomfilterOnRedis.py: -------------------------------------------------------------------------------- 1 | # encoding=utf-8 2 | 3 | class SimpleHash(object): 4 | def __init__(self, cap, seed): 5 | self.cap = cap 6 | self.seed = seed 7 | 8 | def hash(self, value): 9 | ret = 0 10 | for i in range(len(value)): 11 | ret += self.seed * ret + ord(value[i]) 12 | return (self.cap - 1) & ret 13 | 14 | 15 | class BloomFilter(object): 16 | def __init__(self, server, key, blockNum=1): 17 | self.bit_size = 1 << 31 # Redis的String类型最大容量为512M，现使用256M 18 | # self.seeds = [5, 7, 11, 13, 31] 19 | self.seeds = [5, 7, 11, 13, 31, 37, 61] 20 | self.server = server 21 | self.key = key 22 | self.blockNum = blockNum 23 | self.hashfunc = [] 24 | for seed in self.seeds: 25 | self.hashfunc.append(SimpleHash(self.bit_size, seed)) 26 | 27 | def isContains(self, str_input): 28 | if not str_input: 29 | return False 30 | ret = True 31 | 32 | name = self.key + str(int(str_input[0:2], 16) % self.blockNum) 33 | for f in self.hashfunc: 34 | loc = f.hash(str_input) 35 | ret = ret & self.server.getbit(name, loc) 36 | return ret 37 | 38 | def insert(self, str_input): 39 | name = self.key + str(int(str_input[0:2], 16) % self.blockNum) 40 | for f in self.hashfunc: 41 | loc = f.hash(str_input) 42 | self.server.setbit(name, loc, 1) -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/page-target_urls.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 |
5 |

目标ULRS

6 |
7 |
8 | 11 |
12 |
13 |
14 | 18 |
19 | 20 |
21 | 22 |
23 |
24 |
25 | 26 | 27 | 28 | 29 | 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/dist/css/alt/AdminLTE-fullcalendar.min.css: -------------------------------------------------------------------------------- 1 | .fc-button{background:#f4f4f4;background-image:none;color:#444;border-color:#ddd;border-bottom-color:#ddd}.fc-button:hover,.fc-button:active,.fc-button.hover{background-color:#e9e9e9}.fc-header-title h2{font-size:15px;line-height:1.6em;color:#666;margin-left:10px}.fc-header-right{padding-right:10px}.fc-header-left{padding-left:10px}.fc-widget-header{background:#fafafa}.fc-grid{width:100%;border:0}.fc-widget-header:first-of-type,.fc-widget-content:first-of-type{border-left:0;border-right:0}.fc-widget-header:last-of-type,.fc-widget-content:last-of-type{border-right:0}.fc-toolbar{padding:10px;margin:0}.fc-day-number{font-size:20px;font-weight:300;padding-right:10px}.fc-color-picker{list-style:none;margin:0;padding:0}.fc-color-picker>li{float:left;font-size:30px;margin-right:5px;line-height:30px}.fc-color-picker>li .fa{-webkit-transition:-webkit-transform linear .3s;-moz-transition:-moz-transform linear .3s;-o-transition:-o-transform linear .3s;transition:transform linear .3s}.fc-color-picker>li .fa:hover{-webkit-transform:rotate(30deg);-ms-transform:rotate(30deg);-o-transform:rotate(30deg);transform:rotate(30deg)}#add-new-event{-webkit-transition:all linear .3s;-o-transition:all linear .3s;transition:all linear .3s}.external-event{padding:5px 10px;font-weight:bold;margin-bottom:4px;box-shadow:0 1px 1px rgba(0,0,0,0.1);text-shadow:0 1px 1px rgba(0,0,0,0.1);border-radius:3px;cursor:move}.external-event:hover{box-shadow:inset 0 0 90px rgba(0,0,0,0.2)} -------------------------------------------------------------------------------- /mi/settings.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 核心redis数据库 3 | # 地址 4 | CORE_REDIS_HOST = '127.0.0.1' 5 | # 端口 6 | CORE_REDIS_PORT = 6379 7 | 8 | 9 | ############################################################################################## 10 | # redis 数据库分布 11 | # 用于存储调度队列 ———— 的redis数据据库编号（0～15） 12 | FILTER_DB = 0 13 | 14 | # 用于存储记号变量 ———— 的redis数据据库编号（0～15） 15 | SYMBOL_DB = 1 16 | 17 | # 用于存储task(带处理, 已处理) ———— 的redis数据据库编号（0～15） 18 | TASK_DB = 2 19 | 20 | # 用于进行task调度的有序集合 ———— 的redis数据据库编号（0～15） 21 | DISPATCH_DB = 3 22 | 23 | # 用于存储主任务 ———— 的redis数据据库编号（0～15） 24 | MISSION_DB = 4 25 | 26 | # 用于存储子任务 ———— 的redis数据据库编号（0～15） 27 | SUBMISSION_DB = 5 28 | 29 | # 用于存储爬虫配置信息 ———— 的redis数据据库编号（0～15） 30 | SETTINGS_DB = 6 31 | 32 | # 用于存储資源(REDIS服务器)的信息 ———— 的redis数据据库编号（0～15） 33 | RESOURCES_REDIS_DB = 7 34 | 35 | # 用于存储資源(MYSQL服务器)的信息 ———— 的redis数据据库编号（0～15） 36 | RESOURCES_MYSQL_DB = 8 37 | 38 | # 用于存储資源(MONGO服务器)的信息 ———— 的redis数据据库编号（0～15） 39 | RESOURCES_MONGO_DB = 9 40 | 41 | # 用于存储代理ip ———— 的redis数据据库编号（0～15） 42 | PROXY_DB = 10 43 | 44 | # 用于暂存爬虫运行时数据 ———— 的redis数据据库编号（0～15） 45 | RUNNINGDATA_DB = 11 46 | 47 | # 用于存储Cookie数据 ———— 的redis数据据库编号（0～15） 48 | COOKIES_DB = 12 49 | 50 | # 用于存储新闻类爬虫配置参数 ———— 的redis数据据库编号（0～15） 51 | SPIDERS_DB = 13 52 | 53 | # 用于分类爬虫(共三个类别： Whitelist, Fuzzy, Ecommerce) ———— 的redis数据据库编号（0～15） 54 | CLASSIFIER_DB = 14 55 | 56 | # 用于存储Monitor数据 ———— 的redis数据据库编号（0～15） 57 | MONITOR_DB = 15 58 | ############################################################################################## -------------------------------------------------------------------------------- /mi_manager/daemon/model/mission.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | class Mission(object): 3 | ''' 4 | def __init__(self, mission_name, start_time, end_time, submission_list, resource_dic, weight, state): 5 | self.mission_name = mission_name 6 | self.start_time = start_time 7 | self.end_time = end_time 8 | self.submission_list = submission_list 9 | self.resource_dic = resource_dic 10 | self.weight = weight 11 | self.state = state 12 | ''' 13 | 14 | 15 | def __init__(self, name, detail): 16 | dic = eval(detail) 17 | self.mission_name = name 18 | self.start_time = dic['start_time'] 19 | self.end_time = dic['end_time'] 20 | self.submission_list = list(dic['submission_list']) 21 | self.resource_dic = dic['resource_dic'] 22 | self.weight = dic['weight'] 23 | self.state = dic['state'] 24 | 25 | def get_detail(self): 26 | dic = {} 27 | dic['start_time'] = self.start_time 28 | dic['end_time'] = self.end_time 29 | dic['submission_list'] = self.submission_list 30 | dic['resource_dic'] = self.resource_dic 31 | dic['weight'] = self.weight 32 | dic['state'] = self.state 33 | return dic 34 | 35 | def get_submission_list(self): 36 | return self.submission_list 37 | 38 | def get_resource_dic(self): 39 | return self.resource_dic 40 | 41 | 42 | def get_name(self): 43 | return self.mission_name 44 | 45 | def set_all_submission(self): 46 | pass -------------------------------------------------------------------------------- /mi_manager/monitor/classifier.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | 3 | import os 4 | import re 5 | import sys 6 | import settings 7 | from itertools import izip 8 | from sklearn.externals import joblib 9 | from sklearn.datasets.base import Bunch 10 | 11 | from collections import OrderedDict 12 | 13 | CATEGORIES = OrderedDict([ 14 | ['business', []], 15 | ['politics', []], 16 | ['health', []], 17 | ['law', []], 18 | ['technology', []], 19 | ['entertainment', []], 20 | ['military', []], 21 | ['finance', []], 22 | ['sport', []]]) 23 | 24 | def clean(text): 25 | text = re.sub(r'\W', ' ', text) 26 | text = re.sub(r'\s+', ' ', text) 27 | return text.strip() 28 | 29 | def get_data(contents): 30 | all_data = [] 31 | for content in contents: 32 | all_data.extend([clean(content)]) 33 | return Bunch(categories=CATEGORIES.keys(), 34 | values=None, 35 | data=all_data) 36 | 37 | 38 | def get_type_of_news_by_content(content): 39 | filename = os.getcwd() + settings.TEMP_PATH +'/training/1498569115.pkl' 40 | if not filename: 41 | print "No models found in %s" % 'training' 42 | sys.exit(1) 43 | # Load the models using the already generated .pkl file 44 | model = joblib.load(filename) 45 | data = get_data([content]) 46 | print data.data 47 | data_weighted = model['vectorizer'].transform(data.data) 48 | data_weighted = model['feature_selection'].transform(data_weighted) 49 | prediction = model['clf'].predict(data_weighted) 50 | for text, prediction in izip(data.data, prediction): 51 | return CATEGORIES.keys()[prediction] -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/flat/flat.css: -------------------------------------------------------------------------------- 1 | /* iCheck plugin flat skin, black 2 | ----------------------------------- */ 3 | .icheckbox_flat, 4 | .iradio_flat { 5 | display: inline-block; 6 | *display: inline; 7 | vertical-align: middle; 8 | margin: 0; 9 | padding: 0; 10 | width: 20px; 11 | height: 20px; 12 | background: url(flat.png) no-repeat; 13 | border: none; 14 | cursor: pointer; 15 | } 16 | 17 | .icheckbox_flat { 18 | background-position: 0 0; 19 | } 20 | .icheckbox_flat.checked { 21 | background-position: -22px 0; 22 | } 23 | .icheckbox_flat.disabled { 24 | background-position: -44px 0; 25 | cursor: default; 26 | } 27 | .icheckbox_flat.checked.disabled { 28 | background-position: -66px 0; 29 | } 30 | 31 | .iradio_flat { 32 | background-position: -88px 0; 33 | } 34 | .iradio_flat.checked { 35 | background-position: -110px 0; 36 | } 37 | .iradio_flat.disabled { 38 | background-position: -132px 0; 39 | cursor: default; 40 | } 41 | .iradio_flat.checked.disabled { 42 | background-position: -154px 0; 43 | } 44 | 45 | /* Retina support */ 46 | @media only screen and (-webkit-min-device-pixel-ratio: 1.5), 47 | only screen and (-moz-min-device-pixel-ratio: 1.5), 48 | only screen and (-o-min-device-pixel-ratio: 3/2), 49 | only screen and (min-device-pixel-ratio: 1.5) { 50 | .icheckbox_flat, 51 | .iradio_flat { 52 | background-image: url(flat@2x.png); 53 | -webkit-background-size: 176px 22px; 54 | background-size: 176px 22px; 55 | } 56 | } -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/flat/red.css: -------------------------------------------------------------------------------- 1 | /* iCheck plugin Flat skin, red 2 | ----------------------------------- */ 3 | .icheckbox_flat-red, 4 | .iradio_flat-red { 5 | display: inline-block; 6 | *display: inline; 7 | vertical-align: middle; 8 | margin: 0; 9 | padding: 0; 10 | width: 20px; 11 | height: 20px; 12 | background: url(red.png) no-repeat; 13 | border: none; 14 | cursor: pointer; 15 | } 16 | 17 | .icheckbox_flat-red { 18 | background-position: 0 0; 19 | } 20 | .icheckbox_flat-red.checked { 21 | background-position: -22px 0; 22 | } 23 | .icheckbox_flat-red.disabled { 24 | background-position: -44px 0; 25 | cursor: default; 26 | } 27 | .icheckbox_flat-red.checked.disabled { 28 | background-position: -66px 0; 29 | } 30 | 31 | .iradio_flat-red { 32 | background-position: -88px 0; 33 | } 34 | .iradio_flat-red.checked { 35 | background-position: -110px 0; 36 | } 37 | .iradio_flat-red.disabled { 38 | background-position: -132px 0; 39 | cursor: default; 40 | } 41 | .iradio_flat-red.checked.disabled { 42 | background-position: -154px 0; 43 | } 44 | 45 | /* Retina support */ 46 | @media only screen and (-webkit-min-device-pixel-ratio: 1.5), 47 | only screen and (-moz-min-device-pixel-ratio: 1.5), 48 | only screen and (-o-min-device-pixel-ratio: 3/2), 49 | only screen and (min-device-pixel-ratio: 1.5) { 50 | .icheckbox_flat-red, 51 | .iradio_flat-red { 52 | background-image: url(red@2x.png); 53 | -webkit-background-size: 176px 22px; 54 | background-size: 176px 22px; 55 | } 56 | } -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/flat/aero.css: -------------------------------------------------------------------------------- 1 | /* iCheck plugin Flat skin, aero 2 | ----------------------------------- */ 3 | .icheckbox_flat-aero, 4 | .iradio_flat-aero { 5 | display: inline-block; 6 | *display: inline; 7 | vertical-align: middle; 8 | margin: 0; 9 | padding: 0; 10 | width: 20px; 11 | height: 20px; 12 | background: url(aero.png) no-repeat; 13 | border: none; 14 | cursor: pointer; 15 | } 16 | 17 | .icheckbox_flat-aero { 18 | background-position: 0 0; 19 | } 20 | .icheckbox_flat-aero.checked { 21 | background-position: -22px 0; 22 | } 23 | .icheckbox_flat-aero.disabled { 24 | background-position: -44px 0; 25 | cursor: default; 26 | } 27 | .icheckbox_flat-aero.checked.disabled { 28 | background-position: -66px 0; 29 | } 30 | 31 | .iradio_flat-aero { 32 | background-position: -88px 0; 33 | } 34 | .iradio_flat-aero.checked { 35 | background-position: -110px 0; 36 | } 37 | .iradio_flat-aero.disabled { 38 | background-position: -132px 0; 39 | cursor: default; 40 | } 41 | .iradio_flat-aero.checked.disabled { 42 | background-position: -154px 0; 43 | } 44 | 45 | /* Retina support */ 46 | @media only screen and (-webkit-min-device-pixel-ratio: 1.5), 47 | only screen and (-moz-min-device-pixel-ratio: 1.5), 48 | only screen and (-o-min-device-pixel-ratio: 3/2), 49 | only screen and (min-device-pixel-ratio: 1.5) { 50 | .icheckbox_flat-aero, 51 | .iradio_flat-aero { 52 | background-image: url(aero@2x.png); 53 | -webkit-background-size: 176px 22px; 54 | background-size: 176px 22px; 55 | } 56 | } -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/flat/blue.css: -------------------------------------------------------------------------------- 1 | /* iCheck plugin Flat skin, blue 2 | ----------------------------------- */ 3 | .icheckbox_flat-blue, 4 | .iradio_flat-blue { 5 | display: inline-block; 6 | *display: inline; 7 | vertical-align: middle; 8 | margin: 0; 9 | padding: 0; 10 | width: 20px; 11 | height: 20px; 12 | background: url(blue.png) no-repeat; 13 | border: none; 14 | cursor: pointer; 15 | } 16 | 17 | .icheckbox_flat-blue { 18 | background-position: 0 0; 19 | } 20 | .icheckbox_flat-blue.checked { 21 | background-position: -22px 0; 22 | } 23 | .icheckbox_flat-blue.disabled { 24 | background-position: -44px 0; 25 | cursor: default; 26 | } 27 | .icheckbox_flat-blue.checked.disabled { 28 | background-position: -66px 0; 29 | } 30 | 31 | .iradio_flat-blue { 32 | background-position: -88px 0; 33 | } 34 | .iradio_flat-blue.checked { 35 | background-position: -110px 0; 36 | } 37 | .iradio_flat-blue.disabled { 38 | background-position: -132px 0; 39 | cursor: default; 40 | } 41 | .iradio_flat-blue.checked.disabled { 42 | background-position: -154px 0; 43 | } 44 | 45 | /* Retina support */ 46 | @media only screen and (-webkit-min-device-pixel-ratio: 1.5), 47 | only screen and (-moz-min-device-pixel-ratio: 1.5), 48 | only screen and (-o-min-device-pixel-ratio: 3/2), 49 | only screen and (min-device-pixel-ratio: 1.5) { 50 | .icheckbox_flat-blue, 51 | .iradio_flat-blue { 52 | background-image: url(blue@2x.png); 53 | -webkit-background-size: 176px 22px; 54 | background-size: 176px 22px; 55 | } 56 | } -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/flat/grey.css: -------------------------------------------------------------------------------- 1 | /* iCheck plugin Flat skin, grey 2 | ----------------------------------- */ 3 | .icheckbox_flat-grey, 4 | .iradio_flat-grey { 5 | display: inline-block; 6 | *display: inline; 7 | vertical-align: middle; 8 | margin: 0; 9 | padding: 0; 10 | width: 20px; 11 | height: 20px; 12 | background: url(grey.png) no-repeat; 13 | border: none; 14 | cursor: pointer; 15 | } 16 | 17 | .icheckbox_flat-grey { 18 | background-position: 0 0; 19 | } 20 | .icheckbox_flat-grey.checked { 21 | background-position: -22px 0; 22 | } 23 | .icheckbox_flat-grey.disabled { 24 | background-position: -44px 0; 25 | cursor: default; 26 | } 27 | .icheckbox_flat-grey.checked.disabled { 28 | background-position: -66px 0; 29 | } 30 | 31 | .iradio_flat-grey { 32 | background-position: -88px 0; 33 | } 34 | .iradio_flat-grey.checked { 35 | background-position: -110px 0; 36 | } 37 | .iradio_flat-grey.disabled { 38 | background-position: -132px 0; 39 | cursor: default; 40 | } 41 | .iradio_flat-grey.checked.disabled { 42 | background-position: -154px 0; 43 | } 44 | 45 | /* Retina support */ 46 | @media only screen and (-webkit-min-device-pixel-ratio: 1.5), 47 | only screen and (-moz-min-device-pixel-ratio: 1.5), 48 | only screen and (-o-min-device-pixel-ratio: 3/2), 49 | only screen and (min-device-pixel-ratio: 1.5) { 50 | .icheckbox_flat-grey, 51 | .iradio_flat-grey { 52 | background-image: url(grey@2x.png); 53 | -webkit-background-size: 176px 22px; 54 | background-size: 176px 22px; 55 | } 56 | } -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/flat/pink.css: -------------------------------------------------------------------------------- 1 | /* iCheck plugin Flat skin, pink 2 | ----------------------------------- */ 3 | .icheckbox_flat-pink, 4 | .iradio_flat-pink { 5 | display: inline-block; 6 | *display: inline; 7 | vertical-align: middle; 8 | margin: 0; 9 | padding: 0; 10 | width: 20px; 11 | height: 20px; 12 | background: url(pink.png) no-repeat; 13 | border: none; 14 | cursor: pointer; 15 | } 16 | 17 | .icheckbox_flat-pink { 18 | background-position: 0 0; 19 | } 20 | .icheckbox_flat-pink.checked { 21 | background-position: -22px 0; 22 | } 23 | .icheckbox_flat-pink.disabled { 24 | background-position: -44px 0; 25 | cursor: default; 26 | } 27 | .icheckbox_flat-pink.checked.disabled { 28 | background-position: -66px 0; 29 | } 30 | 31 | .iradio_flat-pink { 32 | background-position: -88px 0; 33 | } 34 | .iradio_flat-pink.checked { 35 | background-position: -110px 0; 36 | } 37 | .iradio_flat-pink.disabled { 38 | background-position: -132px 0; 39 | cursor: default; 40 | } 41 | .iradio_flat-pink.checked.disabled { 42 | background-position: -154px 0; 43 | } 44 | 45 | /* Retina support */ 46 | @media only screen and (-webkit-min-device-pixel-ratio: 1.5), 47 | only screen and (-moz-min-device-pixel-ratio: 1.5), 48 | only screen and (-o-min-device-pixel-ratio: 3/2), 49 | only screen and (min-device-pixel-ratio: 1.5) { 50 | .icheckbox_flat-pink, 51 | .iradio_flat-pink { 52 | background-image: url(pink@2x.png); 53 | -webkit-background-size: 176px 22px; 54 | background-size: 176px 22px; 55 | } 56 | } -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/futurico/futurico.css: -------------------------------------------------------------------------------- 1 | /* iCheck plugin Futurico skin 2 | ----------------------------------- */ 3 | .icheckbox_futurico, 4 | .iradio_futurico { 5 | display: inline-block; 6 | *display: inline; 7 | vertical-align: middle; 8 | margin: 0; 9 | padding: 0; 10 | width: 16px; 11 | height: 17px; 12 | background: url(futurico.png) no-repeat; 13 | border: none; 14 | cursor: pointer; 15 | } 16 | 17 | .icheckbox_futurico { 18 | background-position: 0 0; 19 | } 20 | .icheckbox_futurico.checked { 21 | background-position: -18px 0; 22 | } 23 | .icheckbox_futurico.disabled { 24 | background-position: -36px 0; 25 | cursor: default; 26 | } 27 | .icheckbox_futurico.checked.disabled { 28 | background-position: -54px 0; 29 | } 30 | 31 | .iradio_futurico { 32 | background-position: -72px 0; 33 | } 34 | .iradio_futurico.checked { 35 | background-position: -90px 0; 36 | } 37 | .iradio_futurico.disabled { 38 | background-position: -108px 0; 39 | cursor: default; 40 | } 41 | .iradio_futurico.checked.disabled { 42 | background-position: -126px 0; 43 | } 44 | 45 | /* Retina support */ 46 | @media only screen and (-webkit-min-device-pixel-ratio: 1.5), 47 | only screen and (-moz-min-device-pixel-ratio: 1.5), 48 | only screen and (-o-min-device-pixel-ratio: 3/2), 49 | only screen and (min-device-pixel-ratio: 1.5) { 50 | .icheckbox_futurico, 51 | .iradio_futurico { 52 | background-image: url(futurico@2x.png); 53 | -webkit-background-size: 144px 19px; 54 | background-size: 144px 19px; 55 | } 56 | } -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/flat/green.css: -------------------------------------------------------------------------------- 1 | /* iCheck plugin Flat skin, green 2 | ----------------------------------- */ 3 | .icheckbox_flat-green, 4 | .iradio_flat-green { 5 | display: inline-block; 6 | *display: inline; 7 | vertical-align: middle; 8 | margin: 0; 9 | padding: 0; 10 | width: 20px; 11 | height: 20px; 12 | background: url(green.png) no-repeat; 13 | border: none; 14 | cursor: pointer; 15 | } 16 | 17 | .icheckbox_flat-green { 18 | background-position: 0 0; 19 | } 20 | .icheckbox_flat-green.checked { 21 | background-position: -22px 0; 22 | } 23 | .icheckbox_flat-green.disabled { 24 | background-position: -44px 0; 25 | cursor: default; 26 | } 27 | .icheckbox_flat-green.checked.disabled { 28 | background-position: -66px 0; 29 | } 30 | 31 | .iradio_flat-green { 32 | background-position: -88px 0; 33 | } 34 | .iradio_flat-green.checked { 35 | background-position: -110px 0; 36 | } 37 | .iradio_flat-green.disabled { 38 | background-position: -132px 0; 39 | cursor: default; 40 | } 41 | .iradio_flat-green.checked.disabled { 42 | background-position: -154px 0; 43 | } 44 | 45 | /* Retina support */ 46 | @media only screen and (-webkit-min-device-pixel-ratio: 1.5), 47 | only screen and (-moz-min-device-pixel-ratio: 1.5), 48 | only screen and (-o-min-device-pixel-ratio: 3/2), 49 | only screen and (min-device-pixel-ratio: 1.5) { 50 | .icheckbox_flat-green, 51 | .iradio_flat-green { 52 | background-image: url(green@2x.png); 53 | -webkit-background-size: 176px 22px; 54 | background-size: 176px 22px; 55 | } 56 | } -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/flat/orange.css: -------------------------------------------------------------------------------- 1 | /* iCheck plugin Flat skin, orange 2 | ----------------------------------- */ 3 | .icheckbox_flat-orange, 4 | .iradio_flat-orange { 5 | display: inline-block; 6 | *display: inline; 7 | vertical-align: middle; 8 | margin: 0; 9 | padding: 0; 10 | width: 20px; 11 | height: 20px; 12 | background: url(orange.png) no-repeat; 13 | border: none; 14 | cursor: pointer; 15 | } 16 | 17 | .icheckbox_flat-orange { 18 | background-position: 0 0; 19 | } 20 | .icheckbox_flat-orange.checked { 21 | background-position: -22px 0; 22 | } 23 | .icheckbox_flat-orange.disabled { 24 | background-position: -44px 0; 25 | cursor: default; 26 | } 27 | .icheckbox_flat-orange.checked.disabled { 28 | background-position: -66px 0; 29 | } 30 | 31 | .iradio_flat-orange { 32 | background-position: -88px 0; 33 | } 34 | .iradio_flat-orange.checked { 35 | background-position: -110px 0; 36 | } 37 | .iradio_flat-orange.disabled { 38 | background-position: -132px 0; 39 | cursor: default; 40 | } 41 | .iradio_flat-orange.checked.disabled { 42 | background-position: -154px 0; 43 | } 44 | 45 | /* Retina support */ 46 | @media only screen and (-webkit-min-device-pixel-ratio: 1.5), 47 | only screen and (-moz-min-device-pixel-ratio: 1.5), 48 | only screen and (-o-min-device-pixel-ratio: 3/2), 49 | only screen and (min-device-pixel-ratio: 1.5) { 50 | .icheckbox_flat-orange, 51 | .iradio_flat-orange { 52 | background-image: url(orange@2x.png); 53 | -webkit-background-size: 176px 22px; 54 | background-size: 176px 22px; 55 | } 56 | } -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/flat/purple.css: -------------------------------------------------------------------------------- 1 | /* iCheck plugin Flat skin, purple 2 | ----------------------------------- */ 3 | .icheckbox_flat-purple, 4 | .iradio_flat-purple { 5 | display: inline-block; 6 | *display: inline; 7 | vertical-align: middle; 8 | margin: 0; 9 | padding: 0; 10 | width: 20px; 11 | height: 20px; 12 | background: url(purple.png) no-repeat; 13 | border: none; 14 | cursor: pointer; 15 | } 16 | 17 | .icheckbox_flat-purple { 18 | background-position: 0 0; 19 | } 20 | .icheckbox_flat-purple.checked { 21 | background-position: -22px 0; 22 | } 23 | .icheckbox_flat-purple.disabled { 24 | background-position: -44px 0; 25 | cursor: default; 26 | } 27 | .icheckbox_flat-purple.checked.disabled { 28 | background-position: -66px 0; 29 | } 30 | 31 | .iradio_flat-purple { 32 | background-position: -88px 0; 33 | } 34 | .iradio_flat-purple.checked { 35 | background-position: -110px 0; 36 | } 37 | .iradio_flat-purple.disabled { 38 | background-position: -132px 0; 39 | cursor: default; 40 | } 41 | .iradio_flat-purple.checked.disabled { 42 | background-position: -154px 0; 43 | } 44 | 45 | /* Retina support */ 46 | @media only screen and (-webkit-min-device-pixel-ratio: 1.5), 47 | only screen and (-moz-min-device-pixel-ratio: 1.5), 48 | only screen and (-o-min-device-pixel-ratio: 3/2), 49 | only screen and (min-device-pixel-ratio: 1.5) { 50 | .icheckbox_flat-purple, 51 | .iradio_flat-purple { 52 | background-image: url(purple@2x.png); 53 | -webkit-background-size: 176px 22px; 54 | background-size: 176px 22px; 55 | } 56 | } -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/flat/yellow.css: -------------------------------------------------------------------------------- 1 | /* iCheck plugin Flat skin, yellow 2 | ----------------------------------- */ 3 | .icheckbox_flat-yellow, 4 | .iradio_flat-yellow { 5 | display: inline-block; 6 | *display: inline; 7 | vertical-align: middle; 8 | margin: 0; 9 | padding: 0; 10 | width: 20px; 11 | height: 20px; 12 | background: url(yellow.png) no-repeat; 13 | border: none; 14 | cursor: pointer; 15 | } 16 | 17 | .icheckbox_flat-yellow { 18 | background-position: 0 0; 19 | } 20 | .icheckbox_flat-yellow.checked { 21 | background-position: -22px 0; 22 | } 23 | .icheckbox_flat-yellow.disabled { 24 | background-position: -44px 0; 25 | cursor: default; 26 | } 27 | .icheckbox_flat-yellow.checked.disabled { 28 | background-position: -66px 0; 29 | } 30 | 31 | .iradio_flat-yellow { 32 | background-position: -88px 0; 33 | } 34 | .iradio_flat-yellow.checked { 35 | background-position: -110px 0; 36 | } 37 | .iradio_flat-yellow.disabled { 38 | background-position: -132px 0; 39 | cursor: default; 40 | } 41 | .iradio_flat-yellow.checked.disabled { 42 | background-position: -154px 0; 43 | } 44 | 45 | /* Retina support */ 46 | @media only screen and (-webkit-min-device-pixel-ratio: 1.5), 47 | only screen and (-moz-min-device-pixel-ratio: 1.5), 48 | only screen and (-o-min-device-pixel-ratio: 3/2), 49 | only screen and (min-device-pixel-ratio: 1.5) { 50 | .icheckbox_flat-yellow, 51 | .iradio_flat-yellow { 52 | background-image: url(yellow@2x.png); 53 | -webkit-background-size: 176px 22px; 54 | background-size: 176px 22px; 55 | } 56 | } -------------------------------------------------------------------------------- /mi/mi/pipelines/pipeline_mongo.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import pymongo 3 | import jieba.analyse 4 | from mi.items import ArticleItem 5 | from mi.items import DomTreeItem 6 | class MongoPipeline(object): 7 | # 集合名 使用之前会赋值 8 | hp_collection_name = '' 9 | 10 | # 初始化该数据库 11 | def __init__(self, mongo_host, mongo_port, mongo_db): 12 | self.mongo_host = mongo_host 13 | self.mongo_port = mongo_port 14 | self.mongo_db = mongo_db 15 | 16 | @classmethod # 指定该方法为类方法 17 | def from_crawler(cls, crawler): 18 | return cls( 19 | mongo_host = crawler.settings.get('MONGO_HOST', 'items'), 20 | mongo_port = crawler.settings.get('MONGO_PORT', 'items'), 21 | mongo_db = crawler.settings.get('MONGO_DATABASE', 'items'), 22 | ) 23 | 24 | # 数据库连接 25 | def open_spider(self, spider): 26 | self.client = pymongo.MongoClient(self.mongo_host, self.mongo_port) 27 | self.db = self.client[self.mongo_db] 28 | 29 | # 数据库关闭 30 | def close_spider(self, spider): 31 | self.client.close() 32 | 33 | # 将数据存入到数据库中 34 | def process_item(self, item, spider): 35 | if isinstance(item, ArticleItem): 36 | if len(item['articleContent']) > 0: 37 | tags = jieba.analyse.extract_tags(item['articleContent'], topK=20, withWeight=5) 38 | item['articleFirstTag'] = ''.join(tags[0][0]) 39 | item['articleSecondTag'] = ''.join(tags[1][0]) 40 | item['articleThirdTag'] = ''.join(tags[2][0]) 41 | self.db['Article'].insert(dict(item)) # 存入数据库原始数据 42 | if isinstance(item, DomTreeItem): 43 | self.db["DomTree"].insert(dict(item)) # 存入数据库原始数据 44 | return item -------------------------------------------------------------------------------- /mi/.idea/mi.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 27 | -------------------------------------------------------------------------------- /mi/mi/spiders_of_news_need_fuzzymatching/spider_sinacomcn.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from goose import Goose 3 | from goose.text import StopWordsChinese 4 | from scrapy_redis.spiders import RedisCrawlSpider 5 | from mi.items import ArticleItem 6 | from scrapy.spiders import Rule 7 | from scrapy.linkextractors import LinkExtractor 8 | from mi.tools.fuzzyMatching import calc_score 9 | class Spider_sinacomcn(RedisCrawlSpider): 10 | name = 'sina.com.cn' 11 | redis_key = 'sina.com.cn:start_urls' 12 | allowed_domains = ['sina.com.cn'] 13 | rules = [ 14 | Rule(LinkExtractor(allow=('sina.com.cn'),deny=()),callback='processArticle',follow=True) 15 | ] 16 | 17 | def processArticle(self,response): 18 | url = response.url 19 | score = calc_score(url) 20 | if score >= 3: 21 | try: 22 | print 'this url maybe a news_url' 23 | item = ArticleItem() 24 | g = Goose() 25 | article = g.extract(url = url) 26 | title = article.title 27 | content = article.cleaned_text 28 | if len(content) == 0: 29 | print 'news in chinese' 30 | g = Goose({'stopwords_class': StopWordsChinese}) 31 | article = g.extract(url=url) 32 | content = article.cleaned_text 33 | item['articleTitle'] = title 34 | item['articleUrl'] = url 35 | item['articleContent'] = content 36 | yield item 37 | except: 38 | self.logger.info('item in article failed') 39 | 40 | else: 41 | print 'this url maybe not a news_url, ' + ' score only ' + str(score) 42 | print 'you can check this url: ' + url 43 | return 44 | -------------------------------------------------------------------------------- /mi/mi/spiders_of_news_need_fuzzymatching/spider_peoplecomcn.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from goose import Goose 3 | from goose.text import StopWordsChinese 4 | from scrapy_redis.spiders import RedisCrawlSpider 5 | from mi.items import ArticleItem 6 | from scrapy.spiders import Rule 7 | from scrapy.linkextractors import LinkExtractor 8 | from mi.tools.fuzzyMatching import calc_score 9 | class Spider_peoplecomcn(RedisCrawlSpider): 10 | name = 'people.com.cn' 11 | redis_key = 'people.com.cn:start_urls' 12 | allowed_domains = ['people.com.cn'] 13 | rules = [ 14 | Rule(LinkExtractor(allow=('people.com.cn'),deny=()),callback='processArticle',follow=True) 15 | ] 16 | 17 | def processArticle(self,response): 18 | url = response.url 19 | score = calc_score(url) 20 | if score >= 3: 21 | try: 22 | print 'this url maybe a news_url' 23 | item = ArticleItem() 24 | g = Goose() 25 | article = g.extract(url = url) 26 | title = article.title 27 | content = article.cleaned_text 28 | if len(content) == 0: 29 | print 'news in chinese' 30 | g = Goose({'stopwords_class': StopWordsChinese}) 31 | article = g.extract(url=url) 32 | content = article.cleaned_text 33 | item['articleTitle'] = title 34 | item['articleUrl'] = url 35 | item['articleContent'] = content 36 | yield item 37 | except: 38 | self.logger.info('item in article failed') 39 | 40 | else: 41 | print 'this url maybe not a news_url, ' + ' score only ' + str(score) 42 | print 'you can check this url: ' + url 43 | return 44 | -------------------------------------------------------------------------------- /mi_manager/daemon/settings.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # mesos和marathon的地址 (请确保zookeeper+mesos+marathon框架正常运行) 3 | MESOS_URL = 'http://127.0.0.1:5050' 4 | MARATHON_URL = 'http://127.0.0.1:18082' 5 | 6 | # 核心redis服务器 7 | REDIS_HOST = '47.98.124.149' 8 | REDIS_PORT = 6379 9 | 10 | # mi的版本号 11 | MI_VERSION = 'v10' 12 | 13 | # 开启一个工作容器(mi:v8)所需要的CUP数量 14 | NEED_CPU = 1 15 | NEED_MEM = 256 16 | NEED_DISK = 256 17 | 18 | ############################################################################################## 19 | # 用于存储调度队列 ———— 的redis数据据库编号（0～15） 20 | FILTER_DB = 0 21 | 22 | # 用于存储记号变量 ———— 的redis数据据库编号（0～15） 23 | SYMBOL_DB = 1 24 | 25 | # 用于存储task(带处理, 已处理) ———— 的redis数据据库编号（0～15） 26 | TASK_DB = 2 27 | 28 | # 用于进行task调度的有序集合 ———— 的redis数据据库编号（0～15） 29 | DISPATCH_DB = 3 30 | 31 | # 用于存储主任务 ———— 的redis数据据库编号（0～15） 32 | MISSION_DB = 4 33 | 34 | # 用于存储子任务 ———— 的redis数据据库编号（0～15） 35 | SUBMISSION_DB = 5 36 | 37 | # 用于存储爬虫配置信息 ———— 的redis数据据库编号（0～15） 38 | SETTINGS_DB = 6 39 | 40 | # 用于存储資源(REDIS服务器)的信息 ———— 的redis数据据库编号（0～15） 41 | RESOURCES_REDIS_DB = 7 42 | 43 | # 用于存储資源(MYSQL服务器)的信息 ———— 的redis数据据库编号（0～15） 44 | RESOURCES_MYSQL_DB = 8 45 | 46 | # 用于存储資源(MONGO服务器)的信息 ———— 的redis数据据库编号（0～15） 47 | RESOURCES_MONGO_DB = 9 48 | 49 | # 用于存储代理ip ———— 的redis数据据库编号（0～15） 50 | PROXY_DB = 10 51 | 52 | # 用于暂存爬虫运行时数据 ———— 的redis数据据库编号（0～15） 53 | RUNNINGDATA_DB = 11 54 | 55 | # 用于存储Cookie数据 ———— 的redis数据据库编号（0～15） 56 | COOKIES_DB = 12 57 | 58 | # 用于存储新闻类爬虫配置参数 ———— 的redis数据据库编号（0～15） 59 | SPIDERS_DB = 13 60 | 61 | # 用于分类爬虫(共三个类别： Whitelist, Fuzzy, Ecommerce) ———— 的redis数据据库编号（0～15） 62 | CLASSIFIER_DB = 14 63 | 64 | # 用于存储Monitor数据 ———— 的redis数据据库编号（0～15） 65 | MONITOR_DB = 15 66 | ############################################################################################## 67 | 68 | # 临时路径,用于解决开发环境和生产环境中路径当前路径不一直的问题（开发环境中设为空） 69 | TEMP_PATH = '/daemon' -------------------------------------------------------------------------------- /mi/mi/spiders_of_news_need_fuzzymatching/spider_mengzichengcn.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from goose import Goose 3 | from goose.text import StopWordsChinese 4 | from scrapy_redis.spiders import RedisCrawlSpider 5 | from mi.items import ArticleItem 6 | from scrapy.spiders import Rule 7 | from scrapy.linkextractors import LinkExtractor 8 | from mi.tools.fuzzyMatching import calc_score 9 | class Spider_mengzichengcn(RedisCrawlSpider): 10 | name = 'mengzicheng.cn' 11 | redis_key = 'mengzicheng.cn:start_urls' 12 | allowed_domains = ['mengzicheng.cn'] 13 | rules = [ 14 | Rule(LinkExtractor(allow=('mengzicheng.cn'),deny=()),callback='processArticle',follow=True) 15 | ] 16 | 17 | def processArticle(self,response): 18 | url = response.url 19 | score = calc_score(url) 20 | if score >= 3: 21 | try: 22 | print 'this url maybe a news_url' 23 | item = ArticleItem() 24 | g = Goose() 25 | article = g.extract(url = url) 26 | title = article.title 27 | content = article.cleaned_text 28 | if len(content) == 0: 29 | print 'news in chinese' 30 | g = Goose({'stopwords_class': StopWordsChinese}) 31 | article = g.extract(url=url) 32 | content = article.cleaned_text 33 | item['articleTitle'] = title 34 | item['articleUrl'] = url 35 | item['articleContent'] = content 36 | yield item 37 | except: 38 | self.logger.info('item in article failed') 39 | 40 | else: 41 | print 'this url maybe not a news_url, ' + ' score only ' + str(score) 42 | print 'you can check this url: ' + url 43 | return 44 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/square/square.css: -------------------------------------------------------------------------------- 1 | /* iCheck plugin Square skin, black 2 | ----------------------------------- */ 3 | .icheckbox_square, 4 | .iradio_square { 5 | display: inline-block; 6 | *display: inline; 7 | vertical-align: middle; 8 | margin: 0; 9 | padding: 0; 10 | width: 22px; 11 | height: 22px; 12 | background: url(square.png) no-repeat; 13 | border: none; 14 | cursor: pointer; 15 | } 16 | 17 | .icheckbox_square { 18 | background-position: 0 0; 19 | } 20 | .icheckbox_square.hover { 21 | background-position: -24px 0; 22 | } 23 | .icheckbox_square.checked { 24 | background-position: -48px 0; 25 | } 26 | .icheckbox_square.disabled { 27 | background-position: -72px 0; 28 | cursor: default; 29 | } 30 | .icheckbox_square.checked.disabled { 31 | background-position: -96px 0; 32 | } 33 | 34 | .iradio_square { 35 | background-position: -120px 0; 36 | } 37 | .iradio_square.hover { 38 | background-position: -144px 0; 39 | } 40 | .iradio_square.checked { 41 | background-position: -168px 0; 42 | } 43 | .iradio_square.disabled { 44 | background-position: -192px 0; 45 | cursor: default; 46 | } 47 | .iradio_square.checked.disabled { 48 | background-position: -216px 0; 49 | } 50 | 51 | /* Retina support */ 52 | @media only screen and (-webkit-min-device-pixel-ratio: 1.5), 53 | only screen and (-moz-min-device-pixel-ratio: 1.5), 54 | only screen and (-o-min-device-pixel-ratio: 3/2), 55 | only screen and (min-device-pixel-ratio: 1.5) { 56 | .icheckbox_square, 57 | .iradio_square { 58 | background-image: url(square@2x.png); 59 | -webkit-background-size: 240px 24px; 60 | background-size: 240px 24px; 61 | } 62 | } -------------------------------------------------------------------------------- /mi/mi/spiders_of_news_need_fuzzymatching/spider_chinadailycomcn.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from goose import Goose 3 | from goose.text import StopWordsChinese 4 | from scrapy_redis.spiders import RedisCrawlSpider 5 | from mi.items import ArticleItem 6 | from scrapy.spiders import Rule 7 | from scrapy.linkextractors import LinkExtractor 8 | from mi.tools.fuzzyMatching import calc_score 9 | class Spider_chinadailycomcn(RedisCrawlSpider): 10 | name = 'chinadaily.com.cn' 11 | redis_key = 'chinadaily.com.cn:start_urls' 12 | allowed_domains = ['chinadaily.com.cn'] 13 | rules = [ 14 | Rule(LinkExtractor(allow=('chinadaily.com.cn'),deny=()),callback='processArticle',follow=True) 15 | ] 16 | 17 | def processArticle(self,response): 18 | url = response.url 19 | score = calc_score(url) 20 | if score >= 3: 21 | try: 22 | print 'this url maybe a news_url' 23 | item = ArticleItem() 24 | g = Goose() 25 | article = g.extract(url = url) 26 | title = article.title 27 | content = article.cleaned_text 28 | if len(content) == 0: 29 | print 'news in chinese' 30 | g = Goose({'stopwords_class': StopWordsChinese}) 31 | article = g.extract(url=url) 32 | content = article.cleaned_text 33 | item['articleTitle'] = title 34 | item['articleUrl'] = url 35 | item['articleContent'] = content 36 | yield item 37 | except: 38 | self.logger.info('item in article failed') 39 | 40 | else: 41 | print 'this url maybe not a news_url, ' + ' score only ' + str(score) 42 | print 'you can check this url: ' + url 43 | return 44 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/all.css: -------------------------------------------------------------------------------- 1 | /* iCheck plugin skins 2 | ----------------------------------- */ 3 | @import url("minimal/_all.css"); 4 | /* 5 | @import url("minimal/minimal.css"); 6 | @import url("minimal/red.css"); 7 | @import url("minimal/green.css"); 8 | @import url("minimal/blue.css"); 9 | @import url("minimal/aero.css"); 10 | @import url("minimal/grey.css"); 11 | @import url("minimal/orange.css"); 12 | @import url("minimal/yellow.css"); 13 | @import url("minimal/pink.css"); 14 | @import url("minimal/purple.css"); 15 | */ 16 | 17 | @import url("square/_all.css"); 18 | /* 19 | @import url("square/square.css"); 20 | @import url("square/red.css"); 21 | @import url("square/green.css"); 22 | @import url("square/blue.css"); 23 | @import url("square/aero.css"); 24 | @import url("square/grey.css"); 25 | @import url("square/orange.css"); 26 | @import url("square/yellow.css"); 27 | @import url("square/pink.css"); 28 | @import url("square/purple.css"); 29 | */ 30 | 31 | @import url("flat/_all.css"); 32 | /* 33 | @import url("flat/flat.css"); 34 | @import url("flat/red.css"); 35 | @import url("flat/green.css"); 36 | @import url("flat/blue.css"); 37 | @import url("flat/aero.css"); 38 | @import url("flat/grey.css"); 39 | @import url("flat/orange.css"); 40 | @import url("flat/yellow.css"); 41 | @import url("flat/pink.css"); 42 | @import url("flat/purple.css"); 43 | */ 44 | 45 | @import url("line/_all.css"); 46 | /* 47 | @import url("line/line.css"); 48 | @import url("line/red.css"); 49 | @import url("line/green.css"); 50 | @import url("line/blue.css"); 51 | @import url("line/aero.css"); 52 | @import url("line/grey.css"); 53 | @import url("line/orange.css"); 54 | @import url("line/yellow.css"); 55 | @import url("line/pink.css"); 56 | @import url("line/purple.css"); 57 | */ 58 | 59 | @import url("polaris/polaris.css"); 60 | 61 | @import url("futurico/futurico.css"); -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/polaris/polaris.css: -------------------------------------------------------------------------------- 1 | /* iCheck plugin Polaris skin 2 | ----------------------------------- */ 3 | .icheckbox_polaris, 4 | .iradio_polaris { 5 | display: inline-block; 6 | *display: inline; 7 | vertical-align: middle; 8 | margin: 0; 9 | padding: 0; 10 | width: 29px; 11 | height: 29px; 12 | background: url(polaris.png) no-repeat; 13 | border: none; 14 | cursor: pointer; 15 | } 16 | 17 | .icheckbox_polaris { 18 | background-position: 0 0; 19 | } 20 | .icheckbox_polaris.hover { 21 | background-position: -31px 0; 22 | } 23 | .icheckbox_polaris.checked { 24 | background-position: -62px 0; 25 | } 26 | .icheckbox_polaris.disabled { 27 | background-position: -93px 0; 28 | cursor: default; 29 | } 30 | .icheckbox_polaris.checked.disabled { 31 | background-position: -124px 0; 32 | } 33 | 34 | .iradio_polaris { 35 | background-position: -155px 0; 36 | } 37 | .iradio_polaris.hover { 38 | background-position: -186px 0; 39 | } 40 | .iradio_polaris.checked { 41 | background-position: -217px 0; 42 | } 43 | .iradio_polaris.disabled { 44 | background-position: -248px 0; 45 | cursor: default; 46 | } 47 | .iradio_polaris.checked.disabled { 48 | background-position: -279px 0; 49 | } 50 | 51 | /* Retina support */ 52 | @media only screen and (-webkit-min-device-pixel-ratio: 1.5), 53 | only screen and (-moz-min-device-pixel-ratio: 1.5), 54 | only screen and (-o-min-device-pixel-ratio: 3/2), 55 | only screen and (min-device-pixel-ratio: 1.5) { 56 | .icheckbox_polaris, 57 | .iradio_polaris { 58 | background-image: url(polaris@2x.png); 59 | -webkit-background-size: 310px 31px; 60 | background-size: 310px 31px; 61 | } 62 | } -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/minimal/minimal.css: -------------------------------------------------------------------------------- 1 | /* iCheck plugin Minimal skin, black 2 | ----------------------------------- */ 3 | .icheckbox_minimal, 4 | .iradio_minimal { 5 | display: inline-block; 6 | *display: inline; 7 | vertical-align: middle; 8 | margin: 0; 9 | padding: 0; 10 | width: 18px; 11 | height: 18px; 12 | background: url(minimal.png) no-repeat; 13 | border: none; 14 | cursor: pointer; 15 | } 16 | 17 | .icheckbox_minimal { 18 | background-position: 0 0; 19 | } 20 | .icheckbox_minimal.hover { 21 | background-position: -20px 0; 22 | } 23 | .icheckbox_minimal.checked { 24 | background-position: -40px 0; 25 | } 26 | .icheckbox_minimal.disabled { 27 | background-position: -60px 0; 28 | cursor: default; 29 | } 30 | .icheckbox_minimal.checked.disabled { 31 | background-position: -80px 0; 32 | } 33 | 34 | .iradio_minimal { 35 | background-position: -100px 0; 36 | } 37 | .iradio_minimal.hover { 38 | background-position: -120px 0; 39 | } 40 | .iradio_minimal.checked { 41 | background-position: -140px 0; 42 | } 43 | .iradio_minimal.disabled { 44 | background-position: -160px 0; 45 | cursor: default; 46 | } 47 | .iradio_minimal.checked.disabled { 48 | background-position: -180px 0; 49 | } 50 | 51 | /* Retina support */ 52 | @media only screen and (-webkit-min-device-pixel-ratio: 1.5), 53 | only screen and (-moz-min-device-pixel-ratio: 1.5), 54 | only screen and (-o-min-device-pixel-ratio: 3/2), 55 | only screen and (min-device-pixel-ratio: 1.5) { 56 | .icheckbox_minimal, 57 | .iradio_minimal { 58 | background-image: url(minimal@2x.png); 59 | -webkit-background-size: 200px 20px; 60 | background-size: 200px 20px; 61 | } 62 | } -------------------------------------------------------------------------------- /mi/mi/scrapy_redis/pipelines.py: -------------------------------------------------------------------------------- 1 | from scrapy.utils.misc import load_object 2 | from scrapy.utils.serialize import ScrapyJSONEncoder 3 | from twisted.internet.threads import deferToThread 4 | 5 | from . import connection, defaults 6 | 7 | 8 | default_serialize = ScrapyJSONEncoder().encode 9 | 10 | 11 | class RedisPipeline(object): 12 | 13 | def __init__(self, server, 14 | key=defaults.PIPELINE_KEY, 15 | serialize_func=default_serialize): 16 | self.server = server 17 | self.key = key 18 | self.serialize = serialize_func 19 | 20 | @classmethod 21 | def from_settings(cls, settings): 22 | params = { 23 | 'server': connection.from_settings(settings), 24 | } 25 | if settings.get('REDIS_ITEMS_KEY'): 26 | params['key'] = settings['REDIS_ITEMS_KEY'] 27 | if settings.get('REDIS_ITEMS_SERIALIZER'): 28 | params['serialize_func'] = load_object( 29 | settings['REDIS_ITEMS_SERIALIZER'] 30 | ) 31 | 32 | return cls(**params) 33 | 34 | @classmethod 35 | def from_crawler(cls, crawler): 36 | return cls.from_settings(crawler.settings) 37 | 38 | def process_item(self, item, spider): 39 | try: 40 | return deferToThread(self._process_item, item, spider) 41 | except: 42 | print '处理item出错' 43 | 44 | def _process_item(self, item, spider): 45 | key = self.item_key(item, spider) 46 | data = self.serialize(item) 47 | self.server.rpush(key, data) 48 | return item 49 | 50 | def item_key(self, item, spider): 51 | """Returns redis key based on given spider. 52 | 53 | Override this function to use a different key depending on the item 54 | and/or spider. 55 | 56 | """ 57 | return self.key % {'spider': spider.name} 58 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/square/red.css: -------------------------------------------------------------------------------- 1 | /* iCheck plugin Square skin, red 2 | ----------------------------------- */ 3 | .icheckbox_square-red, 4 | .iradio_square-red { 5 | display: inline-block; 6 | *display: inline; 7 | vertical-align: middle; 8 | margin: 0; 9 | padding: 0; 10 | width: 22px; 11 | height: 22px; 12 | background: url(red.png) no-repeat; 13 | border: none; 14 | cursor: pointer; 15 | } 16 | 17 | .icheckbox_square-red { 18 | background-position: 0 0; 19 | } 20 | .icheckbox_square-red.hover { 21 | background-position: -24px 0; 22 | } 23 | .icheckbox_square-red.checked { 24 | background-position: -48px 0; 25 | } 26 | .icheckbox_square-red.disabled { 27 | background-position: -72px 0; 28 | cursor: default; 29 | } 30 | .icheckbox_square-red.checked.disabled { 31 | background-position: -96px 0; 32 | } 33 | 34 | .iradio_square-red { 35 | background-position: -120px 0; 36 | } 37 | .iradio_square-red.hover { 38 | background-position: -144px 0; 39 | } 40 | .iradio_square-red.checked { 41 | background-position: -168px 0; 42 | } 43 | .iradio_square-red.disabled { 44 | background-position: -192px 0; 45 | cursor: default; 46 | } 47 | .iradio_square-red.checked.disabled { 48 | background-position: -216px 0; 49 | } 50 | 51 | /* Retina support */ 52 | @media only screen and (-webkit-min-device-pixel-ratio: 1.5), 53 | only screen and (-moz-min-device-pixel-ratio: 1.5), 54 | only screen and (-o-min-device-pixel-ratio: 3/2), 55 | only screen and (min-device-pixel-ratio: 1.5) { 56 | .icheckbox_square-red, 57 | .iradio_square-red { 58 | background-image: url(red@2x.png); 59 | -webkit-background-size: 240px 24px; 60 | background-size: 240px 24px; 61 | } 62 | } -------------------------------------------------------------------------------- /mi/mi/tools/gen_spiderFile_in_whiteList.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import os 3 | def arr2str(arr): 4 | return ', '.join(map(lambda x: "'" + x + "'", arr)) 5 | 6 | #新闻类爬虫模板 7 | spider_template = \ 8 | """# -*- coding: utf-8 -*- 9 | from scrapy_redis.spiders import RedisCrawlSpider 10 | from mi.items import ArticleItem 11 | from scrapy.spiders import Rule 12 | from scrapy.linkextractors import LinkExtractor 13 | class Spider_%s(RedisCrawlSpider): 14 | name = '%s' 15 | redis_key = '%s:start_urls' 16 | allowed_domains = [%s] 17 | rules = [ 18 | Rule(LinkExtractor(allow=(%s),deny=()),callback='processArticle',follow=True) 19 | ] 20 | 21 | def processArticle(self,response): 22 | try: 23 | item = ArticleItem() 24 | title = response.xpath('''%s''').extract()[0] 25 | content = ''.join(response.xpath('''%s''').extract()) 26 | item['articleTitle'] = title 27 | item['articleUrl'] = ''.join(response.url) 28 | item['articleContent'] = content 29 | yield item 30 | except: 31 | self.logger.info('item in article failed') 32 | """ 33 | # 传入字典格式的字符串 34 | def generate_spider(spidername, jsonfile): 35 | try: 36 | dic = eval(jsonfile) 37 | arr = ( 38 | spidername.split('.')[0], 39 | spidername, 40 | spidername, 41 | arr2str(dic['allowed_domains']), 42 | arr2str(dic['rule_allow']), 43 | dic['xpath_title'][0], 44 | dic['xpath_content'][0]) 45 | ok = spider_template % arr 46 | filename = os.getcwd() + '/mi/spiders_of_news_in_whiteList/spider_'+ spidername.replace('.', '') + '.py' 47 | with open(filename, 'w') as f: 48 | f.write(ok) 49 | print 'success' 50 | except: 51 | print 'fall' -------------------------------------------------------------------------------- /mi/mi/scrapy_redis/dupefilter.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from scrapy.dupefilters import BaseDupeFilter 4 | from scrapy.utils.request import request_fingerprint 5 | from BloomfilterOnRedis import BloomFilter 6 | 7 | from . import connection 8 | 9 | 10 | class RFPDupeFilter(BaseDupeFilter): 11 | """Redis-based request duplication filter""" 12 | 13 | def __init__(self, server, key): 14 | """Initialize duplication filter 15 | 16 | Parameters 17 | ---------- 18 | server : Redis instance 19 | key : str 20 | Where to store fingerprints 21 | """ 22 | self.server = server 23 | self.key = key 24 | self.bf = BloomFilter(server, key, blockNum=1) # you can increase blockNum if your are filtering too many urls 25 | 26 | @classmethod 27 | def from_settings(cls, settings): 28 | server = connection.from_settings_filter(settings) 29 | # create one-time key. needed to support to use this 30 | # class as standalone dupefilter with scrapy's default scheduler 31 | # if scrapy passes spider on open() method this wouldn't be needed 32 | key = "dupefilter:%s" % int(time.time()) 33 | return cls(server, key) 34 | 35 | @classmethod 36 | def from_crawler(cls, crawler): 37 | return cls.from_settings(crawler.settings) 38 | 39 | def request_seen(self, request): 40 | fp = request_fingerprint(request) 41 | if self.bf.isContains(fp): 42 | return True 43 | else: 44 | self.bf.insert(fp) 45 | return False 46 | # added = self.server.sadd(self.key, fp) 47 | # return not added 48 | 49 | def close(self, reason): 50 | """Delete data on close. Called by scrapy's scheduler""" 51 | self.clear() 52 | 53 | def clear(self): 54 | """Clears fingerprints data""" 55 | self.server.delete(self.key) 56 | -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/minimal/red.css: -------------------------------------------------------------------------------- 1 | /* iCheck plugin Minimal skin, red 2 | ----------------------------------- */ 3 | .icheckbox_minimal-red, 4 | .iradio_minimal-red { 5 | display: inline-block; 6 | *display: inline; 7 | vertical-align: middle; 8 | margin: 0; 9 | padding: 0; 10 | width: 18px; 11 | height: 18px; 12 | background: url(red.png) no-repeat; 13 | border: none; 14 | cursor: pointer; 15 | } 16 | 17 | .icheckbox_minimal-red { 18 | background-position: 0 0; 19 | } 20 | .icheckbox_minimal-red.hover { 21 | background-position: -20px 0; 22 | } 23 | .icheckbox_minimal-red.checked { 24 | background-position: -40px 0; 25 | } 26 | .icheckbox_minimal-red.disabled { 27 | background-position: -60px 0; 28 | cursor: default; 29 | } 30 | .icheckbox_minimal-red.checked.disabled { 31 | background-position: -80px 0; 32 | } 33 | 34 | .iradio_minimal-red { 35 | background-position: -100px 0; 36 | } 37 | .iradio_minimal-red.hover { 38 | background-position: -120px 0; 39 | } 40 | .iradio_minimal-red.checked { 41 | background-position: -140px 0; 42 | } 43 | .iradio_minimal-red.disabled { 44 | background-position: -160px 0; 45 | cursor: default; 46 | } 47 | .iradio_minimal-red.checked.disabled { 48 | background-position: -180px 0; 49 | } 50 | 51 | /* Retina support */ 52 | @media only screen and (-webkit-min-device-pixel-ratio: 1.5), 53 | only screen and (-moz-min-device-pixel-ratio: 1.5), 54 | only screen and (-o-min-device-pixel-ratio: 1.5), 55 | only screen and (min-device-pixel-ratio: 1.5) { 56 | .icheckbox_minimal-red, 57 | .iradio_minimal-red { 58 | background-image: url(red@2x.png); 59 | -webkit-background-size: 200px 20px; 60 | background-size: 200px 20px; 61 | } 62 | } -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/square/aero.css: -------------------------------------------------------------------------------- 1 | /* iCheck plugin Square skin, aero 2 | ----------------------------------- */ 3 | .icheckbox_square-aero, 4 | .iradio_square-aero { 5 | display: inline-block; 6 | *display: inline; 7 | vertical-align: middle; 8 | margin: 0; 9 | padding: 0; 10 | width: 22px; 11 | height: 22px; 12 | background: url(aero.png) no-repeat; 13 | border: none; 14 | cursor: pointer; 15 | } 16 | 17 | .icheckbox_square-aero { 18 | background-position: 0 0; 19 | } 20 | .icheckbox_square-aero.hover { 21 | background-position: -24px 0; 22 | } 23 | .icheckbox_square-aero.checked { 24 | background-position: -48px 0; 25 | } 26 | .icheckbox_square-aero.disabled { 27 | background-position: -72px 0; 28 | cursor: default; 29 | } 30 | .icheckbox_square-aero.checked.disabled { 31 | background-position: -96px 0; 32 | } 33 | 34 | .iradio_square-aero { 35 | background-position: -120px 0; 36 | } 37 | .iradio_square-aero.hover { 38 | background-position: -144px 0; 39 | } 40 | .iradio_square-aero.checked { 41 | background-position: -168px 0; 42 | } 43 | .iradio_square-aero.disabled { 44 | background-position: -192px 0; 45 | cursor: default; 46 | } 47 | .iradio_square-aero.checked.disabled { 48 | background-position: -216px 0; 49 | } 50 | 51 | /* Retina support */ 52 | @media only screen and (-webkit-min-device-pixel-ratio: 1.5), 53 | only screen and (-moz-min-device-pixel-ratio: 1.5), 54 | only screen and (-o-min-device-pixel-ratio: 3/2), 55 | only screen and (min-device-pixel-ratio: 1.5) { 56 | .icheckbox_square-aero, 57 | .iradio_square-aero { 58 | background-image: url(aero@2x.png); 59 | -webkit-background-size: 240px 24px; 60 | background-size: 240px 24px; 61 | } 62 | } -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/square/blue.css: -------------------------------------------------------------------------------- 1 | /* iCheck plugin Square skin, blue 2 | ----------------------------------- */ 3 | .icheckbox_square-blue, 4 | .iradio_square-blue { 5 | display: inline-block; 6 | *display: inline; 7 | vertical-align: middle; 8 | margin: 0; 9 | padding: 0; 10 | width: 22px; 11 | height: 22px; 12 | background: url(blue.png) no-repeat; 13 | border: none; 14 | cursor: pointer; 15 | } 16 | 17 | .icheckbox_square-blue { 18 | background-position: 0 0; 19 | } 20 | .icheckbox_square-blue.hover { 21 | background-position: -24px 0; 22 | } 23 | .icheckbox_square-blue.checked { 24 | background-position: -48px 0; 25 | } 26 | .icheckbox_square-blue.disabled { 27 | background-position: -72px 0; 28 | cursor: default; 29 | } 30 | .icheckbox_square-blue.checked.disabled { 31 | background-position: -96px 0; 32 | } 33 | 34 | .iradio_square-blue { 35 | background-position: -120px 0; 36 | } 37 | .iradio_square-blue.hover { 38 | background-position: -144px 0; 39 | } 40 | .iradio_square-blue.checked { 41 | background-position: -168px 0; 42 | } 43 | .iradio_square-blue.disabled { 44 | background-position: -192px 0; 45 | cursor: default; 46 | } 47 | .iradio_square-blue.checked.disabled { 48 | background-position: -216px 0; 49 | } 50 | 51 | /* Retina support */ 52 | @media only screen and (-webkit-min-device-pixel-ratio: 1.5), 53 | only screen and (-moz-min-device-pixel-ratio: 1.5), 54 | only screen and (-o-min-device-pixel-ratio: 3/2), 55 | only screen and (min-device-pixel-ratio: 1.5) { 56 | .icheckbox_square-blue, 57 | .iradio_square-blue { 58 | background-image: url(blue@2x.png); 59 | -webkit-background-size: 240px 24px; 60 | background-size: 240px 24px; 61 | } 62 | } -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/square/grey.css: -------------------------------------------------------------------------------- 1 | /* iCheck plugin Square skin, grey 2 | ----------------------------------- */ 3 | .icheckbox_square-grey, 4 | .iradio_square-grey { 5 | display: inline-block; 6 | *display: inline; 7 | vertical-align: middle; 8 | margin: 0; 9 | padding: 0; 10 | width: 22px; 11 | height: 22px; 12 | background: url(grey.png) no-repeat; 13 | border: none; 14 | cursor: pointer; 15 | } 16 | 17 | .icheckbox_square-grey { 18 | background-position: 0 0; 19 | } 20 | .icheckbox_square-grey.hover { 21 | background-position: -24px 0; 22 | } 23 | .icheckbox_square-grey.checked { 24 | background-position: -48px 0; 25 | } 26 | .icheckbox_square-grey.disabled { 27 | background-position: -72px 0; 28 | cursor: default; 29 | } 30 | .icheckbox_square-grey.checked.disabled { 31 | background-position: -96px 0; 32 | } 33 | 34 | .iradio_square-grey { 35 | background-position: -120px 0; 36 | } 37 | .iradio_square-grey.hover { 38 | background-position: -144px 0; 39 | } 40 | .iradio_square-grey.checked { 41 | background-position: -168px 0; 42 | } 43 | .iradio_square-grey.disabled { 44 | background-position: -192px 0; 45 | cursor: default; 46 | } 47 | .iradio_square-grey.checked.disabled { 48 | background-position: -216px 0; 49 | } 50 | 51 | /* Retina support */ 52 | @media only screen and (-webkit-min-device-pixel-ratio: 1.5), 53 | only screen and (-moz-min-device-pixel-ratio: 1.5), 54 | only screen and (-o-min-device-pixel-ratio: 3/2), 55 | only screen and (min-device-pixel-ratio: 1.5) { 56 | .icheckbox_square-grey, 57 | .iradio_square-grey { 58 | background-image: url(grey@2x.png); 59 | -webkit-background-size: 240px 24px; 60 | background-size: 240px 24px; 61 | } 62 | } -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/square/pink.css: -------------------------------------------------------------------------------- 1 | /* iCheck plugin Square skin, pink 2 | ----------------------------------- */ 3 | .icheckbox_square-pink, 4 | .iradio_square-pink { 5 | display: inline-block; 6 | *display: inline; 7 | vertical-align: middle; 8 | margin: 0; 9 | padding: 0; 10 | width: 22px; 11 | height: 22px; 12 | background: url(pink.png) no-repeat; 13 | border: none; 14 | cursor: pointer; 15 | } 16 | 17 | .icheckbox_square-pink { 18 | background-position: 0 0; 19 | } 20 | .icheckbox_square-pink.hover { 21 | background-position: -24px 0; 22 | } 23 | .icheckbox_square-pink.checked { 24 | background-position: -48px 0; 25 | } 26 | .icheckbox_square-pink.disabled { 27 | background-position: -72px 0; 28 | cursor: default; 29 | } 30 | .icheckbox_square-pink.checked.disabled { 31 | background-position: -96px 0; 32 | } 33 | 34 | .iradio_square-pink { 35 | background-position: -120px 0; 36 | } 37 | .iradio_square-pink.hover { 38 | background-position: -144px 0; 39 | } 40 | .iradio_square-pink.checked { 41 | background-position: -168px 0; 42 | } 43 | .iradio_square-pink.disabled { 44 | background-position: -192px 0; 45 | cursor: default; 46 | } 47 | .iradio_square-pink.checked.disabled { 48 | background-position: -216px 0; 49 | } 50 | 51 | /* Retina support */ 52 | @media only screen and (-webkit-min-device-pixel-ratio: 1.5), 53 | only screen and (-moz-min-device-pixel-ratio: 1.5), 54 | only screen and (-o-min-device-pixel-ratio: 3/2), 55 | only screen and (min-device-pixel-ratio: 1.5) { 56 | .icheckbox_square-pink, 57 | .iradio_square-pink { 58 | background-image: url(pink@2x.png); 59 | -webkit-background-size: 240px 24px; 60 | background-size: 240px 24px; 61 | } 62 | } -------------------------------------------------------------------------------- /mi_manager/monitor/settings.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # mi_manager 3 | APP_HOST = '127.0.0.1' 4 | APP_PORT = 5020 5 | # 核心redis服务器 6 | REDIS_HOST = '47.98.124.149' 7 | REDIS_PORT = 6379 8 | 9 | # 监控器相关参数 10 | TIMEINTERVAL = 2000#监控台刷新时间间隔，单位毫秒 11 | POINTINTERVAL = 10#图上各点之间间隔，越小则表示点越密集 12 | POINTLENGTH = 2000#图上点的数量，越大则表示图上时间跨度越长 13 | # 存储爬虫运行数据的四个队列 14 | STATS_KEYS = ['downloader/request_count', 'downloader/response_count', 'downloader/response_status_count/200', 'item_scraped_count'] 15 | 16 | ############################################################################################## 17 | # 用于存储调度队列 ———— 的redis数据据库编号（0～15） 18 | FILTER_DB = 0 19 | 20 | # 用于存储记号变量 ———— 的redis数据据库编号（0～15） 21 | SYMBOL_DB = 1 22 | 23 | # 用于存储task(带处理, 已处理) ———— 的redis数据据库编号（0～15） 24 | TASK_DB = 2 25 | 26 | # 用于进行task调度的有序集合 ———— 的redis数据据库编号（0～15） 27 | DISPATCH_DB = 3 28 | 29 | # 用于存储主任务 ———— 的redis数据据库编号（0～15） 30 | MISSION_DB = 4 31 | 32 | # 用于存储子任务 ———— 的redis数据据库编号（0～15） 33 | SUBMISSION_DB = 5 34 | 35 | # 用于存储爬虫配置信息 ———— 的redis数据据库编号（0～15） 36 | SETTINGS_DB = 6 37 | 38 | # 用于存储資源(REDIS服务器)的信息 ———— 的redis数据据库编号（0～15） 39 | RESOURCES_REDIS_DB = 7 40 | 41 | # 用于存储資源(MYSQL服务器)的信息 ———— 的redis数据据库编号（0～15） 42 | RESOURCES_MYSQL_DB = 8 43 | 44 | # 用于存储資源(MONGO服务器)的信息 ———— 的redis数据据库编号（0～15） 45 | RESOURCES_MONGO_DB = 9 46 | 47 | # 用于存储代理ip ———— 的redis数据据库编号（0～15） 48 | PROXY_DB = 10 49 | 50 | # 用于暂存爬虫运行时数据 ———— 的redis数据据库编号（0～15） 51 | RUNNINGDATA_DB = 11 52 | 53 | # 用于存储Cookie数据 ———— 的redis数据据库编号（0～15） 54 | COOKIES_DB = 12 55 | 56 | # 用于存储新闻类爬虫配置参数 ———— 的redis数据据库编号（0～15） 57 | SPIDERS_DB = 13 58 | 59 | # 用于分类爬虫(共三个类别： Whitelist, Fuzzy, Ecommerce) ———— 的redis数据据库编号（0～15） 60 | CLASSIFIER_DB = 14 61 | 62 | # 用于存储Monitor数据 ———— 的redis数据据库编号（0～15） 63 | MONITOR_DB = 15 64 | ############################################################################################## 65 | 66 | # 临时路径,用于解决开发环境和生产环境中路径当前路径不一直的问题（开发环境中设为空） 67 | TEMP_PATH = '/monitor' -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/minimal/aero.css: -------------------------------------------------------------------------------- 1 | /* iCheck plugin Minimal skin, aero 2 | ----------------------------------- */ 3 | .icheckbox_minimal-aero, 4 | .iradio_minimal-aero { 5 | display: inline-block; 6 | *display: inline; 7 | vertical-align: middle; 8 | margin: 0; 9 | padding: 0; 10 | width: 18px; 11 | height: 18px; 12 | background: url(aero.png) no-repeat; 13 | border: none; 14 | cursor: pointer; 15 | } 16 | 17 | .icheckbox_minimal-aero { 18 | background-position: 0 0; 19 | } 20 | .icheckbox_minimal-aero.hover { 21 | background-position: -20px 0; 22 | } 23 | .icheckbox_minimal-aero.checked { 24 | background-position: -40px 0; 25 | } 26 | .icheckbox_minimal-aero.disabled { 27 | background-position: -60px 0; 28 | cursor: default; 29 | } 30 | .icheckbox_minimal-aero.checked.disabled { 31 | background-position: -80px 0; 32 | } 33 | 34 | .iradio_minimal-aero { 35 | background-position: -100px 0; 36 | } 37 | .iradio_minimal-aero.hover { 38 | background-position: -120px 0; 39 | } 40 | .iradio_minimal-aero.checked { 41 | background-position: -140px 0; 42 | } 43 | .iradio_minimal-aero.disabled { 44 | background-position: -160px 0; 45 | cursor: default; 46 | } 47 | .iradio_minimal-aero.checked.disabled { 48 | background-position: -180px 0; 49 | } 50 | 51 | /* Retina support */ 52 | @media only screen and (-webkit-min-device-pixel-ratio: 1.5), 53 | only screen and (-moz-min-device-pixel-ratio: 1.5), 54 | only screen and (-o-min-device-pixel-ratio: 3/2), 55 | only screen and (min-device-pixel-ratio: 1.5) { 56 | .icheckbox_minimal-aero, 57 | .iradio_minimal-aero { 58 | background-image: url(aero@2x.png); 59 | -webkit-background-size: 200px 20px; 60 | background-size: 200px 20px; 61 | } 62 | } -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/minimal/blue.css: -------------------------------------------------------------------------------- 1 | /* iCheck plugin Minimal skin, blue 2 | ----------------------------------- */ 3 | .icheckbox_minimal-blue, 4 | .iradio_minimal-blue { 5 | display: inline-block; 6 | *display: inline; 7 | vertical-align: middle; 8 | margin: 0; 9 | padding: 0; 10 | width: 18px; 11 | height: 18px; 12 | background: url(blue.png) no-repeat; 13 | border: none; 14 | cursor: pointer; 15 | } 16 | 17 | .icheckbox_minimal-blue { 18 | background-position: 0 0; 19 | } 20 | .icheckbox_minimal-blue.hover { 21 | background-position: -20px 0; 22 | } 23 | .icheckbox_minimal-blue.checked { 24 | background-position: -40px 0; 25 | } 26 | .icheckbox_minimal-blue.disabled { 27 | background-position: -60px 0; 28 | cursor: default; 29 | } 30 | .icheckbox_minimal-blue.checked.disabled { 31 | background-position: -80px 0; 32 | } 33 | 34 | .iradio_minimal-blue { 35 | background-position: -100px 0; 36 | } 37 | .iradio_minimal-blue.hover { 38 | background-position: -120px 0; 39 | } 40 | .iradio_minimal-blue.checked { 41 | background-position: -140px 0; 42 | } 43 | .iradio_minimal-blue.disabled { 44 | background-position: -160px 0; 45 | cursor: default; 46 | } 47 | .iradio_minimal-blue.checked.disabled { 48 | background-position: -180px 0; 49 | } 50 | 51 | /* Retina support */ 52 | @media only screen and (-webkit-min-device-pixel-ratio: 1.5), 53 | only screen and (-moz-min-device-pixel-ratio: 1.5), 54 | only screen and (-o-min-device-pixel-ratio: 3/2), 55 | only screen and (min-device-pixel-ratio: 1.5) { 56 | .icheckbox_minimal-blue, 57 | .iradio_minimal-blue { 58 | background-image: url(blue@2x.png); 59 | -webkit-background-size: 200px 20px; 60 | background-size: 200px 20px; 61 | } 62 | } -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/minimal/grey.css: -------------------------------------------------------------------------------- 1 | /* iCheck plugin Minimal skin, grey 2 | ----------------------------------- */ 3 | .icheckbox_minimal-grey, 4 | .iradio_minimal-grey { 5 | display: inline-block; 6 | *display: inline; 7 | vertical-align: middle; 8 | margin: 0; 9 | padding: 0; 10 | width: 18px; 11 | height: 18px; 12 | background: url(grey.png) no-repeat; 13 | border: none; 14 | cursor: pointer; 15 | } 16 | 17 | .icheckbox_minimal-grey { 18 | background-position: 0 0; 19 | } 20 | .icheckbox_minimal-grey.hover { 21 | background-position: -20px 0; 22 | } 23 | .icheckbox_minimal-grey.checked { 24 | background-position: -40px 0; 25 | } 26 | .icheckbox_minimal-grey.disabled { 27 | background-position: -60px 0; 28 | cursor: default; 29 | } 30 | .icheckbox_minimal-grey.checked.disabled { 31 | background-position: -80px 0; 32 | } 33 | 34 | .iradio_minimal-grey { 35 | background-position: -100px 0; 36 | } 37 | .iradio_minimal-grey.hover { 38 | background-position: -120px 0; 39 | } 40 | .iradio_minimal-grey.checked { 41 | background-position: -140px 0; 42 | } 43 | .iradio_minimal-grey.disabled { 44 | background-position: -160px 0; 45 | cursor: default; 46 | } 47 | .iradio_minimal-grey.checked.disabled { 48 | background-position: -180px 0; 49 | } 50 | 51 | /* Retina support */ 52 | @media only screen and (-webkit-min-device-pixel-ratio: 1.5), 53 | only screen and (-moz-min-device-pixel-ratio: 1.5), 54 | only screen and (-o-min-device-pixel-ratio: 1.5), 55 | only screen and (min-device-pixel-ratio: 1.5) { 56 | .icheckbox_minimal-grey, 57 | .iradio_minimal-grey { 58 | background-image: url(grey@2x.png); 59 | -webkit-background-size: 200px 20px; 60 | background-size: 200px 20px; 61 | } 62 | } -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/minimal/pink.css: -------------------------------------------------------------------------------- 1 | /* iCheck plugin Minimal skin, pink 2 | ----------------------------------- */ 3 | .icheckbox_minimal-pink, 4 | .iradio_minimal-pink { 5 | display: inline-block; 6 | *display: inline; 7 | vertical-align: middle; 8 | margin: 0; 9 | padding: 0; 10 | width: 18px; 11 | height: 18px; 12 | background: url(pink.png) no-repeat; 13 | border: none; 14 | cursor: pointer; 15 | } 16 | 17 | .icheckbox_minimal-pink { 18 | background-position: 0 0; 19 | } 20 | .icheckbox_minimal-pink.hover { 21 | background-position: -20px 0; 22 | } 23 | .icheckbox_minimal-pink.checked { 24 | background-position: -40px 0; 25 | } 26 | .icheckbox_minimal-pink.disabled { 27 | background-position: -60px 0; 28 | cursor: default; 29 | } 30 | .icheckbox_minimal-pink.checked.disabled { 31 | background-position: -80px 0; 32 | } 33 | 34 | .iradio_minimal-pink { 35 | background-position: -100px 0; 36 | } 37 | .iradio_minimal-pink.hover { 38 | background-position: -120px 0; 39 | } 40 | .iradio_minimal-pink.checked { 41 | background-position: -140px 0; 42 | } 43 | .iradio_minimal-pink.disabled { 44 | background-position: -160px 0; 45 | cursor: default; 46 | } 47 | .iradio_minimal-pink.checked.disabled { 48 | background-position: -180px 0; 49 | } 50 | 51 | /* Retina support */ 52 | @media only screen and (-webkit-min-device-pixel-ratio: 1.5), 53 | only screen and (-moz-min-device-pixel-ratio: 1.5), 54 | only screen and (-o-min-device-pixel-ratio: 1.5), 55 | only screen and (min-device-pixel-ratio: 1.5) { 56 | .icheckbox_minimal-pink, 57 | .iradio_minimal-pink { 58 | background-image: url(pink@2x.png); 59 | -webkit-background-size: 200px 20px; 60 | background-size: 200px 20px; 61 | } 62 | } -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/square/green.css: -------------------------------------------------------------------------------- 1 | /* iCheck plugin Square skin, green 2 | ----------------------------------- */ 3 | .icheckbox_square-green, 4 | .iradio_square-green { 5 | display: inline-block; 6 | *display: inline; 7 | vertical-align: middle; 8 | margin: 0; 9 | padding: 0; 10 | width: 22px; 11 | height: 22px; 12 | background: url(green.png) no-repeat; 13 | border: none; 14 | cursor: pointer; 15 | } 16 | 17 | .icheckbox_square-green { 18 | background-position: 0 0; 19 | } 20 | .icheckbox_square-green.hover { 21 | background-position: -24px 0; 22 | } 23 | .icheckbox_square-green.checked { 24 | background-position: -48px 0; 25 | } 26 | .icheckbox_square-green.disabled { 27 | background-position: -72px 0; 28 | cursor: default; 29 | } 30 | .icheckbox_square-green.checked.disabled { 31 | background-position: -96px 0; 32 | } 33 | 34 | .iradio_square-green { 35 | background-position: -120px 0; 36 | } 37 | .iradio_square-green.hover { 38 | background-position: -144px 0; 39 | } 40 | .iradio_square-green.checked { 41 | background-position: -168px 0; 42 | } 43 | .iradio_square-green.disabled { 44 | background-position: -192px 0; 45 | cursor: default; 46 | } 47 | .iradio_square-green.checked.disabled { 48 | background-position: -216px 0; 49 | } 50 | 51 | /* Retina support */ 52 | @media only screen and (-webkit-min-device-pixel-ratio: 1.5), 53 | only screen and (-moz-min-device-pixel-ratio: 1.5), 54 | only screen and (-o-min-device-pixel-ratio: 3/2), 55 | only screen and (min-device-pixel-ratio: 1.5) { 56 | .icheckbox_square-green, 57 | .iradio_square-green { 58 | background-image: url(green@2x.png); 59 | -webkit-background-size: 240px 24px; 60 | background-size: 240px 24px; 61 | } 62 | } -------------------------------------------------------------------------------- /mi_manager/settings.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # marathon的地址 (请确保zookeeper+mesos+marathon框架正常运行) 3 | MESOS_URL = 'http://127.0.0.1:5050' 4 | MARATHON_URL = 'http://127.0.0.1:18082' 5 | 6 | # 核心redis数据库 7 | # 默认地址 8 | CORE_REDIS_HOST = '127.0.0.1' 9 | # 默认端口 10 | CORE_REDIS_PORT = 6379 11 | 12 | # 可用内存 (单位G, 建议1G以上) 13 | CORE_REDIS_MAX_MEMORY = 2 14 | 15 | 16 | # mi_manager web前端的地址及端口 17 | APP_HOST = '127.0.0.1' 18 | APP_PORT = 5020 19 | 20 | # 监控模块的相对路径 21 | MONITOR_TEMP_PATH = '/monitor' 22 | 23 | # 监控模块的相对路径 24 | DAEMON_TEMP_PATH = '/daemon' 25 | 26 | # 已知的可处理的电商名单 27 | 28 | ############################################################################################## 29 | # redis 数据库分布 30 | # 用于存储调度队列 ———— 的redis数据据库编号（0～15） 31 | FILTER_DB = 0 32 | 33 | # 用于存储记号变量 ———— 的redis数据据库编号（0～15） 34 | SYMBOL_DB = 1 35 | 36 | # 用于存储task(带处理, 已处理) ———— 的redis数据据库编号（0～15） 37 | TASK_DB = 2 38 | 39 | # 用于进行task调度的有序集合 ———— 的redis数据据库编号（0～15） 40 | DISPATCH_DB = 3 41 | 42 | # 用于存储主任务 ———— 的redis数据据库编号（0～15） 43 | MISSION_DB = 4 44 | 45 | # 用于存储子任务 ———— 的redis数据据库编号（0～15） 46 | SUBMISSION_DB = 5 47 | 48 | # 用于存储爬虫配置信息 ———— 的redis数据据库编号（0～15） 49 | SETTINGS_DB = 6 50 | 51 | # 用于存储資源(REDIS服务器)的信息 ———— 的redis数据据库编号（0～15） 52 | RESOURCES_REDIS_DB = 7 53 | 54 | # 用于存储資源(MYSQL服务器)的信息 ———— 的redis数据据库编号（0～15） 55 | RESOURCES_MYSQL_DB = 8 56 | 57 | # 用于存储資源(MONGO服务器)的信息 ———— 的redis数据据库编号（0～15） 58 | RESOURCES_MONGO_DB = 9 59 | 60 | # 用于存储代理ip ———— 的redis数据据库编号（0～15） 61 | PROXY_DB = 10 62 | 63 | # 用于暂存爬虫运行时数据 ———— 的redis数据据库编号（0～15） 64 | RUNNINGDATA_DB = 11 65 | 66 | # 用于存储Cookie数据 ———— 的redis数据据库编号（0～15） 67 | COOKIES_DB = 12 68 | 69 | # 用于存储新闻类爬虫配置参数 ———— 的redis数据据库编号（0～15） 70 | SPIDERS_DB = 13 71 | 72 | # 用于分类爬虫(共三个类别： Whitelist, Fuzzy, Ecommerce) ———— 的redis数据据库编号（0～15） 73 | CLASSIFIER_DB = 14 74 | 75 | # 用于存储Monitor数据 ———— 的redis数据据库编号（0～15） 76 | MONITOR_DB = 15 77 | ############################################################################################## -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/minimal/green.css: -------------------------------------------------------------------------------- 1 | /* iCheck plugin Minimal skin, green 2 | ----------------------------------- */ 3 | .icheckbox_minimal-green, 4 | .iradio_minimal-green { 5 | display: inline-block; 6 | *display: inline; 7 | vertical-align: middle; 8 | margin: 0; 9 | padding: 0; 10 | width: 18px; 11 | height: 18px; 12 | background: url(green.png) no-repeat; 13 | border: none; 14 | cursor: pointer; 15 | } 16 | 17 | .icheckbox_minimal-green { 18 | background-position: 0 0; 19 | } 20 | .icheckbox_minimal-green.hover { 21 | background-position: -20px 0; 22 | } 23 | .icheckbox_minimal-green.checked { 24 | background-position: -40px 0; 25 | } 26 | .icheckbox_minimal-green.disabled { 27 | background-position: -60px 0; 28 | cursor: default; 29 | } 30 | .icheckbox_minimal-green.checked.disabled { 31 | background-position: -80px 0; 32 | } 33 | 34 | .iradio_minimal-green { 35 | background-position: -100px 0; 36 | } 37 | .iradio_minimal-green.hover { 38 | background-position: -120px 0; 39 | } 40 | .iradio_minimal-green.checked { 41 | background-position: -140px 0; 42 | } 43 | .iradio_minimal-green.disabled { 44 | background-position: -160px 0; 45 | cursor: default; 46 | } 47 | .iradio_minimal-green.checked.disabled { 48 | background-position: -180px 0; 49 | } 50 | 51 | /* Retina support */ 52 | @media only screen and (-webkit-min-device-pixel-ratio: 1.5), 53 | only screen and (-moz-min-device-pixel-ratio: 1.5), 54 | only screen and (-o-min-device-pixel-ratio: 1.5), 55 | only screen and (min-device-pixel-ratio: 1.5) { 56 | .icheckbox_minimal-green, 57 | .iradio_minimal-green { 58 | background-image: url(green@2x.png); 59 | -webkit-background-size: 200px 20px; 60 | background-size: 200px 20px; 61 | } 62 | } -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/square/orange.css: -------------------------------------------------------------------------------- 1 | /* iCheck plugin Square skin, orange 2 | ----------------------------------- */ 3 | .icheckbox_square-orange, 4 | .iradio_square-orange { 5 | display: inline-block; 6 | *display: inline; 7 | vertical-align: middle; 8 | margin: 0; 9 | padding: 0; 10 | width: 22px; 11 | height: 22px; 12 | background: url(orange.png) no-repeat; 13 | border: none; 14 | cursor: pointer; 15 | } 16 | 17 | .icheckbox_square-orange { 18 | background-position: 0 0; 19 | } 20 | .icheckbox_square-orange.hover { 21 | background-position: -24px 0; 22 | } 23 | .icheckbox_square-orange.checked { 24 | background-position: -48px 0; 25 | } 26 | .icheckbox_square-orange.disabled { 27 | background-position: -72px 0; 28 | cursor: default; 29 | } 30 | .icheckbox_square-orange.checked.disabled { 31 | background-position: -96px 0; 32 | } 33 | 34 | .iradio_square-orange { 35 | background-position: -120px 0; 36 | } 37 | .iradio_square-orange.hover { 38 | background-position: -144px 0; 39 | } 40 | .iradio_square-orange.checked { 41 | background-position: -168px 0; 42 | } 43 | .iradio_square-orange.disabled { 44 | background-position: -192px 0; 45 | cursor: default; 46 | } 47 | .iradio_square-orange.checked.disabled { 48 | background-position: -216px 0; 49 | } 50 | 51 | /* Retina support */ 52 | @media only screen and (-webkit-min-device-pixel-ratio: 1.5), 53 | only screen and (-moz-min-device-pixel-ratio: 1.5), 54 | only screen and (-o-min-device-pixel-ratio: 3/2), 55 | only screen and (min-device-pixel-ratio: 1.5) { 56 | .icheckbox_square-orange, 57 | .iradio_square-orange { 58 | background-image: url(orange@2x.png); 59 | -webkit-background-size: 240px 24px; 60 | background-size: 240px 24px; 61 | } 62 | } -------------------------------------------------------------------------------- /mi_manager/monitor/static/v2/plugins/iCheck/square/purple.css: -------------------------------------------------------------------------------- 1 | /* iCheck plugin Square skin, purple 2 | ----------------------------------- */ 3 | .icheckbox_square-purple, 4 | .iradio_square-purple { 5 | display: inline-block; 6 | *display: inline; 7 | vertical-align: middle; 8 | margin: 0; 9 | padding: 0; 10 | width: 22px; 11 | height: 22px; 12 | background: url(purple.png) no-repeat; 13 | border: none; 14 | cursor: pointer; 15 | } 16 | 17 | .icheckbox_square-purple { 18 | background-position: 0 0; 19 | } 20 | .icheckbox_square-purple.hover { 21 | background-position: -24px 0; 22 | } 23 | .icheckbox_square-purple.checked { 24 | background-position: -48px 0; 25 | } 26 | .icheckbox_square-purple.disabled { 27 | background-position: -72px 0; 28 | cursor: default; 29 | } 30 | .icheckbox_square-purple.checked.disabled { 31 | background-position: -96px 0; 32 | } 33 | 34 | .iradio_square-purple { 35 | background-position: -120px 0; 36 | } 37 | .iradio_square-purple.hover { 38 | background-position: -144px 0; 39 | } 40 | .iradio_square-purple.checked { 41 | background-position: -168px 0; 42 | } 43 | .iradio_square-purple.disabled { 44 | background-position: -192px 0; 45 | cursor: default; 46 | } 47 | .iradio_square-purple.checked.disabled { 48 | background-position: -216px 0; 49 | } 50 | 51 | /* Retina support */ 52 | @media only screen and (-webkit-min-device-pixel-ratio: 1.5), 53 | only screen and (-moz-min-device-pixel-ratio: 1.5), 54 | only screen and (-o-min-device-pixel-ratio: 3/2), 55 | only screen and (min-device-pixel-ratio: 1.5) { 56 | .icheckbox_square-purple, 57 | .iradio_square-purple { 58 | background-image: url(purple@2x.png); 59 | -webkit-background-size: 240px 24px; 60 | background-size: 240px 24px; 61 | } 62 | } --------------------------------------------------------------------------------