├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── core ├── common │ ├── com_interfaces │ │ └── com_interfaces.go │ ├── config │ │ └── config.go │ ├── etc_config │ │ └── etc_config.go │ ├── mlog │ │ ├── filelog.go │ │ ├── mlog.go │ │ └── strace.go │ ├── page │ │ └── page.go │ ├── page_items │ │ └── page_items.go │ ├── request │ │ └── request.go │ ├── resource_manage │ │ ├── resource_manage.go │ │ ├── resource_manage_chan.go │ │ └── resource_manage_test.go │ └── util │ │ └── util.go ├── downloader │ ├── downloader.go │ ├── downloader_http.go │ └── downloader_test.go ├── page_processer │ └── page_processer.go ├── pipeline │ ├── collect_pipeline_page_items.go │ ├── pipeline.go │ ├── pipeline_console.go │ └── pipeline_file.go ├── scheduler │ ├── scheduler.go │ ├── scheduler_queue.go │ ├── scheduler_simple.go │ └── scheduler_test.go └── spider │ └── spider.go ├── example ├── baidu_baike_page_processor │ └── main.go ├── github_repo_page_processor │ └── main.go ├── login_profile_page_processor │ └── main.go ├── redis_scheduler │ └── redis_scheduler_example.go ├── sina_stock_json_processor │ └── main.go ├── sohu_gossip_page_json_processor │ └── main.go └── weixin_sogou_cookie_processor │ ├── main.go │ └── weixin.sogou.com.json └── extension ├── downloader └── contribution_downloader_extension_here ├── other └── contribution_other_tool_here ├── pipeline └── contribution_pipeline_extension_here └── scheduler ├── contribution_scheduler_extension_here └── redis_scheduler.go /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/.gitignore -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/.travis.yml -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/README.md -------------------------------------------------------------------------------- /core/common/com_interfaces/com_interfaces.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/core/common/com_interfaces/com_interfaces.go -------------------------------------------------------------------------------- /core/common/config/config.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/core/common/config/config.go -------------------------------------------------------------------------------- /core/common/etc_config/etc_config.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/core/common/etc_config/etc_config.go -------------------------------------------------------------------------------- /core/common/mlog/filelog.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/core/common/mlog/filelog.go -------------------------------------------------------------------------------- /core/common/mlog/mlog.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/core/common/mlog/mlog.go -------------------------------------------------------------------------------- /core/common/mlog/strace.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/core/common/mlog/strace.go -------------------------------------------------------------------------------- /core/common/page/page.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/core/common/page/page.go -------------------------------------------------------------------------------- /core/common/page_items/page_items.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/core/common/page_items/page_items.go -------------------------------------------------------------------------------- /core/common/request/request.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/core/common/request/request.go -------------------------------------------------------------------------------- /core/common/resource_manage/resource_manage.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/core/common/resource_manage/resource_manage.go -------------------------------------------------------------------------------- /core/common/resource_manage/resource_manage_chan.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/core/common/resource_manage/resource_manage_chan.go -------------------------------------------------------------------------------- /core/common/resource_manage/resource_manage_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/core/common/resource_manage/resource_manage_test.go -------------------------------------------------------------------------------- /core/common/util/util.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/core/common/util/util.go -------------------------------------------------------------------------------- /core/downloader/downloader.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/core/downloader/downloader.go -------------------------------------------------------------------------------- /core/downloader/downloader_http.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/core/downloader/downloader_http.go -------------------------------------------------------------------------------- /core/downloader/downloader_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/core/downloader/downloader_test.go -------------------------------------------------------------------------------- /core/page_processer/page_processer.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/core/page_processer/page_processer.go -------------------------------------------------------------------------------- /core/pipeline/collect_pipeline_page_items.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/core/pipeline/collect_pipeline_page_items.go -------------------------------------------------------------------------------- /core/pipeline/pipeline.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/core/pipeline/pipeline.go -------------------------------------------------------------------------------- /core/pipeline/pipeline_console.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/core/pipeline/pipeline_console.go -------------------------------------------------------------------------------- /core/pipeline/pipeline_file.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/core/pipeline/pipeline_file.go -------------------------------------------------------------------------------- /core/scheduler/scheduler.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/core/scheduler/scheduler.go -------------------------------------------------------------------------------- /core/scheduler/scheduler_queue.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/core/scheduler/scheduler_queue.go -------------------------------------------------------------------------------- /core/scheduler/scheduler_simple.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/core/scheduler/scheduler_simple.go -------------------------------------------------------------------------------- /core/scheduler/scheduler_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/core/scheduler/scheduler_test.go -------------------------------------------------------------------------------- /core/spider/spider.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/core/spider/spider.go -------------------------------------------------------------------------------- /example/baidu_baike_page_processor/main.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/example/baidu_baike_page_processor/main.go -------------------------------------------------------------------------------- /example/github_repo_page_processor/main.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/example/github_repo_page_processor/main.go -------------------------------------------------------------------------------- /example/login_profile_page_processor/main.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/example/login_profile_page_processor/main.go -------------------------------------------------------------------------------- /example/redis_scheduler/redis_scheduler_example.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/example/redis_scheduler/redis_scheduler_example.go -------------------------------------------------------------------------------- /example/sina_stock_json_processor/main.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/example/sina_stock_json_processor/main.go -------------------------------------------------------------------------------- /example/sohu_gossip_page_json_processor/main.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/example/sohu_gossip_page_json_processor/main.go -------------------------------------------------------------------------------- /example/weixin_sogou_cookie_processor/main.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/example/weixin_sogou_cookie_processor/main.go -------------------------------------------------------------------------------- /example/weixin_sogou_cookie_processor/weixin.sogou.com.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/example/weixin_sogou_cookie_processor/weixin.sogou.com.json -------------------------------------------------------------------------------- /extension/downloader/contribution_downloader_extension_here: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /extension/other/contribution_other_tool_here: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /extension/pipeline/contribution_pipeline_extension_here: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /extension/scheduler/contribution_scheduler_extension_here: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /extension/scheduler/redis_scheduler.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu17889/go_spider/HEAD/extension/scheduler/redis_scheduler.go --------------------------------------------------------------------------------