├── .github └── workflows │ └── go.yml ├── .gitignore ├── Disclaimer.md ├── LICENSE ├── Makefile ├── README.md ├── README_zh-cn.md ├── cmd └── crawlergo │ ├── flag.go │ └── main.go ├── dockerfile ├── examples ├── host_binding.py ├── request_with_cookie.py ├── subprocess_call.py └── zombie_clean.py ├── get_chrome.sh ├── go.mod ├── go.sum ├── imgs ├── bypass.png ├── chrome_path.png ├── demo.gif └── skp.png └── pkg ├── config ├── config.go └── config_test.go ├── domain_collect.go ├── engine ├── after_dom_tasks.go ├── after_loaded_tasks.go ├── browser.go ├── collect_links.go ├── intercept_request.go ├── tab.go └── tab_test.go ├── filter ├── filter.go ├── simple_filter.go ├── smart_filter.go └── smart_filter_test.go ├── js └── javascript.go ├── logger └── logger.go ├── model ├── request.go ├── url.go └── url_test.go ├── path_expansion.go ├── task_main.go ├── taskconfig.go ├── taskconfig_test.go └── tools ├── common.go ├── random.go └── requests ├── requests.go ├── response.go └── utils.go /.github/workflows/go.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/.github/workflows/go.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/.gitignore -------------------------------------------------------------------------------- /Disclaimer.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/Disclaimer.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/LICENSE -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/Makefile -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/README.md -------------------------------------------------------------------------------- /README_zh-cn.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/README_zh-cn.md -------------------------------------------------------------------------------- /cmd/crawlergo/flag.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/cmd/crawlergo/flag.go -------------------------------------------------------------------------------- /cmd/crawlergo/main.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/cmd/crawlergo/main.go -------------------------------------------------------------------------------- /dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/dockerfile -------------------------------------------------------------------------------- /examples/host_binding.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/examples/host_binding.py -------------------------------------------------------------------------------- /examples/request_with_cookie.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/examples/request_with_cookie.py -------------------------------------------------------------------------------- /examples/subprocess_call.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/examples/subprocess_call.py -------------------------------------------------------------------------------- /examples/zombie_clean.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/examples/zombie_clean.py -------------------------------------------------------------------------------- /get_chrome.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/get_chrome.sh -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/go.mod -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/go.sum -------------------------------------------------------------------------------- /imgs/bypass.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/imgs/bypass.png -------------------------------------------------------------------------------- /imgs/chrome_path.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/imgs/chrome_path.png -------------------------------------------------------------------------------- /imgs/demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/imgs/demo.gif -------------------------------------------------------------------------------- /imgs/skp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/imgs/skp.png -------------------------------------------------------------------------------- /pkg/config/config.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/pkg/config/config.go -------------------------------------------------------------------------------- /pkg/config/config_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/pkg/config/config_test.go -------------------------------------------------------------------------------- /pkg/domain_collect.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/pkg/domain_collect.go -------------------------------------------------------------------------------- /pkg/engine/after_dom_tasks.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/pkg/engine/after_dom_tasks.go -------------------------------------------------------------------------------- /pkg/engine/after_loaded_tasks.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/pkg/engine/after_loaded_tasks.go -------------------------------------------------------------------------------- /pkg/engine/browser.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/pkg/engine/browser.go -------------------------------------------------------------------------------- /pkg/engine/collect_links.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/pkg/engine/collect_links.go -------------------------------------------------------------------------------- /pkg/engine/intercept_request.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/pkg/engine/intercept_request.go -------------------------------------------------------------------------------- /pkg/engine/tab.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/pkg/engine/tab.go -------------------------------------------------------------------------------- /pkg/engine/tab_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/pkg/engine/tab_test.go -------------------------------------------------------------------------------- /pkg/filter/filter.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/pkg/filter/filter.go -------------------------------------------------------------------------------- /pkg/filter/simple_filter.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/pkg/filter/simple_filter.go -------------------------------------------------------------------------------- /pkg/filter/smart_filter.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/pkg/filter/smart_filter.go -------------------------------------------------------------------------------- /pkg/filter/smart_filter_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/pkg/filter/smart_filter_test.go -------------------------------------------------------------------------------- /pkg/js/javascript.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/pkg/js/javascript.go -------------------------------------------------------------------------------- /pkg/logger/logger.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/pkg/logger/logger.go -------------------------------------------------------------------------------- /pkg/model/request.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/pkg/model/request.go -------------------------------------------------------------------------------- /pkg/model/url.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/pkg/model/url.go -------------------------------------------------------------------------------- /pkg/model/url_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/pkg/model/url_test.go -------------------------------------------------------------------------------- /pkg/path_expansion.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/pkg/path_expansion.go -------------------------------------------------------------------------------- /pkg/task_main.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/pkg/task_main.go -------------------------------------------------------------------------------- /pkg/taskconfig.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/pkg/taskconfig.go -------------------------------------------------------------------------------- /pkg/taskconfig_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/pkg/taskconfig_test.go -------------------------------------------------------------------------------- /pkg/tools/common.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/pkg/tools/common.go -------------------------------------------------------------------------------- /pkg/tools/random.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/pkg/tools/random.go -------------------------------------------------------------------------------- /pkg/tools/requests/requests.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/pkg/tools/requests/requests.go -------------------------------------------------------------------------------- /pkg/tools/requests/response.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/pkg/tools/requests/response.go -------------------------------------------------------------------------------- /pkg/tools/requests/utils.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qianlitp/crawlergo/HEAD/pkg/tools/requests/utils.go --------------------------------------------------------------------------------