├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── circle.yml ├── crawler.go ├── crawler_options.go ├── docs-templates └── nsq │ ├── consumer │ └── README.md │ └── crawl-schedule │ └── README.md ├── examples ├── imdb │ ├── main.go │ └── spider │ │ └── spider.go └── nsq-skeleton │ ├── .gitignore │ ├── Dockerfile │ ├── Makefile │ ├── circle.yml │ └── main.go ├── forms └── forms.go ├── html_utils.go ├── nsq ├── README.md ├── consumer │ ├── README.md │ ├── app.go │ └── app_options.go ├── crawl-schedule │ ├── Dockerfile │ ├── README.md │ └── main.go └── nsqcrawl │ └── queue.go ├── open └── open.go ├── proxy.go ├── proxy_test.go ├── queue.go ├── queue_memory.go ├── request.go ├── request_error.go ├── response.go └── response_utils.go /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crackcomm/crawl/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crackcomm/crawl/HEAD/LICENSE -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crackcomm/crawl/HEAD/Makefile -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crackcomm/crawl/HEAD/README.md -------------------------------------------------------------------------------- /circle.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crackcomm/crawl/HEAD/circle.yml -------------------------------------------------------------------------------- /crawler.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crackcomm/crawl/HEAD/crawler.go -------------------------------------------------------------------------------- /crawler_options.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crackcomm/crawl/HEAD/crawler_options.go -------------------------------------------------------------------------------- /docs-templates/nsq/consumer/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crackcomm/crawl/HEAD/docs-templates/nsq/consumer/README.md -------------------------------------------------------------------------------- /docs-templates/nsq/crawl-schedule/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crackcomm/crawl/HEAD/docs-templates/nsq/crawl-schedule/README.md -------------------------------------------------------------------------------- /examples/imdb/main.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crackcomm/crawl/HEAD/examples/imdb/main.go -------------------------------------------------------------------------------- /examples/imdb/spider/spider.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crackcomm/crawl/HEAD/examples/imdb/spider/spider.go -------------------------------------------------------------------------------- /examples/nsq-skeleton/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | dist/ 3 | -------------------------------------------------------------------------------- /examples/nsq-skeleton/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crackcomm/crawl/HEAD/examples/nsq-skeleton/Dockerfile -------------------------------------------------------------------------------- /examples/nsq-skeleton/Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crackcomm/crawl/HEAD/examples/nsq-skeleton/Makefile -------------------------------------------------------------------------------- /examples/nsq-skeleton/circle.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crackcomm/crawl/HEAD/examples/nsq-skeleton/circle.yml -------------------------------------------------------------------------------- /examples/nsq-skeleton/main.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crackcomm/crawl/HEAD/examples/nsq-skeleton/main.go -------------------------------------------------------------------------------- /forms/forms.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crackcomm/crawl/HEAD/forms/forms.go -------------------------------------------------------------------------------- /html_utils.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crackcomm/crawl/HEAD/html_utils.go -------------------------------------------------------------------------------- /nsq/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crackcomm/crawl/HEAD/nsq/README.md -------------------------------------------------------------------------------- /nsq/consumer/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crackcomm/crawl/HEAD/nsq/consumer/README.md -------------------------------------------------------------------------------- /nsq/consumer/app.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crackcomm/crawl/HEAD/nsq/consumer/app.go -------------------------------------------------------------------------------- /nsq/consumer/app_options.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crackcomm/crawl/HEAD/nsq/consumer/app_options.go -------------------------------------------------------------------------------- /nsq/crawl-schedule/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crackcomm/crawl/HEAD/nsq/crawl-schedule/Dockerfile -------------------------------------------------------------------------------- /nsq/crawl-schedule/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crackcomm/crawl/HEAD/nsq/crawl-schedule/README.md -------------------------------------------------------------------------------- /nsq/crawl-schedule/main.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crackcomm/crawl/HEAD/nsq/crawl-schedule/main.go -------------------------------------------------------------------------------- /nsq/nsqcrawl/queue.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crackcomm/crawl/HEAD/nsq/nsqcrawl/queue.go -------------------------------------------------------------------------------- /open/open.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crackcomm/crawl/HEAD/open/open.go -------------------------------------------------------------------------------- /proxy.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crackcomm/crawl/HEAD/proxy.go -------------------------------------------------------------------------------- /proxy_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crackcomm/crawl/HEAD/proxy_test.go -------------------------------------------------------------------------------- /queue.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crackcomm/crawl/HEAD/queue.go -------------------------------------------------------------------------------- /queue_memory.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crackcomm/crawl/HEAD/queue_memory.go -------------------------------------------------------------------------------- /request.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crackcomm/crawl/HEAD/request.go -------------------------------------------------------------------------------- /request_error.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crackcomm/crawl/HEAD/request_error.go -------------------------------------------------------------------------------- /response.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crackcomm/crawl/HEAD/response.go -------------------------------------------------------------------------------- /response_utils.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crackcomm/crawl/HEAD/response_utils.go --------------------------------------------------------------------------------