├── .github └── workflows │ └── docker-image.yml ├── .gitignore ├── .golangci.yml ├── CHANGELOG.md ├── Dockerfile ├── Makefile ├── README.md ├── auth └── auth.go ├── cmd ├── cmd.go ├── master │ └── master.go └── worker │ └── worker.go ├── collect └── collect.go ├── config.toml ├── docker-compose.yml ├── engine ├── option.go └── schedule.go ├── extensions └── randomua.go ├── generator └── generator.go ├── go.mod ├── go.sum ├── kubernetes ├── configmap.yaml ├── crawl-master-service.yaml ├── crawl-master.yaml ├── crawl-worker-service.yaml ├── crawl-worker.yaml └── ingress.yaml ├── limiter └── limiter.go ├── log ├── default.go ├── file_test.go └── log.go ├── main.go ├── master ├── master.go └── option.go ├── parse ├── doubanbook │ └── book.go ├── doubangroup │ └── group.go └── doubangroupjs │ └── groupjs.go ├── proto ├── crawler │ ├── crawler.pb.go │ ├── crawler.pb.gw.go │ ├── crawler.pb.micro.go │ ├── crawler.proto │ └── crawler_grpc.pb.go └── greeter │ ├── hello.pb.go │ ├── hello.pb.gw.go │ ├── hello.pb.micro.go │ ├── hello.proto │ └── hello_grpc.pb.go ├── proxy ├── proxy.go └── proxy_test.go ├── spider ├── option.go ├── parse.go ├── parsejs.go ├── request.go ├── storage.go ├── task.go └── temp.go ├── sqldb ├── option.go ├── sqldb.go └── sqldb_test.go ├── storage ├── sqlstorage │ ├── option.go │ ├── sqlstorage.go │ └── sqlstorage_test.go └── storage.go └── version └── version.go /.github/workflows/docker-image.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/.github/workflows/docker-image.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/.gitignore -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/.golangci.yml -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/CHANGELOG.md -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/Dockerfile -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/Makefile -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/README.md -------------------------------------------------------------------------------- /auth/auth.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/auth/auth.go -------------------------------------------------------------------------------- /cmd/cmd.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/cmd/cmd.go -------------------------------------------------------------------------------- /cmd/master/master.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/cmd/master/master.go -------------------------------------------------------------------------------- /cmd/worker/worker.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/cmd/worker/worker.go -------------------------------------------------------------------------------- /collect/collect.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/collect/collect.go -------------------------------------------------------------------------------- /config.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/config.toml -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/docker-compose.yml -------------------------------------------------------------------------------- /engine/option.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/engine/option.go -------------------------------------------------------------------------------- /engine/schedule.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/engine/schedule.go -------------------------------------------------------------------------------- /extensions/randomua.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/extensions/randomua.go -------------------------------------------------------------------------------- /generator/generator.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/generator/generator.go -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/go.mod -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/go.sum -------------------------------------------------------------------------------- /kubernetes/configmap.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/kubernetes/configmap.yaml -------------------------------------------------------------------------------- /kubernetes/crawl-master-service.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/kubernetes/crawl-master-service.yaml -------------------------------------------------------------------------------- /kubernetes/crawl-master.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/kubernetes/crawl-master.yaml -------------------------------------------------------------------------------- /kubernetes/crawl-worker-service.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/kubernetes/crawl-worker-service.yaml -------------------------------------------------------------------------------- /kubernetes/crawl-worker.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/kubernetes/crawl-worker.yaml -------------------------------------------------------------------------------- /kubernetes/ingress.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/kubernetes/ingress.yaml -------------------------------------------------------------------------------- /limiter/limiter.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/limiter/limiter.go -------------------------------------------------------------------------------- /log/default.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/log/default.go -------------------------------------------------------------------------------- /log/file_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/log/file_test.go -------------------------------------------------------------------------------- /log/log.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/log/log.go -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/main.go -------------------------------------------------------------------------------- /master/master.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/master/master.go -------------------------------------------------------------------------------- /master/option.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/master/option.go -------------------------------------------------------------------------------- /parse/doubanbook/book.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/parse/doubanbook/book.go -------------------------------------------------------------------------------- /parse/doubangroup/group.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/parse/doubangroup/group.go -------------------------------------------------------------------------------- /parse/doubangroupjs/groupjs.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/parse/doubangroupjs/groupjs.go -------------------------------------------------------------------------------- /proto/crawler/crawler.pb.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/proto/crawler/crawler.pb.go -------------------------------------------------------------------------------- /proto/crawler/crawler.pb.gw.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/proto/crawler/crawler.pb.gw.go -------------------------------------------------------------------------------- /proto/crawler/crawler.pb.micro.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/proto/crawler/crawler.pb.micro.go -------------------------------------------------------------------------------- /proto/crawler/crawler.proto: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/proto/crawler/crawler.proto -------------------------------------------------------------------------------- /proto/crawler/crawler_grpc.pb.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/proto/crawler/crawler_grpc.pb.go -------------------------------------------------------------------------------- /proto/greeter/hello.pb.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/proto/greeter/hello.pb.go -------------------------------------------------------------------------------- /proto/greeter/hello.pb.gw.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/proto/greeter/hello.pb.gw.go -------------------------------------------------------------------------------- /proto/greeter/hello.pb.micro.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/proto/greeter/hello.pb.micro.go -------------------------------------------------------------------------------- /proto/greeter/hello.proto: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/proto/greeter/hello.proto -------------------------------------------------------------------------------- /proto/greeter/hello_grpc.pb.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/proto/greeter/hello_grpc.pb.go -------------------------------------------------------------------------------- /proxy/proxy.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/proxy/proxy.go -------------------------------------------------------------------------------- /proxy/proxy_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/proxy/proxy_test.go -------------------------------------------------------------------------------- /spider/option.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/spider/option.go -------------------------------------------------------------------------------- /spider/parse.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/spider/parse.go -------------------------------------------------------------------------------- /spider/parsejs.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/spider/parsejs.go -------------------------------------------------------------------------------- /spider/request.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/spider/request.go -------------------------------------------------------------------------------- /spider/storage.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/spider/storage.go -------------------------------------------------------------------------------- /spider/task.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/spider/task.go -------------------------------------------------------------------------------- /spider/temp.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/spider/temp.go -------------------------------------------------------------------------------- /sqldb/option.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/sqldb/option.go -------------------------------------------------------------------------------- /sqldb/sqldb.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/sqldb/sqldb.go -------------------------------------------------------------------------------- /sqldb/sqldb_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/sqldb/sqldb_test.go -------------------------------------------------------------------------------- /storage/sqlstorage/option.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/storage/sqlstorage/option.go -------------------------------------------------------------------------------- /storage/sqlstorage/sqlstorage.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/storage/sqlstorage/sqlstorage.go -------------------------------------------------------------------------------- /storage/sqlstorage/sqlstorage_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/storage/sqlstorage/sqlstorage_test.go -------------------------------------------------------------------------------- /storage/storage.go: -------------------------------------------------------------------------------- 1 | package storage 2 | -------------------------------------------------------------------------------- /version/version.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamerjackson/crawler/HEAD/version/version.go --------------------------------------------------------------------------------