├── .dockerignore ├── .gitattributes ├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── assets └── Readability.js ├── cmd └── main.go ├── common ├── config │ ├── init.go │ └── types.go ├── constants.go ├── initializer │ └── init.go ├── logger │ ├── init.go │ └── trace.go └── utils │ ├── string.go │ └── ua.go ├── config └── temp.yaml ├── docker-compose.yml ├── go.mod ├── go.sum ├── router ├── controller │ ├── common.go │ ├── demo_api.go │ ├── scrape_api.go │ └── scrape_types.go ├── demo.go ├── health.go ├── init.go ├── middleware │ └── logger.go ├── scrape.go └── utils │ └── gin.go ├── service ├── headless │ ├── init.go │ └── run.go └── scrape │ ├── direct.go │ ├── filter.go │ ├── headless.go │ ├── init.go │ ├── scrape.go │ └── types.go └── static ├── error.html └── index.html /.dockerignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/.dockerignore -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | assets/Readability.js linguist-vendored=true 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/.gitignore -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/Dockerfile -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/LICENSE -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/Makefile -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/README.md -------------------------------------------------------------------------------- /assets/Readability.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/assets/Readability.js -------------------------------------------------------------------------------- /cmd/main.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/cmd/main.go -------------------------------------------------------------------------------- /common/config/init.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/common/config/init.go -------------------------------------------------------------------------------- /common/config/types.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/common/config/types.go -------------------------------------------------------------------------------- /common/constants.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/common/constants.go -------------------------------------------------------------------------------- /common/initializer/init.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/common/initializer/init.go -------------------------------------------------------------------------------- /common/logger/init.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/common/logger/init.go -------------------------------------------------------------------------------- /common/logger/trace.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/common/logger/trace.go -------------------------------------------------------------------------------- /common/utils/string.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/common/utils/string.go -------------------------------------------------------------------------------- /common/utils/ua.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/common/utils/ua.go -------------------------------------------------------------------------------- /config/temp.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/config/temp.yaml -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/docker-compose.yml -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/go.mod -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/go.sum -------------------------------------------------------------------------------- /router/controller/common.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/router/controller/common.go -------------------------------------------------------------------------------- /router/controller/demo_api.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/router/controller/demo_api.go -------------------------------------------------------------------------------- /router/controller/scrape_api.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/router/controller/scrape_api.go -------------------------------------------------------------------------------- /router/controller/scrape_types.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/router/controller/scrape_types.go -------------------------------------------------------------------------------- /router/demo.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/router/demo.go -------------------------------------------------------------------------------- /router/health.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/router/health.go -------------------------------------------------------------------------------- /router/init.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/router/init.go -------------------------------------------------------------------------------- /router/middleware/logger.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/router/middleware/logger.go -------------------------------------------------------------------------------- /router/scrape.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/router/scrape.go -------------------------------------------------------------------------------- /router/utils/gin.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/router/utils/gin.go -------------------------------------------------------------------------------- /service/headless/init.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/service/headless/init.go -------------------------------------------------------------------------------- /service/headless/run.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/service/headless/run.go -------------------------------------------------------------------------------- /service/scrape/direct.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/service/scrape/direct.go -------------------------------------------------------------------------------- /service/scrape/filter.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/service/scrape/filter.go -------------------------------------------------------------------------------- /service/scrape/headless.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/service/scrape/headless.go -------------------------------------------------------------------------------- /service/scrape/init.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/service/scrape/init.go -------------------------------------------------------------------------------- /service/scrape/scrape.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/service/scrape/scrape.go -------------------------------------------------------------------------------- /service/scrape/types.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/service/scrape/types.go -------------------------------------------------------------------------------- /static/error.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/static/error.html -------------------------------------------------------------------------------- /static/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzzgydi/webscraper/HEAD/static/index.html --------------------------------------------------------------------------------