├── .idea ├── .gitignore ├── Venom-Crawler.iml ├── modules.xml └── vcs.xml ├── README.md ├── cmd ├── cmd.go ├── crawlergo.go └── katana.go ├── go.mod ├── go.sum ├── internal ├── runner │ ├── executer.go │ ├── options.go │ └── runner.go ├── testutils │ ├── helper.go │ ├── integration.go │ └── testutils.go └── utils │ └── utils.go └── pkg ├── crawlergo ├── config │ └── config.go ├── domain_collect.go ├── engine │ ├── after_dom_tasks.go │ ├── after_loaded_tasks.go │ ├── browser.go │ ├── collect_links.go │ ├── intercept_request.go │ └── tab.go ├── filter │ ├── filter.go │ ├── simple_filter.go │ ├── smart_filter.go │ └── smart_filter_test.go ├── js │ └── javascript.go ├── model │ ├── request.go │ ├── url.go │ └── url_test.go ├── path_expansion.go ├── task_main.go ├── taskconfig.go └── tools │ ├── common.go │ ├── random.go │ └── requests │ ├── requests.go │ ├── response.go │ └── utils.go └── katana ├── engine ├── common │ ├── base.go │ ├── error.go │ └── http.go ├── engine.go ├── hybrid │ ├── crawl.go │ ├── doc.go │ ├── hijack.go │ └── hybrid.go ├── parser │ ├── files │ │ ├── request.go │ │ ├── robotstxt.go │ │ └── sitemapxml.go │ └── parser.go └── standard │ ├── crawl.go │ ├── doc.go │ └── standard.go ├── navigation ├── request.go └── response.go ├── output ├── custom_field.go ├── error.go ├── fields.go ├── fields_test.go ├── file_writer.go ├── format_json.go ├── format_screen.go ├── options.go ├── output.go ├── responses.go └── result.go ├── types ├── crawler_options.go └── options.go └── utils ├── extensions └── extensions.go ├── filters ├── filters.go └── simple.go ├── formfill.go ├── queue ├── priority_queue.go ├── queue.go ├── stack.go └── stragegy.go ├── regex.go ├── scope └── scope.go └── utils.go /.idea/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/.idea/.gitignore -------------------------------------------------------------------------------- /.idea/Venom-Crawler.iml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/.idea/Venom-Crawler.iml -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/.idea/modules.xml -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/.idea/vcs.xml -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/README.md -------------------------------------------------------------------------------- /cmd/cmd.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/cmd/cmd.go -------------------------------------------------------------------------------- /cmd/crawlergo.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/cmd/crawlergo.go -------------------------------------------------------------------------------- /cmd/katana.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/cmd/katana.go -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/go.mod -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/go.sum -------------------------------------------------------------------------------- /internal/runner/executer.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/internal/runner/executer.go -------------------------------------------------------------------------------- /internal/runner/options.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/internal/runner/options.go -------------------------------------------------------------------------------- /internal/runner/runner.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/internal/runner/runner.go -------------------------------------------------------------------------------- /internal/testutils/helper.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/internal/testutils/helper.go -------------------------------------------------------------------------------- /internal/testutils/integration.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/internal/testutils/integration.go -------------------------------------------------------------------------------- /internal/testutils/testutils.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/internal/testutils/testutils.go -------------------------------------------------------------------------------- /internal/utils/utils.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/internal/utils/utils.go -------------------------------------------------------------------------------- /pkg/crawlergo/config/config.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/crawlergo/config/config.go -------------------------------------------------------------------------------- /pkg/crawlergo/domain_collect.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/crawlergo/domain_collect.go -------------------------------------------------------------------------------- /pkg/crawlergo/engine/after_dom_tasks.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/crawlergo/engine/after_dom_tasks.go -------------------------------------------------------------------------------- /pkg/crawlergo/engine/after_loaded_tasks.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/crawlergo/engine/after_loaded_tasks.go -------------------------------------------------------------------------------- /pkg/crawlergo/engine/browser.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/crawlergo/engine/browser.go -------------------------------------------------------------------------------- /pkg/crawlergo/engine/collect_links.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/crawlergo/engine/collect_links.go -------------------------------------------------------------------------------- /pkg/crawlergo/engine/intercept_request.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/crawlergo/engine/intercept_request.go -------------------------------------------------------------------------------- /pkg/crawlergo/engine/tab.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/crawlergo/engine/tab.go -------------------------------------------------------------------------------- /pkg/crawlergo/filter/filter.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/crawlergo/filter/filter.go -------------------------------------------------------------------------------- /pkg/crawlergo/filter/simple_filter.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/crawlergo/filter/simple_filter.go -------------------------------------------------------------------------------- /pkg/crawlergo/filter/smart_filter.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/crawlergo/filter/smart_filter.go -------------------------------------------------------------------------------- /pkg/crawlergo/filter/smart_filter_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/crawlergo/filter/smart_filter_test.go -------------------------------------------------------------------------------- /pkg/crawlergo/js/javascript.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/crawlergo/js/javascript.go -------------------------------------------------------------------------------- /pkg/crawlergo/model/request.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/crawlergo/model/request.go -------------------------------------------------------------------------------- /pkg/crawlergo/model/url.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/crawlergo/model/url.go -------------------------------------------------------------------------------- /pkg/crawlergo/model/url_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/crawlergo/model/url_test.go -------------------------------------------------------------------------------- /pkg/crawlergo/path_expansion.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/crawlergo/path_expansion.go -------------------------------------------------------------------------------- /pkg/crawlergo/task_main.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/crawlergo/task_main.go -------------------------------------------------------------------------------- /pkg/crawlergo/taskconfig.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/crawlergo/taskconfig.go -------------------------------------------------------------------------------- /pkg/crawlergo/tools/common.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/crawlergo/tools/common.go -------------------------------------------------------------------------------- /pkg/crawlergo/tools/random.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/crawlergo/tools/random.go -------------------------------------------------------------------------------- /pkg/crawlergo/tools/requests/requests.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/crawlergo/tools/requests/requests.go -------------------------------------------------------------------------------- /pkg/crawlergo/tools/requests/response.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/crawlergo/tools/requests/response.go -------------------------------------------------------------------------------- /pkg/crawlergo/tools/requests/utils.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/crawlergo/tools/requests/utils.go -------------------------------------------------------------------------------- /pkg/katana/engine/common/base.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/engine/common/base.go -------------------------------------------------------------------------------- /pkg/katana/engine/common/error.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/engine/common/error.go -------------------------------------------------------------------------------- /pkg/katana/engine/common/http.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/engine/common/http.go -------------------------------------------------------------------------------- /pkg/katana/engine/engine.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/engine/engine.go -------------------------------------------------------------------------------- /pkg/katana/engine/hybrid/crawl.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/engine/hybrid/crawl.go -------------------------------------------------------------------------------- /pkg/katana/engine/hybrid/doc.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/engine/hybrid/doc.go -------------------------------------------------------------------------------- /pkg/katana/engine/hybrid/hijack.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/engine/hybrid/hijack.go -------------------------------------------------------------------------------- /pkg/katana/engine/hybrid/hybrid.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/engine/hybrid/hybrid.go -------------------------------------------------------------------------------- /pkg/katana/engine/parser/files/request.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/engine/parser/files/request.go -------------------------------------------------------------------------------- /pkg/katana/engine/parser/files/robotstxt.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/engine/parser/files/robotstxt.go -------------------------------------------------------------------------------- /pkg/katana/engine/parser/files/sitemapxml.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/engine/parser/files/sitemapxml.go -------------------------------------------------------------------------------- /pkg/katana/engine/parser/parser.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/engine/parser/parser.go -------------------------------------------------------------------------------- /pkg/katana/engine/standard/crawl.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/engine/standard/crawl.go -------------------------------------------------------------------------------- /pkg/katana/engine/standard/doc.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/engine/standard/doc.go -------------------------------------------------------------------------------- /pkg/katana/engine/standard/standard.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/engine/standard/standard.go -------------------------------------------------------------------------------- /pkg/katana/navigation/request.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/navigation/request.go -------------------------------------------------------------------------------- /pkg/katana/navigation/response.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/navigation/response.go -------------------------------------------------------------------------------- /pkg/katana/output/custom_field.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/output/custom_field.go -------------------------------------------------------------------------------- /pkg/katana/output/error.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/output/error.go -------------------------------------------------------------------------------- /pkg/katana/output/fields.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/output/fields.go -------------------------------------------------------------------------------- /pkg/katana/output/fields_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/output/fields_test.go -------------------------------------------------------------------------------- /pkg/katana/output/file_writer.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/output/file_writer.go -------------------------------------------------------------------------------- /pkg/katana/output/format_json.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/output/format_json.go -------------------------------------------------------------------------------- /pkg/katana/output/format_screen.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/output/format_screen.go -------------------------------------------------------------------------------- /pkg/katana/output/options.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/output/options.go -------------------------------------------------------------------------------- /pkg/katana/output/output.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/output/output.go -------------------------------------------------------------------------------- /pkg/katana/output/responses.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/output/responses.go -------------------------------------------------------------------------------- /pkg/katana/output/result.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/output/result.go -------------------------------------------------------------------------------- /pkg/katana/types/crawler_options.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/types/crawler_options.go -------------------------------------------------------------------------------- /pkg/katana/types/options.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/types/options.go -------------------------------------------------------------------------------- /pkg/katana/utils/extensions/extensions.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/utils/extensions/extensions.go -------------------------------------------------------------------------------- /pkg/katana/utils/filters/filters.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/utils/filters/filters.go -------------------------------------------------------------------------------- /pkg/katana/utils/filters/simple.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/utils/filters/simple.go -------------------------------------------------------------------------------- /pkg/katana/utils/formfill.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/utils/formfill.go -------------------------------------------------------------------------------- /pkg/katana/utils/queue/priority_queue.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/utils/queue/priority_queue.go -------------------------------------------------------------------------------- /pkg/katana/utils/queue/queue.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/utils/queue/queue.go -------------------------------------------------------------------------------- /pkg/katana/utils/queue/stack.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/utils/queue/stack.go -------------------------------------------------------------------------------- /pkg/katana/utils/queue/stragegy.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/utils/queue/stragegy.go -------------------------------------------------------------------------------- /pkg/katana/utils/regex.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/utils/regex.go -------------------------------------------------------------------------------- /pkg/katana/utils/scope/scope.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/utils/scope/scope.go -------------------------------------------------------------------------------- /pkg/katana/utils/utils.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-bool/Venom-Crawler/HEAD/pkg/katana/utils/utils.go --------------------------------------------------------------------------------