├── .formatter.exs ├── .gitignore ├── LICENSE ├── LICENSE.md ├── README.md ├── config ├── config.exs ├── dev.exs └── test.exs ├── lib ├── spider_man.ex └── spider_man │ ├── application.ex │ ├── common_spider.ex │ ├── component │ ├── component.ex │ ├── downloader.ex │ ├── item_processor.ex │ └── spider.ex │ ├── configuration.ex │ ├── engine.ex │ ├── middleware │ └── user_agent.ex │ ├── pipeline │ ├── counter.ex │ ├── debug.ex │ ├── duplicate_filter.ex │ ├── json_encode.ex │ ├── json_encode2file.ex │ ├── pipeline.ex │ ├── save_to_file.ex │ ├── set_cookie.ex │ └── splash.ex │ ├── producer │ ├── ets.ex │ └── producer.ex │ ├── requester │ ├── dynamic_finch.ex │ ├── finch.ex │ ├── hackney.ex │ └── requester.ex │ ├── stats.ex │ ├── storage │ ├── csv.ex │ ├── ets.ex │ ├── json_lines.ex │ ├── log.ex │ ├── multi.ex │ └── storage.ex │ ├── struct │ ├── item.ex │ ├── request.ex │ └── response.ex │ ├── telemetry.ex │ └── utils.ex ├── mix.exs ├── mix.lock ├── notebooks └── elixirjobs.livemd └── test ├── common_spider_test.exs ├── engine_test.exs ├── middleware └── user_agent_test.exs ├── pipeline ├── duplicate_filter_test.exs ├── pipeline_test.exs └── set_cookie_test.exs ├── spider_man_test.exs ├── spider_test.exs ├── storage ├── csv_test.exs ├── ets_test.exs ├── json_lines_test.exs └── storage_test.exs ├── support ├── just_return.ex ├── pipelines.ex └── spider_modules.ex └── test_helper.exs /.formatter.exs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/.formatter.exs -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/LICENSE -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/LICENSE.md -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/README.md -------------------------------------------------------------------------------- /config/config.exs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/config/config.exs -------------------------------------------------------------------------------- /config/dev.exs: -------------------------------------------------------------------------------- 1 | import Config 2 | -------------------------------------------------------------------------------- /config/test.exs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/config/test.exs -------------------------------------------------------------------------------- /lib/spider_man.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/lib/spider_man.ex -------------------------------------------------------------------------------- /lib/spider_man/application.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/lib/spider_man/application.ex -------------------------------------------------------------------------------- /lib/spider_man/common_spider.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/lib/spider_man/common_spider.ex -------------------------------------------------------------------------------- /lib/spider_man/component/component.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/lib/spider_man/component/component.ex -------------------------------------------------------------------------------- /lib/spider_man/component/downloader.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/lib/spider_man/component/downloader.ex -------------------------------------------------------------------------------- /lib/spider_man/component/item_processor.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/lib/spider_man/component/item_processor.ex -------------------------------------------------------------------------------- /lib/spider_man/component/spider.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/lib/spider_man/component/spider.ex -------------------------------------------------------------------------------- /lib/spider_man/configuration.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/lib/spider_man/configuration.ex -------------------------------------------------------------------------------- /lib/spider_man/engine.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/lib/spider_man/engine.ex -------------------------------------------------------------------------------- /lib/spider_man/middleware/user_agent.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/lib/spider_man/middleware/user_agent.ex -------------------------------------------------------------------------------- /lib/spider_man/pipeline/counter.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/lib/spider_man/pipeline/counter.ex -------------------------------------------------------------------------------- /lib/spider_man/pipeline/debug.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/lib/spider_man/pipeline/debug.ex -------------------------------------------------------------------------------- /lib/spider_man/pipeline/duplicate_filter.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/lib/spider_man/pipeline/duplicate_filter.ex -------------------------------------------------------------------------------- /lib/spider_man/pipeline/json_encode.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/lib/spider_man/pipeline/json_encode.ex -------------------------------------------------------------------------------- /lib/spider_man/pipeline/json_encode2file.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/lib/spider_man/pipeline/json_encode2file.ex -------------------------------------------------------------------------------- /lib/spider_man/pipeline/pipeline.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/lib/spider_man/pipeline/pipeline.ex -------------------------------------------------------------------------------- /lib/spider_man/pipeline/save_to_file.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/lib/spider_man/pipeline/save_to_file.ex -------------------------------------------------------------------------------- /lib/spider_man/pipeline/set_cookie.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/lib/spider_man/pipeline/set_cookie.ex -------------------------------------------------------------------------------- /lib/spider_man/pipeline/splash.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/lib/spider_man/pipeline/splash.ex -------------------------------------------------------------------------------- /lib/spider_man/producer/ets.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/lib/spider_man/producer/ets.ex -------------------------------------------------------------------------------- /lib/spider_man/producer/producer.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/lib/spider_man/producer/producer.ex -------------------------------------------------------------------------------- /lib/spider_man/requester/dynamic_finch.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/lib/spider_man/requester/dynamic_finch.ex -------------------------------------------------------------------------------- /lib/spider_man/requester/finch.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/lib/spider_man/requester/finch.ex -------------------------------------------------------------------------------- /lib/spider_man/requester/hackney.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/lib/spider_man/requester/hackney.ex -------------------------------------------------------------------------------- /lib/spider_man/requester/requester.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/lib/spider_man/requester/requester.ex -------------------------------------------------------------------------------- /lib/spider_man/stats.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/lib/spider_man/stats.ex -------------------------------------------------------------------------------- /lib/spider_man/storage/csv.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/lib/spider_man/storage/csv.ex -------------------------------------------------------------------------------- /lib/spider_man/storage/ets.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/lib/spider_man/storage/ets.ex -------------------------------------------------------------------------------- /lib/spider_man/storage/json_lines.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/lib/spider_man/storage/json_lines.ex -------------------------------------------------------------------------------- /lib/spider_man/storage/log.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/lib/spider_man/storage/log.ex -------------------------------------------------------------------------------- /lib/spider_man/storage/multi.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/lib/spider_man/storage/multi.ex -------------------------------------------------------------------------------- /lib/spider_man/storage/storage.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/lib/spider_man/storage/storage.ex -------------------------------------------------------------------------------- /lib/spider_man/struct/item.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/lib/spider_man/struct/item.ex -------------------------------------------------------------------------------- /lib/spider_man/struct/request.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/lib/spider_man/struct/request.ex -------------------------------------------------------------------------------- /lib/spider_man/struct/response.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/lib/spider_man/struct/response.ex -------------------------------------------------------------------------------- /lib/spider_man/telemetry.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/lib/spider_man/telemetry.ex -------------------------------------------------------------------------------- /lib/spider_man/utils.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/lib/spider_man/utils.ex -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/mix.exs -------------------------------------------------------------------------------- /mix.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/mix.lock -------------------------------------------------------------------------------- /notebooks/elixirjobs.livemd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/notebooks/elixirjobs.livemd -------------------------------------------------------------------------------- /test/common_spider_test.exs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/test/common_spider_test.exs -------------------------------------------------------------------------------- /test/engine_test.exs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/test/engine_test.exs -------------------------------------------------------------------------------- /test/middleware/user_agent_test.exs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/test/middleware/user_agent_test.exs -------------------------------------------------------------------------------- /test/pipeline/duplicate_filter_test.exs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/test/pipeline/duplicate_filter_test.exs -------------------------------------------------------------------------------- /test/pipeline/pipeline_test.exs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/test/pipeline/pipeline_test.exs -------------------------------------------------------------------------------- /test/pipeline/set_cookie_test.exs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/test/pipeline/set_cookie_test.exs -------------------------------------------------------------------------------- /test/spider_man_test.exs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/test/spider_man_test.exs -------------------------------------------------------------------------------- /test/spider_test.exs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/test/spider_test.exs -------------------------------------------------------------------------------- /test/storage/csv_test.exs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/test/storage/csv_test.exs -------------------------------------------------------------------------------- /test/storage/ets_test.exs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/test/storage/ets_test.exs -------------------------------------------------------------------------------- /test/storage/json_lines_test.exs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/test/storage/json_lines_test.exs -------------------------------------------------------------------------------- /test/storage/storage_test.exs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/test/storage/storage_test.exs -------------------------------------------------------------------------------- /test/support/just_return.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/test/support/just_return.ex -------------------------------------------------------------------------------- /test/support/pipelines.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/test/support/pipelines.ex -------------------------------------------------------------------------------- /test/support/spider_modules.ex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feng19/spider_man/HEAD/test/support/spider_modules.ex -------------------------------------------------------------------------------- /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | System.put_env("TMPDIR", "tmp") 2 | ExUnit.start(seed: 0) 3 | --------------------------------------------------------------------------------