├── .github └── workflows │ ├── main.yml │ └── publish.yml ├── .gitignore ├── .npmignore ├── .npmrc.sample ├── API.md ├── LICENSE.md ├── README.md ├── add-job.js ├── bin.js ├── client ├── app.css ├── components │ └── App.js ├── index.html ├── index.old.html ├── package.json ├── preact.js ├── queue.example.json ├── style.css └── web_modules │ ├── import-map.json │ └── preact.js ├── index.js ├── lib ├── browser │ ├── create-browser.js │ ├── get-puppeteer-options.js │ ├── get-puppeteer-options.test.js │ ├── index.js │ ├── prepare-page.js │ ├── set-stealth.js │ └── take-screenshot.js ├── create-server.js ├── queue.get-queue-id.test.js ├── queue │ └── index.js ├── scrapers │ ├── page.js │ ├── page.test.js │ └── with-queue.js ├── storage │ ├── cache.js │ ├── init-stats-cache.js │ └── scrape-cache.js ├── url-logics │ ├── url.getHostname.test.js │ ├── url.js │ └── url.normalizeUrl.test.js └── util │ ├── keys-to-lower.js │ ├── keys-to-lower.test.js │ └── random-ua.js ├── migrate.js ├── migrations ├── 01.folders-cleanup.js └── config.json ├── package.json ├── parse-options.js └── playground.js /.github/workflows/main.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/.github/workflows/main.yml -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/.github/workflows/publish.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/.gitignore -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/.npmignore -------------------------------------------------------------------------------- /.npmrc.sample: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/.npmrc.sample -------------------------------------------------------------------------------- /API.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/API.md -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/LICENSE.md -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/README.md -------------------------------------------------------------------------------- /add-job.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/add-job.js -------------------------------------------------------------------------------- /bin.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/bin.js -------------------------------------------------------------------------------- /client/app.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/client/app.css -------------------------------------------------------------------------------- /client/components/App.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/client/components/App.js -------------------------------------------------------------------------------- /client/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/client/index.html -------------------------------------------------------------------------------- /client/index.old.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/client/index.old.html -------------------------------------------------------------------------------- /client/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/client/package.json -------------------------------------------------------------------------------- /client/preact.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/client/preact.js -------------------------------------------------------------------------------- /client/queue.example.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/client/queue.example.json -------------------------------------------------------------------------------- /client/style.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/client/style.css -------------------------------------------------------------------------------- /client/web_modules/import-map.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/client/web_modules/import-map.json -------------------------------------------------------------------------------- /client/web_modules/preact.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/client/web_modules/preact.js -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/index.js -------------------------------------------------------------------------------- /lib/browser/create-browser.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/lib/browser/create-browser.js -------------------------------------------------------------------------------- /lib/browser/get-puppeteer-options.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/lib/browser/get-puppeteer-options.js -------------------------------------------------------------------------------- /lib/browser/get-puppeteer-options.test.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/lib/browser/get-puppeteer-options.test.js -------------------------------------------------------------------------------- /lib/browser/index.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/lib/browser/index.js -------------------------------------------------------------------------------- /lib/browser/prepare-page.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/lib/browser/prepare-page.js -------------------------------------------------------------------------------- /lib/browser/set-stealth.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/lib/browser/set-stealth.js -------------------------------------------------------------------------------- /lib/browser/take-screenshot.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/lib/browser/take-screenshot.js -------------------------------------------------------------------------------- /lib/create-server.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/lib/create-server.js -------------------------------------------------------------------------------- /lib/queue.get-queue-id.test.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/lib/queue.get-queue-id.test.js -------------------------------------------------------------------------------- /lib/queue/index.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/lib/queue/index.js -------------------------------------------------------------------------------- /lib/scrapers/page.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/lib/scrapers/page.js -------------------------------------------------------------------------------- /lib/scrapers/page.test.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/lib/scrapers/page.test.js -------------------------------------------------------------------------------- /lib/scrapers/with-queue.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/lib/scrapers/with-queue.js -------------------------------------------------------------------------------- /lib/storage/cache.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/lib/storage/cache.js -------------------------------------------------------------------------------- /lib/storage/init-stats-cache.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/lib/storage/init-stats-cache.js -------------------------------------------------------------------------------- /lib/storage/scrape-cache.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/lib/storage/scrape-cache.js -------------------------------------------------------------------------------- /lib/url-logics/url.getHostname.test.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/lib/url-logics/url.getHostname.test.js -------------------------------------------------------------------------------- /lib/url-logics/url.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/lib/url-logics/url.js -------------------------------------------------------------------------------- /lib/url-logics/url.normalizeUrl.test.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/lib/url-logics/url.normalizeUrl.test.js -------------------------------------------------------------------------------- /lib/util/keys-to-lower.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/lib/util/keys-to-lower.js -------------------------------------------------------------------------------- /lib/util/keys-to-lower.test.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/lib/util/keys-to-lower.test.js -------------------------------------------------------------------------------- /lib/util/random-ua.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/lib/util/random-ua.js -------------------------------------------------------------------------------- /migrate.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/migrate.js -------------------------------------------------------------------------------- /migrations/01.folders-cleanup.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/migrations/01.folders-cleanup.js -------------------------------------------------------------------------------- /migrations/config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/migrations/config.json -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/package.json -------------------------------------------------------------------------------- /parse-options.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/parse-options.js -------------------------------------------------------------------------------- /playground.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-fei/mega-scraper/HEAD/playground.js --------------------------------------------------------------------------------