├── .gitattributes ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── archive │ ├── js-sdk.yml │ ├── publish-js-sdk.yml │ ├── publish-python-sdk.yml │ ├── publish-rust-sdk.yml │ ├── python-sdk.yml │ └── rust-sdk.yml ├── dependabot.yml ├── scripts │ ├── check_version_has_incremented.py │ └── requirements.txt └── workflows │ ├── build-docker-images.yml │ ├── check-queues.yml │ └── clean-before-24h-complete-jobs.yml ├── .gitignore ├── .gitmodules ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── SELF_HOST.md ├── apps ├── api │ ├── .dockerignore │ ├── .env.example │ ├── .env.local │ ├── .gitattributes │ ├── .gitignore │ ├── Dockerfile │ ├── jest.config.js │ ├── jest.setup.js │ ├── package.json │ ├── pnpm-lock.yaml │ ├── requests.http │ ├── src │ │ ├── __tests__ │ │ │ ├── e2e_full_withAuth │ │ │ │ └── index.test.ts │ │ │ ├── e2e_noAuth │ │ │ │ └── index.test.ts │ │ │ ├── e2e_v1_withAuth │ │ │ │ └── index.test.ts │ │ │ └── e2e_withAuth │ │ │ │ └── index.test.ts │ │ ├── controllers │ │ │ ├── __tests__ │ │ │ │ └── crawl.test.ts │ │ │ ├── auth.ts │ │ │ ├── v0 │ │ │ │ ├── admin │ │ │ │ │ ├── queue.ts │ │ │ │ │ └── redis-health.ts │ │ │ │ ├── crawl-cancel.ts │ │ │ │ ├── crawl-status.ts │ │ │ │ ├── crawl.ts │ │ │ │ ├── crawlPreview.ts │ │ │ │ ├── keyAuth.ts │ │ │ │ ├── scrape.ts │ │ │ │ └── status.ts │ │ │ └── v1 │ │ │ │ ├── __tests__ │ │ │ │ └── urlValidation.test.ts │ │ │ │ ├── crawl-cancel.ts │ │ │ │ ├── crawl-status.ts │ │ │ │ ├── crawl.ts │ │ │ │ ├── liveness.ts │ │ │ │ ├── map.ts │ │ │ │ ├── readiness.ts │ │ │ │ ├── scrape-status.ts │ │ │ │ ├── scrape.ts │ │ │ │ └── types.ts │ │ ├── index.ts │ │ ├── lib │ │ │ ├── __tests__ │ │ │ │ ├── html-to-markdown.test.ts │ │ │ │ └── job-priority.test.ts │ │ │ ├── batch-process.ts │ │ │ ├── crawl-redis.ts │ │ │ ├── custom-error.ts │ │ │ ├── default-values.ts │ │ │ ├── entities.ts │ │ │ ├── go-html-to-md │ │ │ │ ├── README.md │ │ │ │ ├── go.mod │ │ │ │ ├── go.sum │ │ │ │ └── html-to-markdown.go │ │ │ ├── html-to-markdown.ts │ │ │ ├── job-priority.ts │ │ │ ├── logger.ts │ │ │ ├── parse-mode.ts │ │ │ ├── parseApi.ts │ │ │ ├── timeout.ts │ │ │ ├── validateUrl.test.ts │ │ │ ├── validateUrl.ts │ │ │ └── withAuth.ts │ │ ├── main │ │ │ └── runWebScraper.ts │ │ ├── openapi │ │ │ └── index.ts │ │ ├── routes │ │ │ ├── admin.ts │ │ │ └── v1.ts │ │ ├── run-req.ts │ │ ├── scraper │ │ │ └── WebScraper │ │ │ │ ├── __tests__ │ │ │ │ ├── crawler.test.ts │ │ │ │ └── dns.test.ts │ │ │ │ ├── crawler.ts │ │ │ │ ├── global.ts │ │ │ │ ├── index.ts │ │ │ │ ├── scrapers │ │ │ │ ├── fetch.ts │ │ │ │ └── playwright.ts │ │ │ │ ├── single_url.ts │ │ │ │ ├── sitemap.ts │ │ │ │ └── utils │ │ │ │ ├── __tests__ │ │ │ │ ├── maxDepthUtils.test.ts │ │ │ │ ├── parseTable.test.ts │ │ │ │ ├── removeUnwantedElements.test.ts │ │ │ │ └── replacePaths.test.ts │ │ │ │ ├── custom │ │ │ │ └── website_params.ts │ │ │ │ ├── excludeTags.ts │ │ │ │ ├── maxDepthUtils.ts │ │ │ │ ├── metadata.ts │ │ │ │ ├── parseTable.ts │ │ │ │ ├── removeUnwantedElements.ts │ │ │ │ ├── replacePaths.ts │ │ │ │ └── utils.ts │ │ ├── scripts │ │ │ └── generate-openapi.ts │ │ ├── services │ │ │ ├── alerts │ │ │ │ └── index.ts │ │ │ ├── idempotency │ │ │ │ └── create.ts │ │ │ ├── logtail.ts │ │ │ ├── queue-jobs.ts │ │ │ ├── queue-service.ts │ │ │ ├── queue-worker.ts │ │ │ ├── rate-limiter.test.ts │ │ │ ├── rate-limiter.ts │ │ │ ├── redis.ts │ │ │ ├── redlock.ts │ │ │ └── system-monitor.ts │ │ ├── strings.ts │ │ ├── supabase_types.ts │ │ └── types.ts │ ├── tsconfig.json │ └── v1-openapi.json ├── puppeteer-service-ts │ ├── .dockerignore │ ├── Dockerfile │ ├── README.md │ ├── api.ts │ ├── helpers │ │ └── get_error.ts │ ├── openapi │ │ └── index.ts │ ├── package.json │ ├── pnpm-lock.yaml │ └── tsconfig.json └── test-suite │ ├── .env.example │ ├── README.md │ ├── data │ ├── crawl.json │ └── scrape.json │ ├── jest.config.js │ ├── jest.setup.js │ ├── load-test-results │ ├── tests-1-5 │ │ ├── assets │ │ │ ├── CPU-utilization-report-test-1.png │ │ │ ├── memory-utilization-report-test-1.png │ │ │ ├── metrics-test-2.png │ │ │ ├── metrics-test-3.png │ │ │ ├── metrics-test-4.png │ │ │ └── metrics-test-5.png │ │ ├── load-test-1.md │ │ ├── load-test-2.md │ │ ├── load-test-3.md │ │ ├── load-test-4.md │ │ └── load-test-5.md │ └── tests-6-7 │ │ ├── assets │ │ ├── metrics-fire-engine-test-7-2.png │ │ ├── metrics-fire-engine-test-7.png │ │ ├── metrics-fire-engine-test-8.png │ │ ├── metrics-test-6.png │ │ ├── metrics-test-7.png │ │ └── metrics-test-8.png │ │ └── load-test-6.md │ ├── load-test.yml │ ├── package.json │ ├── pnpm-lock.yaml │ ├── tsconfig.json │ └── utils │ ├── misc.ts │ └── types.ts ├── docker-compose.yaml └── img └── firecrawl_logo.png /.gitattributes: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/.gitattributes -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/.github/ISSUE_TEMPLATE/bug_report.md -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/.github/ISSUE_TEMPLATE/feature_request.md -------------------------------------------------------------------------------- /.github/archive/js-sdk.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/.github/archive/js-sdk.yml -------------------------------------------------------------------------------- /.github/archive/publish-js-sdk.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/.github/archive/publish-js-sdk.yml -------------------------------------------------------------------------------- /.github/archive/publish-python-sdk.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/.github/archive/publish-python-sdk.yml -------------------------------------------------------------------------------- /.github/archive/publish-rust-sdk.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/.github/archive/publish-rust-sdk.yml -------------------------------------------------------------------------------- /.github/archive/python-sdk.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/.github/archive/python-sdk.yml -------------------------------------------------------------------------------- /.github/archive/rust-sdk.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/.github/archive/rust-sdk.yml -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/.github/dependabot.yml -------------------------------------------------------------------------------- /.github/scripts/check_version_has_incremented.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/.github/scripts/check_version_has_incremented.py -------------------------------------------------------------------------------- /.github/scripts/requirements.txt: -------------------------------------------------------------------------------- 1 | requests 2 | packaging 3 | toml -------------------------------------------------------------------------------- /.github/workflows/build-docker-images.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/.github/workflows/build-docker-images.yml -------------------------------------------------------------------------------- /.github/workflows/check-queues.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/.github/workflows/check-queues.yml -------------------------------------------------------------------------------- /.github/workflows/clean-before-24h-complete-jobs.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/.github/workflows/clean-before-24h-complete-jobs.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/.gitignore -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/.gitmodules -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/CONTRIBUTING.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/README.md -------------------------------------------------------------------------------- /SELF_HOST.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/SELF_HOST.md -------------------------------------------------------------------------------- /apps/api/.dockerignore: -------------------------------------------------------------------------------- 1 | /node_modules/ 2 | /dist/ 3 | .env 4 | *.csv 5 | -------------------------------------------------------------------------------- /apps/api/.env.example: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/.env.example -------------------------------------------------------------------------------- /apps/api/.env.local: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/.env.local -------------------------------------------------------------------------------- /apps/api/.gitattributes: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/.gitattributes -------------------------------------------------------------------------------- /apps/api/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/.gitignore -------------------------------------------------------------------------------- /apps/api/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/Dockerfile -------------------------------------------------------------------------------- /apps/api/jest.config.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/jest.config.js -------------------------------------------------------------------------------- /apps/api/jest.setup.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/jest.setup.js -------------------------------------------------------------------------------- /apps/api/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/package.json -------------------------------------------------------------------------------- /apps/api/pnpm-lock.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/pnpm-lock.yaml -------------------------------------------------------------------------------- /apps/api/requests.http: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/requests.http -------------------------------------------------------------------------------- /apps/api/src/__tests__/e2e_full_withAuth/index.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/__tests__/e2e_full_withAuth/index.test.ts -------------------------------------------------------------------------------- /apps/api/src/__tests__/e2e_noAuth/index.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/__tests__/e2e_noAuth/index.test.ts -------------------------------------------------------------------------------- /apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts -------------------------------------------------------------------------------- /apps/api/src/__tests__/e2e_withAuth/index.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/__tests__/e2e_withAuth/index.test.ts -------------------------------------------------------------------------------- /apps/api/src/controllers/__tests__/crawl.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/controllers/__tests__/crawl.test.ts -------------------------------------------------------------------------------- /apps/api/src/controllers/auth.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/controllers/auth.ts -------------------------------------------------------------------------------- /apps/api/src/controllers/v0/admin/queue.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/controllers/v0/admin/queue.ts -------------------------------------------------------------------------------- /apps/api/src/controllers/v0/admin/redis-health.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/controllers/v0/admin/redis-health.ts -------------------------------------------------------------------------------- /apps/api/src/controllers/v0/crawl-cancel.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/controllers/v0/crawl-cancel.ts -------------------------------------------------------------------------------- /apps/api/src/controllers/v0/crawl-status.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/controllers/v0/crawl-status.ts -------------------------------------------------------------------------------- /apps/api/src/controllers/v0/crawl.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/controllers/v0/crawl.ts -------------------------------------------------------------------------------- /apps/api/src/controllers/v0/crawlPreview.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/controllers/v0/crawlPreview.ts -------------------------------------------------------------------------------- /apps/api/src/controllers/v0/keyAuth.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/controllers/v0/keyAuth.ts -------------------------------------------------------------------------------- /apps/api/src/controllers/v0/scrape.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/controllers/v0/scrape.ts -------------------------------------------------------------------------------- /apps/api/src/controllers/v0/status.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/controllers/v0/status.ts -------------------------------------------------------------------------------- /apps/api/src/controllers/v1/__tests__/urlValidation.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/controllers/v1/__tests__/urlValidation.test.ts -------------------------------------------------------------------------------- /apps/api/src/controllers/v1/crawl-cancel.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/controllers/v1/crawl-cancel.ts -------------------------------------------------------------------------------- /apps/api/src/controllers/v1/crawl-status.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/controllers/v1/crawl-status.ts -------------------------------------------------------------------------------- /apps/api/src/controllers/v1/crawl.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/controllers/v1/crawl.ts -------------------------------------------------------------------------------- /apps/api/src/controllers/v1/liveness.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/controllers/v1/liveness.ts -------------------------------------------------------------------------------- /apps/api/src/controllers/v1/map.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/controllers/v1/map.ts -------------------------------------------------------------------------------- /apps/api/src/controllers/v1/readiness.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/controllers/v1/readiness.ts -------------------------------------------------------------------------------- /apps/api/src/controllers/v1/scrape-status.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/controllers/v1/scrape-status.ts -------------------------------------------------------------------------------- /apps/api/src/controllers/v1/scrape.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/controllers/v1/scrape.ts -------------------------------------------------------------------------------- /apps/api/src/controllers/v1/types.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/controllers/v1/types.ts -------------------------------------------------------------------------------- /apps/api/src/index.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/index.ts -------------------------------------------------------------------------------- /apps/api/src/lib/__tests__/html-to-markdown.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/lib/__tests__/html-to-markdown.test.ts -------------------------------------------------------------------------------- /apps/api/src/lib/__tests__/job-priority.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/lib/__tests__/job-priority.test.ts -------------------------------------------------------------------------------- /apps/api/src/lib/batch-process.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/lib/batch-process.ts -------------------------------------------------------------------------------- /apps/api/src/lib/crawl-redis.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/lib/crawl-redis.ts -------------------------------------------------------------------------------- /apps/api/src/lib/custom-error.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/lib/custom-error.ts -------------------------------------------------------------------------------- /apps/api/src/lib/default-values.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/lib/default-values.ts -------------------------------------------------------------------------------- /apps/api/src/lib/entities.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/lib/entities.ts -------------------------------------------------------------------------------- /apps/api/src/lib/go-html-to-md/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/lib/go-html-to-md/README.md -------------------------------------------------------------------------------- /apps/api/src/lib/go-html-to-md/go.mod: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/lib/go-html-to-md/go.mod -------------------------------------------------------------------------------- /apps/api/src/lib/go-html-to-md/go.sum: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/lib/go-html-to-md/go.sum -------------------------------------------------------------------------------- /apps/api/src/lib/go-html-to-md/html-to-markdown.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/lib/go-html-to-md/html-to-markdown.go -------------------------------------------------------------------------------- /apps/api/src/lib/html-to-markdown.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/lib/html-to-markdown.ts -------------------------------------------------------------------------------- /apps/api/src/lib/job-priority.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/lib/job-priority.ts -------------------------------------------------------------------------------- /apps/api/src/lib/logger.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/lib/logger.ts -------------------------------------------------------------------------------- /apps/api/src/lib/parse-mode.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/lib/parse-mode.ts -------------------------------------------------------------------------------- /apps/api/src/lib/parseApi.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/lib/parseApi.ts -------------------------------------------------------------------------------- /apps/api/src/lib/timeout.ts: -------------------------------------------------------------------------------- 1 | export const axiosTimeout = 3000; -------------------------------------------------------------------------------- /apps/api/src/lib/validateUrl.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/lib/validateUrl.test.ts -------------------------------------------------------------------------------- /apps/api/src/lib/validateUrl.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/lib/validateUrl.ts -------------------------------------------------------------------------------- /apps/api/src/lib/withAuth.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/lib/withAuth.ts -------------------------------------------------------------------------------- /apps/api/src/main/runWebScraper.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/main/runWebScraper.ts -------------------------------------------------------------------------------- /apps/api/src/openapi/index.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/openapi/index.ts -------------------------------------------------------------------------------- /apps/api/src/routes/admin.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/routes/admin.ts -------------------------------------------------------------------------------- /apps/api/src/routes/v1.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/routes/v1.ts -------------------------------------------------------------------------------- /apps/api/src/run-req.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/run-req.ts -------------------------------------------------------------------------------- /apps/api/src/scraper/WebScraper/__tests__/crawler.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/scraper/WebScraper/__tests__/crawler.test.ts -------------------------------------------------------------------------------- /apps/api/src/scraper/WebScraper/__tests__/dns.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/scraper/WebScraper/__tests__/dns.test.ts -------------------------------------------------------------------------------- /apps/api/src/scraper/WebScraper/crawler.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/scraper/WebScraper/crawler.ts -------------------------------------------------------------------------------- /apps/api/src/scraper/WebScraper/global.ts: -------------------------------------------------------------------------------- 1 | export const universalTimeout = 60000; -------------------------------------------------------------------------------- /apps/api/src/scraper/WebScraper/index.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/scraper/WebScraper/index.ts -------------------------------------------------------------------------------- /apps/api/src/scraper/WebScraper/scrapers/fetch.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/scraper/WebScraper/scrapers/fetch.ts -------------------------------------------------------------------------------- /apps/api/src/scraper/WebScraper/scrapers/playwright.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/scraper/WebScraper/scrapers/playwright.ts -------------------------------------------------------------------------------- /apps/api/src/scraper/WebScraper/single_url.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/scraper/WebScraper/single_url.ts -------------------------------------------------------------------------------- /apps/api/src/scraper/WebScraper/sitemap.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/scraper/WebScraper/sitemap.ts -------------------------------------------------------------------------------- /apps/api/src/scraper/WebScraper/utils/__tests__/maxDepthUtils.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/scraper/WebScraper/utils/__tests__/maxDepthUtils.test.ts -------------------------------------------------------------------------------- /apps/api/src/scraper/WebScraper/utils/__tests__/parseTable.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/scraper/WebScraper/utils/__tests__/parseTable.test.ts -------------------------------------------------------------------------------- /apps/api/src/scraper/WebScraper/utils/__tests__/removeUnwantedElements.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/scraper/WebScraper/utils/__tests__/removeUnwantedElements.test.ts -------------------------------------------------------------------------------- /apps/api/src/scraper/WebScraper/utils/__tests__/replacePaths.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/scraper/WebScraper/utils/__tests__/replacePaths.test.ts -------------------------------------------------------------------------------- /apps/api/src/scraper/WebScraper/utils/custom/website_params.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts -------------------------------------------------------------------------------- /apps/api/src/scraper/WebScraper/utils/excludeTags.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/scraper/WebScraper/utils/excludeTags.ts -------------------------------------------------------------------------------- /apps/api/src/scraper/WebScraper/utils/maxDepthUtils.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/scraper/WebScraper/utils/maxDepthUtils.ts -------------------------------------------------------------------------------- /apps/api/src/scraper/WebScraper/utils/metadata.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/scraper/WebScraper/utils/metadata.ts -------------------------------------------------------------------------------- /apps/api/src/scraper/WebScraper/utils/parseTable.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/scraper/WebScraper/utils/parseTable.ts -------------------------------------------------------------------------------- /apps/api/src/scraper/WebScraper/utils/removeUnwantedElements.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/scraper/WebScraper/utils/removeUnwantedElements.ts -------------------------------------------------------------------------------- /apps/api/src/scraper/WebScraper/utils/replacePaths.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/scraper/WebScraper/utils/replacePaths.ts -------------------------------------------------------------------------------- /apps/api/src/scraper/WebScraper/utils/utils.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/scraper/WebScraper/utils/utils.ts -------------------------------------------------------------------------------- /apps/api/src/scripts/generate-openapi.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/scripts/generate-openapi.ts -------------------------------------------------------------------------------- /apps/api/src/services/alerts/index.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/services/alerts/index.ts -------------------------------------------------------------------------------- /apps/api/src/services/idempotency/create.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/services/idempotency/create.ts -------------------------------------------------------------------------------- /apps/api/src/services/logtail.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/services/logtail.ts -------------------------------------------------------------------------------- /apps/api/src/services/queue-jobs.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/services/queue-jobs.ts -------------------------------------------------------------------------------- /apps/api/src/services/queue-service.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/services/queue-service.ts -------------------------------------------------------------------------------- /apps/api/src/services/queue-worker.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/services/queue-worker.ts -------------------------------------------------------------------------------- /apps/api/src/services/rate-limiter.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/services/rate-limiter.test.ts -------------------------------------------------------------------------------- /apps/api/src/services/rate-limiter.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/services/rate-limiter.ts -------------------------------------------------------------------------------- /apps/api/src/services/redis.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/services/redis.ts -------------------------------------------------------------------------------- /apps/api/src/services/redlock.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/services/redlock.ts -------------------------------------------------------------------------------- /apps/api/src/services/system-monitor.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/services/system-monitor.ts -------------------------------------------------------------------------------- /apps/api/src/strings.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/strings.ts -------------------------------------------------------------------------------- /apps/api/src/supabase_types.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/supabase_types.ts -------------------------------------------------------------------------------- /apps/api/src/types.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/src/types.ts -------------------------------------------------------------------------------- /apps/api/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/tsconfig.json -------------------------------------------------------------------------------- /apps/api/v1-openapi.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/api/v1-openapi.json -------------------------------------------------------------------------------- /apps/puppeteer-service-ts/.dockerignore: -------------------------------------------------------------------------------- 1 | dist/ 2 | node_modules/ 3 | -------------------------------------------------------------------------------- /apps/puppeteer-service-ts/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/puppeteer-service-ts/Dockerfile -------------------------------------------------------------------------------- /apps/puppeteer-service-ts/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/puppeteer-service-ts/README.md -------------------------------------------------------------------------------- /apps/puppeteer-service-ts/api.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/puppeteer-service-ts/api.ts -------------------------------------------------------------------------------- /apps/puppeteer-service-ts/helpers/get_error.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/puppeteer-service-ts/helpers/get_error.ts -------------------------------------------------------------------------------- /apps/puppeteer-service-ts/openapi/index.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/puppeteer-service-ts/openapi/index.ts -------------------------------------------------------------------------------- /apps/puppeteer-service-ts/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/puppeteer-service-ts/package.json -------------------------------------------------------------------------------- /apps/puppeteer-service-ts/pnpm-lock.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/puppeteer-service-ts/pnpm-lock.yaml -------------------------------------------------------------------------------- /apps/puppeteer-service-ts/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/puppeteer-service-ts/tsconfig.json -------------------------------------------------------------------------------- /apps/test-suite/.env.example: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/test-suite/.env.example -------------------------------------------------------------------------------- /apps/test-suite/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/test-suite/README.md -------------------------------------------------------------------------------- /apps/test-suite/data/crawl.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/test-suite/data/crawl.json -------------------------------------------------------------------------------- /apps/test-suite/data/scrape.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/test-suite/data/scrape.json -------------------------------------------------------------------------------- /apps/test-suite/jest.config.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/test-suite/jest.config.js -------------------------------------------------------------------------------- /apps/test-suite/jest.setup.js: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /apps/test-suite/load-test-results/tests-1-5/assets/CPU-utilization-report-test-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/test-suite/load-test-results/tests-1-5/assets/CPU-utilization-report-test-1.png -------------------------------------------------------------------------------- /apps/test-suite/load-test-results/tests-1-5/assets/memory-utilization-report-test-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/test-suite/load-test-results/tests-1-5/assets/memory-utilization-report-test-1.png -------------------------------------------------------------------------------- /apps/test-suite/load-test-results/tests-1-5/assets/metrics-test-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/test-suite/load-test-results/tests-1-5/assets/metrics-test-2.png -------------------------------------------------------------------------------- /apps/test-suite/load-test-results/tests-1-5/assets/metrics-test-3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/test-suite/load-test-results/tests-1-5/assets/metrics-test-3.png -------------------------------------------------------------------------------- /apps/test-suite/load-test-results/tests-1-5/assets/metrics-test-4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/test-suite/load-test-results/tests-1-5/assets/metrics-test-4.png -------------------------------------------------------------------------------- /apps/test-suite/load-test-results/tests-1-5/assets/metrics-test-5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/test-suite/load-test-results/tests-1-5/assets/metrics-test-5.png -------------------------------------------------------------------------------- /apps/test-suite/load-test-results/tests-1-5/load-test-1.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/test-suite/load-test-results/tests-1-5/load-test-1.md -------------------------------------------------------------------------------- /apps/test-suite/load-test-results/tests-1-5/load-test-2.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/test-suite/load-test-results/tests-1-5/load-test-2.md -------------------------------------------------------------------------------- /apps/test-suite/load-test-results/tests-1-5/load-test-3.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/test-suite/load-test-results/tests-1-5/load-test-3.md -------------------------------------------------------------------------------- /apps/test-suite/load-test-results/tests-1-5/load-test-4.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/test-suite/load-test-results/tests-1-5/load-test-4.md -------------------------------------------------------------------------------- /apps/test-suite/load-test-results/tests-1-5/load-test-5.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/test-suite/load-test-results/tests-1-5/load-test-5.md -------------------------------------------------------------------------------- /apps/test-suite/load-test-results/tests-6-7/assets/metrics-fire-engine-test-7-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/test-suite/load-test-results/tests-6-7/assets/metrics-fire-engine-test-7-2.png -------------------------------------------------------------------------------- /apps/test-suite/load-test-results/tests-6-7/assets/metrics-fire-engine-test-7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/test-suite/load-test-results/tests-6-7/assets/metrics-fire-engine-test-7.png -------------------------------------------------------------------------------- /apps/test-suite/load-test-results/tests-6-7/assets/metrics-fire-engine-test-8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/test-suite/load-test-results/tests-6-7/assets/metrics-fire-engine-test-8.png -------------------------------------------------------------------------------- /apps/test-suite/load-test-results/tests-6-7/assets/metrics-test-6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/test-suite/load-test-results/tests-6-7/assets/metrics-test-6.png -------------------------------------------------------------------------------- /apps/test-suite/load-test-results/tests-6-7/assets/metrics-test-7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/test-suite/load-test-results/tests-6-7/assets/metrics-test-7.png -------------------------------------------------------------------------------- /apps/test-suite/load-test-results/tests-6-7/assets/metrics-test-8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/test-suite/load-test-results/tests-6-7/assets/metrics-test-8.png -------------------------------------------------------------------------------- /apps/test-suite/load-test-results/tests-6-7/load-test-6.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/test-suite/load-test-results/tests-6-7/load-test-6.md -------------------------------------------------------------------------------- /apps/test-suite/load-test.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/test-suite/load-test.yml -------------------------------------------------------------------------------- /apps/test-suite/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/test-suite/package.json -------------------------------------------------------------------------------- /apps/test-suite/pnpm-lock.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/test-suite/pnpm-lock.yaml -------------------------------------------------------------------------------- /apps/test-suite/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/test-suite/tsconfig.json -------------------------------------------------------------------------------- /apps/test-suite/utils/misc.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/test-suite/utils/misc.ts -------------------------------------------------------------------------------- /apps/test-suite/utils/types.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/apps/test-suite/utils/types.ts -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/docker-compose.yaml -------------------------------------------------------------------------------- /img/firecrawl_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devflowinc/firecrawl-simple/HEAD/img/firecrawl_logo.png --------------------------------------------------------------------------------