├── .github └── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── .npmignore ├── .readme-resources ├── architecture.png ├── kendra-screenshot.png └── state-machine.png ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── NOTICE ├── README.md ├── bin ├── build-chrome-layer.sh ├── crawl.ts ├── infrastructure.ts └── local-crawl.ts ├── cdk.json ├── crawl ├── deploy ├── destroy ├── jest.config.js ├── local-crawl ├── package.json ├── src ├── constructs │ ├── kendra │ │ ├── kendra-data-source-iam-role.ts │ │ ├── kendra-index-iam-role.ts │ │ ├── kendra-index.ts │ │ └── kendra-s3-data-source.ts │ └── webcrawler │ │ ├── chrome-lambda-layer.ts │ │ ├── constants.ts │ │ ├── web-crawler-lambda.ts │ │ ├── web-crawler-state-machine.ts │ │ └── web-crawler-step-lambdas.ts ├── lambda │ ├── crawler │ │ ├── core.ts │ │ ├── local.ts │ │ └── types.ts │ ├── index.ts │ ├── steps │ │ ├── 1_startCrawl.ts │ │ ├── 2_readQueuedUrls.ts │ │ ├── 3_crawlPageAndQueueUrls.ts │ │ ├── 4_continueExecution.ts │ │ └── 5_completeCrawl.ts │ └── utils │ │ ├── contextTable.ts │ │ ├── env.ts │ │ ├── historyTable.ts │ │ ├── pagination.ts │ │ └── queuedPaths.ts └── stacks │ ├── kendra-stack.ts │ └── web-crawler-stack.ts ├── test └── infrastructure.test.ts └── tsconfig.json /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/.github/PULL_REQUEST_TEMPLATE.md -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/.gitignore -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/.npmignore -------------------------------------------------------------------------------- /.readme-resources/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/.readme-resources/architecture.png -------------------------------------------------------------------------------- /.readme-resources/kendra-screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/.readme-resources/kendra-screenshot.png -------------------------------------------------------------------------------- /.readme-resources/state-machine.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/.readme-resources/state-machine.png -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/CODE_OF_CONDUCT.md -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/CONTRIBUTING.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/LICENSE -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/NOTICE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/README.md -------------------------------------------------------------------------------- /bin/build-chrome-layer.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/bin/build-chrome-layer.sh -------------------------------------------------------------------------------- /bin/crawl.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/bin/crawl.ts -------------------------------------------------------------------------------- /bin/infrastructure.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/bin/infrastructure.ts -------------------------------------------------------------------------------- /bin/local-crawl.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/bin/local-crawl.ts -------------------------------------------------------------------------------- /cdk.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/cdk.json -------------------------------------------------------------------------------- /crawl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/crawl -------------------------------------------------------------------------------- /deploy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/deploy -------------------------------------------------------------------------------- /destroy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/destroy -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/jest.config.js -------------------------------------------------------------------------------- /local-crawl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/local-crawl -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/package.json -------------------------------------------------------------------------------- /src/constructs/kendra/kendra-data-source-iam-role.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/src/constructs/kendra/kendra-data-source-iam-role.ts -------------------------------------------------------------------------------- /src/constructs/kendra/kendra-index-iam-role.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/src/constructs/kendra/kendra-index-iam-role.ts -------------------------------------------------------------------------------- /src/constructs/kendra/kendra-index.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/src/constructs/kendra/kendra-index.ts -------------------------------------------------------------------------------- /src/constructs/kendra/kendra-s3-data-source.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/src/constructs/kendra/kendra-s3-data-source.ts -------------------------------------------------------------------------------- /src/constructs/webcrawler/chrome-lambda-layer.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/src/constructs/webcrawler/chrome-lambda-layer.ts -------------------------------------------------------------------------------- /src/constructs/webcrawler/constants.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/src/constructs/webcrawler/constants.ts -------------------------------------------------------------------------------- /src/constructs/webcrawler/web-crawler-lambda.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/src/constructs/webcrawler/web-crawler-lambda.ts -------------------------------------------------------------------------------- /src/constructs/webcrawler/web-crawler-state-machine.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/src/constructs/webcrawler/web-crawler-state-machine.ts -------------------------------------------------------------------------------- /src/constructs/webcrawler/web-crawler-step-lambdas.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/src/constructs/webcrawler/web-crawler-step-lambdas.ts -------------------------------------------------------------------------------- /src/lambda/crawler/core.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/src/lambda/crawler/core.ts -------------------------------------------------------------------------------- /src/lambda/crawler/local.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/src/lambda/crawler/local.ts -------------------------------------------------------------------------------- /src/lambda/crawler/types.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/src/lambda/crawler/types.ts -------------------------------------------------------------------------------- /src/lambda/index.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/src/lambda/index.ts -------------------------------------------------------------------------------- /src/lambda/steps/1_startCrawl.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/src/lambda/steps/1_startCrawl.ts -------------------------------------------------------------------------------- /src/lambda/steps/2_readQueuedUrls.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/src/lambda/steps/2_readQueuedUrls.ts -------------------------------------------------------------------------------- /src/lambda/steps/3_crawlPageAndQueueUrls.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/src/lambda/steps/3_crawlPageAndQueueUrls.ts -------------------------------------------------------------------------------- /src/lambda/steps/4_continueExecution.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/src/lambda/steps/4_continueExecution.ts -------------------------------------------------------------------------------- /src/lambda/steps/5_completeCrawl.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/src/lambda/steps/5_completeCrawl.ts -------------------------------------------------------------------------------- /src/lambda/utils/contextTable.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/src/lambda/utils/contextTable.ts -------------------------------------------------------------------------------- /src/lambda/utils/env.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/src/lambda/utils/env.ts -------------------------------------------------------------------------------- /src/lambda/utils/historyTable.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/src/lambda/utils/historyTable.ts -------------------------------------------------------------------------------- /src/lambda/utils/pagination.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/src/lambda/utils/pagination.ts -------------------------------------------------------------------------------- /src/lambda/utils/queuedPaths.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/src/lambda/utils/queuedPaths.ts -------------------------------------------------------------------------------- /src/stacks/kendra-stack.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/src/stacks/kendra-stack.ts -------------------------------------------------------------------------------- /src/stacks/web-crawler-stack.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/src/stacks/web-crawler-stack.ts -------------------------------------------------------------------------------- /test/infrastructure.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/test/infrastructure.test.ts -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-step-functions-kendra-web-crawler-search-engine/HEAD/tsconfig.json --------------------------------------------------------------------------------