├── .gitignore ├── Assets ├── LLM-eval-LLM.png ├── LLM-eval-methods.png ├── LLM-eval-pipeline.png ├── auto-llm-eval-pipeline-v6-ragas-br.drawio.png ├── auto-llm-eval-pipeline-v6-ragas-br.jpg ├── eval-metrics.png ├── eval-prompt-ops-update.png ├── eval-prompt-ops.png ├── example-metric-results.png ├── llm-metric-eval.png ├── stepfunction-newbranch2.png └── stepfunction-pipeline.png ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── evaluation_artifacts ├── Amazon_SageMaker_Developer_Guide.pdf ├── evaluation_prompt.csv └── models.json ├── frontend ├── .env-example ├── Dockerfile ├── README.md ├── backend │ └── server.mjs ├── copilot │ ├── .workspace │ ├── environments │ │ └── test │ │ │ └── manifest.yml │ └── eval-app-genai │ │ ├── addons │ │ └── additional_policy.yaml │ │ └── manifest.yml ├── package-lock.json ├── package.json ├── public │ ├── favicon.ico │ ├── index.html │ ├── logo192.png │ ├── logo512.png │ ├── manifest.json │ └── robots.txt ├── resolve ├── run.sh ├── src │ ├── App.css │ ├── App.js │ ├── components.js │ ├── constants.js │ ├── drawer.js │ ├── functions.js │ ├── index.css │ ├── index.js │ ├── logo.svg │ ├── prompt_modal.js │ ├── reportWebVitals.js │ ├── setupTests.js │ └── utils.js └── start_servers.sh ├── infrastructure ├── api-layer-ddb-data.json ├── custom_resource │ ├── lambda_function.py │ └── requirements.txt ├── llmetersample │ ├── lambda_function.py │ └── requirements.txt ├── opensearch.yaml ├── requirements_lambda.txt ├── samsetup.sh ├── setup.md ├── setup.sh ├── solutionddb-data.json ├── template.yaml └── test.yaml └── lambda ├── eval_metrics.py ├── evaluate_llm_handler.py ├── evaluate_llm_input_validation.py ├── evaluate_llm_result_summary.py ├── evaluate_ragas_handler.py ├── fmeval_input_preprocessor.py ├── handlers └── utils │ └── evaluation_util.py ├── invoke_llm_convert_to_json.py ├── invoke_llm_handler.py ├── invoke_llm_result_summary.py ├── llm_api ├── __init__.py └── invoke_llm.py ├── log └── logger.py ├── metrics ├── base.py ├── cosine.py └── survey.py ├── prompts ├── __init__.py ├── store.py └── template.py ├── ragaseval_input_preprocess.py ├── ragasutils ├── __init__.py ├── generate_answer_from_kb.py ├── prepare_environment.py └── ragas_evaluation.py ├── src ├── processing.py ├── requirements.txt └── runproc.sh └── start_sf.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/.gitignore -------------------------------------------------------------------------------- /Assets/LLM-eval-LLM.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/Assets/LLM-eval-LLM.png -------------------------------------------------------------------------------- /Assets/LLM-eval-methods.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/Assets/LLM-eval-methods.png -------------------------------------------------------------------------------- /Assets/LLM-eval-pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/Assets/LLM-eval-pipeline.png -------------------------------------------------------------------------------- /Assets/auto-llm-eval-pipeline-v6-ragas-br.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/Assets/auto-llm-eval-pipeline-v6-ragas-br.drawio.png -------------------------------------------------------------------------------- /Assets/auto-llm-eval-pipeline-v6-ragas-br.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/Assets/auto-llm-eval-pipeline-v6-ragas-br.jpg -------------------------------------------------------------------------------- /Assets/eval-metrics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/Assets/eval-metrics.png -------------------------------------------------------------------------------- /Assets/eval-prompt-ops-update.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/Assets/eval-prompt-ops-update.png -------------------------------------------------------------------------------- /Assets/eval-prompt-ops.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/Assets/eval-prompt-ops.png -------------------------------------------------------------------------------- /Assets/example-metric-results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/Assets/example-metric-results.png -------------------------------------------------------------------------------- /Assets/llm-metric-eval.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/Assets/llm-metric-eval.png -------------------------------------------------------------------------------- /Assets/stepfunction-newbranch2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/Assets/stepfunction-newbranch2.png -------------------------------------------------------------------------------- /Assets/stepfunction-pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/Assets/stepfunction-pipeline.png -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/CODE_OF_CONDUCT.md -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/CONTRIBUTING.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/README.md -------------------------------------------------------------------------------- /evaluation_artifacts/Amazon_SageMaker_Developer_Guide.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/evaluation_artifacts/Amazon_SageMaker_Developer_Guide.pdf -------------------------------------------------------------------------------- /evaluation_artifacts/evaluation_prompt.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/evaluation_artifacts/evaluation_prompt.csv -------------------------------------------------------------------------------- /evaluation_artifacts/models.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/evaluation_artifacts/models.json -------------------------------------------------------------------------------- /frontend/.env-example: -------------------------------------------------------------------------------- 1 | REACT_APP_BACKEND_API_HOST=URL of backend api 2 | -------------------------------------------------------------------------------- /frontend/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/frontend/Dockerfile -------------------------------------------------------------------------------- /frontend/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/frontend/README.md -------------------------------------------------------------------------------- /frontend/backend/server.mjs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/frontend/backend/server.mjs -------------------------------------------------------------------------------- /frontend/copilot/.workspace: -------------------------------------------------------------------------------- 1 | application: eval-app-genai 2 | -------------------------------------------------------------------------------- /frontend/copilot/environments/test/manifest.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/frontend/copilot/environments/test/manifest.yml -------------------------------------------------------------------------------- /frontend/copilot/eval-app-genai/addons/additional_policy.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/frontend/copilot/eval-app-genai/addons/additional_policy.yaml -------------------------------------------------------------------------------- /frontend/copilot/eval-app-genai/manifest.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/frontend/copilot/eval-app-genai/manifest.yml -------------------------------------------------------------------------------- /frontend/package-lock.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/frontend/package-lock.json -------------------------------------------------------------------------------- /frontend/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/frontend/package.json -------------------------------------------------------------------------------- /frontend/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/frontend/public/favicon.ico -------------------------------------------------------------------------------- /frontend/public/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/frontend/public/index.html -------------------------------------------------------------------------------- /frontend/public/logo192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/frontend/public/logo192.png -------------------------------------------------------------------------------- /frontend/public/logo512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/frontend/public/logo512.png -------------------------------------------------------------------------------- /frontend/public/manifest.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/frontend/public/manifest.json -------------------------------------------------------------------------------- /frontend/public/robots.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/frontend/public/robots.txt -------------------------------------------------------------------------------- /frontend/resolve: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /frontend/run.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/frontend/run.sh -------------------------------------------------------------------------------- /frontend/src/App.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/frontend/src/App.css -------------------------------------------------------------------------------- /frontend/src/App.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/frontend/src/App.js -------------------------------------------------------------------------------- /frontend/src/components.js: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /frontend/src/constants.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/frontend/src/constants.js -------------------------------------------------------------------------------- /frontend/src/drawer.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/frontend/src/drawer.js -------------------------------------------------------------------------------- /frontend/src/functions.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/frontend/src/functions.js -------------------------------------------------------------------------------- /frontend/src/index.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/frontend/src/index.css -------------------------------------------------------------------------------- /frontend/src/index.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/frontend/src/index.js -------------------------------------------------------------------------------- /frontend/src/logo.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/frontend/src/logo.svg -------------------------------------------------------------------------------- /frontend/src/prompt_modal.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/frontend/src/prompt_modal.js -------------------------------------------------------------------------------- /frontend/src/reportWebVitals.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/frontend/src/reportWebVitals.js -------------------------------------------------------------------------------- /frontend/src/setupTests.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/frontend/src/setupTests.js -------------------------------------------------------------------------------- /frontend/src/utils.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/frontend/src/utils.js -------------------------------------------------------------------------------- /frontend/start_servers.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/frontend/start_servers.sh -------------------------------------------------------------------------------- /infrastructure/api-layer-ddb-data.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/infrastructure/api-layer-ddb-data.json -------------------------------------------------------------------------------- /infrastructure/custom_resource/lambda_function.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/infrastructure/custom_resource/lambda_function.py -------------------------------------------------------------------------------- /infrastructure/custom_resource/requirements.txt: -------------------------------------------------------------------------------- 1 | requests_aws4auth 2 | cfnresponse -------------------------------------------------------------------------------- /infrastructure/llmetersample/lambda_function.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/infrastructure/llmetersample/lambda_function.py -------------------------------------------------------------------------------- /infrastructure/llmetersample/requirements.txt: -------------------------------------------------------------------------------- 1 | llmeter==0.1.3 -------------------------------------------------------------------------------- /infrastructure/opensearch.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/infrastructure/opensearch.yaml -------------------------------------------------------------------------------- /infrastructure/requirements_lambda.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/infrastructure/requirements_lambda.txt -------------------------------------------------------------------------------- /infrastructure/samsetup.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/infrastructure/samsetup.sh -------------------------------------------------------------------------------- /infrastructure/setup.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/infrastructure/setup.md -------------------------------------------------------------------------------- /infrastructure/setup.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/infrastructure/setup.sh -------------------------------------------------------------------------------- /infrastructure/solutionddb-data.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/infrastructure/solutionddb-data.json -------------------------------------------------------------------------------- /infrastructure/template.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/infrastructure/template.yaml -------------------------------------------------------------------------------- /infrastructure/test.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/infrastructure/test.yaml -------------------------------------------------------------------------------- /lambda/eval_metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/lambda/eval_metrics.py -------------------------------------------------------------------------------- /lambda/evaluate_llm_handler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/lambda/evaluate_llm_handler.py -------------------------------------------------------------------------------- /lambda/evaluate_llm_input_validation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/lambda/evaluate_llm_input_validation.py -------------------------------------------------------------------------------- /lambda/evaluate_llm_result_summary.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/lambda/evaluate_llm_result_summary.py -------------------------------------------------------------------------------- /lambda/evaluate_ragas_handler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/lambda/evaluate_ragas_handler.py -------------------------------------------------------------------------------- /lambda/fmeval_input_preprocessor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/lambda/fmeval_input_preprocessor.py -------------------------------------------------------------------------------- /lambda/handlers/utils/evaluation_util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/lambda/handlers/utils/evaluation_util.py -------------------------------------------------------------------------------- /lambda/invoke_llm_convert_to_json.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/lambda/invoke_llm_convert_to_json.py -------------------------------------------------------------------------------- /lambda/invoke_llm_handler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/lambda/invoke_llm_handler.py -------------------------------------------------------------------------------- /lambda/invoke_llm_result_summary.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/lambda/invoke_llm_result_summary.py -------------------------------------------------------------------------------- /lambda/llm_api/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lambda/llm_api/invoke_llm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/lambda/llm_api/invoke_llm.py -------------------------------------------------------------------------------- /lambda/log/logger.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/lambda/log/logger.py -------------------------------------------------------------------------------- /lambda/metrics/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/lambda/metrics/base.py -------------------------------------------------------------------------------- /lambda/metrics/cosine.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/lambda/metrics/cosine.py -------------------------------------------------------------------------------- /lambda/metrics/survey.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/lambda/metrics/survey.py -------------------------------------------------------------------------------- /lambda/prompts/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lambda/prompts/store.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/lambda/prompts/store.py -------------------------------------------------------------------------------- /lambda/prompts/template.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/lambda/prompts/template.py -------------------------------------------------------------------------------- /lambda/ragaseval_input_preprocess.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/lambda/ragaseval_input_preprocess.py -------------------------------------------------------------------------------- /lambda/ragasutils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lambda/ragasutils/generate_answer_from_kb.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/lambda/ragasutils/generate_answer_from_kb.py -------------------------------------------------------------------------------- /lambda/ragasutils/prepare_environment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/lambda/ragasutils/prepare_environment.py -------------------------------------------------------------------------------- /lambda/ragasutils/ragas_evaluation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/lambda/ragasutils/ragas_evaluation.py -------------------------------------------------------------------------------- /lambda/src/processing.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/lambda/src/processing.py -------------------------------------------------------------------------------- /lambda/src/requirements.txt: -------------------------------------------------------------------------------- 1 | boto3 2 | fmeval -------------------------------------------------------------------------------- /lambda/src/runproc.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/lambda/src/runproc.sh -------------------------------------------------------------------------------- /lambda/start_sf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/build-an-automated-large-language-model-evaluation-pipeline-on-aws/HEAD/lambda/start_sf.py --------------------------------------------------------------------------------