├── .gitignore ├── .pre-commit-config.yaml ├── CITATION.cff ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── MANIFEST.in ├── README.md ├── THIRD_PARTY_LICENSES.txt ├── analytics ├── analytics.py ├── sagemaker_cost_rpm_plot.py ├── sagemaker_metrics_plot.py └── scratchpad.ipynb ├── check_s3_content.sh ├── copy_s3_content.sh ├── create_manifest.py ├── debug.sh ├── docs ├── accuracy.md ├── advanced.md ├── analytics.md ├── announcement.md ├── benchmarking.md ├── benchmarking_multimodal_models_on_bedrock.md ├── benchmarking_on_bedrock.md ├── benchmarking_on_ec2.md ├── benchmarking_on_eks.md ├── benchmarking_on_sagemaker.md ├── build.md ├── byo_dataset.md ├── byo_rest_predictor.md ├── byoe.md ├── cli.md ├── configs ├── customize_config_files.md ├── deepseek.md ├── ec2.md ├── evaluation.md ├── features.md ├── gettingstarted.md ├── img │ ├── ML-FMBT-cloudformation-launch-stack.png │ ├── accuracy_trajectory_per_payload.png │ ├── business_summary.png │ ├── config-structure.png │ ├── ec2-iam.png │ ├── ec2connect1.png │ ├── ec2connect10.png │ ├── ec2connect2.png │ ├── ec2connect3.png │ ├── ec2connect4.png │ ├── ec2connect5.png │ ├── ec2connect5a.png │ ├── ec2connect6.png │ ├── ec2connect7.png │ ├── ec2connect8.png │ ├── ec2connect9.png │ ├── ec2tag.png │ ├── fmbench-deployment1.png │ ├── fmbench-deployment2.png │ ├── fmbench-thumbnail.png │ ├── fmbench-website.png │ ├── fmbench.drawio │ ├── fmbt-small.png │ ├── fmbt.png │ ├── ground_truth_info.png │ ├── heatmap.png │ ├── hf_ds_keys.png │ ├── latency_vs_tokens.png │ ├── overall_candidate_model_majority_voting_accuracy.png │ ├── results.gif │ └── website.png ├── index.md ├── manifest.md ├── misc │ ├── ec2_instance_creation_steps.md │ ├── eks_cluster-creation_steps.md │ └── the-diy-version-w-gory-details.md ├── mm_copies.md ├── neuron.md ├── quarto.md ├── quickstart.md ├── releases.md ├── resources.md ├── results.md ├── run_as_container.md ├── simplified_config_files.md ├── website.md └── workflow.md ├── ec2_template.yaml ├── fmbench ├── 0_setup.ipynb ├── 1_generate_data.ipynb ├── 2_deploy_model.ipynb ├── 3_run_inference.ipynb ├── 4_get_evaluations.ipynb ├── 5_model_metric_analysis.ipynb ├── 6_cleanup.ipynb ├── __init__.py ├── bring_your_own_dataset.ipynb ├── config_filepath.txt ├── configs │ ├── NousResearchHermes70B │ │ ├── config-Nous-Hermes3-1-70b-g5.48xl-tp-8-mc-max-djl.yml │ │ ├── config-Nous-Hermes3-1-70b-inf2.48xl-triton-tp24.yml │ │ └── config-Nous-Hermes3-1-70b-trn1.32xl-deploy-ec2-triton.yml │ ├── bedrock │ │ ├── config-bedrock-all-anthropic-models-longbench-data.yml │ │ ├── config-bedrock-anthropic-models-OpenOrca.yml │ │ ├── config-bedrock-claude.yml │ │ ├── config-bedrock-evals-only-conc-1.yml │ │ ├── config-bedrock-haiku-sonnet-majority-voting.yml │ │ ├── config-bedrock-llama3-1-70b-streaming.yml │ │ ├── config-bedrock-llama3-1-8b-streaming.yml │ │ ├── config-bedrock-llama3-1-no-streaming.yml │ │ ├── config-bedrock-llama3-1.yml │ │ ├── config-bedrock-llama3-streaming.yml │ │ ├── config-bedrock-models-OpenOrca.yml │ │ ├── config-bedrock-titan-text-express.yml │ │ ├── config-bedrock.yml │ │ ├── config-claude-3-5-sonnet-v2.yml │ │ ├── config-claude-dolly-dataset.yml │ │ ├── config-llama-3-2-11b-databricks-dolly-15k.yml │ │ ├── config-llama-3-2-1b-3b-no-evals.yml │ │ ├── config-llama-3-2-1b-3b.yml │ │ ├── config-llama-3-2-all-models-longbench-hf-version.yml │ │ ├── config-llama-3-2-all-models.yml │ │ ├── config-llama-3-3-all-models-open-orca.yml │ │ ├── config-llama-3-3-all-models.yml │ │ ├── config-nova-all-models-convfinqa.yml │ │ ├── config-nova-all-models-dolly-dataset.yml │ │ ├── config-nova-all-models-openarca.yml │ │ └── config-nova-all-models.yml │ ├── bert │ │ └── config-distilbert-base-uncased.yml │ ├── byoe │ │ ├── config-byo-custom-rest-predictor-tinyllama.yml │ │ ├── config-byo-custom-rest-predictor.yml │ │ └── config-model-byo-sagemaker-endpoint.yml │ ├── deepseek │ │ ├── config-deepseek-r1-ollama.yml │ │ ├── config-deepseek-r1-quant1.58-longbench-byoe.yml │ │ ├── config-deepseek-r1-sglang.yml │ │ ├── config-deepseek-r1-vllm-convfinqa.yml │ │ ├── config-deepseek-r1-vllm-longbench.yml │ │ └── config-deepseek-r1-vllm-openorca.yml │ ├── eks_manifests │ │ ├── llama3-ray-service.yaml │ │ └── mistral-ray-service.yaml │ ├── embeddings │ │ ├── bge-base-en-v1-5-c5-embeddings.yml │ │ ├── bge-base-en-v1-5-g5-embeddings.yml │ │ └── bge-base-en-v1-5-g5-g4dn-c7-embeddings.yml │ ├── gemma │ │ └── config-gemma-2b-g5.yml │ ├── generic │ │ └── ec2 │ │ │ ├── Qwen2.5_djl.yml │ │ │ ├── djl.yml │ │ │ └── llama3.1_djl.yml │ ├── llama2 │ │ ├── 13b │ │ │ ├── config-bedrock-sagemaker-llama2.yml │ │ │ ├── config-byo-rest-ep-llama2-13b.yml │ │ │ ├── config-llama2-13b-inf2-g5-p4d.yml │ │ │ └── config-llama2-13b-inf2-g5.yml │ │ ├── 70b │ │ │ ├── config-ec2-llama2-70b.yml │ │ │ ├── config-llama2-70b-g5-p4d-tgi.yml │ │ │ ├── config-llama2-70b-g5-p4d-trt.yml │ │ │ └── config-llama2-70b-inf2-g5.yml │ │ └── 7b │ │ │ ├── config-llama2-7b-byo-sagemaker-endpoint.yml │ │ │ ├── config-llama2-7b-g4dn-g5-trt.yml │ │ │ ├── config-llama2-7b-g5-no-s3-quick.yml │ │ │ ├── config-llama2-7b-g5-quick.yml │ │ │ └── config-llama2-7b-inf2-g5.yml │ ├── llama3.1 │ │ ├── 70b │ │ │ ├── config-ec2-llama3-1-70b-inf2-48xl-deploy-ec2-djl.yml │ │ │ ├── config-ec2-llama3-1-70b-inf2-deploy-sm.yml │ │ │ ├── config-ec2-llama3-1-70b-p4de.24xl-deploy-ec2-large-prompts.yml │ │ │ ├── config-ec2-llama3-1-70b-p4de.24xl-deploy-ec2-longbench.yml │ │ │ ├── config-ec2-llama3-1-70b-p4de.24xl-deploy-ec2-summarization.yml │ │ │ ├── config-llama3-1-70b-g5.48xl-tp-8-mc-max-djl.yml │ │ │ ├── config-llama3-1-70b-g6.48xl-tp-8-mc-max-djl.yml │ │ │ ├── config-llama3-1-70b-g6e.24xl-tp-4-mc-max-djl.yml │ │ │ ├── config-llama3-1-70b-g6e.48xl-tp-8-mc-max-djl.yml │ │ │ ├── config-llama3-1-70b-inf2.48xl-deploy-ec2-triton.yml │ │ │ ├── config-llama3-1-70b-inf2.48xl-triton-tp24.yml │ │ │ ├── config-llama3-1-70b-p5-djl-lmi.yml │ │ │ ├── config-llama3-1-70b-trn1.32xl-deploy-ec2-triton.yml │ │ │ └── config-llama3-1-7b-inf2.48xl-triton-ec2.yml │ │ └── 8b │ │ │ ├── client-config-ec2-llama3-1-8b.yml │ │ │ ├── config-ec2-llama3-1-8b-g6e-2xlarge-byoe-ollama.yml │ │ │ ├── config-ec2-llama3-1-8b-inf2-48xl-deploy-ec2-tp24-bs12.yml │ │ │ ├── config-ec2-llama3-1-8b-inf2-48xl-deploy-ec2.yml │ │ │ ├── config-ec2-llama3-1-8b-inf2.yml │ │ │ ├── config-ec2-llama3-1-8b-p4-tp-2-mc-max.yml │ │ │ ├── config-ec2-llama3-1-8b-p4-tp-4-mc-max.yml │ │ │ ├── config-ec2-llama3-1-8b-p4-tp-8-mc-max.yml │ │ │ ├── config-ec2-llama3-1-8b-p5-tp-2-mc-max.yml │ │ │ ├── config-ec2-llama3-1-8b-tp-8-mc-auto-p5.yml │ │ │ ├── config-ec2-llama3-1-8b-trn1-32xl-deploy-ec2-tp32-bs8.yml │ │ │ ├── config-llama3-1-8b-g5.12xlarge-djl-lmi-sm.yml │ │ │ ├── config-llama3-1-8b-p5en-djl.yml │ │ │ ├── config-llama3.1-8b-g5-ec2.yml │ │ │ ├── config-llama3.1-8b-g5.12xl-tp-2-mc-auto-ec2.yml │ │ │ ├── config-llama3.1-8b-g5.12xl-tp-2-mc-max-ec2.yml │ │ │ ├── config-llama3.1-8b-g5.12xl-tp-4-mc-max-ec2.yml │ │ │ ├── config-llama3.1-8b-g5.24xl-tp-2-mc-auto-ec2.yml │ │ │ ├── config-llama3.1-8b-g5.24xl-tp-2-mc-max-ec2.yml │ │ │ ├── config-llama3.1-8b-g5.24xl-tp-4-mc-max-ec2.yml │ │ │ ├── config-llama3.1-8b-g5.2xl-g5.4xl-sm.yml │ │ │ ├── config-llama3.1-8b-g5.2xl-tp-1-mc-max-ec2-conc-1-2.yml │ │ │ ├── config-llama3.1-8b-g5.2xl-tp-1-mc-max-ec2.yml │ │ │ ├── config-llama3.1-8b-g5.48xl-tp-2-mc-auto-ec2.yml │ │ │ ├── config-llama3.1-8b-g5.48xl-tp-2-mc-max-ec2.yml │ │ │ ├── config-llama3.1-8b-g5.48xl-tp-4-mc-max-ec2.yml │ │ │ ├── config-llama3.1-8b-g5.48xl-tp-8-mc-max-ec2.yml │ │ │ ├── config-llama3.1-8b-g5.xl-tp-1-mc-max-ec2-conc-1-2.yml │ │ │ ├── config-llama3.1-8b-g5.yml │ │ │ ├── config-llama3.1-8b-g6e.12xl-tp-2-mc-max-djl.yml │ │ │ ├── config-llama3.1-8b-g6e.12xl-tp-4-mc-max-djl.yml │ │ │ ├── config-llama3.1-8b-g6e.24xl-tp-2-mc-max-djl.yml │ │ │ ├── config-llama3.1-8b-g6e.24xl-tp-4-mc-max-djl.yml │ │ │ ├── config-llama3.1-8b-g6e.2xl-ollama.yml │ │ │ ├── config-llama3.1-8b-g6e.2xl-tp-1-mc-max-djl.yml │ │ │ ├── config-llama3.1-8b-g6e.48xl-tp-2-mc-max-djl.yml │ │ │ ├── config-llama3.1-8b-g6e.48xl-tp-4-mc-max-djl.yml │ │ │ ├── config-llama3.1-8b-g6e.48xl-tp-8-mc-max-djl.yml │ │ │ ├── config-llama3.1-8b-g6e.4xl-tp-1-mc-max-djl.yml │ │ │ ├── config-llama3.1-8b-inf2-48xl-deploy-tp-24-ec2.yml │ │ │ ├── config-llama3.1-8b-inf2-48xl-deploy-tp-8-ec2.yml │ │ │ ├── config-llama3.1-8b-trn1-32xl-deploy-tp-8-ec2.yml │ │ │ ├── config-llama3.1-8b-trn32xl-triton-vllm.yml │ │ │ └── server-config-ec2-llama3-1-8b-inf2-48xl-deploy-ec2.yml │ ├── llama3.2 │ │ ├── 11b │ │ │ └── config-llama3.2-11b-g6e-2xl-tp-1-mc-max-djl-vllm-ec2.yml │ │ ├── 1b │ │ │ ├── config-llama3.2-1b-g5.2xl-summarization-500-50.yml │ │ │ ├── config-llama3.2-1b-g5.2xl-tp-1-mc-max-djl-ec2.yml │ │ │ ├── config-llama3.2-1b-g5.4xl-tp-1-mc-max-djl-ec2.yml │ │ │ ├── config-llama3.2-1b-g6e.2xl-tp-1-mc-max-djl.yml │ │ │ ├── config-llama3.2-1b-m5-16xlarge-ec2.yml │ │ │ ├── config-llama3.2-1b-m7a-16xlarge-ec2.yml │ │ │ ├── config-llama3.2-1b-m7a-24xlarge-ec2-summarization.yml │ │ │ ├── config-llama3.2-1b-m7a-24xlarge-ec2.yml │ │ │ └── config-llama3.2-1b-m7i-12xlarge-ec2.yml │ │ └── 3b │ │ │ └── config-llama3.2-3b-g5.4xl-tp-1-mc-max-djl-ec2.yml │ ├── llama3 │ │ ├── 70b │ │ │ ├── config-bedrock.yml │ │ │ ├── config-ec2-llama3-70b-instruct.yml │ │ │ ├── config-ec2-neuron-llama3-70b-inf2-48xl-deploy-sm.yml │ │ │ ├── config-llama3-70b-instruct-g5-48xl.yml │ │ │ ├── config-llama3-70b-instruct-g5-p4d.yml │ │ │ └── config-llama3-70b-instruct-p4d.yml │ │ └── 8b │ │ │ ├── config-bedrock.yml │ │ │ ├── config-ec2-llama3-8b-c5-18xlarge.yml │ │ │ ├── config-ec2-llama3-8b-c8g-24xlarge.yml │ │ │ ├── config-ec2-llama3-8b-g6e-2xlarge.yml │ │ │ ├── config-ec2-llama3-8b-inf2-48xl.yml │ │ │ ├── config-ec2-llama3-8b-m5-16xlarge.yml │ │ │ ├── config-ec2-llama3-8b-m7a-16xlarge.yml │ │ │ ├── config-ec2-llama3-8b-m7a-24xlarge.yml │ │ │ ├── config-ec2-llama3-8b-m7i-12xlarge.yml │ │ │ ├── config-ec2-llama3-8b-m7i-16xlarge.yml │ │ │ ├── config-ec2-llama3-8b-m7i-24xlarge.yml │ │ │ ├── config-ec2-llama3-8b-neuron-trn1-32xl-tp16-sm.yml │ │ │ ├── config-ec2-llama3-8b-p4d-tp-2-mc-max.yml │ │ │ ├── config-ec2-llama3-8b-p4d-tp-4-mc-max.yml │ │ │ ├── config-ec2-llama3-8b-p4d-tp-8-mc-max.yml │ │ │ ├── config-ec2-llama3-8b-p4de-tp-2-mc-max.yml │ │ │ ├── config-ec2-llama3-8b-p4de-tp-4-mc-max.yml │ │ │ ├── config-ec2-llama3-8b-p4de-tp-8-mc-max.yml │ │ │ ├── config-ec2-llama3-8b-p5-tp-2-mc-max.yml │ │ │ ├── config-ec2-llama3-8b-p5-tp-8-mc-auto.yml │ │ │ ├── config-ec2-llama3-8b.yml │ │ │ ├── config-ec2-neuron-llama3-8b-inf2-24xl-deploy-sm.yml │ │ │ ├── config-ec2-neuron-llama3-8b-inf2-48xl-deploy-sm.yml │ │ │ ├── config-llama3-8b-eks-inf2.yml │ │ │ ├── config-llama3-8b-g5-streaming.yml │ │ │ ├── config-llama3-8b-g5.12xl-tp-2-mc-max-djl-ec2.yml │ │ │ ├── config-llama3-8b-g5.12xl-tp-2-mc-max-triton-ec2.yml │ │ │ ├── config-llama3-8b-g5.12xl-tp-4-mc-max-djl-ec2.yml │ │ │ ├── config-llama3-8b-g5.12xl-tp-4-mc-max-triton-ec2.yml │ │ │ ├── config-llama3-8b-g5.48xl-tp-8-mc-max-triton-ec2.yml │ │ │ ├── config-llama3-8b-g5.4xl-tp-1-mc-max-triton-ec2.yml │ │ │ ├── config-llama3-8b-g6e.12xl-tp-2-mc-max-djl-ec2.yml │ │ │ ├── config-llama3-8b-g6e.12xl-tp-2-mc-max-triton-ec2.yml │ │ │ ├── config-llama3-8b-g6e.12xl-tp-4-mc-max-djl-ec2.yml │ │ │ ├── config-llama3-8b-g6e.12xl-tp-4-mc-max-triton-ec2.yml │ │ │ ├── config-llama3-8b-g6e.24xl-tp-2-mc-max-djl-ec2.yml │ │ │ ├── config-llama3-8b-g6e.24xl-tp-2-mc-max-triton-ec2.yml │ │ │ ├── config-llama3-8b-g6e.24xl-tp-4-mc-max-djl-ec2.yml │ │ │ ├── config-llama3-8b-g6e.24xl-tp-4-mc-max-triton-ec2.yml │ │ │ ├── config-llama3-8b-g6e.2xl-tp-1-mc-max-triton-ec2.yml │ │ │ ├── config-llama3-8b-g6e.48xl-tp-2-mc-max-djl-ec2.yml │ │ │ ├── config-llama3-8b-g6e.48xl-tp-2-mc-max-triton-ec2.yml │ │ │ ├── config-llama3-8b-g6e.48xl-tp-4-mc-max-djl-ec2.yml │ │ │ ├── config-llama3-8b-g6e.48xl-tp-4-mc-max-triton-ec2.yml │ │ │ ├── config-llama3-8b-g6e.48xl-tp-8-mc-max-djl-ec2.yml │ │ │ ├── config-llama3-8b-g6e.48xl-tp-8-mc-max-triton-ec2.yml │ │ │ ├── config-llama3-8b-g6e.4xl-tp-1-mc-max-djl-ec2.yml │ │ │ ├── config-llama3-8b-g6e.4xl-tp-1-mc-max-triton-ec2.yml │ │ │ ├── config-llama3-8b-g6e.xl-tp-1-mc-max-triton-ec2.yml │ │ │ ├── config-llama3-8b-inf2-24xl-tp=8-bs=4-byoe.yml │ │ │ ├── config-llama3-8b-inf2-48xl-tp=8-bs=4-byoe.yml │ │ │ ├── config-llama3-8b-inf2-48xlarge-triton-djl.yml │ │ │ ├── config-llama3-8b-inf2-g5-byoe-w-openorca.yml │ │ │ ├── config-llama3-8b-inf2-g5.yml │ │ │ ├── config-llama3-8b-instruct-all.yml │ │ │ ├── config-llama3-8b-instruct-g5-12xl-4-instances.yml │ │ │ ├── config-llama3-8b-instruct-g5-12xl.yml │ │ │ ├── config-llama3-8b-instruct-g5-24xl.yml │ │ │ ├── config-llama3-8b-instruct-g5-2xl.yml │ │ │ ├── config-llama3-8b-instruct-g5-48xl.yml │ │ │ ├── config-llama3-8b-instruct-g5-p4d.yml │ │ │ ├── config-llama3-8b-instruct-g6-12xl.yml │ │ │ ├── config-llama3-8b-instruct-g6-24xl.yml │ │ │ ├── config-llama3-8b-instruct-g6-48xl.yml │ │ │ ├── config-llama3-8b-instruct-p4d-djl-lmi-dist.yml │ │ │ ├── config-llama3-8b-instruct-p4d-djl-vllm.yml │ │ │ ├── config-llama3-8b-instruct-p5-djl-lmi-dist.yml │ │ │ ├── config-llama3-8b-trn1-32xl-tp-16-bs-4-byoe.yml │ │ │ ├── config-llama3-8b-trn1-32xl-tp-8-bs-4-byoe.yml │ │ │ ├── config-llama3-8b-trn1-32xl-tp16-bs-4-ec2.yml │ │ │ ├── config-llama3-8b-trn1-32xlarge-triton-djl.yml │ │ │ ├── config-llama3-8b-trn1-32xlarge-triton-vllm.yml │ │ │ ├── config-llama3-8b-trn1.yml │ │ │ ├── llama3-8b-inf2-24xl-byoe-g5-12xl.yml │ │ │ ├── llama3-8b-inf2-48xl-byoe-g5-24xl.yml │ │ │ └── llama3-8b-trn1-32xl-byoe-g5-24xl.yml │ ├── llama4 │ │ ├── config-llama4-scout-17b-g6e-longbench-vllm.yml │ │ ├── config-llama4-scout-17b-g6e-meetings-vllm.yml │ │ └── config-llama4-scout-17b-g6e-openorca-vllm.yml │ ├── mistral │ │ ├── config-mistral-7b-eks-inf2.yml │ │ ├── config-mistral-7b-tgi-g5.yml │ │ ├── config-mistral-7b-trn1-32xl-triton.yml │ │ ├── config-mistral-instruct-AWQ-p4d.yml │ │ ├── config-mistral-instruct-AWQ-p5-byo-ep.yml │ │ ├── config-mistral-instruct-AWQ-p5.yml │ │ ├── config-mistral-instruct-p4d.yml │ │ ├── config-mistral-instruct-v1-p5-trtllm.yml │ │ ├── config-mistral-instruct-v2-p4d-lmi-dist.yml │ │ ├── config-mistral-instruct-v2-p4d-trtllm.yml │ │ ├── config-mistral-instruct-v2-p5-lmi-dist.yml │ │ ├── config-mistral-instruct-v2-p5-trtllm.yml │ │ ├── config-mistral-trn1-32xl-deploy-ec2-tp32.yml │ │ └── config-mistral-v3-inf2-48xl-deploy-ec2-tp24.yml │ ├── mixtral │ │ └── config-mixtral-8x7b-g6e.48xl-ec2.yml │ ├── model_eval_all_info.yml │ ├── multimodal │ │ └── bedrock │ │ │ ├── config-claude-scienceqa.yml │ │ │ ├── config-llama-3-2-11b-vision-instruct-marqo-GS-10M.yml │ │ │ ├── config-llama-3-2-11b-vision-instruct-scienceqa.yml │ │ │ └── config-llama-3-2-claude-models-scienceqa.yml │ ├── phi │ │ └── config-phi-3-g5.yml │ ├── pricing.yml │ └── pricing_fallback.yml ├── defaults.py ├── globals.py ├── img │ └── llm_eval_flowchart.png ├── main.py ├── prompt_template │ ├── eval_criteria │ │ ├── claude_eval_prompt_templates │ │ │ └── claude_eval_majority_vote.txt │ │ ├── cohere_eval_prompt_templates │ │ │ └── cohere_eval_majority_vote.txt │ │ ├── evaluation_instructions_majority_vote.txt │ │ ├── llama3_eval_prompt_templates │ │ │ └── llama3_eval_majority_vote.txt │ │ └── mistral_eval_prompt_templates │ │ │ └── mistral_eval_majority_vote.txt │ ├── prompt_template_bert.txt │ ├── prompt_template_claude.txt │ ├── prompt_template_claude_OpenOrca.txt │ ├── prompt_template_claude_dolly_dataset.txt │ ├── prompt_template_claude_images_ScienceQA.txt │ ├── prompt_template_convfinqa.txt │ ├── prompt_template_deepseek_longbench.txt │ ├── prompt_template_gemma.txt │ ├── prompt_template_llama2.txt │ ├── prompt_template_llama2_Mistral_OpenOrca.txt │ ├── prompt_template_llama3.2_vision.txt │ ├── prompt_template_llama3.txt │ ├── prompt_template_llama3_OpenOrca.txt │ ├── prompt_template_llama3_OpenOrca_accuracy.txt │ ├── prompt_template_llama3_databricks-dolly-15k.txt │ ├── prompt_template_llama3_images_ScienceQA.txt │ ├── prompt_template_llama3_images_marqo-GS-10M.txt │ ├── prompt_template_llama3_meeting_summarization.txt │ ├── prompt_template_llama3_summarization.txt │ ├── prompt_template_mistral.txt │ ├── prompt_template_nova.txt │ ├── prompt_template_nova_convfinqa.txt │ ├── prompt_template_nova_dolly_dataset.txt │ ├── prompt_template_nova_open_orca.txt │ ├── prompt_template_open_orca.txt │ ├── prompt_template_qwen.txt │ └── prompt_template_titan_text.txt ├── requirements.txt ├── scripts │ ├── bedrock_predictor.py │ ├── bedrock_predictor_converseAPI.py │ ├── compile-llm-for-aws-silicon │ │ ├── requirements.txt │ │ ├── scripts │ │ │ ├── compile.py │ │ │ ├── download_compile_deploy.sh │ │ │ └── split_and_save.py │ │ └── smep-with-lmi │ │ │ └── deploy.py │ ├── constants.py │ ├── custom_rest_predictor.py │ ├── deploy_w_djl_serving.py │ ├── deploy_w_hf_tgi.py │ ├── ec2_deploy.py │ ├── ec2_metrics.py │ ├── ec2_predictor.py │ ├── eks_deploy.py │ ├── eks_predictor.py │ ├── fmbench_predictor.py │ ├── inference_containers │ │ ├── djl.py │ │ ├── ollama.py │ │ ├── sglang.py │ │ ├── triton.py │ │ ├── triton_serve_model.sh │ │ ├── utils.py │ │ ├── vllm.py │ │ └── vllm_gpu.py │ ├── jumpstart.py │ ├── neuron_deploy.py │ ├── neuron_metrics.py │ ├── ollama_predictor.py │ ├── prepare_for_multi_model_containers.py │ ├── pricing.py │ ├── rest_predictor.py │ ├── sagemaker_deploy.py │ ├── sagemaker_metrics.py │ ├── sagemaker_predictor.py │ ├── stream_responses.py │ └── triton │ │ ├── Dockerfile_triton │ │ ├── build_and_push_triton.sh │ │ ├── djl │ │ ├── config.pbtxt │ │ ├── model.py │ │ └── triton-djl-python-neuronx.sh │ │ └── vllm │ │ ├── config.pbtxt │ │ ├── triton-vllm-neuronx.sh │ │ └── vllm-neuron-issue-1.patch ├── tokenizer │ ├── .keep │ ├── config.json │ └── tokenizer.json └── utils.py ├── img ├── ML-FMBT-cloudformation-launch-stack.png ├── accuracy_trajectory_per_payload.png ├── business_summary.png ├── ec2-iam.png ├── ec2connect1.png ├── ec2connect10.png ├── ec2connect2.png ├── ec2connect3.png ├── ec2connect4.png ├── ec2connect5.png ├── ec2connect5a.png ├── ec2connect6.png ├── ec2connect7.png ├── ec2connect8.png ├── ec2connect9.png ├── ec2tag.png ├── fmbench-thumbnail.png ├── fmbench-website.png ├── fmbt-small.png ├── fmbt.png ├── overall_candidate_model_majority_voting_accuracy.png ├── results.gif └── tags.png ├── manifest.txt ├── misc ├── ec2_instance_creation_steps.md ├── eks_cluster-creation_steps.md └── the-diy-version-w-gory-details.md ├── mkdocs.yml ├── poetry.lock ├── pyproject.toml ├── release_history.md ├── render_fmbench_website.py ├── template.yml ├── tests ├── __init__.py └── test_pricing.py ├── uv.lock └── website ├── create_fmbench_website.py ├── index.md ├── mkdocs_template.yml └── nginx.conf.template /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/.gitignore -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/.pre-commit-config.yaml -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/CITATION.cff -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/CODE_OF_CONDUCT.md -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/CONTRIBUTING.md -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/Dockerfile -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/LICENSE -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/MANIFEST.in -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/README.md -------------------------------------------------------------------------------- /THIRD_PARTY_LICENSES.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/THIRD_PARTY_LICENSES.txt -------------------------------------------------------------------------------- /analytics/analytics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/analytics/analytics.py -------------------------------------------------------------------------------- /analytics/sagemaker_cost_rpm_plot.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/analytics/sagemaker_cost_rpm_plot.py -------------------------------------------------------------------------------- /analytics/sagemaker_metrics_plot.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/analytics/sagemaker_metrics_plot.py -------------------------------------------------------------------------------- /analytics/scratchpad.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/analytics/scratchpad.ipynb -------------------------------------------------------------------------------- /check_s3_content.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/check_s3_content.sh -------------------------------------------------------------------------------- /copy_s3_content.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/copy_s3_content.sh -------------------------------------------------------------------------------- /create_manifest.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/create_manifest.py -------------------------------------------------------------------------------- /debug.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/debug.sh -------------------------------------------------------------------------------- /docs/accuracy.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/accuracy.md -------------------------------------------------------------------------------- /docs/advanced.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/advanced.md -------------------------------------------------------------------------------- /docs/analytics.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/analytics.md -------------------------------------------------------------------------------- /docs/announcement.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/announcement.md -------------------------------------------------------------------------------- /docs/benchmarking.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/benchmarking.md -------------------------------------------------------------------------------- /docs/benchmarking_multimodal_models_on_bedrock.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/benchmarking_multimodal_models_on_bedrock.md -------------------------------------------------------------------------------- /docs/benchmarking_on_bedrock.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/benchmarking_on_bedrock.md -------------------------------------------------------------------------------- /docs/benchmarking_on_ec2.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/benchmarking_on_ec2.md -------------------------------------------------------------------------------- /docs/benchmarking_on_eks.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/benchmarking_on_eks.md -------------------------------------------------------------------------------- /docs/benchmarking_on_sagemaker.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/benchmarking_on_sagemaker.md -------------------------------------------------------------------------------- /docs/build.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/build.md -------------------------------------------------------------------------------- /docs/byo_dataset.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/byo_dataset.md -------------------------------------------------------------------------------- /docs/byo_rest_predictor.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/byo_rest_predictor.md -------------------------------------------------------------------------------- /docs/byoe.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/byoe.md -------------------------------------------------------------------------------- /docs/cli.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/cli.md -------------------------------------------------------------------------------- /docs/configs: -------------------------------------------------------------------------------- 1 | ../fmbench/configs -------------------------------------------------------------------------------- /docs/customize_config_files.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/customize_config_files.md -------------------------------------------------------------------------------- /docs/deepseek.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/deepseek.md -------------------------------------------------------------------------------- /docs/ec2.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/ec2.md -------------------------------------------------------------------------------- /docs/evaluation.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/features.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/features.md -------------------------------------------------------------------------------- /docs/gettingstarted.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/gettingstarted.md -------------------------------------------------------------------------------- /docs/img/ML-FMBT-cloudformation-launch-stack.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/img/ML-FMBT-cloudformation-launch-stack.png -------------------------------------------------------------------------------- /docs/img/accuracy_trajectory_per_payload.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/img/accuracy_trajectory_per_payload.png -------------------------------------------------------------------------------- /docs/img/business_summary.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/img/business_summary.png -------------------------------------------------------------------------------- /docs/img/config-structure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/img/config-structure.png -------------------------------------------------------------------------------- /docs/img/ec2-iam.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/img/ec2-iam.png -------------------------------------------------------------------------------- /docs/img/ec2connect1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/img/ec2connect1.png -------------------------------------------------------------------------------- /docs/img/ec2connect10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/img/ec2connect10.png -------------------------------------------------------------------------------- /docs/img/ec2connect2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/img/ec2connect2.png -------------------------------------------------------------------------------- /docs/img/ec2connect3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/img/ec2connect3.png -------------------------------------------------------------------------------- /docs/img/ec2connect4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/img/ec2connect4.png -------------------------------------------------------------------------------- /docs/img/ec2connect5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/img/ec2connect5.png -------------------------------------------------------------------------------- /docs/img/ec2connect5a.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/img/ec2connect5a.png -------------------------------------------------------------------------------- /docs/img/ec2connect6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/img/ec2connect6.png -------------------------------------------------------------------------------- /docs/img/ec2connect7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/img/ec2connect7.png -------------------------------------------------------------------------------- /docs/img/ec2connect8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/img/ec2connect8.png -------------------------------------------------------------------------------- /docs/img/ec2connect9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/img/ec2connect9.png -------------------------------------------------------------------------------- /docs/img/ec2tag.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/img/ec2tag.png -------------------------------------------------------------------------------- /docs/img/fmbench-deployment1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/img/fmbench-deployment1.png -------------------------------------------------------------------------------- /docs/img/fmbench-deployment2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/img/fmbench-deployment2.png -------------------------------------------------------------------------------- /docs/img/fmbench-thumbnail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/img/fmbench-thumbnail.png -------------------------------------------------------------------------------- /docs/img/fmbench-website.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/img/fmbench-website.png -------------------------------------------------------------------------------- /docs/img/fmbench.drawio: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/img/fmbench.drawio -------------------------------------------------------------------------------- /docs/img/fmbt-small.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/img/fmbt-small.png -------------------------------------------------------------------------------- /docs/img/fmbt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/img/fmbt.png -------------------------------------------------------------------------------- /docs/img/ground_truth_info.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/img/ground_truth_info.png -------------------------------------------------------------------------------- /docs/img/heatmap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/img/heatmap.png -------------------------------------------------------------------------------- /docs/img/hf_ds_keys.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/img/hf_ds_keys.png -------------------------------------------------------------------------------- /docs/img/latency_vs_tokens.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/img/latency_vs_tokens.png -------------------------------------------------------------------------------- /docs/img/overall_candidate_model_majority_voting_accuracy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/img/overall_candidate_model_majority_voting_accuracy.png -------------------------------------------------------------------------------- /docs/img/results.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/img/results.gif -------------------------------------------------------------------------------- /docs/img/website.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/img/website.png -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/index.md -------------------------------------------------------------------------------- /docs/manifest.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/manifest.md -------------------------------------------------------------------------------- /docs/misc/ec2_instance_creation_steps.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/misc/ec2_instance_creation_steps.md -------------------------------------------------------------------------------- /docs/misc/eks_cluster-creation_steps.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/misc/eks_cluster-creation_steps.md -------------------------------------------------------------------------------- /docs/misc/the-diy-version-w-gory-details.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/misc/the-diy-version-w-gory-details.md -------------------------------------------------------------------------------- /docs/mm_copies.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/mm_copies.md -------------------------------------------------------------------------------- /docs/neuron.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/neuron.md -------------------------------------------------------------------------------- /docs/quarto.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/quarto.md -------------------------------------------------------------------------------- /docs/quickstart.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/quickstart.md -------------------------------------------------------------------------------- /docs/releases.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/releases.md -------------------------------------------------------------------------------- /docs/resources.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/resources.md -------------------------------------------------------------------------------- /docs/results.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/results.md -------------------------------------------------------------------------------- /docs/run_as_container.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/run_as_container.md -------------------------------------------------------------------------------- /docs/simplified_config_files.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/simplified_config_files.md -------------------------------------------------------------------------------- /docs/website.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/website.md -------------------------------------------------------------------------------- /docs/workflow.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/docs/workflow.md -------------------------------------------------------------------------------- /ec2_template.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/ec2_template.yaml -------------------------------------------------------------------------------- /fmbench/0_setup.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/0_setup.ipynb -------------------------------------------------------------------------------- /fmbench/1_generate_data.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/1_generate_data.ipynb -------------------------------------------------------------------------------- /fmbench/2_deploy_model.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/2_deploy_model.ipynb -------------------------------------------------------------------------------- /fmbench/3_run_inference.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/3_run_inference.ipynb -------------------------------------------------------------------------------- /fmbench/4_get_evaluations.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/4_get_evaluations.ipynb -------------------------------------------------------------------------------- /fmbench/5_model_metric_analysis.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/5_model_metric_analysis.ipynb -------------------------------------------------------------------------------- /fmbench/6_cleanup.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/6_cleanup.ipynb -------------------------------------------------------------------------------- /fmbench/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/__init__.py -------------------------------------------------------------------------------- /fmbench/bring_your_own_dataset.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/bring_your_own_dataset.ipynb -------------------------------------------------------------------------------- /fmbench/config_filepath.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/config_filepath.txt -------------------------------------------------------------------------------- /fmbench/configs/NousResearchHermes70B/config-Nous-Hermes3-1-70b-g5.48xl-tp-8-mc-max-djl.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/NousResearchHermes70B/config-Nous-Hermes3-1-70b-g5.48xl-tp-8-mc-max-djl.yml -------------------------------------------------------------------------------- /fmbench/configs/NousResearchHermes70B/config-Nous-Hermes3-1-70b-inf2.48xl-triton-tp24.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/NousResearchHermes70B/config-Nous-Hermes3-1-70b-inf2.48xl-triton-tp24.yml -------------------------------------------------------------------------------- /fmbench/configs/NousResearchHermes70B/config-Nous-Hermes3-1-70b-trn1.32xl-deploy-ec2-triton.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/NousResearchHermes70B/config-Nous-Hermes3-1-70b-trn1.32xl-deploy-ec2-triton.yml -------------------------------------------------------------------------------- /fmbench/configs/bedrock/config-bedrock-all-anthropic-models-longbench-data.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/bedrock/config-bedrock-all-anthropic-models-longbench-data.yml -------------------------------------------------------------------------------- /fmbench/configs/bedrock/config-bedrock-anthropic-models-OpenOrca.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/bedrock/config-bedrock-anthropic-models-OpenOrca.yml -------------------------------------------------------------------------------- /fmbench/configs/bedrock/config-bedrock-claude.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/bedrock/config-bedrock-claude.yml -------------------------------------------------------------------------------- /fmbench/configs/bedrock/config-bedrock-evals-only-conc-1.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/bedrock/config-bedrock-evals-only-conc-1.yml -------------------------------------------------------------------------------- /fmbench/configs/bedrock/config-bedrock-haiku-sonnet-majority-voting.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/bedrock/config-bedrock-haiku-sonnet-majority-voting.yml -------------------------------------------------------------------------------- /fmbench/configs/bedrock/config-bedrock-llama3-1-70b-streaming.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/bedrock/config-bedrock-llama3-1-70b-streaming.yml -------------------------------------------------------------------------------- /fmbench/configs/bedrock/config-bedrock-llama3-1-8b-streaming.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/bedrock/config-bedrock-llama3-1-8b-streaming.yml -------------------------------------------------------------------------------- /fmbench/configs/bedrock/config-bedrock-llama3-1-no-streaming.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/bedrock/config-bedrock-llama3-1-no-streaming.yml -------------------------------------------------------------------------------- /fmbench/configs/bedrock/config-bedrock-llama3-1.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/bedrock/config-bedrock-llama3-1.yml -------------------------------------------------------------------------------- /fmbench/configs/bedrock/config-bedrock-llama3-streaming.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/bedrock/config-bedrock-llama3-streaming.yml -------------------------------------------------------------------------------- /fmbench/configs/bedrock/config-bedrock-models-OpenOrca.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/bedrock/config-bedrock-models-OpenOrca.yml -------------------------------------------------------------------------------- /fmbench/configs/bedrock/config-bedrock-titan-text-express.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/bedrock/config-bedrock-titan-text-express.yml -------------------------------------------------------------------------------- /fmbench/configs/bedrock/config-bedrock.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/bedrock/config-bedrock.yml -------------------------------------------------------------------------------- /fmbench/configs/bedrock/config-claude-3-5-sonnet-v2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/bedrock/config-claude-3-5-sonnet-v2.yml -------------------------------------------------------------------------------- /fmbench/configs/bedrock/config-claude-dolly-dataset.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/bedrock/config-claude-dolly-dataset.yml -------------------------------------------------------------------------------- /fmbench/configs/bedrock/config-llama-3-2-11b-databricks-dolly-15k.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/bedrock/config-llama-3-2-11b-databricks-dolly-15k.yml -------------------------------------------------------------------------------- /fmbench/configs/bedrock/config-llama-3-2-1b-3b-no-evals.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/bedrock/config-llama-3-2-1b-3b-no-evals.yml -------------------------------------------------------------------------------- /fmbench/configs/bedrock/config-llama-3-2-1b-3b.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/bedrock/config-llama-3-2-1b-3b.yml -------------------------------------------------------------------------------- /fmbench/configs/bedrock/config-llama-3-2-all-models-longbench-hf-version.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/bedrock/config-llama-3-2-all-models-longbench-hf-version.yml -------------------------------------------------------------------------------- /fmbench/configs/bedrock/config-llama-3-2-all-models.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/bedrock/config-llama-3-2-all-models.yml -------------------------------------------------------------------------------- /fmbench/configs/bedrock/config-llama-3-3-all-models-open-orca.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/bedrock/config-llama-3-3-all-models-open-orca.yml -------------------------------------------------------------------------------- /fmbench/configs/bedrock/config-llama-3-3-all-models.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/bedrock/config-llama-3-3-all-models.yml -------------------------------------------------------------------------------- /fmbench/configs/bedrock/config-nova-all-models-convfinqa.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/bedrock/config-nova-all-models-convfinqa.yml -------------------------------------------------------------------------------- /fmbench/configs/bedrock/config-nova-all-models-dolly-dataset.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/bedrock/config-nova-all-models-dolly-dataset.yml -------------------------------------------------------------------------------- /fmbench/configs/bedrock/config-nova-all-models-openarca.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/bedrock/config-nova-all-models-openarca.yml -------------------------------------------------------------------------------- /fmbench/configs/bedrock/config-nova-all-models.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/bedrock/config-nova-all-models.yml -------------------------------------------------------------------------------- /fmbench/configs/bert/config-distilbert-base-uncased.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/bert/config-distilbert-base-uncased.yml -------------------------------------------------------------------------------- /fmbench/configs/byoe/config-byo-custom-rest-predictor-tinyllama.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/byoe/config-byo-custom-rest-predictor-tinyllama.yml -------------------------------------------------------------------------------- /fmbench/configs/byoe/config-byo-custom-rest-predictor.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/byoe/config-byo-custom-rest-predictor.yml -------------------------------------------------------------------------------- /fmbench/configs/byoe/config-model-byo-sagemaker-endpoint.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/byoe/config-model-byo-sagemaker-endpoint.yml -------------------------------------------------------------------------------- /fmbench/configs/deepseek/config-deepseek-r1-ollama.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/deepseek/config-deepseek-r1-ollama.yml -------------------------------------------------------------------------------- /fmbench/configs/deepseek/config-deepseek-r1-quant1.58-longbench-byoe.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/deepseek/config-deepseek-r1-quant1.58-longbench-byoe.yml -------------------------------------------------------------------------------- /fmbench/configs/deepseek/config-deepseek-r1-sglang.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/deepseek/config-deepseek-r1-sglang.yml -------------------------------------------------------------------------------- /fmbench/configs/deepseek/config-deepseek-r1-vllm-convfinqa.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/deepseek/config-deepseek-r1-vllm-convfinqa.yml -------------------------------------------------------------------------------- /fmbench/configs/deepseek/config-deepseek-r1-vllm-longbench.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/deepseek/config-deepseek-r1-vllm-longbench.yml -------------------------------------------------------------------------------- /fmbench/configs/deepseek/config-deepseek-r1-vllm-openorca.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/deepseek/config-deepseek-r1-vllm-openorca.yml -------------------------------------------------------------------------------- /fmbench/configs/eks_manifests/llama3-ray-service.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/eks_manifests/llama3-ray-service.yaml -------------------------------------------------------------------------------- /fmbench/configs/eks_manifests/mistral-ray-service.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/eks_manifests/mistral-ray-service.yaml -------------------------------------------------------------------------------- /fmbench/configs/embeddings/bge-base-en-v1-5-c5-embeddings.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/embeddings/bge-base-en-v1-5-c5-embeddings.yml -------------------------------------------------------------------------------- /fmbench/configs/embeddings/bge-base-en-v1-5-g5-embeddings.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/embeddings/bge-base-en-v1-5-g5-embeddings.yml -------------------------------------------------------------------------------- /fmbench/configs/embeddings/bge-base-en-v1-5-g5-g4dn-c7-embeddings.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/embeddings/bge-base-en-v1-5-g5-g4dn-c7-embeddings.yml -------------------------------------------------------------------------------- /fmbench/configs/gemma/config-gemma-2b-g5.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/gemma/config-gemma-2b-g5.yml -------------------------------------------------------------------------------- /fmbench/configs/generic/ec2/Qwen2.5_djl.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/generic/ec2/Qwen2.5_djl.yml -------------------------------------------------------------------------------- /fmbench/configs/generic/ec2/djl.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/generic/ec2/djl.yml -------------------------------------------------------------------------------- /fmbench/configs/generic/ec2/llama3.1_djl.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/generic/ec2/llama3.1_djl.yml -------------------------------------------------------------------------------- /fmbench/configs/llama2/13b/config-bedrock-sagemaker-llama2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama2/13b/config-bedrock-sagemaker-llama2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama2/13b/config-byo-rest-ep-llama2-13b.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama2/13b/config-byo-rest-ep-llama2-13b.yml -------------------------------------------------------------------------------- /fmbench/configs/llama2/13b/config-llama2-13b-inf2-g5-p4d.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama2/13b/config-llama2-13b-inf2-g5-p4d.yml -------------------------------------------------------------------------------- /fmbench/configs/llama2/13b/config-llama2-13b-inf2-g5.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama2/13b/config-llama2-13b-inf2-g5.yml -------------------------------------------------------------------------------- /fmbench/configs/llama2/70b/config-ec2-llama2-70b.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama2/70b/config-ec2-llama2-70b.yml -------------------------------------------------------------------------------- /fmbench/configs/llama2/70b/config-llama2-70b-g5-p4d-tgi.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama2/70b/config-llama2-70b-g5-p4d-tgi.yml -------------------------------------------------------------------------------- /fmbench/configs/llama2/70b/config-llama2-70b-g5-p4d-trt.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama2/70b/config-llama2-70b-g5-p4d-trt.yml -------------------------------------------------------------------------------- /fmbench/configs/llama2/70b/config-llama2-70b-inf2-g5.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama2/70b/config-llama2-70b-inf2-g5.yml -------------------------------------------------------------------------------- /fmbench/configs/llama2/7b/config-llama2-7b-byo-sagemaker-endpoint.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama2/7b/config-llama2-7b-byo-sagemaker-endpoint.yml -------------------------------------------------------------------------------- /fmbench/configs/llama2/7b/config-llama2-7b-g4dn-g5-trt.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama2/7b/config-llama2-7b-g4dn-g5-trt.yml -------------------------------------------------------------------------------- /fmbench/configs/llama2/7b/config-llama2-7b-g5-no-s3-quick.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama2/7b/config-llama2-7b-g5-no-s3-quick.yml -------------------------------------------------------------------------------- /fmbench/configs/llama2/7b/config-llama2-7b-g5-quick.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama2/7b/config-llama2-7b-g5-quick.yml -------------------------------------------------------------------------------- /fmbench/configs/llama2/7b/config-llama2-7b-inf2-g5.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama2/7b/config-llama2-7b-inf2-g5.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/70b/config-ec2-llama3-1-70b-inf2-48xl-deploy-ec2-djl.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/70b/config-ec2-llama3-1-70b-inf2-48xl-deploy-ec2-djl.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/70b/config-ec2-llama3-1-70b-inf2-deploy-sm.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/70b/config-ec2-llama3-1-70b-inf2-deploy-sm.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/70b/config-ec2-llama3-1-70b-p4de.24xl-deploy-ec2-large-prompts.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/70b/config-ec2-llama3-1-70b-p4de.24xl-deploy-ec2-large-prompts.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/70b/config-ec2-llama3-1-70b-p4de.24xl-deploy-ec2-longbench.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/70b/config-ec2-llama3-1-70b-p4de.24xl-deploy-ec2-longbench.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/70b/config-ec2-llama3-1-70b-p4de.24xl-deploy-ec2-summarization.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/70b/config-ec2-llama3-1-70b-p4de.24xl-deploy-ec2-summarization.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/70b/config-llama3-1-70b-g5.48xl-tp-8-mc-max-djl.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/70b/config-llama3-1-70b-g5.48xl-tp-8-mc-max-djl.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/70b/config-llama3-1-70b-g6.48xl-tp-8-mc-max-djl.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/70b/config-llama3-1-70b-g6.48xl-tp-8-mc-max-djl.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/70b/config-llama3-1-70b-g6e.24xl-tp-4-mc-max-djl.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/70b/config-llama3-1-70b-g6e.24xl-tp-4-mc-max-djl.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/70b/config-llama3-1-70b-g6e.48xl-tp-8-mc-max-djl.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/70b/config-llama3-1-70b-g6e.48xl-tp-8-mc-max-djl.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/70b/config-llama3-1-70b-inf2.48xl-deploy-ec2-triton.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/70b/config-llama3-1-70b-inf2.48xl-deploy-ec2-triton.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/70b/config-llama3-1-70b-inf2.48xl-triton-tp24.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/70b/config-llama3-1-70b-inf2.48xl-triton-tp24.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/70b/config-llama3-1-70b-p5-djl-lmi.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/70b/config-llama3-1-70b-p5-djl-lmi.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/70b/config-llama3-1-70b-trn1.32xl-deploy-ec2-triton.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/70b/config-llama3-1-70b-trn1.32xl-deploy-ec2-triton.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/70b/config-llama3-1-7b-inf2.48xl-triton-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/70b/config-llama3-1-7b-inf2.48xl-triton-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/client-config-ec2-llama3-1-8b.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/client-config-ec2-llama3-1-8b.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-ec2-llama3-1-8b-g6e-2xlarge-byoe-ollama.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-ec2-llama3-1-8b-g6e-2xlarge-byoe-ollama.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-ec2-llama3-1-8b-inf2-48xl-deploy-ec2-tp24-bs12.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-ec2-llama3-1-8b-inf2-48xl-deploy-ec2-tp24-bs12.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-ec2-llama3-1-8b-inf2-48xl-deploy-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-ec2-llama3-1-8b-inf2-48xl-deploy-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-ec2-llama3-1-8b-inf2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-ec2-llama3-1-8b-inf2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-ec2-llama3-1-8b-p4-tp-2-mc-max.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-ec2-llama3-1-8b-p4-tp-2-mc-max.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-ec2-llama3-1-8b-p4-tp-4-mc-max.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-ec2-llama3-1-8b-p4-tp-4-mc-max.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-ec2-llama3-1-8b-p4-tp-8-mc-max.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-ec2-llama3-1-8b-p4-tp-8-mc-max.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-ec2-llama3-1-8b-p5-tp-2-mc-max.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-ec2-llama3-1-8b-p5-tp-2-mc-max.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-ec2-llama3-1-8b-tp-8-mc-auto-p5.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-ec2-llama3-1-8b-tp-8-mc-auto-p5.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-ec2-llama3-1-8b-trn1-32xl-deploy-ec2-tp32-bs8.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-ec2-llama3-1-8b-trn1-32xl-deploy-ec2-tp32-bs8.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-llama3-1-8b-g5.12xlarge-djl-lmi-sm.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-llama3-1-8b-g5.12xlarge-djl-lmi-sm.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-llama3-1-8b-p5en-djl.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-llama3-1-8b-p5en-djl.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.12xl-tp-2-mc-auto-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.12xl-tp-2-mc-auto-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.12xl-tp-2-mc-max-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.12xl-tp-2-mc-max-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.12xl-tp-4-mc-max-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.12xl-tp-4-mc-max-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.24xl-tp-2-mc-auto-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.24xl-tp-2-mc-auto-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.24xl-tp-2-mc-max-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.24xl-tp-2-mc-max-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.24xl-tp-4-mc-max-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.24xl-tp-4-mc-max-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.2xl-g5.4xl-sm.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.2xl-g5.4xl-sm.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.2xl-tp-1-mc-max-ec2-conc-1-2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.2xl-tp-1-mc-max-ec2-conc-1-2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.2xl-tp-1-mc-max-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.2xl-tp-1-mc-max-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.48xl-tp-2-mc-auto-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.48xl-tp-2-mc-auto-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.48xl-tp-2-mc-max-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.48xl-tp-2-mc-max-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.48xl-tp-4-mc-max-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.48xl-tp-4-mc-max-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.48xl-tp-8-mc-max-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.48xl-tp-8-mc-max-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.xl-tp-1-mc-max-ec2-conc-1-2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.xl-tp-1-mc-max-ec2-conc-1-2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-llama3.1-8b-g6e.12xl-tp-2-mc-max-djl.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-llama3.1-8b-g6e.12xl-tp-2-mc-max-djl.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-llama3.1-8b-g6e.12xl-tp-4-mc-max-djl.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-llama3.1-8b-g6e.12xl-tp-4-mc-max-djl.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-llama3.1-8b-g6e.24xl-tp-2-mc-max-djl.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-llama3.1-8b-g6e.24xl-tp-2-mc-max-djl.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-llama3.1-8b-g6e.24xl-tp-4-mc-max-djl.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-llama3.1-8b-g6e.24xl-tp-4-mc-max-djl.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-llama3.1-8b-g6e.2xl-ollama.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-llama3.1-8b-g6e.2xl-ollama.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-llama3.1-8b-g6e.2xl-tp-1-mc-max-djl.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-llama3.1-8b-g6e.2xl-tp-1-mc-max-djl.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-llama3.1-8b-g6e.48xl-tp-2-mc-max-djl.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-llama3.1-8b-g6e.48xl-tp-2-mc-max-djl.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-llama3.1-8b-g6e.48xl-tp-4-mc-max-djl.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-llama3.1-8b-g6e.48xl-tp-4-mc-max-djl.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-llama3.1-8b-g6e.48xl-tp-8-mc-max-djl.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-llama3.1-8b-g6e.48xl-tp-8-mc-max-djl.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-llama3.1-8b-g6e.4xl-tp-1-mc-max-djl.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-llama3.1-8b-g6e.4xl-tp-1-mc-max-djl.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-llama3.1-8b-inf2-48xl-deploy-tp-24-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-llama3.1-8b-inf2-48xl-deploy-tp-24-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-llama3.1-8b-inf2-48xl-deploy-tp-8-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-llama3.1-8b-inf2-48xl-deploy-tp-8-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-llama3.1-8b-trn1-32xl-deploy-tp-8-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-llama3.1-8b-trn1-32xl-deploy-tp-8-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/config-llama3.1-8b-trn32xl-triton-vllm.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/config-llama3.1-8b-trn32xl-triton-vllm.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.1/8b/server-config-ec2-llama3-1-8b-inf2-48xl-deploy-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.1/8b/server-config-ec2-llama3-1-8b-inf2-48xl-deploy-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.2/11b/config-llama3.2-11b-g6e-2xl-tp-1-mc-max-djl-vllm-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.2/11b/config-llama3.2-11b-g6e-2xl-tp-1-mc-max-djl-vllm-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.2/1b/config-llama3.2-1b-g5.2xl-summarization-500-50.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.2/1b/config-llama3.2-1b-g5.2xl-summarization-500-50.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.2/1b/config-llama3.2-1b-g5.2xl-tp-1-mc-max-djl-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.2/1b/config-llama3.2-1b-g5.2xl-tp-1-mc-max-djl-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.2/1b/config-llama3.2-1b-g5.4xl-tp-1-mc-max-djl-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.2/1b/config-llama3.2-1b-g5.4xl-tp-1-mc-max-djl-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.2/1b/config-llama3.2-1b-g6e.2xl-tp-1-mc-max-djl.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.2/1b/config-llama3.2-1b-g6e.2xl-tp-1-mc-max-djl.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.2/1b/config-llama3.2-1b-m5-16xlarge-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.2/1b/config-llama3.2-1b-m5-16xlarge-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.2/1b/config-llama3.2-1b-m7a-16xlarge-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.2/1b/config-llama3.2-1b-m7a-16xlarge-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.2/1b/config-llama3.2-1b-m7a-24xlarge-ec2-summarization.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.2/1b/config-llama3.2-1b-m7a-24xlarge-ec2-summarization.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.2/1b/config-llama3.2-1b-m7a-24xlarge-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.2/1b/config-llama3.2-1b-m7a-24xlarge-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.2/1b/config-llama3.2-1b-m7i-12xlarge-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.2/1b/config-llama3.2-1b-m7i-12xlarge-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3.2/3b/config-llama3.2-3b-g5.4xl-tp-1-mc-max-djl-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3.2/3b/config-llama3.2-3b-g5.4xl-tp-1-mc-max-djl-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/70b/config-bedrock.yml: -------------------------------------------------------------------------------- 1 | ../../bedrock/config-bedrock.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/70b/config-ec2-llama3-70b-instruct.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/70b/config-ec2-llama3-70b-instruct.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/70b/config-ec2-neuron-llama3-70b-inf2-48xl-deploy-sm.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/70b/config-ec2-neuron-llama3-70b-inf2-48xl-deploy-sm.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/70b/config-llama3-70b-instruct-g5-48xl.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/70b/config-llama3-70b-instruct-g5-48xl.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/70b/config-llama3-70b-instruct-g5-p4d.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/70b/config-llama3-70b-instruct-g5-p4d.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/70b/config-llama3-70b-instruct-p4d.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/70b/config-llama3-70b-instruct-p4d.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-bedrock.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-bedrock.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-ec2-llama3-8b-c5-18xlarge.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-ec2-llama3-8b-c5-18xlarge.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-ec2-llama3-8b-c8g-24xlarge.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-ec2-llama3-8b-c8g-24xlarge.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-ec2-llama3-8b-g6e-2xlarge.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-ec2-llama3-8b-g6e-2xlarge.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-ec2-llama3-8b-inf2-48xl.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-ec2-llama3-8b-inf2-48xl.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-ec2-llama3-8b-m5-16xlarge.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-ec2-llama3-8b-m5-16xlarge.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-ec2-llama3-8b-m7a-16xlarge.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-ec2-llama3-8b-m7a-16xlarge.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-ec2-llama3-8b-m7a-24xlarge.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-ec2-llama3-8b-m7a-24xlarge.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-ec2-llama3-8b-m7i-12xlarge.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-ec2-llama3-8b-m7i-12xlarge.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-ec2-llama3-8b-m7i-16xlarge.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-ec2-llama3-8b-m7i-16xlarge.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-ec2-llama3-8b-m7i-24xlarge.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-ec2-llama3-8b-m7i-24xlarge.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-ec2-llama3-8b-neuron-trn1-32xl-tp16-sm.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-ec2-llama3-8b-neuron-trn1-32xl-tp16-sm.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-ec2-llama3-8b-p4d-tp-2-mc-max.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-ec2-llama3-8b-p4d-tp-2-mc-max.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-ec2-llama3-8b-p4d-tp-4-mc-max.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-ec2-llama3-8b-p4d-tp-4-mc-max.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-ec2-llama3-8b-p4d-tp-8-mc-max.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-ec2-llama3-8b-p4d-tp-8-mc-max.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-ec2-llama3-8b-p4de-tp-2-mc-max.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-ec2-llama3-8b-p4de-tp-2-mc-max.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-ec2-llama3-8b-p4de-tp-4-mc-max.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-ec2-llama3-8b-p4de-tp-4-mc-max.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-ec2-llama3-8b-p4de-tp-8-mc-max.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-ec2-llama3-8b-p4de-tp-8-mc-max.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-ec2-llama3-8b-p5-tp-2-mc-max.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-ec2-llama3-8b-p5-tp-2-mc-max.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-ec2-llama3-8b-p5-tp-8-mc-auto.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-ec2-llama3-8b-p5-tp-8-mc-auto.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-ec2-llama3-8b.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-ec2-llama3-8b.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-ec2-neuron-llama3-8b-inf2-24xl-deploy-sm.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-ec2-neuron-llama3-8b-inf2-24xl-deploy-sm.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-ec2-neuron-llama3-8b-inf2-48xl-deploy-sm.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-ec2-neuron-llama3-8b-inf2-48xl-deploy-sm.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-eks-inf2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-eks-inf2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-g5-streaming.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-g5-streaming.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-g5.12xl-tp-2-mc-max-djl-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-g5.12xl-tp-2-mc-max-djl-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-g5.12xl-tp-2-mc-max-triton-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-g5.12xl-tp-2-mc-max-triton-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-g5.12xl-tp-4-mc-max-djl-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-g5.12xl-tp-4-mc-max-djl-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-g5.12xl-tp-4-mc-max-triton-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-g5.12xl-tp-4-mc-max-triton-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-g5.48xl-tp-8-mc-max-triton-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-g5.48xl-tp-8-mc-max-triton-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-g5.4xl-tp-1-mc-max-triton-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-g5.4xl-tp-1-mc-max-triton-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-g6e.12xl-tp-2-mc-max-djl-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-g6e.12xl-tp-2-mc-max-djl-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-g6e.12xl-tp-2-mc-max-triton-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-g6e.12xl-tp-2-mc-max-triton-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-g6e.12xl-tp-4-mc-max-djl-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-g6e.12xl-tp-4-mc-max-djl-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-g6e.12xl-tp-4-mc-max-triton-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-g6e.12xl-tp-4-mc-max-triton-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-g6e.24xl-tp-2-mc-max-djl-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-g6e.24xl-tp-2-mc-max-djl-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-g6e.24xl-tp-2-mc-max-triton-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-g6e.24xl-tp-2-mc-max-triton-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-g6e.24xl-tp-4-mc-max-djl-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-g6e.24xl-tp-4-mc-max-djl-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-g6e.24xl-tp-4-mc-max-triton-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-g6e.24xl-tp-4-mc-max-triton-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-g6e.2xl-tp-1-mc-max-triton-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-g6e.2xl-tp-1-mc-max-triton-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-g6e.48xl-tp-2-mc-max-djl-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-g6e.48xl-tp-2-mc-max-djl-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-g6e.48xl-tp-2-mc-max-triton-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-g6e.48xl-tp-2-mc-max-triton-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-g6e.48xl-tp-4-mc-max-djl-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-g6e.48xl-tp-4-mc-max-djl-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-g6e.48xl-tp-4-mc-max-triton-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-g6e.48xl-tp-4-mc-max-triton-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-g6e.48xl-tp-8-mc-max-djl-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-g6e.48xl-tp-8-mc-max-djl-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-g6e.48xl-tp-8-mc-max-triton-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-g6e.48xl-tp-8-mc-max-triton-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-g6e.4xl-tp-1-mc-max-djl-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-g6e.4xl-tp-1-mc-max-djl-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-g6e.4xl-tp-1-mc-max-triton-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-g6e.4xl-tp-1-mc-max-triton-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-g6e.xl-tp-1-mc-max-triton-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-g6e.xl-tp-1-mc-max-triton-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-inf2-24xl-tp=8-bs=4-byoe.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-inf2-24xl-tp=8-bs=4-byoe.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-inf2-48xl-tp=8-bs=4-byoe.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-inf2-48xl-tp=8-bs=4-byoe.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-inf2-48xlarge-triton-djl.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-inf2-48xlarge-triton-djl.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-inf2-g5-byoe-w-openorca.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-inf2-g5-byoe-w-openorca.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-inf2-g5.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-inf2-g5.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-instruct-all.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-instruct-all.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-instruct-g5-12xl-4-instances.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-instruct-g5-12xl-4-instances.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-instruct-g5-12xl.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-instruct-g5-12xl.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-instruct-g5-24xl.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-instruct-g5-24xl.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-instruct-g5-2xl.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-instruct-g5-2xl.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-instruct-g5-48xl.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-instruct-g5-48xl.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-instruct-g5-p4d.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-instruct-g5-p4d.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-instruct-g6-12xl.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-instruct-g6-12xl.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-instruct-g6-24xl.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-instruct-g6-24xl.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-instruct-g6-48xl.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-instruct-g6-48xl.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-instruct-p4d-djl-lmi-dist.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-instruct-p4d-djl-lmi-dist.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-instruct-p4d-djl-vllm.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-instruct-p4d-djl-vllm.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-instruct-p5-djl-lmi-dist.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-instruct-p5-djl-lmi-dist.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-trn1-32xl-tp-16-bs-4-byoe.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-trn1-32xl-tp-16-bs-4-byoe.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-trn1-32xl-tp-8-bs-4-byoe.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-trn1-32xl-tp-8-bs-4-byoe.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-trn1-32xl-tp16-bs-4-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-trn1-32xl-tp16-bs-4-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-trn1-32xlarge-triton-djl.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-trn1-32xlarge-triton-djl.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-trn1-32xlarge-triton-vllm.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-trn1-32xlarge-triton-vllm.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/config-llama3-8b-trn1.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/config-llama3-8b-trn1.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/llama3-8b-inf2-24xl-byoe-g5-12xl.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/llama3-8b-inf2-24xl-byoe-g5-12xl.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/llama3-8b-inf2-48xl-byoe-g5-24xl.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/llama3-8b-inf2-48xl-byoe-g5-24xl.yml -------------------------------------------------------------------------------- /fmbench/configs/llama3/8b/llama3-8b-trn1-32xl-byoe-g5-24xl.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama3/8b/llama3-8b-trn1-32xl-byoe-g5-24xl.yml -------------------------------------------------------------------------------- /fmbench/configs/llama4/config-llama4-scout-17b-g6e-longbench-vllm.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama4/config-llama4-scout-17b-g6e-longbench-vllm.yml -------------------------------------------------------------------------------- /fmbench/configs/llama4/config-llama4-scout-17b-g6e-meetings-vllm.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama4/config-llama4-scout-17b-g6e-meetings-vllm.yml -------------------------------------------------------------------------------- /fmbench/configs/llama4/config-llama4-scout-17b-g6e-openorca-vllm.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/llama4/config-llama4-scout-17b-g6e-openorca-vllm.yml -------------------------------------------------------------------------------- /fmbench/configs/mistral/config-mistral-7b-eks-inf2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/mistral/config-mistral-7b-eks-inf2.yml -------------------------------------------------------------------------------- /fmbench/configs/mistral/config-mistral-7b-tgi-g5.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/mistral/config-mistral-7b-tgi-g5.yml -------------------------------------------------------------------------------- /fmbench/configs/mistral/config-mistral-7b-trn1-32xl-triton.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/mistral/config-mistral-7b-trn1-32xl-triton.yml -------------------------------------------------------------------------------- /fmbench/configs/mistral/config-mistral-instruct-AWQ-p4d.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/mistral/config-mistral-instruct-AWQ-p4d.yml -------------------------------------------------------------------------------- /fmbench/configs/mistral/config-mistral-instruct-AWQ-p5-byo-ep.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/mistral/config-mistral-instruct-AWQ-p5-byo-ep.yml -------------------------------------------------------------------------------- /fmbench/configs/mistral/config-mistral-instruct-AWQ-p5.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/mistral/config-mistral-instruct-AWQ-p5.yml -------------------------------------------------------------------------------- /fmbench/configs/mistral/config-mistral-instruct-p4d.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/mistral/config-mistral-instruct-p4d.yml -------------------------------------------------------------------------------- /fmbench/configs/mistral/config-mistral-instruct-v1-p5-trtllm.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/mistral/config-mistral-instruct-v1-p5-trtllm.yml -------------------------------------------------------------------------------- /fmbench/configs/mistral/config-mistral-instruct-v2-p4d-lmi-dist.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/mistral/config-mistral-instruct-v2-p4d-lmi-dist.yml -------------------------------------------------------------------------------- /fmbench/configs/mistral/config-mistral-instruct-v2-p4d-trtllm.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/mistral/config-mistral-instruct-v2-p4d-trtllm.yml -------------------------------------------------------------------------------- /fmbench/configs/mistral/config-mistral-instruct-v2-p5-lmi-dist.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/mistral/config-mistral-instruct-v2-p5-lmi-dist.yml -------------------------------------------------------------------------------- /fmbench/configs/mistral/config-mistral-instruct-v2-p5-trtllm.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/mistral/config-mistral-instruct-v2-p5-trtllm.yml -------------------------------------------------------------------------------- /fmbench/configs/mistral/config-mistral-trn1-32xl-deploy-ec2-tp32.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/mistral/config-mistral-trn1-32xl-deploy-ec2-tp32.yml -------------------------------------------------------------------------------- /fmbench/configs/mistral/config-mistral-v3-inf2-48xl-deploy-ec2-tp24.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/mistral/config-mistral-v3-inf2-48xl-deploy-ec2-tp24.yml -------------------------------------------------------------------------------- /fmbench/configs/mixtral/config-mixtral-8x7b-g6e.48xl-ec2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/mixtral/config-mixtral-8x7b-g6e.48xl-ec2.yml -------------------------------------------------------------------------------- /fmbench/configs/model_eval_all_info.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/model_eval_all_info.yml -------------------------------------------------------------------------------- /fmbench/configs/multimodal/bedrock/config-claude-scienceqa.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/multimodal/bedrock/config-claude-scienceqa.yml -------------------------------------------------------------------------------- /fmbench/configs/multimodal/bedrock/config-llama-3-2-11b-vision-instruct-marqo-GS-10M.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/multimodal/bedrock/config-llama-3-2-11b-vision-instruct-marqo-GS-10M.yml -------------------------------------------------------------------------------- /fmbench/configs/multimodal/bedrock/config-llama-3-2-11b-vision-instruct-scienceqa.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/multimodal/bedrock/config-llama-3-2-11b-vision-instruct-scienceqa.yml -------------------------------------------------------------------------------- /fmbench/configs/multimodal/bedrock/config-llama-3-2-claude-models-scienceqa.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/multimodal/bedrock/config-llama-3-2-claude-models-scienceqa.yml -------------------------------------------------------------------------------- /fmbench/configs/phi/config-phi-3-g5.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/phi/config-phi-3-g5.yml -------------------------------------------------------------------------------- /fmbench/configs/pricing.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/pricing.yml -------------------------------------------------------------------------------- /fmbench/configs/pricing_fallback.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/configs/pricing_fallback.yml -------------------------------------------------------------------------------- /fmbench/defaults.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/defaults.py -------------------------------------------------------------------------------- /fmbench/globals.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/globals.py -------------------------------------------------------------------------------- /fmbench/img/llm_eval_flowchart.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/img/llm_eval_flowchart.png -------------------------------------------------------------------------------- /fmbench/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/main.py -------------------------------------------------------------------------------- /fmbench/prompt_template/eval_criteria/claude_eval_prompt_templates/claude_eval_majority_vote.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/prompt_template/eval_criteria/claude_eval_prompt_templates/claude_eval_majority_vote.txt -------------------------------------------------------------------------------- /fmbench/prompt_template/eval_criteria/cohere_eval_prompt_templates/cohere_eval_majority_vote.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/prompt_template/eval_criteria/cohere_eval_prompt_templates/cohere_eval_majority_vote.txt -------------------------------------------------------------------------------- /fmbench/prompt_template/eval_criteria/evaluation_instructions_majority_vote.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/prompt_template/eval_criteria/evaluation_instructions_majority_vote.txt -------------------------------------------------------------------------------- /fmbench/prompt_template/eval_criteria/llama3_eval_prompt_templates/llama3_eval_majority_vote.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/prompt_template/eval_criteria/llama3_eval_prompt_templates/llama3_eval_majority_vote.txt -------------------------------------------------------------------------------- /fmbench/prompt_template/eval_criteria/mistral_eval_prompt_templates/mistral_eval_majority_vote.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/prompt_template/eval_criteria/mistral_eval_prompt_templates/mistral_eval_majority_vote.txt -------------------------------------------------------------------------------- /fmbench/prompt_template/prompt_template_bert.txt: -------------------------------------------------------------------------------- 1 | {text} 2 | -------------------------------------------------------------------------------- /fmbench/prompt_template/prompt_template_claude.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/prompt_template/prompt_template_claude.txt -------------------------------------------------------------------------------- /fmbench/prompt_template/prompt_template_claude_OpenOrca.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/prompt_template/prompt_template_claude_OpenOrca.txt -------------------------------------------------------------------------------- /fmbench/prompt_template/prompt_template_claude_dolly_dataset.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/prompt_template/prompt_template_claude_dolly_dataset.txt -------------------------------------------------------------------------------- /fmbench/prompt_template/prompt_template_claude_images_ScienceQA.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/prompt_template/prompt_template_claude_images_ScienceQA.txt -------------------------------------------------------------------------------- /fmbench/prompt_template/prompt_template_convfinqa.txt: -------------------------------------------------------------------------------- 1 | {input} 2 | -------------------------------------------------------------------------------- /fmbench/prompt_template/prompt_template_deepseek_longbench.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/prompt_template/prompt_template_deepseek_longbench.txt -------------------------------------------------------------------------------- /fmbench/prompt_template/prompt_template_gemma.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/prompt_template/prompt_template_gemma.txt -------------------------------------------------------------------------------- /fmbench/prompt_template/prompt_template_llama2.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/prompt_template/prompt_template_llama2.txt -------------------------------------------------------------------------------- /fmbench/prompt_template/prompt_template_llama2_Mistral_OpenOrca.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/prompt_template/prompt_template_llama2_Mistral_OpenOrca.txt -------------------------------------------------------------------------------- /fmbench/prompt_template/prompt_template_llama3.2_vision.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/prompt_template/prompt_template_llama3.2_vision.txt -------------------------------------------------------------------------------- /fmbench/prompt_template/prompt_template_llama3.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/prompt_template/prompt_template_llama3.txt -------------------------------------------------------------------------------- /fmbench/prompt_template/prompt_template_llama3_OpenOrca.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/prompt_template/prompt_template_llama3_OpenOrca.txt -------------------------------------------------------------------------------- /fmbench/prompt_template/prompt_template_llama3_OpenOrca_accuracy.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/prompt_template/prompt_template_llama3_OpenOrca_accuracy.txt -------------------------------------------------------------------------------- /fmbench/prompt_template/prompt_template_llama3_databricks-dolly-15k.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/prompt_template/prompt_template_llama3_databricks-dolly-15k.txt -------------------------------------------------------------------------------- /fmbench/prompt_template/prompt_template_llama3_images_ScienceQA.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/prompt_template/prompt_template_llama3_images_ScienceQA.txt -------------------------------------------------------------------------------- /fmbench/prompt_template/prompt_template_llama3_images_marqo-GS-10M.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/prompt_template/prompt_template_llama3_images_marqo-GS-10M.txt -------------------------------------------------------------------------------- /fmbench/prompt_template/prompt_template_llama3_meeting_summarization.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/prompt_template/prompt_template_llama3_meeting_summarization.txt -------------------------------------------------------------------------------- /fmbench/prompt_template/prompt_template_llama3_summarization.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/prompt_template/prompt_template_llama3_summarization.txt -------------------------------------------------------------------------------- /fmbench/prompt_template/prompt_template_mistral.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/prompt_template/prompt_template_mistral.txt -------------------------------------------------------------------------------- /fmbench/prompt_template/prompt_template_nova.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/prompt_template/prompt_template_nova.txt -------------------------------------------------------------------------------- /fmbench/prompt_template/prompt_template_nova_convfinqa.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/prompt_template/prompt_template_nova_convfinqa.txt -------------------------------------------------------------------------------- /fmbench/prompt_template/prompt_template_nova_dolly_dataset.txt: -------------------------------------------------------------------------------- 1 | {instruction} -------------------------------------------------------------------------------- /fmbench/prompt_template/prompt_template_nova_open_orca.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/prompt_template/prompt_template_nova_open_orca.txt -------------------------------------------------------------------------------- /fmbench/prompt_template/prompt_template_open_orca.txt: -------------------------------------------------------------------------------- 1 | {question} 2 | -------------------------------------------------------------------------------- /fmbench/prompt_template/prompt_template_qwen.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/prompt_template/prompt_template_qwen.txt -------------------------------------------------------------------------------- /fmbench/prompt_template/prompt_template_titan_text.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/prompt_template/prompt_template_titan_text.txt -------------------------------------------------------------------------------- /fmbench/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/requirements.txt -------------------------------------------------------------------------------- /fmbench/scripts/bedrock_predictor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/bedrock_predictor.py -------------------------------------------------------------------------------- /fmbench/scripts/bedrock_predictor_converseAPI.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/bedrock_predictor_converseAPI.py -------------------------------------------------------------------------------- /fmbench/scripts/compile-llm-for-aws-silicon/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/compile-llm-for-aws-silicon/requirements.txt -------------------------------------------------------------------------------- /fmbench/scripts/compile-llm-for-aws-silicon/scripts/compile.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/compile-llm-for-aws-silicon/scripts/compile.py -------------------------------------------------------------------------------- /fmbench/scripts/compile-llm-for-aws-silicon/scripts/download_compile_deploy.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/compile-llm-for-aws-silicon/scripts/download_compile_deploy.sh -------------------------------------------------------------------------------- /fmbench/scripts/compile-llm-for-aws-silicon/scripts/split_and_save.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/compile-llm-for-aws-silicon/scripts/split_and_save.py -------------------------------------------------------------------------------- /fmbench/scripts/compile-llm-for-aws-silicon/smep-with-lmi/deploy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/compile-llm-for-aws-silicon/smep-with-lmi/deploy.py -------------------------------------------------------------------------------- /fmbench/scripts/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/constants.py -------------------------------------------------------------------------------- /fmbench/scripts/custom_rest_predictor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/custom_rest_predictor.py -------------------------------------------------------------------------------- /fmbench/scripts/deploy_w_djl_serving.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/deploy_w_djl_serving.py -------------------------------------------------------------------------------- /fmbench/scripts/deploy_w_hf_tgi.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/deploy_w_hf_tgi.py -------------------------------------------------------------------------------- /fmbench/scripts/ec2_deploy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/ec2_deploy.py -------------------------------------------------------------------------------- /fmbench/scripts/ec2_metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/ec2_metrics.py -------------------------------------------------------------------------------- /fmbench/scripts/ec2_predictor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/ec2_predictor.py -------------------------------------------------------------------------------- /fmbench/scripts/eks_deploy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/eks_deploy.py -------------------------------------------------------------------------------- /fmbench/scripts/eks_predictor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/eks_predictor.py -------------------------------------------------------------------------------- /fmbench/scripts/fmbench_predictor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/fmbench_predictor.py -------------------------------------------------------------------------------- /fmbench/scripts/inference_containers/djl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/inference_containers/djl.py -------------------------------------------------------------------------------- /fmbench/scripts/inference_containers/ollama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/inference_containers/ollama.py -------------------------------------------------------------------------------- /fmbench/scripts/inference_containers/sglang.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/inference_containers/sglang.py -------------------------------------------------------------------------------- /fmbench/scripts/inference_containers/triton.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/inference_containers/triton.py -------------------------------------------------------------------------------- /fmbench/scripts/inference_containers/triton_serve_model.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/inference_containers/triton_serve_model.sh -------------------------------------------------------------------------------- /fmbench/scripts/inference_containers/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/inference_containers/utils.py -------------------------------------------------------------------------------- /fmbench/scripts/inference_containers/vllm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/inference_containers/vllm.py -------------------------------------------------------------------------------- /fmbench/scripts/inference_containers/vllm_gpu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/inference_containers/vllm_gpu.py -------------------------------------------------------------------------------- /fmbench/scripts/jumpstart.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/jumpstart.py -------------------------------------------------------------------------------- /fmbench/scripts/neuron_deploy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/neuron_deploy.py -------------------------------------------------------------------------------- /fmbench/scripts/neuron_metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/neuron_metrics.py -------------------------------------------------------------------------------- /fmbench/scripts/ollama_predictor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/ollama_predictor.py -------------------------------------------------------------------------------- /fmbench/scripts/prepare_for_multi_model_containers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/prepare_for_multi_model_containers.py -------------------------------------------------------------------------------- /fmbench/scripts/pricing.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/pricing.py -------------------------------------------------------------------------------- /fmbench/scripts/rest_predictor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/rest_predictor.py -------------------------------------------------------------------------------- /fmbench/scripts/sagemaker_deploy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/sagemaker_deploy.py -------------------------------------------------------------------------------- /fmbench/scripts/sagemaker_metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/sagemaker_metrics.py -------------------------------------------------------------------------------- /fmbench/scripts/sagemaker_predictor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/sagemaker_predictor.py -------------------------------------------------------------------------------- /fmbench/scripts/stream_responses.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/stream_responses.py -------------------------------------------------------------------------------- /fmbench/scripts/triton/Dockerfile_triton: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/triton/Dockerfile_triton -------------------------------------------------------------------------------- /fmbench/scripts/triton/build_and_push_triton.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/triton/build_and_push_triton.sh -------------------------------------------------------------------------------- /fmbench/scripts/triton/djl/config.pbtxt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/triton/djl/config.pbtxt -------------------------------------------------------------------------------- /fmbench/scripts/triton/djl/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/triton/djl/model.py -------------------------------------------------------------------------------- /fmbench/scripts/triton/djl/triton-djl-python-neuronx.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/triton/djl/triton-djl-python-neuronx.sh -------------------------------------------------------------------------------- /fmbench/scripts/triton/vllm/config.pbtxt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/triton/vllm/config.pbtxt -------------------------------------------------------------------------------- /fmbench/scripts/triton/vllm/triton-vllm-neuronx.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/triton/vllm/triton-vllm-neuronx.sh -------------------------------------------------------------------------------- /fmbench/scripts/triton/vllm/vllm-neuron-issue-1.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/scripts/triton/vllm/vllm-neuron-issue-1.patch -------------------------------------------------------------------------------- /fmbench/tokenizer/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /fmbench/tokenizer/config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/tokenizer/config.json -------------------------------------------------------------------------------- /fmbench/tokenizer/tokenizer.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/tokenizer/tokenizer.json -------------------------------------------------------------------------------- /fmbench/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/fmbench/utils.py -------------------------------------------------------------------------------- /img/ML-FMBT-cloudformation-launch-stack.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/img/ML-FMBT-cloudformation-launch-stack.png -------------------------------------------------------------------------------- /img/accuracy_trajectory_per_payload.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/img/accuracy_trajectory_per_payload.png -------------------------------------------------------------------------------- /img/business_summary.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/img/business_summary.png -------------------------------------------------------------------------------- /img/ec2-iam.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/img/ec2-iam.png -------------------------------------------------------------------------------- /img/ec2connect1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/img/ec2connect1.png -------------------------------------------------------------------------------- /img/ec2connect10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/img/ec2connect10.png -------------------------------------------------------------------------------- /img/ec2connect2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/img/ec2connect2.png -------------------------------------------------------------------------------- /img/ec2connect3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/img/ec2connect3.png -------------------------------------------------------------------------------- /img/ec2connect4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/img/ec2connect4.png -------------------------------------------------------------------------------- /img/ec2connect5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/img/ec2connect5.png -------------------------------------------------------------------------------- /img/ec2connect5a.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/img/ec2connect5a.png -------------------------------------------------------------------------------- /img/ec2connect6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/img/ec2connect6.png -------------------------------------------------------------------------------- /img/ec2connect7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/img/ec2connect7.png -------------------------------------------------------------------------------- /img/ec2connect8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/img/ec2connect8.png -------------------------------------------------------------------------------- /img/ec2connect9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/img/ec2connect9.png -------------------------------------------------------------------------------- /img/ec2tag.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/img/ec2tag.png -------------------------------------------------------------------------------- /img/fmbench-thumbnail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/img/fmbench-thumbnail.png -------------------------------------------------------------------------------- /img/fmbench-website.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/img/fmbench-website.png -------------------------------------------------------------------------------- /img/fmbt-small.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/img/fmbt-small.png -------------------------------------------------------------------------------- /img/fmbt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/img/fmbt.png -------------------------------------------------------------------------------- /img/overall_candidate_model_majority_voting_accuracy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/img/overall_candidate_model_majority_voting_accuracy.png -------------------------------------------------------------------------------- /img/results.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/img/results.gif -------------------------------------------------------------------------------- /img/tags.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/img/tags.png -------------------------------------------------------------------------------- /manifest.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/manifest.txt -------------------------------------------------------------------------------- /misc/ec2_instance_creation_steps.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/misc/ec2_instance_creation_steps.md -------------------------------------------------------------------------------- /misc/eks_cluster-creation_steps.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/misc/eks_cluster-creation_steps.md -------------------------------------------------------------------------------- /misc/the-diy-version-w-gory-details.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/misc/the-diy-version-w-gory-details.md -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/mkdocs.yml -------------------------------------------------------------------------------- /poetry.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/poetry.lock -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/pyproject.toml -------------------------------------------------------------------------------- /release_history.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/release_history.md -------------------------------------------------------------------------------- /render_fmbench_website.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/render_fmbench_website.py -------------------------------------------------------------------------------- /template.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/template.yml -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/test_pricing.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/tests/test_pricing.py -------------------------------------------------------------------------------- /uv.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/uv.lock -------------------------------------------------------------------------------- /website/create_fmbench_website.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/website/create_fmbench_website.py -------------------------------------------------------------------------------- /website/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/website/index.md -------------------------------------------------------------------------------- /website/mkdocs_template.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/website/mkdocs_template.yml -------------------------------------------------------------------------------- /website/nginx.conf.template: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/foundation-model-benchmarking-tool/HEAD/website/nginx.conf.template --------------------------------------------------------------------------------