LICENSE
MANIFEST.in
README.md
pyproject.toml
fmbench/0_setup.ipynb
fmbench/1_generate_data.ipynb
fmbench/2_deploy_model.ipynb
fmbench/3_run_inference.ipynb
fmbench/4_get_evaluations.ipynb
fmbench/5_model_metric_analysis.ipynb
fmbench/6_cleanup.ipynb
fmbench/__init__.py
fmbench/bring_your_own_dataset.ipynb
fmbench/config_filepath.txt
fmbench/defaults.py
fmbench/globals.py
fmbench/main.py
fmbench/requirements.txt
fmbench/utils.py
fmbench.egg-info/PKG-INFO
fmbench.egg-info/SOURCES.txt
fmbench.egg-info/dependency_links.txt
fmbench.egg-info/entry_points.txt
fmbench.egg-info/requires.txt
fmbench.egg-info/top_level.txt
fmbench/configs/model_eval_all_info.yml
fmbench/configs/pricing.yml
fmbench/configs/pricing_fallback.yml
fmbench/configs/NousResearchHermes70B/config-Nous-Hermes3-1-70b-g5.48xl-tp-8-mc-max-djl.yml
fmbench/configs/NousResearchHermes70B/config-Nous-Hermes3-1-70b-inf2.48xl-triton-tp24.yml
fmbench/configs/NousResearchHermes70B/config-Nous-Hermes3-1-70b-trn1.32xl-deploy-ec2-triton.yml
fmbench/configs/bedrock/config-bedrock-all-anthropic-models-longbench-data.yml
fmbench/configs/bedrock/config-bedrock-anthropic-models-OpenOrca.yml
fmbench/configs/bedrock/config-bedrock-claude.yml
fmbench/configs/bedrock/config-bedrock-evals-only-conc-1.yml
fmbench/configs/bedrock/config-bedrock-haiku-sonnet-majority-voting.yml
fmbench/configs/bedrock/config-bedrock-llama3-1-70b-streaming.yml
fmbench/configs/bedrock/config-bedrock-llama3-1-8b-streaming.yml
fmbench/configs/bedrock/config-bedrock-llama3-1-no-streaming.yml
fmbench/configs/bedrock/config-bedrock-llama3-1.yml
fmbench/configs/bedrock/config-bedrock-llama3-streaming.yml
fmbench/configs/bedrock/config-bedrock-models-OpenOrca.yml
fmbench/configs/bedrock/config-bedrock-titan-text-express.yml
fmbench/configs/bedrock/config-bedrock.yml
fmbench/configs/bedrock/config-claude-3-5-sonnet-v2.yml
fmbench/configs/bedrock/config-claude-dolly-dataset.yml
fmbench/configs/bedrock/config-llama-3-2-11b-databricks-dolly-15k.yml
fmbench/configs/bedrock/config-llama-3-2-1b-3b-no-evals.yml
fmbench/configs/bedrock/config-llama-3-2-1b-3b.yml
fmbench/configs/bedrock/config-llama-3-2-all-models-longbench-hf-version.yml
fmbench/configs/bedrock/config-llama-3-2-all-models.yml
fmbench/configs/bedrock/config-llama-3-3-all-models-open-orca.yml
fmbench/configs/bedrock/config-llama-3-3-all-models.yml
fmbench/configs/bedrock/config-nova-all-models-convfinqa.yml
fmbench/configs/bedrock/config-nova-all-models-dolly-dataset.yml
fmbench/configs/bedrock/config-nova-all-models-openarca.yml
fmbench/configs/bedrock/config-nova-all-models.yml
fmbench/configs/bert/config-distilbert-base-uncased.yml
fmbench/configs/byoe/config-byo-custom-rest-predictor-tinyllama.yml
fmbench/configs/byoe/config-byo-custom-rest-predictor.yml
fmbench/configs/byoe/config-model-byo-sagemaker-endpoint.yml
fmbench/configs/deepseek/config-deepseek-r1-ollama.yml
fmbench/configs/deepseek/config-deepseek-r1-quant1.58-longbench-byoe.yml
fmbench/configs/deepseek/config-deepseek-r1-sglang.yml
fmbench/configs/deepseek/config-deepseek-r1-vllm-convfinqa.yml
fmbench/configs/deepseek/config-deepseek-r1-vllm-longbench.yml
fmbench/configs/deepseek/config-deepseek-r1-vllm-openorca.yml
fmbench/configs/eks_manifests/llama3-ray-service.yaml
fmbench/configs/eks_manifests/mistral-ray-service.yaml
fmbench/configs/embeddings/bge-base-en-v1-5-c5-embeddings.yml
fmbench/configs/embeddings/bge-base-en-v1-5-g5-embeddings.yml
fmbench/configs/embeddings/bge-base-en-v1-5-g5-g4dn-c7-embeddings.yml
fmbench/configs/gemma/config-gemma-2b-g5.yml
fmbench/configs/generic/ec2/Qwen2.5_djl.yml
fmbench/configs/generic/ec2/djl.yml
fmbench/configs/generic/ec2/llama3.1_djl.yml
fmbench/configs/llama2/13b/config-bedrock-sagemaker-llama2.yml
fmbench/configs/llama2/13b/config-byo-rest-ep-llama2-13b.yml
fmbench/configs/llama2/13b/config-llama2-13b-inf2-g5-p4d.yml
fmbench/configs/llama2/13b/config-llama2-13b-inf2-g5.yml
fmbench/configs/llama2/70b/config-ec2-llama2-70b.yml
fmbench/configs/llama2/70b/config-llama2-70b-g5-p4d-tgi.yml
fmbench/configs/llama2/70b/config-llama2-70b-g5-p4d-trt.yml
fmbench/configs/llama2/70b/config-llama2-70b-inf2-g5.yml
fmbench/configs/llama2/7b/config-llama2-7b-byo-sagemaker-endpoint.yml
fmbench/configs/llama2/7b/config-llama2-7b-g4dn-g5-trt.yml
fmbench/configs/llama2/7b/config-llama2-7b-g5-no-s3-quick.yml
fmbench/configs/llama2/7b/config-llama2-7b-g5-quick.yml
fmbench/configs/llama2/7b/config-llama2-7b-inf2-g5.yml
fmbench/configs/llama3.1/70b/config-ec2-llama3-1-70b-inf2-48xl-deploy-ec2-djl.yml
fmbench/configs/llama3.1/70b/config-ec2-llama3-1-70b-inf2-deploy-sm.yml
fmbench/configs/llama3.1/70b/config-ec2-llama3-1-70b-p4de.24xl-deploy-ec2-large-prompts.yml
fmbench/configs/llama3.1/70b/config-ec2-llama3-1-70b-p4de.24xl-deploy-ec2-longbench.yml
fmbench/configs/llama3.1/70b/config-ec2-llama3-1-70b-p4de.24xl-deploy-ec2-summarization.yml
fmbench/configs/llama3.1/70b/config-llama3-1-70b-g5.48xl-tp-8-mc-max-djl.yml
fmbench/configs/llama3.1/70b/config-llama3-1-70b-g6.48xl-tp-8-mc-max-djl.yml
fmbench/configs/llama3.1/70b/config-llama3-1-70b-g6e.24xl-tp-4-mc-max-djl.yml
fmbench/configs/llama3.1/70b/config-llama3-1-70b-g6e.48xl-tp-8-mc-max-djl.yml
fmbench/configs/llama3.1/70b/config-llama3-1-70b-inf2.48xl-deploy-ec2-triton.yml
fmbench/configs/llama3.1/70b/config-llama3-1-70b-inf2.48xl-triton-tp24.yml
fmbench/configs/llama3.1/70b/config-llama3-1-70b-p5-djl-lmi.yml
fmbench/configs/llama3.1/70b/config-llama3-1-70b-trn1.32xl-deploy-ec2-triton.yml
fmbench/configs/llama3.1/70b/config-llama3-1-7b-inf2.48xl-triton-ec2.yml
fmbench/configs/llama3.1/8b/client-config-ec2-llama3-1-8b.yml
fmbench/configs/llama3.1/8b/config-ec2-llama3-1-8b-g6e-2xlarge-byoe-ollama.yml
fmbench/configs/llama3.1/8b/config-ec2-llama3-1-8b-inf2-48xl-deploy-ec2-tp24-bs12.yml
fmbench/configs/llama3.1/8b/config-ec2-llama3-1-8b-inf2-48xl-deploy-ec2.yml
fmbench/configs/llama3.1/8b/config-ec2-llama3-1-8b-inf2.yml
fmbench/configs/llama3.1/8b/config-ec2-llama3-1-8b-p4-tp-2-mc-max.yml
fmbench/configs/llama3.1/8b/config-ec2-llama3-1-8b-p4-tp-4-mc-max.yml
fmbench/configs/llama3.1/8b/config-ec2-llama3-1-8b-p4-tp-8-mc-max.yml
fmbench/configs/llama3.1/8b/config-ec2-llama3-1-8b-p5-tp-2-mc-max.yml
fmbench/configs/llama3.1/8b/config-ec2-llama3-1-8b-tp-8-mc-auto-p5.yml
fmbench/configs/llama3.1/8b/config-ec2-llama3-1-8b-trn1-32xl-deploy-ec2-tp32-bs8.yml
fmbench/configs/llama3.1/8b/config-llama3-1-8b-g5.12xlarge-djl-lmi-sm.yml
fmbench/configs/llama3.1/8b/config-llama3-1-8b-p5en-djl.yml
fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5-ec2.yml
fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.12xl-tp-2-mc-auto-ec2.yml
fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.12xl-tp-2-mc-max-ec2.yml
fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.12xl-tp-4-mc-max-ec2.yml
fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.24xl-tp-2-mc-auto-ec2.yml
fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.24xl-tp-2-mc-max-ec2.yml
fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.24xl-tp-4-mc-max-ec2.yml
fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.2xl-g5.4xl-sm.yml
fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.2xl-tp-1-mc-max-ec2-conc-1-2.yml
fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.2xl-tp-1-mc-max-ec2.yml
fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.48xl-tp-2-mc-auto-ec2.yml
fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.48xl-tp-2-mc-max-ec2.yml
fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.48xl-tp-4-mc-max-ec2.yml
fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.48xl-tp-8-mc-max-ec2.yml
fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.xl-tp-1-mc-max-ec2-conc-1-2.yml
fmbench/configs/llama3.1/8b/config-llama3.1-8b-g5.yml
fmbench/configs/llama3.1/8b/config-llama3.1-8b-g6e.12xl-tp-2-mc-max-djl.yml
fmbench/configs/llama3.1/8b/config-llama3.1-8b-g6e.12xl-tp-4-mc-max-djl.yml
fmbench/configs/llama3.1/8b/config-llama3.1-8b-g6e.24xl-tp-2-mc-max-djl.yml
fmbench/configs/llama3.1/8b/config-llama3.1-8b-g6e.24xl-tp-4-mc-max-djl.yml
fmbench/configs/llama3.1/8b/config-llama3.1-8b-g6e.2xl-ollama.yml
fmbench/configs/llama3.1/8b/config-llama3.1-8b-g6e.2xl-tp-1-mc-max-djl.yml
fmbench/configs/llama3.1/8b/config-llama3.1-8b-g6e.48xl-tp-2-mc-max-djl.yml
fmbench/configs/llama3.1/8b/config-llama3.1-8b-g6e.48xl-tp-4-mc-max-djl.yml
fmbench/configs/llama3.1/8b/config-llama3.1-8b-g6e.48xl-tp-8-mc-max-djl.yml
fmbench/configs/llama3.1/8b/config-llama3.1-8b-g6e.4xl-tp-1-mc-max-djl.yml
fmbench/configs/llama3.1/8b/config-llama3.1-8b-inf2-48xl-deploy-tp-24-ec2.yml
fmbench/configs/llama3.1/8b/config-llama3.1-8b-inf2-48xl-deploy-tp-8-ec2.yml
fmbench/configs/llama3.1/8b/config-llama3.1-8b-trn1-32xl-deploy-tp-8-ec2.yml
fmbench/configs/llama3.1/8b/config-llama3.1-8b-trn32xl-triton-vllm.yml
fmbench/configs/llama3.1/8b/server-config-ec2-llama3-1-8b-inf2-48xl-deploy-ec2.yml
fmbench/configs/llama3.2/11b/config-llama3.2-11b-g6e-2xl-tp-1-mc-max-djl-vllm-ec2.yml
fmbench/configs/llama3.2/1b/config-llama3.2-1b-g5.2xl-summarization-500-50.yml
fmbench/configs/llama3.2/1b/config-llama3.2-1b-g5.2xl-tp-1-mc-max-djl-ec2.yml
fmbench/configs/llama3.2/1b/config-llama3.2-1b-g5.4xl-tp-1-mc-max-djl-ec2.yml
fmbench/configs/llama3.2/1b/config-llama3.2-1b-g6e.2xl-tp-1-mc-max-djl.yml
fmbench/configs/llama3.2/1b/config-llama3.2-1b-m5-16xlarge-ec2.yml
fmbench/configs/llama3.2/1b/config-llama3.2-1b-m7a-16xlarge-ec2.yml
fmbench/configs/llama3.2/1b/config-llama3.2-1b-m7a-24xlarge-ec2-summarization.yml
fmbench/configs/llama3.2/1b/config-llama3.2-1b-m7a-24xlarge-ec2.yml
fmbench/configs/llama3.2/1b/config-llama3.2-1b-m7i-12xlarge-ec2.yml
fmbench/configs/llama3.2/3b/config-llama3.2-3b-g5.4xl-tp-1-mc-max-djl-ec2.yml
fmbench/configs/llama3/70b/config-bedrock.yml
fmbench/configs/llama3/70b/config-ec2-llama3-70b-instruct.yml
fmbench/configs/llama3/70b/config-ec2-neuron-llama3-70b-inf2-48xl-deploy-sm.yml
fmbench/configs/llama3/70b/config-llama3-70b-instruct-g5-48xl.yml
fmbench/configs/llama3/70b/config-llama3-70b-instruct-g5-p4d.yml
fmbench/configs/llama3/70b/config-llama3-70b-instruct-p4d.yml
fmbench/configs/llama3/8b/config-bedrock.yml
fmbench/configs/llama3/8b/config-ec2-llama3-8b-c5-18xlarge.yml
fmbench/configs/llama3/8b/config-ec2-llama3-8b-c8g-24xlarge.yml
fmbench/configs/llama3/8b/config-ec2-llama3-8b-g6e-2xlarge.yml
fmbench/configs/llama3/8b/config-ec2-llama3-8b-inf2-48xl.yml
fmbench/configs/llama3/8b/config-ec2-llama3-8b-m5-16xlarge.yml
fmbench/configs/llama3/8b/config-ec2-llama3-8b-m7a-16xlarge.yml
fmbench/configs/llama3/8b/config-ec2-llama3-8b-m7a-24xlarge.yml
fmbench/configs/llama3/8b/config-ec2-llama3-8b-m7i-12xlarge.yml
fmbench/configs/llama3/8b/config-ec2-llama3-8b-m7i-16xlarge.yml
fmbench/configs/llama3/8b/config-ec2-llama3-8b-m7i-24xlarge.yml
fmbench/configs/llama3/8b/config-ec2-llama3-8b-neuron-trn1-32xl-tp16-sm.yml
fmbench/configs/llama3/8b/config-ec2-llama3-8b-p4d-tp-2-mc-max.yml
fmbench/configs/llama3/8b/config-ec2-llama3-8b-p4d-tp-4-mc-max.yml
fmbench/configs/llama3/8b/config-ec2-llama3-8b-p4d-tp-8-mc-max.yml
fmbench/configs/llama3/8b/config-ec2-llama3-8b-p4de-tp-2-mc-max.yml
fmbench/configs/llama3/8b/config-ec2-llama3-8b-p4de-tp-4-mc-max.yml
fmbench/configs/llama3/8b/config-ec2-llama3-8b-p4de-tp-8-mc-max.yml
fmbench/configs/llama3/8b/config-ec2-llama3-8b-p5-tp-2-mc-max.yml
fmbench/configs/llama3/8b/config-ec2-llama3-8b-p5-tp-8-mc-auto.yml
fmbench/configs/llama3/8b/config-ec2-llama3-8b.yml
fmbench/configs/llama3/8b/config-ec2-neuron-llama3-8b-inf2-24xl-deploy-sm.yml
fmbench/configs/llama3/8b/config-ec2-neuron-llama3-8b-inf2-48xl-deploy-sm.yml
fmbench/configs/llama3/8b/config-llama3-8b-eks-inf2.yml
fmbench/configs/llama3/8b/config-llama3-8b-g5-streaming.yml
fmbench/configs/llama3/8b/config-llama3-8b-g5.12xl-tp-2-mc-max-djl-ec2.yml
fmbench/configs/llama3/8b/config-llama3-8b-g5.12xl-tp-2-mc-max-triton-ec2.yml
fmbench/configs/llama3/8b/config-llama3-8b-g5.12xl-tp-4-mc-max-djl-ec2.yml
fmbench/configs/llama3/8b/config-llama3-8b-g5.12xl-tp-4-mc-max-triton-ec2.yml
fmbench/configs/llama3/8b/config-llama3-8b-g5.48xl-tp-8-mc-max-triton-ec2.yml
fmbench/configs/llama3/8b/config-llama3-8b-g5.4xl-tp-1-mc-max-triton-ec2.yml
fmbench/configs/llama3/8b/config-llama3-8b-g6e.12xl-tp-2-mc-max-djl-ec2.yml
fmbench/configs/llama3/8b/config-llama3-8b-g6e.12xl-tp-2-mc-max-triton-ec2.yml
fmbench/configs/llama3/8b/config-llama3-8b-g6e.12xl-tp-4-mc-max-djl-ec2.yml
fmbench/configs/llama3/8b/config-llama3-8b-g6e.12xl-tp-4-mc-max-triton-ec2.yml
fmbench/configs/llama3/8b/config-llama3-8b-g6e.24xl-tp-2-mc-max-djl-ec2.yml
fmbench/configs/llama3/8b/config-llama3-8b-g6e.24xl-tp-2-mc-max-triton-ec2.yml
fmbench/configs/llama3/8b/config-llama3-8b-g6e.24xl-tp-4-mc-max-djl-ec2.yml
fmbench/configs/llama3/8b/config-llama3-8b-g6e.24xl-tp-4-mc-max-triton-ec2.yml
fmbench/configs/llama3/8b/config-llama3-8b-g6e.2xl-tp-1-mc-max-triton-ec2.yml
fmbench/configs/llama3/8b/config-llama3-8b-g6e.48xl-tp-2-mc-max-djl-ec2.yml
fmbench/configs/llama3/8b/config-llama3-8b-g6e.48xl-tp-2-mc-max-triton-ec2.yml
fmbench/configs/llama3/8b/config-llama3-8b-g6e.48xl-tp-4-mc-max-djl-ec2.yml
fmbench/configs/llama3/8b/config-llama3-8b-g6e.48xl-tp-4-mc-max-triton-ec2.yml
fmbench/configs/llama3/8b/config-llama3-8b-g6e.48xl-tp-8-mc-max-djl-ec2.yml
fmbench/configs/llama3/8b/config-llama3-8b-g6e.48xl-tp-8-mc-max-triton-ec2.yml
fmbench/configs/llama3/8b/config-llama3-8b-g6e.4xl-tp-1-mc-max-djl-ec2.yml
fmbench/configs/llama3/8b/config-llama3-8b-g6e.4xl-tp-1-mc-max-triton-ec2.yml
fmbench/configs/llama3/8b/config-llama3-8b-g6e.xl-tp-1-mc-max-triton-ec2.yml
fmbench/configs/llama3/8b/config-llama3-8b-inf2-24xl-tp=8-bs=4-byoe.yml
fmbench/configs/llama3/8b/config-llama3-8b-inf2-48xl-tp=8-bs=4-byoe.yml
fmbench/configs/llama3/8b/config-llama3-8b-inf2-48xlarge-triton-djl.yml
fmbench/configs/llama3/8b/config-llama3-8b-inf2-g5-byoe-w-openorca.yml
fmbench/configs/llama3/8b/config-llama3-8b-inf2-g5.yml
fmbench/configs/llama3/8b/config-llama3-8b-instruct-all.yml
fmbench/configs/llama3/8b/config-llama3-8b-instruct-g5-12xl-4-instances.yml
fmbench/configs/llama3/8b/config-llama3-8b-instruct-g5-12xl.yml
fmbench/configs/llama3/8b/config-llama3-8b-instruct-g5-24xl.yml
fmbench/configs/llama3/8b/config-llama3-8b-instruct-g5-2xl.yml
fmbench/configs/llama3/8b/config-llama3-8b-instruct-g5-48xl.yml
fmbench/configs/llama3/8b/config-llama3-8b-instruct-g5-p4d.yml
fmbench/configs/llama3/8b/config-llama3-8b-instruct-g6-12xl.yml
fmbench/configs/llama3/8b/config-llama3-8b-instruct-g6-24xl.yml
fmbench/configs/llama3/8b/config-llama3-8b-instruct-g6-48xl.yml
fmbench/configs/llama3/8b/config-llama3-8b-instruct-p4d-djl-lmi-dist.yml
fmbench/configs/llama3/8b/config-llama3-8b-instruct-p4d-djl-vllm.yml
fmbench/configs/llama3/8b/config-llama3-8b-instruct-p5-djl-lmi-dist.yml
fmbench/configs/llama3/8b/config-llama3-8b-trn1-32xl-tp-16-bs-4-byoe.yml
fmbench/configs/llama3/8b/config-llama3-8b-trn1-32xl-tp-8-bs-4-byoe.yml
fmbench/configs/llama3/8b/config-llama3-8b-trn1-32xl-tp16-bs-4-ec2.yml
fmbench/configs/llama3/8b/config-llama3-8b-trn1-32xlarge-triton-djl.yml
fmbench/configs/llama3/8b/config-llama3-8b-trn1-32xlarge-triton-vllm.yml
fmbench/configs/llama3/8b/config-llama3-8b-trn1.yml
fmbench/configs/llama3/8b/llama3-8b-inf2-24xl-byoe-g5-12xl.yml
fmbench/configs/llama3/8b/llama3-8b-inf2-48xl-byoe-g5-24xl.yml
fmbench/configs/llama3/8b/llama3-8b-trn1-32xl-byoe-g5-24xl.yml
fmbench/configs/mistral/config-mistral-7b-eks-inf2.yml
fmbench/configs/mistral/config-mistral-7b-tgi-g5.yml
fmbench/configs/mistral/config-mistral-7b-trn1-32xl-triton.yml
fmbench/configs/mistral/config-mistral-instruct-AWQ-p4d.yml
fmbench/configs/mistral/config-mistral-instruct-AWQ-p5-byo-ep.yml
fmbench/configs/mistral/config-mistral-instruct-AWQ-p5.yml
fmbench/configs/mistral/config-mistral-instruct-p4d.yml
fmbench/configs/mistral/config-mistral-instruct-v1-p5-trtllm.yml
fmbench/configs/mistral/config-mistral-instruct-v2-p4d-lmi-dist.yml
fmbench/configs/mistral/config-mistral-instruct-v2-p4d-trtllm.yml
fmbench/configs/mistral/config-mistral-instruct-v2-p5-lmi-dist.yml
fmbench/configs/mistral/config-mistral-instruct-v2-p5-trtllm.yml
fmbench/configs/mistral/config-mistral-trn1-32xl-deploy-ec2-tp32.yml
fmbench/configs/mistral/config-mistral-v3-inf2-48xl-deploy-ec2-tp24.yml
fmbench/configs/mixtral/config-mixtral-8x7b-g6e.48xl-ec2.yml
fmbench/configs/multimodal/bedrock/config-claude-scienceqa.yml
fmbench/configs/multimodal/bedrock/config-llama-3-2-11b-vision-instruct-marqo-GS-10M.yml
fmbench/configs/multimodal/bedrock/config-llama-3-2-11b-vision-instruct-scienceqa.yml
fmbench/configs/multimodal/bedrock/config-llama-3-2-claude-models-scienceqa.yml
fmbench/configs/phi/config-phi-3-g5.yml
fmbench/img/llm_eval_flowchart.png
fmbench/prompt_template/prompt_template_bert.txt
fmbench/prompt_template/prompt_template_claude.txt
fmbench/prompt_template/prompt_template_claude_OpenOrca.txt
fmbench/prompt_template/prompt_template_claude_dolly_dataset.txt
fmbench/prompt_template/prompt_template_claude_images_ScienceQA.txt
fmbench/prompt_template/prompt_template_convfinqa.txt
fmbench/prompt_template/prompt_template_deepseek_longbench.txt
fmbench/prompt_template/prompt_template_gemma.txt
fmbench/prompt_template/prompt_template_llama2.txt
fmbench/prompt_template/prompt_template_llama2_Mistral_OpenOrca.txt
fmbench/prompt_template/prompt_template_llama3.2_vision.txt
fmbench/prompt_template/prompt_template_llama3.txt
fmbench/prompt_template/prompt_template_llama3_OpenOrca.txt
fmbench/prompt_template/prompt_template_llama3_OpenOrca_accuracy.txt
fmbench/prompt_template/prompt_template_llama3_databricks-dolly-15k.txt
fmbench/prompt_template/prompt_template_llama3_images_ScienceQA.txt
fmbench/prompt_template/prompt_template_llama3_images_marqo-GS-10M.txt
fmbench/prompt_template/prompt_template_llama3_summarization.txt
fmbench/prompt_template/prompt_template_mistral.txt
fmbench/prompt_template/prompt_template_nova.txt
fmbench/prompt_template/prompt_template_nova_convfinqa.txt
fmbench/prompt_template/prompt_template_nova_dolly_dataset.txt
fmbench/prompt_template/prompt_template_nova_open_orca.txt
fmbench/prompt_template/prompt_template_open_orca.txt
fmbench/prompt_template/prompt_template_qwen.txt
fmbench/prompt_template/prompt_template_titan_text.txt
fmbench/prompt_template/eval_criteria/evaluation_instructions_majority_vote.txt
fmbench/prompt_template/eval_criteria/claude_eval_prompt_templates/claude_eval_majority_vote.txt
fmbench/prompt_template/eval_criteria/cohere_eval_prompt_templates/cohere_eval_majority_vote.txt
fmbench/prompt_template/eval_criteria/llama3_eval_prompt_templates/llama3_eval_majority_vote.txt
fmbench/prompt_template/eval_criteria/mistral_eval_prompt_templates/mistral_eval_majority_vote.txt
fmbench/scripts/bedrock_predictor.py
fmbench/scripts/bedrock_predictor_converseAPI.py
fmbench/scripts/constants.py
fmbench/scripts/custom_rest_predictor.py
fmbench/scripts/deploy_w_djl_serving.py
fmbench/scripts/deploy_w_hf_tgi.py
fmbench/scripts/ec2_deploy.py
fmbench/scripts/ec2_metrics.py
fmbench/scripts/ec2_predictor.py
fmbench/scripts/eks_deploy.py
fmbench/scripts/eks_predictor.py
fmbench/scripts/fmbench_predictor.py
fmbench/scripts/jumpstart.py
fmbench/scripts/neuron_deploy.py
fmbench/scripts/neuron_metrics.py
fmbench/scripts/ollama_predictor.py
fmbench/scripts/prepare_for_multi_model_containers.py
fmbench/scripts/pricing.py
fmbench/scripts/rest_predictor.py
fmbench/scripts/sagemaker_deploy.py
fmbench/scripts/sagemaker_metrics.py
fmbench/scripts/sagemaker_predictor.py
fmbench/scripts/stream_responses.py
fmbench/scripts/compile-llm-for-aws-silicon/requirements.txt
fmbench/scripts/compile-llm-for-aws-silicon/scripts/compile.py
fmbench/scripts/compile-llm-for-aws-silicon/scripts/download_compile_deploy.sh
fmbench/scripts/compile-llm-for-aws-silicon/scripts/split_and_save.py
fmbench/scripts/compile-llm-for-aws-silicon/smep-with-lmi/deploy.py
fmbench/scripts/inference_containers/djl.py
fmbench/scripts/inference_containers/ollama.py
fmbench/scripts/inference_containers/sglang.py
fmbench/scripts/inference_containers/triton.py
fmbench/scripts/inference_containers/triton_serve_model.sh
fmbench/scripts/inference_containers/utils.py
fmbench/scripts/inference_containers/vllm.py
fmbench/scripts/inference_containers/vllm_gpu.py
fmbench/scripts/triton/Dockerfile_triton
fmbench/scripts/triton/build_and_push_triton.sh
fmbench/scripts/triton/djl/config.pbtxt
fmbench/scripts/triton/djl/model.py
fmbench/scripts/triton/djl/triton-djl-python-neuronx.sh
fmbench/scripts/triton/vllm/config.pbtxt
fmbench/scripts/triton/vllm/triton-vllm-neuronx.sh
fmbench/scripts/triton/vllm/vllm-neuron-issue-1.patch
fmbench/tokenizer/.keep
fmbench/tokenizer/config.json
fmbench/tokenizer/tokenizer.json
tests/test_pricing.py