├── .gitignore ├── LICENSE ├── README.md ├── config.yaml ├── data ├── .gitkeep ├── multilabel_classification.pkl └── ner.pkl ├── data_sources ├── data_models.py └── generate_dataset.py ├── frameworks ├── __init__.py ├── base.py ├── fructose_framework.py ├── instructor_framework.py ├── llamaindex_framework.py ├── lm_format_enforcer_framework.py ├── marvin_framework.py ├── metrics.py ├── mirascope_framework.py ├── modelsmith_framework.py ├── outlines_framework.py └── vanilla_openai_framework.py ├── main.py ├── requirements.txt └── results ├── .gitkeep ├── multilabel_classification ├── FructoseFramework.pkl ├── InstructorFramework.pkl ├── LMFormatEnforcerFramework.pkl ├── LlamaIndexFramework.pkl ├── MarvinFramework.pkl ├── MirascopeFramework.pkl ├── ModelsmithFramework.pkl ├── OutlinesFramework.pkl └── VanillaOpenAIFramework.pkl ├── ner ├── InstructorFramework.pkl ├── LMFormatEnforcerFramework.pkl ├── LlamaIndexFramework.pkl ├── MarvinFramework.pkl ├── MirascopeFramework.pkl └── VanillaOpenAIFramework.pkl └── synthetic_data_generation ├── InstructorFramework.pkl ├── LMFormatEnforcerFramework.pkl ├── LlamaIndexFramework.pkl ├── MarvinFramework.pkl ├── MirascopeFramework.pkl ├── ModelsmithFramework.pkl ├── OutlinesFramework.pkl └── VanillaOpenAIFramework.pkl /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/README.md -------------------------------------------------------------------------------- /config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/config.yaml -------------------------------------------------------------------------------- /data/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/multilabel_classification.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/data/multilabel_classification.pkl -------------------------------------------------------------------------------- /data/ner.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/data/ner.pkl -------------------------------------------------------------------------------- /data_sources/data_models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/data_sources/data_models.py -------------------------------------------------------------------------------- /data_sources/generate_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/data_sources/generate_dataset.py -------------------------------------------------------------------------------- /frameworks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/frameworks/__init__.py -------------------------------------------------------------------------------- /frameworks/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/frameworks/base.py -------------------------------------------------------------------------------- /frameworks/fructose_framework.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/frameworks/fructose_framework.py -------------------------------------------------------------------------------- /frameworks/instructor_framework.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/frameworks/instructor_framework.py -------------------------------------------------------------------------------- /frameworks/llamaindex_framework.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/frameworks/llamaindex_framework.py -------------------------------------------------------------------------------- /frameworks/lm_format_enforcer_framework.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/frameworks/lm_format_enforcer_framework.py -------------------------------------------------------------------------------- /frameworks/marvin_framework.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/frameworks/marvin_framework.py -------------------------------------------------------------------------------- /frameworks/metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/frameworks/metrics.py -------------------------------------------------------------------------------- /frameworks/mirascope_framework.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/frameworks/mirascope_framework.py -------------------------------------------------------------------------------- /frameworks/modelsmith_framework.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/frameworks/modelsmith_framework.py -------------------------------------------------------------------------------- /frameworks/outlines_framework.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/frameworks/outlines_framework.py -------------------------------------------------------------------------------- /frameworks/vanilla_openai_framework.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/frameworks/vanilla_openai_framework.py -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/main.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/requirements.txt -------------------------------------------------------------------------------- /results/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /results/multilabel_classification/FructoseFramework.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/results/multilabel_classification/FructoseFramework.pkl -------------------------------------------------------------------------------- /results/multilabel_classification/InstructorFramework.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/results/multilabel_classification/InstructorFramework.pkl -------------------------------------------------------------------------------- /results/multilabel_classification/LMFormatEnforcerFramework.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/results/multilabel_classification/LMFormatEnforcerFramework.pkl -------------------------------------------------------------------------------- /results/multilabel_classification/LlamaIndexFramework.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/results/multilabel_classification/LlamaIndexFramework.pkl -------------------------------------------------------------------------------- /results/multilabel_classification/MarvinFramework.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/results/multilabel_classification/MarvinFramework.pkl -------------------------------------------------------------------------------- /results/multilabel_classification/MirascopeFramework.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/results/multilabel_classification/MirascopeFramework.pkl -------------------------------------------------------------------------------- /results/multilabel_classification/ModelsmithFramework.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/results/multilabel_classification/ModelsmithFramework.pkl -------------------------------------------------------------------------------- /results/multilabel_classification/OutlinesFramework.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/results/multilabel_classification/OutlinesFramework.pkl -------------------------------------------------------------------------------- /results/multilabel_classification/VanillaOpenAIFramework.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/results/multilabel_classification/VanillaOpenAIFramework.pkl -------------------------------------------------------------------------------- /results/ner/InstructorFramework.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/results/ner/InstructorFramework.pkl -------------------------------------------------------------------------------- /results/ner/LMFormatEnforcerFramework.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/results/ner/LMFormatEnforcerFramework.pkl -------------------------------------------------------------------------------- /results/ner/LlamaIndexFramework.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/results/ner/LlamaIndexFramework.pkl -------------------------------------------------------------------------------- /results/ner/MarvinFramework.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/results/ner/MarvinFramework.pkl -------------------------------------------------------------------------------- /results/ner/MirascopeFramework.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/results/ner/MirascopeFramework.pkl -------------------------------------------------------------------------------- /results/ner/VanillaOpenAIFramework.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/results/ner/VanillaOpenAIFramework.pkl -------------------------------------------------------------------------------- /results/synthetic_data_generation/InstructorFramework.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/results/synthetic_data_generation/InstructorFramework.pkl -------------------------------------------------------------------------------- /results/synthetic_data_generation/LMFormatEnforcerFramework.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/results/synthetic_data_generation/LMFormatEnforcerFramework.pkl -------------------------------------------------------------------------------- /results/synthetic_data_generation/LlamaIndexFramework.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/results/synthetic_data_generation/LlamaIndexFramework.pkl -------------------------------------------------------------------------------- /results/synthetic_data_generation/MarvinFramework.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/results/synthetic_data_generation/MarvinFramework.pkl -------------------------------------------------------------------------------- /results/synthetic_data_generation/MirascopeFramework.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/results/synthetic_data_generation/MirascopeFramework.pkl -------------------------------------------------------------------------------- /results/synthetic_data_generation/ModelsmithFramework.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/results/synthetic_data_generation/ModelsmithFramework.pkl -------------------------------------------------------------------------------- /results/synthetic_data_generation/OutlinesFramework.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/results/synthetic_data_generation/OutlinesFramework.pkl -------------------------------------------------------------------------------- /results/synthetic_data_generation/VanillaOpenAIFramework.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenleo/llm-structured-output-benchmarks/HEAD/results/synthetic_data_generation/VanillaOpenAIFramework.pkl --------------------------------------------------------------------------------