├── README.md ├── img ├── code.png ├── eng_overview.png └── kor_overview.png ├── lm_cache ├── hf-causal-experimental_pretrained-nlpai-lab-KULLM3.db └── hf-causal-experimental_pretrained-nlpai-lab-kullm-polyglot-12.8b-v2.db ├── lm_eval ├── __init__.py ├── base.py ├── datasets │ └── ko_commongen_v2 │ │ ├── korean_commongen_v2_847_shuffled.jsonl │ │ ├── shuffled_china_1.0.jsonl │ │ ├── shuffled_english_1.0.jsonl │ │ ├── shuffled_espanol_1.0.jsonl │ │ ├── shuffled_japan_1.0.jsonl │ │ ├── shuffled_korean_1.0.jsonl │ │ └── v2_5shot.jsonl ├── evaluator.py ├── metrics.py ├── models │ ├── __init__.py │ ├── anthropic_llms.py │ ├── dummy.py │ ├── gpt2.py │ ├── gpt3.py │ ├── huggingface.py │ └── textsynth.py ├── tasks │ ├── __init__.py │ └── ko_commongen_v2.py └── utils.py ├── main.py ├── requirements.txt ├── results ├── google │ ├── flan-t5-xxl_shot0 │ ├── flan-t5-xxl_shot2 │ └── flan-t5-xxl_shot5 └── nlpai-lab │ └── KULLM3_shot2 ├── setup.py └── test.sh /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/J-Seo/KoCommonGEN-V2/HEAD/README.md -------------------------------------------------------------------------------- /img/code.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/J-Seo/KoCommonGEN-V2/HEAD/img/code.png -------------------------------------------------------------------------------- /img/eng_overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/J-Seo/KoCommonGEN-V2/HEAD/img/eng_overview.png -------------------------------------------------------------------------------- /img/kor_overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/J-Seo/KoCommonGEN-V2/HEAD/img/kor_overview.png -------------------------------------------------------------------------------- /lm_cache/hf-causal-experimental_pretrained-nlpai-lab-KULLM3.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/J-Seo/KoCommonGEN-V2/HEAD/lm_cache/hf-causal-experimental_pretrained-nlpai-lab-KULLM3.db -------------------------------------------------------------------------------- /lm_cache/hf-causal-experimental_pretrained-nlpai-lab-kullm-polyglot-12.8b-v2.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/J-Seo/KoCommonGEN-V2/HEAD/lm_cache/hf-causal-experimental_pretrained-nlpai-lab-kullm-polyglot-12.8b-v2.db -------------------------------------------------------------------------------- /lm_eval/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lm_eval/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/J-Seo/KoCommonGEN-V2/HEAD/lm_eval/base.py -------------------------------------------------------------------------------- /lm_eval/datasets/ko_commongen_v2/korean_commongen_v2_847_shuffled.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/J-Seo/KoCommonGEN-V2/HEAD/lm_eval/datasets/ko_commongen_v2/korean_commongen_v2_847_shuffled.jsonl -------------------------------------------------------------------------------- /lm_eval/datasets/ko_commongen_v2/shuffled_china_1.0.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/J-Seo/KoCommonGEN-V2/HEAD/lm_eval/datasets/ko_commongen_v2/shuffled_china_1.0.jsonl -------------------------------------------------------------------------------- /lm_eval/datasets/ko_commongen_v2/shuffled_english_1.0.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/J-Seo/KoCommonGEN-V2/HEAD/lm_eval/datasets/ko_commongen_v2/shuffled_english_1.0.jsonl -------------------------------------------------------------------------------- /lm_eval/datasets/ko_commongen_v2/shuffled_espanol_1.0.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/J-Seo/KoCommonGEN-V2/HEAD/lm_eval/datasets/ko_commongen_v2/shuffled_espanol_1.0.jsonl -------------------------------------------------------------------------------- /lm_eval/datasets/ko_commongen_v2/shuffled_japan_1.0.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/J-Seo/KoCommonGEN-V2/HEAD/lm_eval/datasets/ko_commongen_v2/shuffled_japan_1.0.jsonl -------------------------------------------------------------------------------- /lm_eval/datasets/ko_commongen_v2/shuffled_korean_1.0.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/J-Seo/KoCommonGEN-V2/HEAD/lm_eval/datasets/ko_commongen_v2/shuffled_korean_1.0.jsonl -------------------------------------------------------------------------------- /lm_eval/datasets/ko_commongen_v2/v2_5shot.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/J-Seo/KoCommonGEN-V2/HEAD/lm_eval/datasets/ko_commongen_v2/v2_5shot.jsonl -------------------------------------------------------------------------------- /lm_eval/evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/J-Seo/KoCommonGEN-V2/HEAD/lm_eval/evaluator.py -------------------------------------------------------------------------------- /lm_eval/metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/J-Seo/KoCommonGEN-V2/HEAD/lm_eval/metrics.py -------------------------------------------------------------------------------- /lm_eval/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/J-Seo/KoCommonGEN-V2/HEAD/lm_eval/models/__init__.py -------------------------------------------------------------------------------- /lm_eval/models/anthropic_llms.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/J-Seo/KoCommonGEN-V2/HEAD/lm_eval/models/anthropic_llms.py -------------------------------------------------------------------------------- /lm_eval/models/dummy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/J-Seo/KoCommonGEN-V2/HEAD/lm_eval/models/dummy.py -------------------------------------------------------------------------------- /lm_eval/models/gpt2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/J-Seo/KoCommonGEN-V2/HEAD/lm_eval/models/gpt2.py -------------------------------------------------------------------------------- /lm_eval/models/gpt3.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/J-Seo/KoCommonGEN-V2/HEAD/lm_eval/models/gpt3.py -------------------------------------------------------------------------------- /lm_eval/models/huggingface.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/J-Seo/KoCommonGEN-V2/HEAD/lm_eval/models/huggingface.py -------------------------------------------------------------------------------- /lm_eval/models/textsynth.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/J-Seo/KoCommonGEN-V2/HEAD/lm_eval/models/textsynth.py -------------------------------------------------------------------------------- /lm_eval/tasks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/J-Seo/KoCommonGEN-V2/HEAD/lm_eval/tasks/__init__.py -------------------------------------------------------------------------------- /lm_eval/tasks/ko_commongen_v2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/J-Seo/KoCommonGEN-V2/HEAD/lm_eval/tasks/ko_commongen_v2.py -------------------------------------------------------------------------------- /lm_eval/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/J-Seo/KoCommonGEN-V2/HEAD/lm_eval/utils.py -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/J-Seo/KoCommonGEN-V2/HEAD/main.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | -e . 2 | -------------------------------------------------------------------------------- /results/google/flan-t5-xxl_shot0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/J-Seo/KoCommonGEN-V2/HEAD/results/google/flan-t5-xxl_shot0 -------------------------------------------------------------------------------- /results/google/flan-t5-xxl_shot2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/J-Seo/KoCommonGEN-V2/HEAD/results/google/flan-t5-xxl_shot2 -------------------------------------------------------------------------------- /results/google/flan-t5-xxl_shot5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/J-Seo/KoCommonGEN-V2/HEAD/results/google/flan-t5-xxl_shot5 -------------------------------------------------------------------------------- /results/nlpai-lab/KULLM3_shot2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/J-Seo/KoCommonGEN-V2/HEAD/results/nlpai-lab/KULLM3_shot2 -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/J-Seo/KoCommonGEN-V2/HEAD/setup.py -------------------------------------------------------------------------------- /test.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/J-Seo/KoCommonGEN-V2/HEAD/test.sh --------------------------------------------------------------------------------