7 | The page you are looking for cannot be found. 8 |
9 |10 | If you just switched documentation versions, it is likely that the page you were on is moved. You can look for it in 11 | the content table left, or go to the homepage. 12 |
13 | 17 | 18 | {% endblock %} 19 | -------------------------------------------------------------------------------- /opencompass/docs/en/_templates/autosummary/class.rst: -------------------------------------------------------------------------------- 1 | .. role:: hidden 2 | :class: hidden-section 3 | .. currentmodule:: {{ module }} 4 | 5 | 6 | {{ name | underline}} 7 | 8 | .. autoclass:: {{ name }} 9 | :members: 10 | 11 | .. 12 | autogenerated from _templates/autosummary/class.rst 13 | note it does not have :inherited-members: 14 | -------------------------------------------------------------------------------- /opencompass/docs/en/_templates/callable.rst: -------------------------------------------------------------------------------- 1 | .. role:: hidden 2 | :class: hidden-section 3 | .. currentmodule:: {{ module }} 4 | 5 | 6 | {{ name | underline}} 7 | 8 | .. autoclass:: {{ name }} 9 | :members: 10 | :special-members: __call__ 11 | 12 | .. 13 | autogenerated from _templates/callable.rst 14 | note it does not have :inherited-members: 15 | -------------------------------------------------------------------------------- /opencompass/docs/en/docutils.conf: -------------------------------------------------------------------------------- 1 | [html writers] 2 | table_style: colwidths-auto 3 | -------------------------------------------------------------------------------- /opencompass/docs/en/prompt/few_shot.md: -------------------------------------------------------------------------------- 1 | # In-context Learning 2 | 3 | Coming soon. 4 | -------------------------------------------------------------------------------- /opencompass/docs/en/prompt/overview.md: -------------------------------------------------------------------------------- 1 | # Prompt Overview 2 | -------------------------------------------------------------------------------- /opencompass/docs/en/prompt/prompt_template.md: -------------------------------------------------------------------------------- 1 | # Prompt Template 2 | 3 | Coming soon. 4 | -------------------------------------------------------------------------------- /opencompass/docs/en/user_guides/framework_overview.md: -------------------------------------------------------------------------------- 1 | # Overview 2 | -------------------------------------------------------------------------------- /opencompass/docs/en/user_guides/metrics.md: -------------------------------------------------------------------------------- 1 | # Metric Calculation 2 | -------------------------------------------------------------------------------- /opencompass/docs/zh_cn/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /opencompass/docs/zh_cn/_static/js/custom.js: -------------------------------------------------------------------------------- 1 | var collapsedSections = ['Advanced Guides', 'Tools', 'User Guides', 'Notes']; 2 | 3 | $(document).ready(function () { 4 | $('.model-summary').DataTable({ 5 | "stateSave": false, 6 | "lengthChange": false, 7 | "pageLength": 20, 8 | "order": [] 9 | }); 10 | }); 11 | -------------------------------------------------------------------------------- /opencompass/docs/zh_cn/_templates/404.html: -------------------------------------------------------------------------------- 1 | {% extends "layout.html" %} 2 | 3 | {% block body %} 4 | 5 |7 | The page you are looking for cannot be found. 8 |
9 |10 | If you just switched documentation versions, it is likely that the page you were on is moved. You can look for it in 11 | the content table left, or go to the homepage. 12 |
13 | 17 | 18 | {% endblock %} 19 | -------------------------------------------------------------------------------- /opencompass/docs/zh_cn/_templates/autosummary/class.rst: -------------------------------------------------------------------------------- 1 | .. role:: hidden 2 | :class: hidden-section 3 | .. currentmodule:: {{ module }} 4 | 5 | 6 | {{ name | underline}} 7 | 8 | .. autoclass:: {{ name }} 9 | :members: 10 | 11 | .. 12 | autogenerated from _templates/autosummary/class.rst 13 | note it does not have :inherited-members: 14 | -------------------------------------------------------------------------------- /opencompass/docs/zh_cn/_templates/callable.rst: -------------------------------------------------------------------------------- 1 | .. role:: hidden 2 | :class: hidden-section 3 | .. currentmodule:: {{ module }} 4 | 5 | 6 | {{ name | underline}} 7 | 8 | .. autoclass:: {{ name }} 9 | :members: 10 | :special-members: __call__ 11 | 12 | .. 13 | autogenerated from _templates/callable.rst 14 | note it does not have :inherited-members: 15 | -------------------------------------------------------------------------------- /opencompass/docs/zh_cn/docutils.conf: -------------------------------------------------------------------------------- 1 | [html writers] 2 | table_style: colwidths-auto 3 | -------------------------------------------------------------------------------- /opencompass/docs/zh_cn/prompt/few_shot.md: -------------------------------------------------------------------------------- 1 | # Few-shot 2 | 3 | Coming soon. 4 | -------------------------------------------------------------------------------- /opencompass/docs/zh_cn/prompt/overview.md: -------------------------------------------------------------------------------- 1 | # Prompt 概括 2 | -------------------------------------------------------------------------------- /opencompass/docs/zh_cn/prompt/prompt_template.md: -------------------------------------------------------------------------------- 1 | # Prompt 模板 2 | 3 | Coming soon. 4 | -------------------------------------------------------------------------------- /opencompass/docs/zh_cn/user_guides/framework_overview.md: -------------------------------------------------------------------------------- 1 | # 整体概括 2 | -------------------------------------------------------------------------------- /opencompass/docs/zh_cn/user_guides/metrics.md: -------------------------------------------------------------------------------- 1 | # 评估指标 2 | 3 | Coming soon. 4 | -------------------------------------------------------------------------------- /opencompass/opencompass/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = '0.1.0' 2 | -------------------------------------------------------------------------------- /opencompass/opencompass/datasets/TheoremQA.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from datasets import load_dataset 4 | 5 | from opencompass.registry import LOAD_DATASET, TEXT_POSTPROCESSORS 6 | 7 | from .base import BaseDataset 8 | 9 | 10 | @LOAD_DATASET.register_module() 11 | class TheoremQADataset(BaseDataset): 12 | 13 | @staticmethod 14 | def load(path: str): 15 | return load_dataset('csv', data_files={'test': path}) 16 | 17 | 18 | @TEXT_POSTPROCESSORS.register_module('TheoremQA') 19 | def TheoremQA_postprocess(text: str) -> str: 20 | text = text.strip() 21 | matches = re.findall(r'answer is ([^\s]+)', text) 22 | if len(matches) == 0: 23 | return text 24 | else: 25 | text = matches[0].strip().strip('.,?!\"\';:') 26 | return text 27 | -------------------------------------------------------------------------------- /opencompass/opencompass/datasets/afqmcd.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from datasets import Dataset 4 | 5 | from opencompass.registry import LOAD_DATASET 6 | 7 | from .base import BaseDataset 8 | 9 | 10 | @LOAD_DATASET.register_module() 11 | class AFQMCDataset_V2(BaseDataset): 12 | 13 | @staticmethod 14 | def load(path): 15 | data = [] 16 | with open(path, 'r') as f: 17 | for line in f: 18 | line = json.loads(line) 19 | line['label'] = 'AB'[int(line['label'])] 20 | data.append(line) 21 | return Dataset.from_list(data) 22 | -------------------------------------------------------------------------------- /opencompass/opencompass/datasets/agieval/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | 3 | from .agieval import * # noqa: F401, F403 4 | -------------------------------------------------------------------------------- /opencompass/opencompass/datasets/agieval/utils.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | import json 3 | 4 | 5 | def read_jsonl(path): 6 | with open(path, encoding='utf8') as fh: 7 | results = [] 8 | for line in fh: 9 | if line is None: 10 | continue 11 | try: 12 | results.append(json.loads(line) if line != 'null' else line) 13 | except Exception as e: 14 | print(e) 15 | print(path) 16 | print(line) 17 | raise e 18 | return results 19 | 20 | 21 | def save_jsonl(lines, directory): 22 | with open(directory, 'w', encoding='utf8') as f: 23 | for line in lines: 24 | f.write(json.dumps(line, ensure_ascii=False) + '\n') 25 | 26 | 27 | def extract_answer(js): 28 | try: 29 | if js is None or js == 'null': 30 | return '' 31 | answer = '' 32 | if isinstance(js, str): 33 | answer = js 34 | elif 'text' in js['choices'][0]: 35 | answer = js['choices'][0]['text'] 36 | else: 37 | answer = js['choices'][0]['message']['content'] 38 | # answer = js[''] 39 | return answer 40 | except Exception as e: 41 | # print(e) 42 | # print(js) 43 | return '' 44 | -------------------------------------------------------------------------------- /opencompass/opencompass/datasets/ax.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from datasets import Dataset 4 | 5 | from opencompass.registry import LOAD_DATASET 6 | 7 | from .base import BaseDataset 8 | 9 | 10 | @LOAD_DATASET.register_module() 11 | class AXDataset_V2(BaseDataset): 12 | 13 | @staticmethod 14 | def load(path: str): 15 | dataset = [] 16 | with open(path, 'r') as f: 17 | for line in f: 18 | line = json.loads(line) 19 | line['label'] = { 20 | 'entailment': 'A', 21 | 'not_entailment': 'B' 22 | }[line['label']] 23 | dataset.append(line) 24 | return Dataset.from_list(dataset) 25 | -------------------------------------------------------------------------------- /opencompass/opencompass/datasets/base.py: -------------------------------------------------------------------------------- 1 | from abc import abstractstaticmethod 2 | from typing import Dict, Optional, Union 3 | 4 | from datasets import Dataset, DatasetDict 5 | 6 | from opencompass.openicl import DatasetReader 7 | 8 | 9 | class BaseDataset: 10 | 11 | def __init__(self, reader_cfg: Optional[Dict] = {}, **kwargs): 12 | self.dataset = self.load(**kwargs) 13 | self._init_reader(**reader_cfg) 14 | 15 | def _init_reader(self, **kwargs): 16 | self.reader = DatasetReader(self.dataset, **kwargs) 17 | 18 | @property 19 | def train(self): 20 | return self.reader.dataset['train'] 21 | 22 | @property 23 | def test(self): 24 | return self.reader.dataset['test'] 25 | 26 | @abstractstaticmethod 27 | def load(**kwargs) -> Union[Dataset, DatasetDict]: 28 | pass 29 | -------------------------------------------------------------------------------- /opencompass/opencompass/datasets/boolq.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from datasets import Dataset, load_dataset 4 | 5 | from opencompass.registry import LOAD_DATASET 6 | 7 | from .base import BaseDataset 8 | 9 | 10 | @LOAD_DATASET.register_module() 11 | class BoolQDataset(BaseDataset): 12 | 13 | @staticmethod 14 | def load(**kwargs): 15 | 16 | dataset = load_dataset(**kwargs) 17 | 18 | def preprocess(example): 19 | if example['label'] == 'true': 20 | example['answer'] = 1 21 | else: 22 | example['answer'] = 0 23 | 24 | return example 25 | 26 | dataset = dataset.map(preprocess) 27 | return dataset 28 | 29 | 30 | @LOAD_DATASET.register_module() 31 | class BoolQDataset_V2(BaseDataset): 32 | 33 | @staticmethod 34 | def load(path): 35 | dataset = [] 36 | with open(path, 'r') as f: 37 | for line in f: 38 | line = json.loads(line) 39 | line['label'] = {'true': 'A', 'false': 'B'}[line['label']] 40 | dataset.append(line) 41 | return Dataset.from_list(dataset) 42 | -------------------------------------------------------------------------------- /opencompass/opencompass/datasets/bustum.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from datasets import Dataset 4 | 5 | from opencompass.registry import LOAD_DATASET 6 | 7 | from .base import BaseDataset 8 | 9 | 10 | @LOAD_DATASET.register_module() 11 | class bustumDataset_V2(BaseDataset): 12 | 13 | @staticmethod 14 | def load(path): 15 | data = [] 16 | with open(path, 'r') as f: 17 | for line in f: 18 | line = json.loads(line) 19 | line['label'] = 'AB'[int(line['label'])] 20 | data.append(line) 21 | return Dataset.from_list(data) 22 | -------------------------------------------------------------------------------- /opencompass/opencompass/datasets/cb.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from datasets import Dataset 4 | 5 | from opencompass.registry import LOAD_DATASET 6 | 7 | from .base import BaseDataset 8 | 9 | 10 | @LOAD_DATASET.register_module() 11 | class CBDataset_V2(BaseDataset): 12 | 13 | @staticmethod 14 | def load(path): 15 | dataset = [] 16 | with open(path, 'r') as f: 17 | for line in f: 18 | line = json.loads(line) 19 | line['label'] = { 20 | 'contradiction': 'A', 21 | 'entailment': 'B', 22 | 'neutral': 'C' 23 | }[line['label']] 24 | dataset.append(line) 25 | return Dataset.from_list(dataset) 26 | -------------------------------------------------------------------------------- /opencompass/opencompass/datasets/civilcomments.py: -------------------------------------------------------------------------------- 1 | from datasets import DatasetDict, load_dataset 2 | 3 | from opencompass.registry import LOAD_DATASET 4 | 5 | from .base import BaseDataset 6 | 7 | 8 | @LOAD_DATASET.register_module() 9 | class CivilCommentsDataset(BaseDataset): 10 | 11 | @staticmethod 12 | def load(**kwargs): 13 | train_dataset = load_dataset(**kwargs, split='train') 14 | test_dataset = load_dataset(**kwargs, split='test') 15 | 16 | def pre_process(example): 17 | example['label'] = int(example['toxicity'] >= 0.5) 18 | example['choices'] = ['no', 'yes'] 19 | return example 20 | 21 | def remove_columns(dataset): 22 | return dataset.remove_columns([ 23 | 'severe_toxicity', 'obscene', 'threat', 'insult', 24 | 'identity_attack', 'sexual_explicit' 25 | ]) 26 | 27 | train_dataset = remove_columns(train_dataset) 28 | test_dataset = remove_columns(test_dataset) 29 | test_dataset = test_dataset.shuffle(seed=42) 30 | test_dataset = test_dataset.select(list(range(10000))) 31 | test_dataset = test_dataset.map(pre_process) 32 | 33 | return DatasetDict({ 34 | 'train': train_dataset, 35 | 'test': test_dataset, 36 | }) 37 | -------------------------------------------------------------------------------- /opencompass/opencompass/datasets/cmnli.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from datasets import Dataset 4 | 5 | from opencompass.registry import LOAD_DATASET 6 | 7 | from .base import BaseDataset 8 | 9 | 10 | @LOAD_DATASET.register_module() 11 | class cmnliDataset_V2(BaseDataset): 12 | 13 | @staticmethod 14 | def load(path): 15 | data = [] 16 | with open(path, 'r') as f: 17 | for line in f: 18 | line = json.loads(line) 19 | if line['label'] == '-': 20 | continue 21 | line['label'] = { 22 | 'entailment': 'A', 23 | 'contradiction': 'B', 24 | 'neutral': 'C', 25 | }[line['label']] 26 | data.append(line) 27 | return Dataset.from_list(data) 28 | -------------------------------------------------------------------------------- /opencompass/opencompass/datasets/commonsenseqa.py: -------------------------------------------------------------------------------- 1 | from datasets import load_dataset 2 | 3 | from opencompass.registry import LOAD_DATASET 4 | 5 | from .base import BaseDataset 6 | 7 | 8 | @LOAD_DATASET.register_module() 9 | class commonsenseqaDataset(BaseDataset): 10 | 11 | @staticmethod 12 | def load(**kwargs): 13 | dataset = load_dataset(**kwargs) 14 | 15 | def pre_process(example): 16 | for i in range(5): 17 | example[chr(ord('A') + i)] = example['choices']['text'][i] 18 | return example 19 | 20 | dataset = dataset.map(pre_process).remove_columns( 21 | ['question_concept', 'id', 'choices']) 22 | return dataset 23 | -------------------------------------------------------------------------------- /opencompass/opencompass/datasets/copa.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from datasets import Dataset 4 | 5 | from opencompass.registry import LOAD_DATASET 6 | 7 | from .base import BaseDataset 8 | 9 | 10 | @LOAD_DATASET.register_module() 11 | class COPADataset_V2(BaseDataset): 12 | 13 | @staticmethod 14 | def load(path): 15 | dataset = [] 16 | with open(path, 'r') as f: 17 | for line in f: 18 | line = json.loads(line) 19 | line['label'] = 'AB'[line['label']] 20 | dataset.append(line) 21 | return Dataset.from_list(dataset) 22 | -------------------------------------------------------------------------------- /opencompass/opencompass/datasets/crowspairs.py: -------------------------------------------------------------------------------- 1 | from datasets import load_dataset 2 | 3 | from opencompass.registry import LOAD_DATASET 4 | 5 | from .base import BaseDataset 6 | 7 | 8 | @LOAD_DATASET.register_module() 9 | class crowspairsDataset(BaseDataset): 10 | 11 | @staticmethod 12 | def load(**kwargs): 13 | 14 | dataset = load_dataset(**kwargs) 15 | 16 | def preprocess(example): 17 | example['label'] = 0 18 | return example 19 | 20 | return dataset.map(preprocess) 21 | 22 | 23 | @LOAD_DATASET.register_module() 24 | class crowspairsDataset_V2(BaseDataset): 25 | 26 | @staticmethod 27 | def load(**kwargs): 28 | dataset = load_dataset(**kwargs) 29 | 30 | def preprocess(example): 31 | example['label'] = 'A' 32 | return example 33 | 34 | return dataset.map(preprocess) 35 | -------------------------------------------------------------------------------- /opencompass/opencompass/datasets/csl.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from datasets import Dataset, load_dataset 4 | 5 | from opencompass.registry import LOAD_DATASET 6 | 7 | from .base import BaseDataset 8 | 9 | 10 | @LOAD_DATASET.register_module() 11 | class CslDataset(BaseDataset): 12 | 13 | @staticmethod 14 | def load(**kwargs): 15 | 16 | dataset = load_dataset(**kwargs) 17 | 18 | def preprocess(example): 19 | keywords = '，'.join(example['keyword']) 20 | example['keywords'] = keywords 21 | 22 | return example 23 | 24 | dataset = dataset.map(preprocess) 25 | return dataset 26 | 27 | 28 | @LOAD_DATASET.register_module() 29 | class CslDataset_V2(BaseDataset): 30 | 31 | @staticmethod 32 | def load(path): 33 | data = [] 34 | with open(path, 'r') as f: 35 | for line in f: 36 | line = json.loads(line) 37 | item = { 38 | 'abst': line['abst'], 39 | 'keywords': '，'.join(line['keyword']), 40 | 'label': 'AB'[int(line['label'])], 41 | } 42 | data.append(item) 43 | return Dataset.from_list(data) 44 | -------------------------------------------------------------------------------- /opencompass/opencompass/datasets/cvalues.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from datasets import load_dataset 4 | 5 | from opencompass.registry import LOAD_DATASET 6 | 7 | from .base import BaseDataset 8 | 9 | 10 | @LOAD_DATASET.register_module() 11 | class CValuesDataset(BaseDataset): 12 | 13 | @staticmethod 14 | def load(path): 15 | 16 | dataset = load_dataset('json', data_files=path) 17 | 18 | def preprocess(example): 19 | example['prompt'] = re.sub('回复1', '回复A', example['prompt']) 20 | example['prompt'] = re.sub('回复2', '回复B', example['prompt']) 21 | example['label'] = re.sub('回复1', 'A', example['label']) 22 | example['label'] = re.sub('回复2', 'B', example['label']) 23 | return example 24 | 25 | return dataset.map(preprocess) 26 | -------------------------------------------------------------------------------- /opencompass/opencompass/datasets/drop.py: -------------------------------------------------------------------------------- 1 | from datasets import DatasetDict, load_dataset 2 | 3 | from opencompass.registry import LOAD_DATASET 4 | 5 | from .base import BaseDataset 6 | 7 | 8 | @LOAD_DATASET.register_module() 9 | class dropDataset(BaseDataset): 10 | 11 | @staticmethod 12 | def load(**kwargs): 13 | dataset = load_dataset(**kwargs, split='validation') 14 | 15 | def pre_process(example): 16 | example['answers'] = example['answers_spans']['spans'] 17 | example['prompt'] = example.pop('passage') 18 | return example 19 | 20 | def only_number(example): 21 | for i in example['answers_spans']['types']: 22 | if i == 'number': 23 | return True 24 | return False 25 | 26 | dataset = dataset.filter(only_number) 27 | dataset = dataset.map(pre_process).remove_columns( 28 | ['section_id', 'query_id']) 29 | return DatasetDict({'validation': dataset}) 30 | -------------------------------------------------------------------------------- /opencompass/opencompass/datasets/eprstmt.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from datasets import Dataset 4 | 5 | from opencompass.registry import LOAD_DATASET 6 | 7 | from .base import BaseDataset 8 | 9 | 10 | @LOAD_DATASET.register_module() 11 | class eprstmtDataset_V2(BaseDataset): 12 | 13 | @staticmethod 14 | def load(path): 15 | data = [] 16 | with open(path, 'r') as f: 17 | for line in f: 18 | line = json.loads(line) 19 | item = { 20 | 'sentence': line['sentence'], 21 | 'label': { 22 | 'Positive': 'A', 23 | 'Negative': 'B', 24 | }[line['label']], 25 | } 26 | data.append(item) 27 | return Dataset.from_list(data) 28 | -------------------------------------------------------------------------------- /opencompass/opencompass/datasets/flores.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from datasets import DatasetDict, load_dataset 4 | 5 | from opencompass.registry import LOAD_DATASET, TEXT_POSTPROCESSORS 6 | 7 | from .base import BaseDataset 8 | 9 | 10 | @LOAD_DATASET.register_module() 11 | class FloresFirst100Dataset(BaseDataset): 12 | 13 | @staticmethod 14 | def load(name): 15 | return DatasetDict({ 16 | 'dev': 17 | load_dataset(path='facebook/flores', name=name, split='dev'), 18 | 'devtest': 19 | load_dataset(path='facebook/flores', 20 | name=name, 21 | split='devtest[:100]') 22 | }) 23 | 24 | 25 | @TEXT_POSTPROCESSORS.register_module('flores') 26 | def flores_postprocess(text: str) -> str: 27 | text = text.strip().split('\n')[0] 28 | return text 29 | 30 | 31 | @TEXT_POSTPROCESSORS.register_module('flores-chinese') 32 | def flores_postprocess_chinese(text: str) -> str: 33 | import jieba 34 | truncated_text = text.strip().split('\n')[0] 35 | cleaned_text = re.sub(r'\s+', ' ', truncated_text).strip() 36 | cleaned_text = ' '.join(jieba.cut(cleaned_text)) 37 | return cleaned_text 38 | -------------------------------------------------------------------------------- /opencompass/opencompass/datasets/gsm8k.py: -------------------------------------------------------------------------------- 1 | from opencompass.registry import TEXT_POSTPROCESSORS 2 | 3 | 4 | @TEXT_POSTPROCESSORS.register_module('gsm8k_dataset') 5 | def gsm8k_dataset_postprocess(text: str) -> str: 6 | return text.split('#### ')[1].replace(',', '') 7 | 8 | 9 | @TEXT_POSTPROCESSORS.register_module('gsm8k') 10 | def gsm8k_postprocess(text: str) -> str: 11 | text = text.split('\n\n')[0] 12 | text = text.split(' ')[::-1] 13 | flag = False 14 | ret = '' 15 | for i in range(len(text)): 16 | s = text[i] 17 | for i in range(len(s)): 18 | if s[i].isdigit(): 19 | flag = True 20 | ret = s 21 | break 22 | if flag: 23 | break 24 | ret1 = '' 25 | for i in range(len(ret)): 26 | if ret[i].isdigit(): 27 | ret1 += ret[i] 28 | return ret1 29 | -------------------------------------------------------------------------------- /opencompass/opencompass/datasets/hellaswag.py: -------------------------------------------------------------------------------- 1 | from datasets import load_dataset 2 | 3 | from opencompass.registry import LOAD_DATASET 4 | 5 | from .base import BaseDataset 6 | 7 | 8 | @LOAD_DATASET.register_module() 9 | class hellaswagDataset(BaseDataset): 10 | 11 | @staticmethod 12 | def load(**kwargs): 13 | dataset = load_dataset(**kwargs) 14 | 15 | def preprocess(example): 16 | for i in range(4): 17 | example[chr(ord('A') + i)] = example['endings'][i] 18 | return example 19 | 20 | dataset = dataset.map(preprocess).remove_columns(['endings']) 21 | return dataset 22 | 23 | 24 | @LOAD_DATASET.register_module() 25 | class hellaswagDataset_V2(BaseDataset): 26 | 27 | @staticmethod 28 | def load(**kwargs): 29 | dataset = load_dataset(**kwargs) 30 | 31 | def preprocess(example): 32 | for i in range(4): 33 | example[chr(ord('A') + i)] = example['endings'][i] 34 | if example['label']: 35 | example['label'] = 'ABCD'[int(example['label'])] 36 | else: 37 | example['label'] = 'NULL' 38 | return example 39 | 40 | dataset = dataset.map(preprocess).remove_columns(['endings']) 41 | return dataset 42 | -------------------------------------------------------------------------------- /opencompass/opencompass/datasets/huggingface.py: -------------------------------------------------------------------------------- 1 | from datasets import load_dataset 2 | 3 | from opencompass.registry import LOAD_DATASET 4 | 5 | from .base import BaseDataset 6 | 7 | 8 | @LOAD_DATASET.register_module() 9 | class HFDataset(BaseDataset): 10 | 11 | @staticmethod 12 | def load(**kwargs): 13 | return load_dataset(**kwargs) 14 | -------------------------------------------------------------------------------- /opencompass/opencompass/datasets/iwslt2017.py: -------------------------------------------------------------------------------- 1 | from datasets import load_dataset 2 | 3 | from opencompass.registry import LOAD_DATASET 4 | 5 | from .base import BaseDataset 6 | 7 | 8 | @LOAD_DATASET.register_module() 9 | class IWSLT2017Dataset(BaseDataset): 10 | 11 | @staticmethod 12 | def load(**kwargs): 13 | dataset = load_dataset(**kwargs) 14 | dataset = dataset.map(lambda example: example['translation'] 15 | ).remove_columns('translation') 16 | return dataset 17 | -------------------------------------------------------------------------------- /opencompass/opencompass/datasets/jigsawmultilingual.py: -------------------------------------------------------------------------------- 1 | import csv 2 | 3 | from datasets import Dataset, DatasetDict 4 | 5 | from opencompass.registry import LOAD_DATASET 6 | 7 | from .base import BaseDataset 8 | 9 | 10 | @LOAD_DATASET.register_module() 11 | class JigsawMultilingualDataset(BaseDataset): 12 | 13 | @staticmethod 14 | def load(path, label, lang): 15 | assert lang in ['es', 'fr', 'it', 'pt', 'ru', 'tr'] 16 | dataset = DatasetDict() 17 | 18 | data_list = list() 19 | idx = 0 20 | with open(path) as file, open(label) as label: 21 | text_reader = csv.reader(file) 22 | label_reader = csv.reader(label) 23 | for text, target in zip(text_reader, label_reader): 24 | if text[2] == lang: 25 | assert text[0] == target[0] 26 | data_list.append({ 27 | 'idx': idx, 28 | 'text': text[1], 29 | 'label': int(target[1]), 30 | 'choices': ['no', 'yes'] 31 | }) 32 | idx += 1 33 | 34 | dataset['test'] = Dataset.from_list(data_list) 35 | return dataset 36 | -------------------------------------------------------------------------------- /opencompass/opencompass/datasets/mmlu.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import os.path as osp 3 | 4 | from datasets import Dataset, DatasetDict 5 | 6 | from opencompass.registry import LOAD_DATASET 7 | 8 | from .base import BaseDataset 9 | 10 | 11 | @LOAD_DATASET.register_module() 12 | class MMLUDataset(BaseDataset): 13 | 14 | @staticmethod 15 | def load(path: str, name: str): 16 | dataset = DatasetDict() 17 | for split in ['dev', 'test']: 18 | raw_data = [] 19 | filename = osp.join(path, split, f'{name}_{split}.csv') 20 | with open(filename, encoding='utf-8') as f: 21 | reader = csv.reader(f) 22 | for row in reader: 23 | assert len(row) == 6 24 | raw_data.append({ 25 | 'input': row[0], 26 | 'A': row[1], 27 | 'B': row[2], 28 | 'C': row[3], 29 | 'D': row[4], 30 | 'target': row[5], 31 | }) 32 | dataset[split] = Dataset.from_list(raw_data) 33 | return dataset 34 | -------------------------------------------------------------------------------- /opencompass/opencompass/datasets/obqa.py: -------------------------------------------------------------------------------- 1 | from datasets import load_dataset 2 | 3 | from opencompass.registry import LOAD_DATASET 4 | 5 | from .base import BaseDataset 6 | 7 | 8 | @LOAD_DATASET.register_module() 9 | class OBQADataset(BaseDataset): 10 | 11 | @staticmethod 12 | def load(**kwargs): 13 | dataset = load_dataset(**kwargs) 14 | 15 | def pre_process(example): 16 | for i in range(4): 17 | example[chr(ord('A') + i)] = example['choices']['text'][i] 18 | return example 19 | 20 | dataset = dataset.map(pre_process).remove_columns(['id', 'choices']) 21 | return dataset 22 | -------------------------------------------------------------------------------- /opencompass/opencompass/datasets/piqa.py: -------------------------------------------------------------------------------- 1 | from datasets import load_dataset 2 | 3 | from opencompass.registry import LOAD_DATASET 4 | 5 | from .base import BaseDataset 6 | 7 | 8 | @LOAD_DATASET.register_module() 9 | class piqaDataset_V2(BaseDataset): 10 | 11 | @staticmethod 12 | def load(**kwargs): 13 | dataset = load_dataset(**kwargs) 14 | 15 | def preprocess(example): 16 | assert isinstance(example['label'], int) 17 | if example['label'] < 0: 18 | example['answer'] = 'NULL' 19 | else: 20 | example['answer'] = 'AB'[example['label']] 21 | example.pop('label') 22 | return example 23 | 24 | dataset = dataset.map(preprocess) 25 | return dataset 26 | -------------------------------------------------------------------------------- /opencompass/opencompass/datasets/race.py: -------------------------------------------------------------------------------- 1 | from datasets import load_dataset 2 | 3 | from opencompass.registry import LOAD_DATASET 4 | 5 | from .base import BaseDataset 6 | 7 | 8 | @LOAD_DATASET.register_module() 9 | class RaceDataset(BaseDataset): 10 | 11 | @staticmethod 12 | def load(path: str, name: str): 13 | dataset = load_dataset(path, name) 14 | 15 | def preprocess(x): 16 | for ans, option in zip(['A', 'B', 'C', 'D'], x['options']): 17 | x[ans] = option 18 | del x['options'] 19 | return x 20 | 21 | return dataset.map(preprocess) 22 | -------------------------------------------------------------------------------- /opencompass/opencompass/datasets/safety.py: -------------------------------------------------------------------------------- 1 | from datasets import Dataset, DatasetDict 2 | 3 | from opencompass.registry import LOAD_DATASET 4 | 5 | from .base import BaseDataset 6 | 7 | 8 | @LOAD_DATASET.register_module() 9 | class SafetyDataset(BaseDataset): 10 | 11 | @staticmethod 12 | def load(path): 13 | dataset = DatasetDict() 14 | 15 | data_list = list() 16 | idx = 0 17 | with open(path, 'r') as f: 18 | for line in f: 19 | if line.strip(): 20 | data_list.append({'idx': idx, 'prompt': line.strip()}) 21 | idx += 1 22 | 23 | dataset['test'] = Dataset.from_list(data_list) 24 | -------------------------------------------------------------------------------- /opencompass/opencompass/datasets/siqa.py: -------------------------------------------------------------------------------- 1 | from datasets import load_dataset 2 | 3 | from opencompass.registry import LOAD_DATASET 4 | 5 | from .base import BaseDataset 6 | 7 | 8 | @LOAD_DATASET.register_module() 9 | class siqaDataset_V2(BaseDataset): 10 | 11 | @staticmethod 12 | def load(**kwargs): 13 | dataset = load_dataset(**kwargs) 14 | 15 | def preprocess(example): 16 | example['label'] = ' ABC'[int(example['label'])] 17 | return example 18 | 19 | dataset = dataset.map(preprocess) 20 | return dataset 21 | -------------------------------------------------------------------------------- /opencompass/opencompass/datasets/strategyqa.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from opencompass.registry import TEXT_POSTPROCESSORS 4 | 5 | 6 | @TEXT_POSTPROCESSORS.register_module('strategyqa') 7 | def strategyqa_pred_postprocess(text: str) -> str: 8 | text = text.split('\n\n')[0] 9 | text = text.split('answer is ')[-1] 10 | match = re.search(r'(yes|no)', text.lower()) 11 | if match: 12 | return match.group(1) 13 | return '' 14 | 15 | 16 | @TEXT_POSTPROCESSORS.register_module('strategyqa_dataset') 17 | def strategyqa_dataset_postprocess(text: str) -> str: 18 | return 'yes' if str(text) == 'True' else 'no' 19 | -------------------------------------------------------------------------------- /opencompass/opencompass/datasets/summedits.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from datasets import Dataset 4 | 5 | from opencompass.registry import LOAD_DATASET 6 | 7 | from .base import BaseDataset 8 | 9 | 10 | @LOAD_DATASET.register_module() 11 | class SummeditsDataset_V2(BaseDataset): 12 | 13 | @staticmethod 14 | def load(path: str): 15 | dataset = [] 16 | with open(path, 'r') as f: 17 | for line in f: 18 | line = json.loads(line) 19 | line['label'] = 'BA'[line['label']] 20 | dataset.append(line) 21 | return Dataset.from_list(dataset) 22 | -------------------------------------------------------------------------------- /opencompass/opencompass/datasets/wic.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from datasets import Dataset, load_dataset 4 | 5 | from opencompass.registry import LOAD_DATASET 6 | 7 | from .base import BaseDataset 8 | 9 | 10 | @LOAD_DATASET.register_module() 11 | class WiCDataset(BaseDataset): 12 | 13 | @staticmethod 14 | def load(**kwargs): 15 | 16 | dataset = load_dataset(**kwargs) 17 | 18 | def preprocess(example): 19 | if example['label'] == 'true': 20 | example['answer'] = 1 21 | else: 22 | example['answer'] = 0 23 | 24 | return example 25 | 26 | dataset = dataset.map(preprocess) 27 | return dataset 28 | 29 | 30 | @LOAD_DATASET.register_module() 31 | class WiCDataset_V2(BaseDataset): 32 | 33 | @staticmethod 34 | def load(path): 35 | dataset = [] 36 | with open(path, 'r') as f: 37 | for line in f: 38 | line = json.loads(line) 39 | line['label'] = {'true': 'A', 'false': 'B'}[line['label']] 40 | dataset.append(line) 41 | return Dataset.from_list(dataset) 42 | -------------------------------------------------------------------------------- /opencompass/opencompass/datasets/winograd.py: -------------------------------------------------------------------------------- 1 | from datasets import load_dataset 2 | 3 | from opencompass.registry import LOAD_DATASET 4 | 5 | from .base import BaseDataset 6 | 7 | 8 | @LOAD_DATASET.register_module() 9 | class winogradDataset(BaseDataset): 10 | 11 | @staticmethod 12 | def load(**kwargs): 13 | dataset = load_dataset(**kwargs) 14 | 15 | def pre_process(example): 16 | example['prompt'] = example.pop('text') 17 | example['opt1'] = example['options'][0] 18 | example['opt2'] = example['options'][1] 19 | return example 20 | 21 | dataset = dataset.map(pre_process).remove_columns( 22 | ['options', 'source']) 23 | return dataset 24 | -------------------------------------------------------------------------------- /opencompass/opencompass/datasets/xcopa.py: -------------------------------------------------------------------------------- 1 | from datasets import concatenate_datasets, load_dataset 2 | 3 | from opencompass.registry import LOAD_DATASET 4 | 5 | from .base import BaseDataset 6 | 7 | 8 | @LOAD_DATASET.register_module() 9 | class XCOPADataset(BaseDataset): 10 | 11 | @staticmethod 12 | def load(**kwargs): 13 | path = kwargs.get('path', None) 14 | lans = [ 15 | 'et', 'ht', 'it', 'id', 'qu', 'sw', 'zh', 'ta', 'th', 'tr', 'vi', 16 | 'translation-et', 'translation-ht', 'translation-it', 17 | 'translation-id', 'translation-sw', 'translation-zh', 18 | 'translation-ta', 'translation-th', 'translation-tr', 19 | 'translation-vi' 20 | ] 21 | 22 | datasets = [] 23 | for lan in lans: 24 | dataset = load_dataset(path, lan)['validation'] 25 | datasets.append(dataset) 26 | 27 | combined_dataset = concatenate_datasets(datasets) 28 | 29 | return combined_dataset 30 | -------------------------------------------------------------------------------- /opencompass/opencompass/datasets/xsum.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from datasets import Dataset 4 | 5 | from opencompass.registry import LOAD_DATASET, TEXT_POSTPROCESSORS 6 | 7 | from .base import BaseDataset 8 | 9 | 10 | @LOAD_DATASET.register_module() 11 | class XsumDataset(BaseDataset): 12 | 13 | @staticmethod 14 | def load(path: str): 15 | with open(path, 'r', errors='ignore') as in_f: 16 | rows = [] 17 | for i, line in enumerate(in_f): 18 | if i == 1000: 19 | break 20 | sample = json.loads(line.strip()) 21 | dialogue = sample['dialogue'] 22 | summary = sample['summary'] 23 | if isinstance(dialogue, float) or isinstance(summary, float): 24 | continue 25 | rows.append({'dialogue': dialogue, 'summary': summary}) 26 | dataset = Dataset.from_dict({ 27 | 'dialogue': [row['dialogue'] for row in rows], 28 | 'summary': [row['summary'] for row in rows] 29 | }) 30 | return dataset 31 | 32 | 33 | @TEXT_POSTPROCESSORS.register_module('Xsum') 34 | def Xsum_postprocess(text: str) -> str: 35 | text = text.strip().split('\n')[0].strip() 36 | return text 37 | -------------------------------------------------------------------------------- /opencompass/opencompass/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import BaseModel, LMTemplateParser # noqa 2 | from .base_api import APITemplateParser, BaseAPIModel # noqa 3 | from .glm import GLM130B # noqa: F401, F403 4 | from .huggingface import HuggingFace # noqa: F401, F403 5 | from .huggingface import HuggingFaceCausalLM, GPTQCausalLM, ExllamaCausalLM 6 | # noqa: F401, F403 7 | from .llama2 import Llama2Chat # noqa: F401, F403 8 | from .openai_api import OpenAI # noqa: F401 9 | -------------------------------------------------------------------------------- /opencompass/opencompass/openicl/__init__.py: -------------------------------------------------------------------------------- 1 | from .icl_dataset_reader import DatasetReader # noqa 2 | from .icl_evaluator import * # noqa 3 | from .icl_inferencer import * # noqa 4 | from .icl_prompt_template import PromptTemplate # noqa 5 | from .icl_retriever import * # noqa 6 | -------------------------------------------------------------------------------- /opencompass/opencompass/openicl/icl_evaluator/__init__.py: -------------------------------------------------------------------------------- 1 | from .icl_aucroc_evaluator import AUCROCEvaluator # noqa 2 | from .icl_base_evaluator import BaseEvaluator # noqa 3 | from .icl_em_evaluator import EMEvaluator # noqa 4 | from .icl_hf_evaluator import * # noqa 5 | from .icl_toxic_evaluator import ToxicEvaluator # noqa 6 | -------------------------------------------------------------------------------- /opencompass/opencompass/openicl/icl_evaluator/icl_base_evaluator.py: -------------------------------------------------------------------------------- 1 | """Base Evaluator.""" 2 | 3 | 4 | class BaseEvaluator: 5 | 6 | def __init__(self) -> None: 7 | pass 8 | 9 | def score(self): 10 | raise NotImplementedError("Method hasn't been implemented yet") 11 | -------------------------------------------------------------------------------- /opencompass/opencompass/openicl/icl_inferencer/__init__.py: -------------------------------------------------------------------------------- 1 | from .icl_base_inferencer import BaseInferencer # noqa 2 | from .icl_clp_inferencer import CLPInferencer # noqa 3 | from .icl_gen_inferencer import GenInferencer # noqa 4 | from .icl_ppl_inferencer import PPLInferencer # noqa 5 | -------------------------------------------------------------------------------- /opencompass/opencompass/openicl/icl_retriever/__init__.py: -------------------------------------------------------------------------------- 1 | from .icl_base_retriever import BaseRetriever # noqa 2 | from .icl_bm25_retriever import BM25Retriever # noqa 3 | from .icl_dpp_retriever import DPPRetriever # noqa 4 | from .icl_fix_k_retriever import FixKRetriever # noqa 5 | from .icl_mdl_retriever import MDLRetriever # noqa 6 | from .icl_random_retriever import RandomRetriever # noqa 7 | from .icl_topk_retriever import TopkRetriever # noqa 8 | from .icl_votek_retriever import VotekRetriever # noqa 9 | from .icl_zero_retriever import ZeroRetriever # noqa 10 | -------------------------------------------------------------------------------- /opencompass/opencompass/openicl/icl_retriever/icl_zero_retriever.py: -------------------------------------------------------------------------------- 1 | """Zeroshot Retriever.""" 2 | 3 | from typing import List, Optional 4 | 5 | from opencompass.openicl.icl_retriever import BaseRetriever 6 | from opencompass.registry import ICL_RETRIEVERS 7 | 8 | 9 | @ICL_RETRIEVERS.register_module() 10 | class ZeroRetriever(BaseRetriever): 11 | """Zeroshot Retriever. The retriever returns empty list for all queries. 12 | 13 | Args: 14 | dataset (`BaseDataset`): Any BaseDataset instances. 15 | Attributes of ``reader``, ``train`` and ``test`` will be used. 16 | ice_eos_token (`Optional[str]`): The end of sentence token for 17 | in-context example template when origin `PromptTemplate` is 18 | provided. Defaults to ''. 19 | """ 20 | 21 | def __init__(self, dataset, ice_eos_token: Optional[str] = '') -> None: 22 | super().__init__(dataset, '', ice_eos_token, 0) 23 | 24 | def retrieve(self) -> List[List]: 25 | rtr_idx_list = [[] for _ in range(len(self.test_ds))] 26 | return rtr_idx_list 27 | -------------------------------------------------------------------------------- /opencompass/opencompass/openicl/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .logging import * # noqa 2 | -------------------------------------------------------------------------------- /opencompass/opencompass/openicl/utils/logging.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import torch.distributed as dist 4 | 5 | LOG_LEVEL = logging.INFO 6 | SUBPROCESS_LOG_LEVEL = logging.ERROR 7 | LOG_FORMATTER = '[%(asctime)s] [%(name)s] [%(levelname)s] %(message)s' 8 | 9 | 10 | def get_logger(name, level=LOG_LEVEL, log_file=None, file_mode='w'): 11 | formatter = logging.Formatter(LOG_FORMATTER) 12 | 13 | logger = logging.getLogger(name) 14 | 15 | for handler in logger.root.handlers: 16 | if type(handler) is logging.StreamHandler: 17 | handler.setLevel(logging.ERROR) 18 | 19 | if dist.is_available() and dist.is_initialized(): 20 | rank = dist.get_rank() 21 | else: 22 | rank = 0 23 | 24 | if rank == 0 and log_file is not None: 25 | file_handler = logging.FileHandler(log_file, file_mode) 26 | file_handler.setFormatter(formatter) 27 | file_handler.setLevel(level) 28 | logger.addHandler(file_handler) 29 | 30 | if rank == 0: 31 | logger.setLevel(level) 32 | else: 33 | logger.setLevel(SUBPROCESS_LOG_LEVEL) 34 | 35 | stream_handler = logging.StreamHandler() 36 | stream_handler.setFormatter(formatter) 37 | stream_handler.setLevel(level) 38 | logger.addHandler(stream_handler) 39 | 40 | return logger 41 | -------------------------------------------------------------------------------- /opencompass/opencompass/partitioners/__init__.py: -------------------------------------------------------------------------------- 1 | from .naive import * # noqa: F401, F403 2 | from .size import * # noqa: F401, F403 3 | -------------------------------------------------------------------------------- /opencompass/opencompass/registry.py: -------------------------------------------------------------------------------- 1 | from mmengine.registry import Registry 2 | 3 | PARTITIONERS = Registry('partitioner', locations=['opencompass.partitioners']) 4 | RUNNERS = Registry('runner', locations=['opencompass.runners']) 5 | TASKS = Registry('task', locations=['opencompass.tasks']) 6 | MODELS = Registry('model', locations=['opencompass.models']) 7 | # TODO: LOAD_DATASET -> DATASETS 8 | LOAD_DATASET = Registry('load_dataset', locations=['opencompass.datasets']) 9 | TEXT_POSTPROCESSORS = Registry( 10 | 'text_postprocessors', locations=['opencompass.utils.text_postprocessors']) 11 | EVALUATORS = Registry('evaluators', locations=['opencompass.evaluators']) 12 | 13 | ICL_INFERENCERS = Registry('icl_inferencers', 14 | locations=['opencompass.openicl.icl_inferencer']) 15 | ICL_RETRIEVERS = Registry('icl_retrievers', 16 | locations=['opencompass.openicl.icl_retriever']) 17 | ICL_DATASET_READERS = Registry( 18 | 'icl_dataset_readers', 19 | locations=['opencompass.openicl.icl_dataset_reader']) 20 | ICL_PROMPT_TEMPLATES = Registry( 21 | 'icl_prompt_templates', 22 | locations=['opencompass.openicl.icl_prompt_template']) 23 | ICL_EVALUATORS = Registry('icl_evaluators', 24 | locations=['opencompass.openicl.icl_evaluator']) 25 | -------------------------------------------------------------------------------- /opencompass/opencompass/runners/__init__.py: -------------------------------------------------------------------------------- 1 | from .dlc import * # noqa: F401, F403 2 | from .local import * # noqa: F401, F403 3 | from .slurm import * # noqa: F401, F403 4 | -------------------------------------------------------------------------------- /opencompass/opencompass/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | from .openicl_eval import * # noqa: F401, F403 2 | from .openicl_infer import * # noqa: F401, F403 3 | -------------------------------------------------------------------------------- /opencompass/opencompass/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .abbr import * # noqa 2 | from .build import * # noqa 3 | from .collect_env import * # noqa 4 | from .fileio import * # noqa 5 | from .git import * # noqa 6 | from .lark import * # noqa 7 | from .logging import * # noqa 8 | from .menu import * # noqa 9 | from .prompt import * # noqa 10 | from .summarizer import * # noqa 11 | from .text_postprocessors import * # noqa 12 | -------------------------------------------------------------------------------- /opencompass/opencompass/utils/build.py: -------------------------------------------------------------------------------- 1 | import copy 2 | 3 | from mmengine.config import ConfigDict 4 | 5 | from opencompass.registry import LOAD_DATASET, MODELS 6 | 7 | 8 | def build_dataset_from_cfg(dataset_cfg: ConfigDict) -> ConfigDict: 9 | dataset_cfg = copy.deepcopy(dataset_cfg) 10 | dataset_cfg.pop('infer_cfg', None) 11 | dataset_cfg.pop('eval_cfg', None) 12 | dataset_cfg.pop('abbr', None) 13 | return LOAD_DATASET.build(dataset_cfg) 14 | 15 | 16 | def build_model_from_cfg(model_cfg: ConfigDict) -> ConfigDict: 17 | model_cfg = copy.deepcopy(model_cfg) 18 | model_cfg.pop('run_cfg', None) 19 | model_cfg.pop('max_out_len', None) 20 | model_cfg.pop('batch_size', None) 21 | model_cfg.pop('abbr', None) 22 | return MODELS.build(model_cfg) 23 | -------------------------------------------------------------------------------- /opencompass/opencompass/utils/collect_env.py: -------------------------------------------------------------------------------- 1 | from mmengine.utils import get_git_hash 2 | from mmengine.utils.dl_utils import collect_env as collect_base_env 3 | 4 | import opencompass 5 | 6 | 7 | def collect_env(): 8 | """Collect the information of the running environments.""" 9 | env_info = collect_base_env() 10 | env_info['opencompass'] = opencompass.__version__ + '+' + get_git_hash( 11 | )[:7] 12 | return env_info 13 | -------------------------------------------------------------------------------- /opencompass/opencompass/utils/git.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | 3 | 4 | def get_git_root() -> str: 5 | cmd = ['git', 'rev-parse', '--show-toplevel'] 6 | result = subprocess.run(cmd, stdout=subprocess.PIPE, check=True) 7 | return result.stdout.decode('utf-8').strip() 8 | 9 | 10 | def get_latest_commit(branch: str) -> str: 11 | cmd = ['git', 'rev-parse', branch] 12 | result = subprocess.run(cmd, stdout=subprocess.PIPE, check=True) 13 | return result.stdout.decode('utf-8').strip() 14 | -------------------------------------------------------------------------------- /opencompass/opencompass/utils/logging.py: -------------------------------------------------------------------------------- 1 | from mmengine.logging import MMLogger 2 | 3 | 4 | def get_logger(log_level='INFO') -> MMLogger: 5 | """Get the logger for OpenCompass. 6 | 7 | Args: 8 | log_level (str): The log level. Default: 'INFO'. Choices are 'DEBUG', 9 | 'INFO', 'WARNING', 'ERROR', 'CRITICAL'. 10 | """ 11 | return MMLogger.get_instance('OpenCompass', 12 | logger_name='OpenCompass', 13 | log_level=log_level) 14 | -------------------------------------------------------------------------------- /opencompass/requirements.txt: -------------------------------------------------------------------------------- 1 | accelerate>=0.19.0 2 | boto3 3 | colossalai 4 | cpm_kernels 5 | datasets>=2.12.0 6 | evaluate>=0.3.0 7 | fairscale 8 | faiss_gpu==1.7.2 9 | jieba 10 | mmengine>=0.8.2 11 | nltk==3.8 12 | numpy==1.23.4 13 | openai 14 | pandas<2.0.0 15 | rank_bm25==0.2.2 16 | requests==2.28.1 17 | scikit_learn==1.2.1 18 | sentence_transformers==2.2.2 19 | tabulate 20 | tiktoken 21 | tokenizers>=0.13.3 22 | torch>=1.13.1 23 | tqdm==4.64.1 24 | transformers>=4.29.1 25 | -------------------------------------------------------------------------------- /opencompass/requirements/docs.txt: -------------------------------------------------------------------------------- 1 | docutils==0.18.1 2 | modelindex 3 | myst-parser 4 | -e git+https://github.com/Ezra-Yu/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme 5 | sphinx==6.1.3 6 | sphinx-copybutton 7 | sphinx-notfound-page 8 | sphinx-tabs 9 | sphinxcontrib-jquery 10 | tabulate 11 | -------------------------------------------------------------------------------- /opencompass/requirements/runtime.txt: -------------------------------------------------------------------------------- 1 | accelerate>=0.19.0 2 | boto3 3 | colossalai 4 | cpm_kernels 5 | datasets>=2.12.0 6 | evaluate>=0.3.0 7 | fairscale 8 | faiss_gpu==1.7.2 9 | jieba 10 | mmengine 11 | nltk==3.8 12 | numpy==1.23.4 13 | openai 14 | pandas<2.0.0 15 | rank_bm25==0.2.2 16 | requests==2.28.1 17 | scikit_learn==1.2.1 18 | sentence_transformers==2.2.2 19 | tabulate 20 | tiktoken 21 | tokenizers>=0.13.3 22 | torch>=1.13.1 23 | tqdm==4.64.1 24 | transformers>=4.29.1 25 | -------------------------------------------------------------------------------- /opencompass/tools/ceval_util.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import os 3 | import argparse 4 | 5 | 6 | def parse_args(): 7 | parser = argparse.ArgumentParser() 8 | parser.add_argument('--path', help='file path', type=str) 9 | parser.add_argument('--key', help='score column name', type=str) 10 | args = parser.parse_args() 11 | return args 12 | 13 | 14 | def main(): 15 | args = parse_args() 16 | df = pd.read_csv(args.path) 17 | print(df.shape) 18 | score = [] 19 | for idx, row in df.iterrows(): 20 | print(row.to_dict()) 21 | if row[args.key] == "-": 22 | continue 23 | if "ceval" not in row["dataset"]: 24 | continue 25 | score.append(float(row[args.key])) 26 | print(f"score: {score}, sum: {sum(score)}, length: {len(score)}") 27 | print(f"average score: {sum(score) / len(score)}") 28 | 29 | 30 | if __name__ == "__main__": 31 | main() 32 | -------------------------------------------------------------------------------- /opencompass/tools/mmlu_util.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import os 3 | import argparse 4 | 5 | 6 | def parse_args(): 7 | parser = argparse.ArgumentParser() 8 | parser.add_argument('--path', help='file path', type=str) 9 | parser.add_argument('--key', help='score column name', type=str) 10 | args = parser.parse_args() 11 | return args 12 | 13 | 14 | def main(): 15 | args = parse_args() 16 | df = pd.read_csv(args.path) 17 | print(df.shape) 18 | score = [] 19 | for idx, row in df.iterrows(): 20 | print(row.to_dict()) 21 | if row[args.key] == "-": 22 | continue 23 | if "mmlu" not in row["dataset"]: 24 | continue 25 | score.append(float(row[args.key])) 26 | print(f"score: {score}, sum: {sum(score)}, length: {len(score)}") 27 | print(f"average score: {sum(score) / len(score)}") 28 | 29 | 30 | if __name__ == "__main__": 31 | main() 32 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | safetensors==0.3.1 2 | datasets==2.10.1 3 | accelerate>=0.20.3 4 | protobuf==3.20.2 5 | transformers>=4.34.0 6 | scikit-learn==1.0.2 7 | torch>=2.0.0 8 | evaluate==0.4.0 9 | texttable==1.6.7 10 | toml==0.10.2 11 | numpy>=1.22.0 12 | sentencepiece==0.1.98 13 | fire==0.5.0 14 | flash-attn==2.1.1 15 | deepspeed==0.9.5 16 | streamlit==1.24.1 17 | -------------------------------------------------------------------------------- /train/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TigerResearch/TigerBot/a80a3fbbdf8536fa81378635ff0baf3bdd3f5b37/train/.DS_Store -------------------------------------------------------------------------------- /train/requirements_qlora.txt: -------------------------------------------------------------------------------- 1 | transformers==4.30.2 2 | peft@git+https://github.com/huggingface/peft.git@eb01b5ee1dfeb6fdacc73dc2fb1dd674bb6868ac 3 | accelerate==0.20.3 4 | einops==0.6.1 5 | bitsandbytes==0.39.0 -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TigerResearch/TigerBot/a80a3fbbdf8536fa81378635ff0baf3bdd3f5b37/utils/__init__.py --------------------------------------------------------------------------------