├── .gitignore ├── LICENSE ├── README.md ├── TrainingParams.md ├── docs ├── MatBERTInference.ipynb ├── PretrainingCorpusWordCloud.png ├── model_2Mpapers_cased_30522_wd.csv └── model_2Mpapers_uncased_30522_wd.csv ├── examples ├── GLUE-submission │ ├── matbert-base-cased.zip │ ├── matbert-base-cased │ │ ├── AX.tsv │ │ ├── CoLA.tsv │ │ ├── MNLI-m.tsv │ │ ├── MNLI-mm.tsv │ │ ├── MRPC.tsv │ │ ├── QNLI.tsv │ │ ├── QQP.tsv │ │ ├── RTE.tsv │ │ ├── SST-2.tsv │ │ ├── STS-B.tsv │ │ └── WNLI.tsv │ ├── matbert-base-uncased.zip │ └── matbert-base-uncased │ │ ├── AX.tsv │ │ ├── CoLA.tsv │ │ ├── MNLI-m.tsv │ │ ├── MNLI-mm.tsv │ │ ├── MRPC.tsv │ │ ├── QNLI.tsv │ │ ├── QQP.tsv │ │ ├── RTE.tsv │ │ ├── SST-2.tsv │ │ ├── STS-B.tsv │ │ └── WNLI.tsv ├── GenerateJobs.ipynb ├── GenerateResults.ipynb └── evaluate_glue.py ├── matbert ├── __init__.py └── training │ ├── __init__.py │ ├── configs │ ├── bert-base-cased-wd.json │ ├── bert-base-cased.json │ ├── bert-base-uncased-wd.json │ └── bert-base-uncased.json │ ├── dataset.py │ ├── options.py │ ├── script_collect.py │ ├── script_tokenize_lmdb.py │ ├── script_train_bert.py │ ├── script_train_tokenizer_bert.py │ └── script_train_tokenizer_gpt2.py ├── requirements.txt ├── requirements_training.txt └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .idea/ 3 | 4 | # Byte-compiled / optimized / DLL files 5 | __pycache__/ 6 | *.py[cod] 7 | *$py.class 8 | 9 | # C extensions 10 | *.so 11 | 12 | # Distribution / packaging 13 | .Python 14 | build/ 15 | develop-eggs/ 16 | dist/ 17 | downloads/ 18 | eggs/ 19 | .eggs/ 20 | lib/ 21 | lib64/ 22 | parts/ 23 | sdist/ 24 | var/ 25 | wheels/ 26 | pip-wheel-metadata/ 27 | share/python-wheels/ 28 | *.egg-info/ 29 | .installed.cfg 30 | *.egg 31 | MANIFEST 32 | 33 | # PyInstaller 34 | # Usually these files are written by a python script from a template 35 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 36 | *.manifest 37 | *.spec 38 | 39 | # Installer logs 40 | pip-log.txt 41 | pip-delete-this-directory.txt 42 | 43 | # Unit test / coverage reports 44 | htmlcov/ 45 | .tox/ 46 | .nox/ 47 | .coverage 48 | .coverage.* 49 | .cache 50 | nosetests.xml 51 | coverage.xml 52 | *.cover 53 | *.py,cover 54 | .hypothesis/ 55 | .pytest_cache/ 56 | 57 | # Translations 58 | *.mo 59 | *.pot 60 | 61 | # Django stuff: 62 | *.log 63 | local_settings.py 64 | db.sqlite3 65 | db.sqlite3-journal 66 | 67 | # Flask stuff: 68 | instance/ 69 | .webassets-cache 70 | 71 | # Scrapy stuff: 72 | .scrapy 73 | 74 | # Sphinx documentation 75 | docs/_build/ 76 | 77 | # PyBuilder 78 | target/ 79 | 80 | # Jupyter Notebook 81 | .ipynb_checkpoints 82 | 83 | # IPython 84 | profile_default/ 85 | ipython_config.py 86 | 87 | # pyenv 88 | .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 98 | __pypackages__/ 99 | 100 | # Celery stuff 101 | celerybeat-schedule 102 | celerybeat.pid 103 | 104 | # SageMath parsed files 105 | *.sage.py 106 | 107 | # Environments 108 | .env 109 | .venv 110 | env/ 111 | venv/ 112 | ENV/ 113 | env.bak/ 114 | venv.bak/ 115 | 116 | # Spyder project settings 117 | .spyderproject 118 | .spyproject 119 | 120 | # Rope project settings 121 | .ropeproject 122 | 123 | # mkdocs documentation 124 | /site 125 | 126 | # mypy 127 | .mypy_cache/ 128 | .dmypy.json 129 | dmypy.json 130 | 131 | # Pyre type checker 132 | .pyre/ 133 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Lawrence Berkeley National Lab + Natural Language Processing = LBNLP 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MatBERT 2 | A pretrained BERT model on materials science literature. MatBERT specializes in understanding 3 | materials science terminologies and paragraph-level scientific reasoning. 4 | 5 | ## Downloading data files 6 | 7 | To use MatBERT, download these files into a folder: 8 | 9 | ``` 10 | export MODEL_PATH="Your path" 11 | mkdir $MODEL_PATH/matbert-base-cased $MODEL_PATH/matbert-base-uncased 12 | 13 | curl -# -o $MODEL_PATH/matbert-base-cased/config.json https://cedergroup-share.s3-us-west-2.amazonaws.com/public/MatBERT/model_2Mpapers_cased_30522_wd/config.json 14 | curl -# -o $MODEL_PATH/matbert-base-cased/vocab.txt https://cedergroup-share.s3-us-west-2.amazonaws.com/public/MatBERT/model_2Mpapers_cased_30522_wd/vocab.txt 15 | curl -# -o $MODEL_PATH/matbert-base-cased/pytorch_model.bin https://cedergroup-share.s3-us-west-2.amazonaws.com/public/MatBERT/model_2Mpapers_cased_30522_wd/pytorch_model.bin 16 | 17 | curl -# -o $MODEL_PATH/matbert-base-uncased/config.json https://cedergroup-share.s3-us-west-2.amazonaws.com/public/MatBERT/model_2Mpapers_uncased_30522_wd/config.json 18 | curl -# -o $MODEL_PATH/matbert-base-uncased/vocab.txt https://cedergroup-share.s3-us-west-2.amazonaws.com/public/MatBERT/model_2Mpapers_uncased_30522_wd/vocab.txt 19 | curl -# -o $MODEL_PATH/matbert-base-uncased/pytorch_model.bin https://cedergroup-share.s3-us-west-2.amazonaws.com/public/MatBERT/model_2Mpapers_uncased_30522_wd/pytorch_model.bin 20 | ``` 21 | 22 | ## Using MatBERT 23 | 24 | ### Tokenizers 25 | 26 | The tokenizer is specifically trained to handle materials science terminologies: 27 | 28 | ```python 29 | >>> from transformers import BertTokenizerFast 30 | >>> tokenizer = BertTokenizerFast.from_pretrained('PATH-TO-MATBERT/matbert-base-cased', do_lower_case=False) 31 | >>> tokenizer_bert = BertTokenizerFast.from_pretrained('bert-base-cased', do_lower_case=False) 32 | 33 | >>> for i in ['Fe(NO3)3• 9H2O', 'La0.85Ag0.15Mn1−yAlyO3']: 34 | >>> print(i) 35 | >>> print('='*100) 36 | >>> print('MatBERT tokenizer:', tokenizer.tokenize(i)) 37 | >>> print('BERT tokenizer:', tokenizer_bert.tokenize(i)) 38 | 39 | Fe(NO3)3• 9H2O 40 | ==================================================================================================== 41 | MatBERT tokenizer: ['Fe', '(', 'NO3', ')', '3', '•', '9H2O'] 42 | BERT tokenizer: ['Fe', '(', 'NO', '##3', ')', '3', '•', '9', '##H', '##2', '##O'] 43 | La0.85Ag0.15Mn1−yAlyO3 44 | ==================================================================================================== 45 | MatBERT tokenizer: ['La0', '.', '85', '##Ag', '##0', '.', '15', '##Mn1', '##−y', '##Al', '##y', '##O3'] 46 | BERT tokenizer: ['La', '##0', '.', '85', '##A', '##g', '##0', '.', '15', '##M', '##n', '##1', '##−', '##y', '##A', '##ly', '##O', '##3'] 47 | ``` 48 | 49 | ### The model 50 | 51 | The model can be loaded using Transformers' unversal loading API. Here, we demonstrate 52 | how MatBERT performs scientific reasoning for the synthesis of Li-ion battery materials. 53 | 54 | ```python 55 | >>> from transformers import BertForMaskedLM, BertTokenizerFast, pipeline 56 | >>> from pprint import pprint 57 | 58 | >>> model = BertForMaskedLM.from_pretrained('PATH-TO-MATBERT/matbert-base-cased') 59 | >>> tokenizer = BertTokenizerFast.from_pretrained('PATH-TO-MATBERT/matbert-base-cased', do_lower_case=False) 60 | >>> unmasker = pipeline('fill-mask', model=model, tokenizer=tokenizer) 61 | >>> pprint(unmasker("Conventional [MASK] synthesis is used to fabricate material LiMn2O4.")) 62 | [{'sequence': '[CLS] Conventional combustion synthesis is used to fabricate material LiMn2O4. [SEP]', 63 | 'score': 0.4971400499343872, 64 | 'token': 5444, 65 | 'token_str': 'combustion'}, 66 | {'sequence': '[CLS] Conventional hydrothermal synthesis is used to fabricate material LiMn2O4. [SEP]', 67 | 'score': 0.2478722780942917, 68 | 'token': 7524, 69 | 'token_str': 'hydrothermal'}, 70 | {'sequence': '[CLS] Conventional chemical synthesis is used to fabricate material LiMn2O4. [SEP]', 71 | 'score': 0.060953784734010696, 72 | 'token': 2868, 73 | 'token_str': 'chemical'}, 74 | {'sequence': '[CLS] Conventional gel synthesis is used to fabricate material LiMn2O4. [SEP]', 75 | 'score': 0.03871171176433563, 76 | 'token': 4003, 77 | 'token_str': 'gel'}, 78 | {'sequence': '[CLS] Conventional solution synthesis is used to fabricate material LiMn2O4. [SEP]', 79 | 'score': 0.019403140991926193, 80 | 'token': 2291, 81 | 'token_str': 'solution'}] 82 | ``` 83 | 84 | ## Evaluation 85 | 86 | ### GLUE 87 | 88 | The [General Language Understanding Evaluation (GLUE) benchmark](https://gluebenchmark.com/) 89 | is a collection of resources for training, evaluating, and analyzing natural language understanding systems. 90 | Note, GLUE evaluates language models' capability to model general purpose language understanding, 91 | which may not align with the capabilities of language models trained on special domains, such as MatBERT. 92 | 93 | | Task | Metric | Score (MatBERT-base-cased) | Score (MatBERT-base-uncased) | 94 | |----------------------------------------|-----------------------|---------------------------:|-----------------------------:| 95 | | The Corpus of Linguistic Acceptability | Matthew's Corr | 26.1 | 26.6 | 96 | | The Stanford Sentiment Treebank | Accuracy | 89.5 | 90.2 | 97 | | Microsoft Research Paraphrase Corpus | F1/Accuracy | 87.0/83.1 | 87.4/82.7 | 98 | | Semantic Textual Similarity Benchmark | Pearson-Spearman Corr | 80.3/79.3 | 81.5/80.2 | 99 | | Quora Question Pairs | F1/Accuracy | 69.8/88.4 | 69.7/88.6 | 100 | | MultiNLI Matched | Accuracy | 79.6 | 80.7 | 101 | | MultiNLI Mismatched | Accuracy | 79.3 | 80.1 | 102 | | Question NLI | Accuracy | 88.4 | 88.5 | 103 | | Recognizing Textual Entailment | Accuracy | 63.0 | 60.2 | 104 | | Winograd NLI | Accuracy | 61.6 | 65.1 | 105 | | Diagnostics Main | Matthew's Corr | 32.6 | 31.9 | 106 | | Average Score | ---- | 72.4 | 72.9 | 107 | 108 | 109 | ## Training details 110 | 111 | Training of all MatBERT models was done using `transformers==3.3.1`. 112 | The corpus of this training contains 2 million papers collected by the 113 | text-mining efforts at [CEDER group](https://ceder.berkeley.edu/). In 114 | total, we had collected 61,253,938 paragraphs, from which around 50 million 115 | paragraphs with 20-510 tokens are filtered and used for training. Two WordPiece 116 | tokenizers (cased and uncased) that are optimized for materials science 117 | literature was trained using these paragraphs. 118 | 119 | The DOIs and titles of the 2 million papers can be found at [this CSV file](https://cedergroup-share.s3-us-west-2.amazonaws.com/public/MatBERT/PretrainingDataDOIs.csv.zip). 120 | To grasp an overview of this corpus, we created a word cloud image [here](docs/PretrainingCorpusWordCloud.png). 121 | 122 | For training MatBERT, the config files we used were 123 | [matbert-base-uncased](matbert/training/configs/bert-base-uncased-wd.json) 124 | and [matbert-base-cased](matbert/training/configs/bert-base-cased-wd.json). 125 | Only the masked language modeling (MLM) task was used to pretrain MatBERT models. 126 | Roughly the batch size is 192 paragraphs per gradient update step and there are 127 | 5 epochs in total. The optimizer used is AdamW with beta1=0.9 and beta2=0.999. 128 | Learning rates start with 5e-5 and decays linearly to zero as the training finishes. 129 | A weight decay of 0.01 was used. All models are trained using FP16 mode and O2 130 | optimization on 8 NVIDIA V100 cards. The loss values during training can be found 131 | at [matbert-base-uncased](docs/model_2Mpapers_uncased_30522_wd.csv) and 132 | [matbert-base-cased](docs/model_2Mpapers_cased_30522_wd.csv). 133 | 134 | ## Citing 135 | 136 | If you used this work in your projects, please consider citing this paper: 137 | 138 | ``` 139 | @article{walker2021impact, 140 | title={The Impact of Domain-Specific Pre-Training on Named Entity Recognition Tasks in Materials Science}, 141 | author={Walker, Nicholas and Trewartha, Amalie and Huo, Haoyan and Lee, Sanghoon and Cruse, Kevin and Dagdelen, John and Dunn, Alexander and Persson, Kristin and Ceder, Gerbrand and Jain, Anubhav}, 142 | journal={Available at SSRN 3950755}, 143 | year={2021} 144 | } 145 | ``` 146 | 147 | This work used the Extreme Science and Engineering Discovery Environment (XSEDE) 148 | GPU resources, specifically the Bridges-2 supercomputer at the Pittsburgh 149 | Supercomputing Center, through allocation TG-DMR970008S. 150 | -------------------------------------------------------------------------------- /TrainingParams.md: -------------------------------------------------------------------------------- 1 | ## Some parameters that could be used on various clusters 2 | 3 | ### Cori GPU nodes 4 | 5 | There are 8 V100 cards, 16GB memory each. 6 | 7 | For base model: 8 | 9 | - Use fp16 training. 10 | - Maximal batch size per card is 16. 11 | - Average training speed is 1-1.5 batch/s. 12 | 13 | ### Bridges GPU K80 14 | 15 | There are 4 K80 cards, 12GB memory each. 16 | 17 | For base model: 18 | 19 | - Use fp16 training. 20 | - Maximal batch size per card is 8. 21 | - Average training speed is 0.2-0.3 batch/s. 22 | -------------------------------------------------------------------------------- /docs/PretrainingCorpusWordCloud.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lbnlp/MatBERT/31592cf5a26c9730dd39585c38f470b9eba6f4ba/docs/PretrainingCorpusWordCloud.png -------------------------------------------------------------------------------- /examples/GLUE-submission/matbert-base-cased.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lbnlp/MatBERT/31592cf5a26c9730dd39585c38f470b9eba6f4ba/examples/GLUE-submission/matbert-base-cased.zip -------------------------------------------------------------------------------- /examples/GLUE-submission/matbert-base-cased/AX.tsv: -------------------------------------------------------------------------------- 1 | index prediction 2 | 0 contradiction 3 | 1 contradiction 4 | 2 entailment 5 | 3 contradiction 6 | 4 contradiction 7 | 5 contradiction 8 | 6 entailment 9 | 7 entailment 10 | 8 contradiction 11 | 9 contradiction 12 | 10 contradiction 13 | 11 entailment 14 | 12 contradiction 15 | 13 contradiction 16 | 14 entailment 17 | 15 contradiction 18 | 16 contradiction 19 | 17 entailment 20 | 18 entailment 21 | 19 neutral 22 | 20 contradiction 23 | 21 contradiction 24 | 22 entailment 25 | 23 entailment 26 | 24 entailment 27 | 25 entailment 28 | 26 entailment 29 | 27 entailment 30 | 28 entailment 31 | 29 entailment 32 | 30 entailment 33 | 31 entailment 34 | 32 entailment 35 | 33 entailment 36 | 34 entailment 37 | 35 entailment 38 | 36 contradiction 39 | 37 contradiction 40 | 38 contradiction 41 | 39 contradiction 42 | 40 entailment 43 | 41 entailment 44 | 42 contradiction 45 | 43 contradiction 46 | 44 entailment 47 | 45 entailment 48 | 46 entailment 49 | 47 entailment 50 | 48 entailment 51 | 49 contradiction 52 | 50 entailment 53 | 51 entailment 54 | 52 entailment 55 | 53 neutral 56 | 54 entailment 57 | 55 neutral 58 | 56 entailment 59 | 57 entailment 60 | 58 contradiction 61 | 59 entailment 62 | 60 neutral 63 | 61 entailment 64 | 62 entailment 65 | 63 entailment 66 | 64 entailment 67 | 65 contradiction 68 | 66 entailment 69 | 67 entailment 70 | 68 contradiction 71 | 69 contradiction 72 | 70 entailment 73 | 71 entailment 74 | 72 neutral 75 | 73 entailment 76 | 74 neutral 77 | 75 entailment 78 | 76 neutral 79 | 77 entailment 80 | 78 neutral 81 | 79 entailment 82 | 80 entailment 83 | 81 entailment 84 | 82 entailment 85 | 83 entailment 86 | 84 entailment 87 | 85 entailment 88 | 86 entailment 89 | 87 contradiction 90 | 88 contradiction 91 | 89 contradiction 92 | 90 entailment 93 | 91 entailment 94 | 92 contradiction 95 | 93 contradiction 96 | 94 entailment 97 | 95 entailment 98 | 96 contradiction 99 | 97 contradiction 100 | 98 entailment 101 | 99 entailment 102 | 100 entailment 103 | 101 entailment 104 | 102 entailment 105 | 103 entailment 106 | 104 entailment 107 | 105 entailment 108 | 106 contradiction 109 | 107 contradiction 110 | 108 entailment 111 | 109 entailment 112 | 110 entailment 113 | 111 entailment 114 | 112 entailment 115 | 113 entailment 116 | 114 entailment 117 | 115 entailment 118 | 116 entailment 119 | 117 neutral 120 | 118 contradiction 121 | 119 contradiction 122 | 120 entailment 123 | 121 entailment 124 | 122 entailment 125 | 123 neutral 126 | 124 entailment 127 | 125 entailment 128 | 126 entailment 129 | 127 entailment 130 | 128 entailment 131 | 129 neutral 132 | 130 entailment 133 | 131 neutral 134 | 132 entailment 135 | 133 neutral 136 | 134 entailment 137 | 135 entailment 138 | 136 entailment 139 | 137 entailment 140 | 138 entailment 141 | 139 entailment 142 | 140 entailment 143 | 141 entailment 144 | 142 contradiction 145 | 143 entailment 146 | 144 entailment 147 | 145 entailment 148 | 146 entailment 149 | 147 entailment 150 | 148 entailment 151 | 149 entailment 152 | 150 entailment 153 | 151 entailment 154 | 152 contradiction 155 | 153 contradiction 156 | 154 entailment 157 | 155 entailment 158 | 156 contradiction 159 | 157 contradiction 160 | 158 entailment 161 | 159 entailment 162 | 160 neutral 163 | 161 entailment 164 | 162 entailment 165 | 163 neutral 166 | 164 entailment 167 | 165 entailment 168 | 166 entailment 169 | 167 entailment 170 | 168 contradiction 171 | 169 entailment 172 | 170 entailment 173 | 171 entailment 174 | 172 contradiction 175 | 173 contradiction 176 | 174 contradiction 177 | 175 entailment 178 | 176 entailment 179 | 177 entailment 180 | 178 entailment 181 | 179 entailment 182 | 180 entailment 183 | 181 entailment 184 | 182 entailment 185 | 183 entailment 186 | 184 entailment 187 | 185 entailment 188 | 186 entailment 189 | 187 entailment 190 | 188 entailment 191 | 189 entailment 192 | 190 entailment 193 | 191 entailment 194 | 192 entailment 195 | 193 entailment 196 | 194 entailment 197 | 195 entailment 198 | 196 entailment 199 | 197 entailment 200 | 198 entailment 201 | 199 entailment 202 | 200 entailment 203 | 201 entailment 204 | 202 contradiction 205 | 203 contradiction 206 | 204 neutral 207 | 205 entailment 208 | 206 entailment 209 | 207 entailment 210 | 208 entailment 211 | 209 entailment 212 | 210 entailment 213 | 211 entailment 214 | 212 entailment 215 | 213 entailment 216 | 214 entailment 217 | 215 entailment 218 | 216 entailment 219 | 217 entailment 220 | 218 entailment 221 | 219 entailment 222 | 220 entailment 223 | 221 entailment 224 | 222 entailment 225 | 223 entailment 226 | 224 entailment 227 | 225 entailment 228 | 226 entailment 229 | 227 entailment 230 | 228 entailment 231 | 229 entailment 232 | 230 entailment 233 | 231 entailment 234 | 232 entailment 235 | 233 neutral 236 | 234 entailment 237 | 235 neutral 238 | 236 entailment 239 | 237 neutral 240 | 238 entailment 241 | 239 neutral 242 | 240 entailment 243 | 241 neutral 244 | 242 entailment 245 | 243 neutral 246 | 244 entailment 247 | 245 neutral 248 | 246 entailment 249 | 247 neutral 250 | 248 entailment 251 | 249 neutral 252 | 250 entailment 253 | 251 neutral 254 | 252 entailment 255 | 253 neutral 256 | 254 entailment 257 | 255 neutral 258 | 256 entailment 259 | 257 neutral 260 | 258 entailment 261 | 259 neutral 262 | 260 entailment 263 | 261 neutral 264 | 262 entailment 265 | 263 neutral 266 | 264 entailment 267 | 265 neutral 268 | 266 entailment 269 | 267 neutral 270 | 268 entailment 271 | 269 neutral 272 | 270 entailment 273 | 271 neutral 274 | 272 entailment 275 | 273 neutral 276 | 274 entailment 277 | 275 neutral 278 | 276 entailment 279 | 277 neutral 280 | 278 entailment 281 | 279 neutral 282 | 280 entailment 283 | 281 neutral 284 | 282 entailment 285 | 283 neutral 286 | 284 entailment 287 | 285 neutral 288 | 286 entailment 289 | 287 neutral 290 | 288 entailment 291 | 289 neutral 292 | 290 entailment 293 | 291 neutral 294 | 292 entailment 295 | 293 contradiction 296 | 294 contradiction 297 | 295 contradiction 298 | 296 contradiction 299 | 297 contradiction 300 | 298 entailment 301 | 299 neutral 302 | 300 contradiction 303 | 301 contradiction 304 | 302 neutral 305 | 303 contradiction 306 | 304 contradiction 307 | 305 contradiction 308 | 306 entailment 309 | 307 contradiction 310 | 308 contradiction 311 | 309 contradiction 312 | 310 entailment 313 | 311 entailment 314 | 312 entailment 315 | 313 entailment 316 | 314 entailment 317 | 315 contradiction 318 | 316 neutral 319 | 317 neutral 320 | 318 neutral 321 | 319 neutral 322 | 320 entailment 323 | 321 neutral 324 | 322 contradiction 325 | 323 neutral 326 | 324 entailment 327 | 325 neutral 328 | 326 entailment 329 | 327 neutral 330 | 328 neutral 331 | 329 neutral 332 | 330 contradiction 333 | 331 contradiction 334 | 332 contradiction 335 | 333 neutral 336 | 334 contradiction 337 | 335 neutral 338 | 336 entailment 339 | 337 neutral 340 | 338 entailment 341 | 339 neutral 342 | 340 contradiction 343 | 341 contradiction 344 | 342 entailment 345 | 343 neutral 346 | 344 entailment 347 | 345 neutral 348 | 346 contradiction 349 | 347 neutral 350 | 348 contradiction 351 | 349 contradiction 352 | 350 neutral 353 | 351 contradiction 354 | 352 neutral 355 | 353 contradiction 356 | 354 entailment 357 | 355 neutral 358 | 356 contradiction 359 | 357 contradiction 360 | 358 entailment 361 | 359 neutral 362 | 360 contradiction 363 | 361 contradiction 364 | 362 entailment 365 | 363 neutral 366 | 364 contradiction 367 | 365 neutral 368 | 366 entailment 369 | 367 neutral 370 | 368 contradiction 371 | 369 neutral 372 | 370 entailment 373 | 371 neutral 374 | 372 contradiction 375 | 373 neutral 376 | 374 contradiction 377 | 375 contradiction 378 | 376 contradiction 379 | 377 contradiction 380 | 378 contradiction 381 | 379 contradiction 382 | 380 entailment 383 | 381 neutral 384 | 382 contradiction 385 | 383 contradiction 386 | 384 contradiction 387 | 385 contradiction 388 | 386 contradiction 389 | 387 contradiction 390 | 388 contradiction 391 | 389 contradiction 392 | 390 entailment 393 | 391 neutral 394 | 392 contradiction 395 | 393 neutral 396 | 394 entailment 397 | 395 neutral 398 | 396 contradiction 399 | 397 contradiction 400 | 398 entailment 401 | 399 neutral 402 | 400 contradiction 403 | 401 neutral 404 | 402 entailment 405 | 403 neutral 406 | 404 contradiction 407 | 405 contradiction 408 | 406 entailment 409 | 407 entailment 410 | 408 neutral 411 | 409 entailment 412 | 410 entailment 413 | 411 neutral 414 | 412 entailment 415 | 413 contradiction 416 | 414 contradiction 417 | 415 contradiction 418 | 416 entailment 419 | 417 entailment 420 | 418 entailment 421 | 419 entailment 422 | 420 contradiction 423 | 421 contradiction 424 | 422 entailment 425 | 423 contradiction 426 | 424 contradiction 427 | 425 contradiction 428 | 426 entailment 429 | 427 entailment 430 | 428 entailment 431 | 429 neutral 432 | 430 entailment 433 | 431 neutral 434 | 432 entailment 435 | 433 neutral 436 | 434 entailment 437 | 435 neutral 438 | 436 entailment 439 | 437 neutral 440 | 438 entailment 441 | 439 neutral 442 | 440 entailment 443 | 441 neutral 444 | 442 entailment 445 | 443 neutral 446 | 444 entailment 447 | 445 neutral 448 | 446 entailment 449 | 447 entailment 450 | 448 entailment 451 | 449 entailment 452 | 450 entailment 453 | 451 entailment 454 | 452 entailment 455 | 453 entailment 456 | 454 neutral 457 | 455 entailment 458 | 456 neutral 459 | 457 entailment 460 | 458 entailment 461 | 459 entailment 462 | 460 neutral 463 | 461 neutral 464 | 462 entailment 465 | 463 neutral 466 | 464 entailment 467 | 465 neutral 468 | 466 contradiction 469 | 467 contradiction 470 | 468 neutral 471 | 469 neutral 472 | 470 contradiction 473 | 471 contradiction 474 | 472 contradiction 475 | 473 entailment 476 | 474 contradiction 477 | 475 entailment 478 | 476 entailment 479 | 477 entailment 480 | 478 entailment 481 | 479 neutral 482 | 480 entailment 483 | 481 neutral 484 | 482 entailment 485 | 483 entailment 486 | 484 entailment 487 | 485 entailment 488 | 486 entailment 489 | 487 entailment 490 | 488 entailment 491 | 489 neutral 492 | 490 neutral 493 | 491 neutral 494 | 492 neutral 495 | 493 entailment 496 | 494 entailment 497 | 495 entailment 498 | 496 entailment 499 | 497 entailment 500 | 498 entailment 501 | 499 entailment 502 | 500 entailment 503 | 501 entailment 504 | 502 contradiction 505 | 503 contradiction 506 | 504 entailment 507 | 505 neutral 508 | 506 entailment 509 | 507 neutral 510 | 508 entailment 511 | 509 entailment 512 | 510 entailment 513 | 511 entailment 514 | 512 contradiction 515 | 513 entailment 516 | 514 contradiction 517 | 515 contradiction 518 | 516 entailment 519 | 517 entailment 520 | 518 contradiction 521 | 519 contradiction 522 | 520 entailment 523 | 521 entailment 524 | 522 contradiction 525 | 523 contradiction 526 | 524 entailment 527 | 525 entailment 528 | 526 entailment 529 | 527 neutral 530 | 528 entailment 531 | 529 neutral 532 | 530 entailment 533 | 531 entailment 534 | 532 contradiction 535 | 533 contradiction 536 | 534 entailment 537 | 535 entailment 538 | 536 entailment 539 | 537 entailment 540 | 538 entailment 541 | 539 entailment 542 | 540 neutral 543 | 541 entailment 544 | 542 contradiction 545 | 543 contradiction 546 | 544 contradiction 547 | 545 contradiction 548 | 546 entailment 549 | 547 neutral 550 | 548 entailment 551 | 549 neutral 552 | 550 entailment 553 | 551 entailment 554 | 552 entailment 555 | 553 entailment 556 | 554 entailment 557 | 555 entailment 558 | 556 contradiction 559 | 557 contradiction 560 | 558 entailment 561 | 559 neutral 562 | 560 neutral 563 | 561 entailment 564 | 562 entailment 565 | 563 entailment 566 | 564 entailment 567 | 565 neutral 568 | 566 contradiction 569 | 567 contradiction 570 | 568 entailment 571 | 569 entailment 572 | 570 neutral 573 | 571 entailment 574 | 572 entailment 575 | 573 entailment 576 | 574 entailment 577 | 575 entailment 578 | 576 entailment 579 | 577 contradiction 580 | 578 entailment 581 | 579 entailment 582 | 580 contradiction 583 | 581 contradiction 584 | 582 entailment 585 | 583 neutral 586 | 584 contradiction 587 | 585 contradiction 588 | 586 entailment 589 | 587 entailment 590 | 588 neutral 591 | 589 entailment 592 | 590 neutral 593 | 591 entailment 594 | 592 contradiction 595 | 593 contradiction 596 | 594 neutral 597 | 595 entailment 598 | 596 entailment 599 | 597 entailment 600 | 598 entailment 601 | 599 entailment 602 | 600 contradiction 603 | 601 contradiction 604 | 602 entailment 605 | 603 entailment 606 | 604 entailment 607 | 605 entailment 608 | 606 entailment 609 | 607 neutral 610 | 608 entailment 611 | 609 neutral 612 | 610 entailment 613 | 611 neutral 614 | 612 entailment 615 | 613 neutral 616 | 614 entailment 617 | 615 neutral 618 | 616 entailment 619 | 617 neutral 620 | 618 entailment 621 | 619 neutral 622 | 620 entailment 623 | 621 neutral 624 | 622 entailment 625 | 623 neutral 626 | 624 entailment 627 | 625 neutral 628 | 626 entailment 629 | 627 neutral 630 | 628 contradiction 631 | 629 contradiction 632 | 630 entailment 633 | 631 neutral 634 | 632 contradiction 635 | 633 entailment 636 | 634 entailment 637 | 635 entailment 638 | 636 entailment 639 | 637 entailment 640 | 638 entailment 641 | 639 neutral 642 | 640 entailment 643 | 641 neutral 644 | 642 entailment 645 | 643 neutral 646 | 644 contradiction 647 | 645 neutral 648 | 646 entailment 649 | 647 neutral 650 | 648 entailment 651 | 649 neutral 652 | 650 neutral 653 | 651 entailment 654 | 652 entailment 655 | 653 neutral 656 | 654 entailment 657 | 655 contradiction 658 | 656 contradiction 659 | 657 contradiction 660 | 658 contradiction 661 | 659 contradiction 662 | 660 contradiction 663 | 661 contradiction 664 | 662 contradiction 665 | 663 entailment 666 | 664 entailment 667 | 665 neutral 668 | 666 contradiction 669 | 667 contradiction 670 | 668 neutral 671 | 669 entailment 672 | 670 entailment 673 | 671 entailment 674 | 672 entailment 675 | 673 neutral 676 | 674 contradiction 677 | 675 contradiction 678 | 676 neutral 679 | 677 neutral 680 | 678 contradiction 681 | 679 contradiction 682 | 680 neutral 683 | 681 contradiction 684 | 682 neutral 685 | 683 neutral 686 | 684 neutral 687 | 685 neutral 688 | 686 entailment 689 | 687 neutral 690 | 688 entailment 691 | 689 neutral 692 | 690 entailment 693 | 691 neutral 694 | 692 entailment 695 | 693 neutral 696 | 694 entailment 697 | 695 entailment 698 | 696 contradiction 699 | 697 contradiction 700 | 698 entailment 701 | 699 neutral 702 | 700 entailment 703 | 701 entailment 704 | 702 contradiction 705 | 703 contradiction 706 | 704 contradiction 707 | 705 contradiction 708 | 706 contradiction 709 | 707 contradiction 710 | 708 contradiction 711 | 709 contradiction 712 | 710 contradiction 713 | 711 contradiction 714 | 712 entailment 715 | 713 neutral 716 | 714 entailment 717 | 715 entailment 718 | 716 entailment 719 | 717 entailment 720 | 718 entailment 721 | 719 entailment 722 | 720 entailment 723 | 721 entailment 724 | 722 entailment 725 | 723 entailment 726 | 724 entailment 727 | 725 neutral 728 | 726 entailment 729 | 727 neutral 730 | 728 contradiction 731 | 729 contradiction 732 | 730 neutral 733 | 731 entailment 734 | 732 contradiction 735 | 733 entailment 736 | 734 neutral 737 | 735 entailment 738 | 736 entailment 739 | 737 neutral 740 | 738 neutral 741 | 739 contradiction 742 | 740 entailment 743 | 741 neutral 744 | 742 entailment 745 | 743 neutral 746 | 744 entailment 747 | 745 neutral 748 | 746 entailment 749 | 747 neutral 750 | 748 entailment 751 | 749 entailment 752 | 750 entailment 753 | 751 entailment 754 | 752 contradiction 755 | 753 neutral 756 | 754 entailment 757 | 755 contradiction 758 | 756 entailment 759 | 757 neutral 760 | 758 neutral 761 | 759 entailment 762 | 760 contradiction 763 | 761 entailment 764 | 762 entailment 765 | 763 entailment 766 | 764 contradiction 767 | 765 entailment 768 | 766 entailment 769 | 767 entailment 770 | 768 entailment 771 | 769 entailment 772 | 770 entailment 773 | 771 entailment 774 | 772 entailment 775 | 773 entailment 776 | 774 entailment 777 | 775 neutral 778 | 776 entailment 779 | 777 neutral 780 | 778 entailment 781 | 779 neutral 782 | 780 entailment 783 | 781 neutral 784 | 782 entailment 785 | 783 entailment 786 | 784 entailment 787 | 785 entailment 788 | 786 entailment 789 | 787 entailment 790 | 788 entailment 791 | 789 neutral 792 | 790 entailment 793 | 791 neutral 794 | 792 entailment 795 | 793 neutral 796 | 794 contradiction 797 | 795 contradiction 798 | 796 entailment 799 | 797 entailment 800 | 798 entailment 801 | 799 contradiction 802 | 800 entailment 803 | 801 entailment 804 | 802 entailment 805 | 803 entailment 806 | 804 entailment 807 | 805 entailment 808 | 806 entailment 809 | 807 entailment 810 | 808 entailment 811 | 809 entailment 812 | 810 entailment 813 | 811 entailment 814 | 812 entailment 815 | 813 neutral 816 | 814 entailment 817 | 815 neutral 818 | 816 neutral 819 | 817 entailment 820 | 818 neutral 821 | 819 entailment 822 | 820 contradiction 823 | 821 contradiction 824 | 822 entailment 825 | 823 neutral 826 | 824 entailment 827 | 825 neutral 828 | 826 contradiction 829 | 827 contradiction 830 | 828 contradiction 831 | 829 entailment 832 | 830 entailment 833 | 831 neutral 834 | 832 entailment 835 | 833 neutral 836 | 834 entailment 837 | 835 entailment 838 | 836 entailment 839 | 837 entailment 840 | 838 entailment 841 | 839 entailment 842 | 840 entailment 843 | 841 entailment 844 | 842 entailment 845 | 843 entailment 846 | 844 entailment 847 | 845 entailment 848 | 846 entailment 849 | 847 contradiction 850 | 848 entailment 851 | 849 entailment 852 | 850 contradiction 853 | 851 contradiction 854 | 852 entailment 855 | 853 neutral 856 | 854 entailment 857 | 855 neutral 858 | 856 entailment 859 | 857 neutral 860 | 858 entailment 861 | 859 entailment 862 | 860 entailment 863 | 861 entailment 864 | 862 entailment 865 | 863 entailment 866 | 864 entailment 867 | 865 entailment 868 | 866 entailment 869 | 867 neutral 870 | 868 entailment 871 | 869 neutral 872 | 870 entailment 873 | 871 neutral 874 | 872 entailment 875 | 873 neutral 876 | 874 entailment 877 | 875 entailment 878 | 876 entailment 879 | 877 entailment 880 | 878 entailment 881 | 879 neutral 882 | 880 entailment 883 | 881 neutral 884 | 882 neutral 885 | 883 entailment 886 | 884 entailment 887 | 885 entailment 888 | 886 entailment 889 | 887 entailment 890 | 888 entailment 891 | 889 entailment 892 | 890 entailment 893 | 891 entailment 894 | 892 neutral 895 | 893 entailment 896 | 894 contradiction 897 | 895 contradiction 898 | 896 entailment 899 | 897 entailment 900 | 898 entailment 901 | 899 neutral 902 | 900 entailment 903 | 901 contradiction 904 | 902 entailment 905 | 903 entailment 906 | 904 neutral 907 | 905 entailment 908 | 906 contradiction 909 | 907 contradiction 910 | 908 contradiction 911 | 909 contradiction 912 | 910 neutral 913 | 911 neutral 914 | 912 entailment 915 | 913 entailment 916 | 914 neutral 917 | 915 entailment 918 | 916 entailment 919 | 917 entailment 920 | 918 entailment 921 | 919 entailment 922 | 920 entailment 923 | 921 entailment 924 | 922 contradiction 925 | 923 entailment 926 | 924 entailment 927 | 925 entailment 928 | 926 entailment 929 | 927 entailment 930 | 928 entailment 931 | 929 neutral 932 | 930 neutral 933 | 931 neutral 934 | 932 entailment 935 | 933 neutral 936 | 934 entailment 937 | 935 contradiction 938 | 936 contradiction 939 | 937 neutral 940 | 938 contradiction 941 | 939 contradiction 942 | 940 neutral 943 | 941 neutral 944 | 942 neutral 945 | 943 neutral 946 | 944 neutral 947 | 945 entailment 948 | 946 neutral 949 | 947 entailment 950 | 948 contradiction 951 | 949 entailment 952 | 950 entailment 953 | 951 neutral 954 | 952 entailment 955 | 953 contradiction 956 | 954 contradiction 957 | 955 contradiction 958 | 956 entailment 959 | 957 entailment 960 | 958 entailment 961 | 959 neutral 962 | 960 neutral 963 | 961 entailment 964 | 962 contradiction 965 | 963 contradiction 966 | 964 contradiction 967 | 965 contradiction 968 | 966 neutral 969 | 967 entailment 970 | 968 entailment 971 | 969 neutral 972 | 970 entailment 973 | 971 neutral 974 | 972 contradiction 975 | 973 neutral 976 | 974 entailment 977 | 975 neutral 978 | 976 entailment 979 | 977 neutral 980 | 978 entailment 981 | 979 neutral 982 | 980 entailment 983 | 981 neutral 984 | 982 entailment 985 | 983 neutral 986 | 984 entailment 987 | 985 entailment 988 | 986 neutral 989 | 987 neutral 990 | 988 entailment 991 | 989 entailment 992 | 990 entailment 993 | 991 neutral 994 | 992 entailment 995 | 993 neutral 996 | 994 entailment 997 | 995 entailment 998 | 996 entailment 999 | 997 entailment 1000 | 998 entailment 1001 | 999 neutral 1002 | 1000 entailment 1003 | 1001 neutral 1004 | 1002 entailment 1005 | 1003 entailment 1006 | 1004 entailment 1007 | 1005 entailment 1008 | 1006 neutral 1009 | 1007 entailment 1010 | 1008 entailment 1011 | 1009 neutral 1012 | 1010 entailment 1013 | 1011 contradiction 1014 | 1012 entailment 1015 | 1013 contradiction 1016 | 1014 entailment 1017 | 1015 contradiction 1018 | 1016 entailment 1019 | 1017 entailment 1020 | 1018 entailment 1021 | 1019 neutral 1022 | 1020 contradiction 1023 | 1021 contradiction 1024 | 1022 neutral 1025 | 1023 neutral 1026 | 1024 neutral 1027 | 1025 neutral 1028 | 1026 neutral 1029 | 1027 neutral 1030 | 1028 entailment 1031 | 1029 neutral 1032 | 1030 entailment 1033 | 1031 entailment 1034 | 1032 neutral 1035 | 1033 contradiction 1036 | 1034 neutral 1037 | 1035 contradiction 1038 | 1036 neutral 1039 | 1037 neutral 1040 | 1038 contradiction 1041 | 1039 contradiction 1042 | 1040 entailment 1043 | 1041 neutral 1044 | 1042 contradiction 1045 | 1043 contradiction 1046 | 1044 entailment 1047 | 1045 neutral 1048 | 1046 entailment 1049 | 1047 entailment 1050 | 1048 entailment 1051 | 1049 entailment 1052 | 1050 contradiction 1053 | 1051 contradiction 1054 | 1052 contradiction 1055 | 1053 contradiction 1056 | 1054 entailment 1057 | 1055 entailment 1058 | 1056 entailment 1059 | 1057 entailment 1060 | 1058 entailment 1061 | 1059 entailment 1062 | 1060 contradiction 1063 | 1061 contradiction 1064 | 1062 entailment 1065 | 1063 entailment 1066 | 1064 contradiction 1067 | 1065 contradiction 1068 | 1066 contradiction 1069 | 1067 contradiction 1070 | 1068 neutral 1071 | 1069 neutral 1072 | 1070 neutral 1073 | 1071 contradiction 1074 | 1072 entailment 1075 | 1073 entailment 1076 | 1074 entailment 1077 | 1075 entailment 1078 | 1076 neutral 1079 | 1077 entailment 1080 | 1078 entailment 1081 | 1079 neutral 1082 | 1080 entailment 1083 | 1081 entailment 1084 | 1082 contradiction 1085 | 1083 entailment 1086 | 1084 entailment 1087 | 1085 neutral 1088 | 1086 entailment 1089 | 1087 entailment 1090 | 1088 entailment 1091 | 1089 entailment 1092 | 1090 contradiction 1093 | 1091 neutral 1094 | 1092 entailment 1095 | 1093 neutral 1096 | 1094 entailment 1097 | 1095 neutral 1098 | 1096 entailment 1099 | 1097 neutral 1100 | 1098 entailment 1101 | 1099 entailment 1102 | 1100 contradiction 1103 | 1101 contradiction 1104 | 1102 neutral 1105 | 1103 neutral 1106 | -------------------------------------------------------------------------------- /examples/GLUE-submission/matbert-base-cased/CoLA.tsv: -------------------------------------------------------------------------------- 1 | index prediction 2 | 0 1 3 | 1 1 4 | 2 0 5 | 3 0 6 | 4 0 7 | 5 1 8 | 6 1 9 | 7 0 10 | 8 0 11 | 9 1 12 | 10 1 13 | 11 0 14 | 12 1 15 | 13 0 16 | 14 1 17 | 15 1 18 | 16 0 19 | 17 1 20 | 18 1 21 | 19 0 22 | 20 1 23 | 21 1 24 | 22 1 25 | 23 1 26 | 24 1 27 | 25 1 28 | 26 1 29 | 27 0 30 | 28 0 31 | 29 1 32 | 30 1 33 | 31 1 34 | 32 1 35 | 33 0 36 | 34 0 37 | 35 0 38 | 36 1 39 | 37 1 40 | 38 0 41 | 39 1 42 | 40 0 43 | 41 0 44 | 42 1 45 | 43 0 46 | 44 0 47 | 45 1 48 | 46 1 49 | 47 1 50 | 48 1 51 | 49 1 52 | 50 1 53 | 51 1 54 | 52 1 55 | 53 1 56 | 54 1 57 | 55 1 58 | 56 1 59 | 57 1 60 | 58 1 61 | 59 1 62 | 60 0 63 | 61 1 64 | 62 1 65 | 63 1 66 | 64 1 67 | 65 1 68 | 66 1 69 | 67 1 70 | 68 1 71 | 69 0 72 | 70 1 73 | 71 0 74 | 72 1 75 | 73 1 76 | 74 1 77 | 75 1 78 | 76 1 79 | 77 1 80 | 78 1 81 | 79 1 82 | 80 1 83 | 81 1 84 | 82 0 85 | 83 1 86 | 84 1 87 | 85 1 88 | 86 1 89 | 87 1 90 | 88 1 91 | 89 1 92 | 90 1 93 | 91 1 94 | 92 1 95 | 93 1 96 | 94 1 97 | 95 1 98 | 96 1 99 | 97 1 100 | 98 1 101 | 99 1 102 | 100 0 103 | 101 1 104 | 102 1 105 | 103 0 106 | 104 1 107 | 105 1 108 | 106 1 109 | 107 1 110 | 108 1 111 | 109 1 112 | 110 1 113 | 111 1 114 | 112 1 115 | 113 1 116 | 114 1 117 | 115 0 118 | 116 1 119 | 117 0 120 | 118 0 121 | 119 1 122 | 120 0 123 | 121 1 124 | 122 1 125 | 123 0 126 | 124 1 127 | 125 1 128 | 126 0 129 | 127 1 130 | 128 1 131 | 129 1 132 | 130 0 133 | 131 1 134 | 132 0 135 | 133 1 136 | 134 1 137 | 135 1 138 | 136 1 139 | 137 1 140 | 138 1 141 | 139 0 142 | 140 0 143 | 141 0 144 | 142 1 145 | 143 1 146 | 144 1 147 | 145 1 148 | 146 1 149 | 147 1 150 | 148 1 151 | 149 1 152 | 150 1 153 | 151 1 154 | 152 1 155 | 153 1 156 | 154 1 157 | 155 0 158 | 156 0 159 | 157 0 160 | 158 0 161 | 159 1 162 | 160 1 163 | 161 1 164 | 162 1 165 | 163 1 166 | 164 1 167 | 165 0 168 | 166 0 169 | 167 1 170 | 168 1 171 | 169 0 172 | 170 1 173 | 171 1 174 | 172 1 175 | 173 1 176 | 174 1 177 | 175 0 178 | 176 1 179 | 177 1 180 | 178 1 181 | 179 1 182 | 180 1 183 | 181 1 184 | 182 0 185 | 183 1 186 | 184 1 187 | 185 1 188 | 186 1 189 | 187 1 190 | 188 0 191 | 189 1 192 | 190 1 193 | 191 1 194 | 192 1 195 | 193 1 196 | 194 1 197 | 195 1 198 | 196 1 199 | 197 0 200 | 198 1 201 | 199 1 202 | 200 1 203 | 201 0 204 | 202 1 205 | 203 0 206 | 204 1 207 | 205 1 208 | 206 1 209 | 207 1 210 | 208 1 211 | 209 1 212 | 210 1 213 | 211 1 214 | 212 1 215 | 213 1 216 | 214 1 217 | 215 0 218 | 216 0 219 | 217 1 220 | 218 1 221 | 219 1 222 | 220 1 223 | 221 0 224 | 222 1 225 | 223 1 226 | 224 0 227 | 225 1 228 | 226 1 229 | 227 1 230 | 228 1 231 | 229 1 232 | 230 1 233 | 231 1 234 | 232 1 235 | 233 1 236 | 234 1 237 | 235 0 238 | 236 0 239 | 237 0 240 | 238 1 241 | 239 1 242 | 240 1 243 | 241 1 244 | 242 0 245 | 243 1 246 | 244 1 247 | 245 1 248 | 246 1 249 | 247 1 250 | 248 1 251 | 249 1 252 | 250 1 253 | 251 1 254 | 252 1 255 | 253 1 256 | 254 1 257 | 255 1 258 | 256 0 259 | 257 1 260 | 258 0 261 | 259 0 262 | 260 1 263 | 261 1 264 | 262 1 265 | 263 1 266 | 264 1 267 | 265 1 268 | 266 1 269 | 267 0 270 | 268 1 271 | 269 1 272 | 270 1 273 | 271 1 274 | 272 1 275 | 273 1 276 | 274 1 277 | 275 1 278 | 276 1 279 | 277 1 280 | 278 1 281 | 279 0 282 | 280 1 283 | 281 1 284 | 282 1 285 | 283 1 286 | 284 0 287 | 285 0 288 | 286 0 289 | 287 1 290 | 288 1 291 | 289 1 292 | 290 1 293 | 291 1 294 | 292 1 295 | 293 0 296 | 294 1 297 | 295 1 298 | 296 1 299 | 297 1 300 | 298 1 301 | 299 1 302 | 300 1 303 | 301 1 304 | 302 1 305 | 303 1 306 | 304 1 307 | 305 1 308 | 306 1 309 | 307 1 310 | 308 1 311 | 309 1 312 | 310 1 313 | 311 1 314 | 312 1 315 | 313 1 316 | 314 0 317 | 315 0 318 | 316 1 319 | 317 0 320 | 318 1 321 | 319 0 322 | 320 1 323 | 321 1 324 | 322 1 325 | 323 0 326 | 324 1 327 | 325 1 328 | 326 1 329 | 327 1 330 | 328 1 331 | 329 1 332 | 330 1 333 | 331 1 334 | 332 1 335 | 333 1 336 | 334 1 337 | 335 1 338 | 336 1 339 | 337 1 340 | 338 1 341 | 339 1 342 | 340 0 343 | 341 1 344 | 342 1 345 | 343 1 346 | 344 0 347 | 345 1 348 | 346 1 349 | 347 1 350 | 348 1 351 | 349 1 352 | 350 1 353 | 351 1 354 | 352 1 355 | 353 0 356 | 354 1 357 | 355 1 358 | 356 0 359 | 357 1 360 | 358 1 361 | 359 1 362 | 360 1 363 | 361 1 364 | 362 1 365 | 363 1 366 | 364 1 367 | 365 1 368 | 366 1 369 | 367 1 370 | 368 1 371 | 369 1 372 | 370 1 373 | 371 1 374 | 372 1 375 | 373 1 376 | 374 1 377 | 375 1 378 | 376 1 379 | 377 1 380 | 378 1 381 | 379 1 382 | 380 1 383 | 381 1 384 | 382 1 385 | 383 1 386 | 384 1 387 | 385 0 388 | 386 1 389 | 387 1 390 | 388 1 391 | 389 0 392 | 390 1 393 | 391 0 394 | 392 0 395 | 393 1 396 | 394 1 397 | 395 1 398 | 396 1 399 | 397 1 400 | 398 1 401 | 399 1 402 | 400 0 403 | 401 1 404 | 402 1 405 | 403 1 406 | 404 1 407 | 405 1 408 | 406 1 409 | 407 1 410 | 408 1 411 | 409 1 412 | 410 1 413 | 411 1 414 | 412 1 415 | 413 1 416 | 414 1 417 | 415 0 418 | 416 1 419 | 417 0 420 | 418 0 421 | 419 0 422 | 420 1 423 | 421 1 424 | 422 1 425 | 423 1 426 | 424 0 427 | 425 1 428 | 426 1 429 | 427 1 430 | 428 1 431 | 429 1 432 | 430 1 433 | 431 1 434 | 432 1 435 | 433 1 436 | 434 1 437 | 435 1 438 | 436 1 439 | 437 1 440 | 438 1 441 | 439 1 442 | 440 1 443 | 441 1 444 | 442 1 445 | 443 1 446 | 444 0 447 | 445 1 448 | 446 1 449 | 447 1 450 | 448 0 451 | 449 0 452 | 450 0 453 | 451 1 454 | 452 1 455 | 453 1 456 | 454 1 457 | 455 1 458 | 456 1 459 | 457 1 460 | 458 1 461 | 459 1 462 | 460 0 463 | 461 1 464 | 462 1 465 | 463 1 466 | 464 1 467 | 465 1 468 | 466 1 469 | 467 1 470 | 468 1 471 | 469 1 472 | 470 1 473 | 471 1 474 | 472 1 475 | 473 0 476 | 474 1 477 | 475 0 478 | 476 0 479 | 477 0 480 | 478 0 481 | 479 0 482 | 480 1 483 | 481 1 484 | 482 0 485 | 483 0 486 | 484 0 487 | 485 1 488 | 486 1 489 | 487 1 490 | 488 0 491 | 489 1 492 | 490 1 493 | 491 0 494 | 492 1 495 | 493 1 496 | 494 1 497 | 495 0 498 | 496 0 499 | 497 1 500 | 498 1 501 | 499 1 502 | 500 1 503 | 501 1 504 | 502 1 505 | 503 1 506 | 504 0 507 | 505 0 508 | 506 1 509 | 507 1 510 | 508 1 511 | 509 1 512 | 510 1 513 | 511 1 514 | 512 1 515 | 513 1 516 | 514 1 517 | 515 1 518 | 516 1 519 | 517 1 520 | 518 1 521 | 519 1 522 | 520 1 523 | 521 1 524 | 522 1 525 | 523 1 526 | 524 0 527 | 525 1 528 | 526 1 529 | 527 1 530 | 528 1 531 | 529 1 532 | 530 1 533 | 531 0 534 | 532 1 535 | 533 1 536 | 534 1 537 | 535 1 538 | 536 1 539 | 537 1 540 | 538 1 541 | 539 0 542 | 540 1 543 | 541 1 544 | 542 1 545 | 543 1 546 | 544 1 547 | 545 1 548 | 546 1 549 | 547 1 550 | 548 1 551 | 549 1 552 | 550 1 553 | 551 1 554 | 552 1 555 | 553 1 556 | 554 1 557 | 555 1 558 | 556 1 559 | 557 1 560 | 558 1 561 | 559 1 562 | 560 1 563 | 561 1 564 | 562 1 565 | 563 0 566 | 564 0 567 | 565 1 568 | 566 1 569 | 567 1 570 | 568 1 571 | 569 1 572 | 570 0 573 | 571 0 574 | 572 1 575 | 573 1 576 | 574 1 577 | 575 1 578 | 576 1 579 | 577 1 580 | 578 1 581 | 579 1 582 | 580 0 583 | 581 1 584 | 582 1 585 | 583 1 586 | 584 1 587 | 585 1 588 | 586 1 589 | 587 1 590 | 588 1 591 | 589 1 592 | 590 1 593 | 591 1 594 | 592 0 595 | 593 1 596 | 594 1 597 | 595 1 598 | 596 1 599 | 597 1 600 | 598 1 601 | 599 1 602 | 600 1 603 | 601 0 604 | 602 1 605 | 603 1 606 | 604 0 607 | 605 1 608 | 606 1 609 | 607 1 610 | 608 1 611 | 609 1 612 | 610 1 613 | 611 0 614 | 612 1 615 | 613 1 616 | 614 0 617 | 615 1 618 | 616 1 619 | 617 1 620 | 618 1 621 | 619 1 622 | 620 1 623 | 621 1 624 | 622 1 625 | 623 1 626 | 624 1 627 | 625 1 628 | 626 1 629 | 627 1 630 | 628 1 631 | 629 1 632 | 630 0 633 | 631 1 634 | 632 1 635 | 633 1 636 | 634 1 637 | 635 0 638 | 636 0 639 | 637 1 640 | 638 1 641 | 639 1 642 | 640 0 643 | 641 1 644 | 642 1 645 | 643 1 646 | 644 1 647 | 645 1 648 | 646 1 649 | 647 1 650 | 648 1 651 | 649 1 652 | 650 1 653 | 651 1 654 | 652 1 655 | 653 1 656 | 654 1 657 | 655 1 658 | 656 0 659 | 657 1 660 | 658 1 661 | 659 1 662 | 660 0 663 | 661 1 664 | 662 1 665 | 663 1 666 | 664 1 667 | 665 1 668 | 666 1 669 | 667 1 670 | 668 1 671 | 669 0 672 | 670 1 673 | 671 1 674 | 672 1 675 | 673 1 676 | 674 1 677 | 675 0 678 | 676 1 679 | 677 1 680 | 678 1 681 | 679 1 682 | 680 1 683 | 681 1 684 | 682 1 685 | 683 1 686 | 684 1 687 | 685 1 688 | 686 1 689 | 687 1 690 | 688 1 691 | 689 0 692 | 690 1 693 | 691 1 694 | 692 0 695 | 693 1 696 | 694 0 697 | 695 1 698 | 696 1 699 | 697 1 700 | 698 1 701 | 699 1 702 | 700 0 703 | 701 0 704 | 702 1 705 | 703 1 706 | 704 1 707 | 705 1 708 | 706 1 709 | 707 0 710 | 708 1 711 | 709 1 712 | 710 1 713 | 711 1 714 | 712 0 715 | 713 1 716 | 714 1 717 | 715 1 718 | 716 0 719 | 717 0 720 | 718 1 721 | 719 1 722 | 720 1 723 | 721 0 724 | 722 0 725 | 723 0 726 | 724 1 727 | 725 1 728 | 726 1 729 | 727 0 730 | 728 1 731 | 729 1 732 | 730 1 733 | 731 1 734 | 732 1 735 | 733 1 736 | 734 1 737 | 735 0 738 | 736 1 739 | 737 1 740 | 738 1 741 | 739 1 742 | 740 0 743 | 741 1 744 | 742 1 745 | 743 0 746 | 744 0 747 | 745 0 748 | 746 1 749 | 747 1 750 | 748 1 751 | 749 1 752 | 750 1 753 | 751 1 754 | 752 1 755 | 753 0 756 | 754 0 757 | 755 1 758 | 756 0 759 | 757 1 760 | 758 1 761 | 759 1 762 | 760 1 763 | 761 1 764 | 762 1 765 | 763 1 766 | 764 1 767 | 765 1 768 | 766 1 769 | 767 1 770 | 768 1 771 | 769 1 772 | 770 1 773 | 771 1 774 | 772 1 775 | 773 1 776 | 774 1 777 | 775 1 778 | 776 1 779 | 777 1 780 | 778 1 781 | 779 0 782 | 780 1 783 | 781 0 784 | 782 1 785 | 783 0 786 | 784 1 787 | 785 1 788 | 786 1 789 | 787 1 790 | 788 1 791 | 789 1 792 | 790 1 793 | 791 1 794 | 792 0 795 | 793 1 796 | 794 1 797 | 795 1 798 | 796 0 799 | 797 1 800 | 798 0 801 | 799 0 802 | 800 1 803 | 801 1 804 | 802 1 805 | 803 1 806 | 804 1 807 | 805 1 808 | 806 1 809 | 807 1 810 | 808 0 811 | 809 1 812 | 810 1 813 | 811 1 814 | 812 1 815 | 813 1 816 | 814 1 817 | 815 0 818 | 816 1 819 | 817 1 820 | 818 1 821 | 819 1 822 | 820 1 823 | 821 1 824 | 822 1 825 | 823 0 826 | 824 1 827 | 825 0 828 | 826 1 829 | 827 1 830 | 828 1 831 | 829 1 832 | 830 1 833 | 831 1 834 | 832 1 835 | 833 1 836 | 834 1 837 | 835 1 838 | 836 0 839 | 837 0 840 | 838 1 841 | 839 1 842 | 840 1 843 | 841 0 844 | 842 1 845 | 843 0 846 | 844 1 847 | 845 1 848 | 846 1 849 | 847 1 850 | 848 1 851 | 849 1 852 | 850 1 853 | 851 1 854 | 852 1 855 | 853 0 856 | 854 1 857 | 855 1 858 | 856 1 859 | 857 1 860 | 858 1 861 | 859 1 862 | 860 1 863 | 861 1 864 | 862 1 865 | 863 1 866 | 864 1 867 | 865 0 868 | 866 1 869 | 867 1 870 | 868 0 871 | 869 1 872 | 870 1 873 | 871 1 874 | 872 1 875 | 873 0 876 | 874 1 877 | 875 1 878 | 876 1 879 | 877 0 880 | 878 1 881 | 879 1 882 | 880 1 883 | 881 1 884 | 882 1 885 | 883 1 886 | 884 1 887 | 885 1 888 | 886 1 889 | 887 0 890 | 888 0 891 | 889 1 892 | 890 0 893 | 891 0 894 | 892 0 895 | 893 1 896 | 894 1 897 | 895 1 898 | 896 1 899 | 897 1 900 | 898 1 901 | 899 1 902 | 900 1 903 | 901 1 904 | 902 1 905 | 903 1 906 | 904 1 907 | 905 1 908 | 906 1 909 | 907 1 910 | 908 1 911 | 909 1 912 | 910 1 913 | 911 1 914 | 912 1 915 | 913 1 916 | 914 1 917 | 915 1 918 | 916 1 919 | 917 1 920 | 918 1 921 | 919 1 922 | 920 1 923 | 921 1 924 | 922 1 925 | 923 1 926 | 924 1 927 | 925 1 928 | 926 1 929 | 927 1 930 | 928 1 931 | 929 0 932 | 930 0 933 | 931 0 934 | 932 1 935 | 933 0 936 | 934 1 937 | 935 1 938 | 936 1 939 | 937 1 940 | 938 0 941 | 939 1 942 | 940 1 943 | 941 1 944 | 942 1 945 | 943 1 946 | 944 1 947 | 945 1 948 | 946 1 949 | 947 1 950 | 948 0 951 | 949 1 952 | 950 1 953 | 951 1 954 | 952 1 955 | 953 1 956 | 954 1 957 | 955 0 958 | 956 1 959 | 957 1 960 | 958 1 961 | 959 1 962 | 960 1 963 | 961 1 964 | 962 1 965 | 963 1 966 | 964 1 967 | 965 1 968 | 966 1 969 | 967 1 970 | 968 1 971 | 969 1 972 | 970 1 973 | 971 1 974 | 972 1 975 | 973 1 976 | 974 1 977 | 975 1 978 | 976 1 979 | 977 1 980 | 978 1 981 | 979 1 982 | 980 1 983 | 981 1 984 | 982 0 985 | 983 0 986 | 984 1 987 | 985 1 988 | 986 1 989 | 987 1 990 | 988 1 991 | 989 1 992 | 990 1 993 | 991 0 994 | 992 1 995 | 993 1 996 | 994 1 997 | 995 1 998 | 996 0 999 | 997 1 1000 | 998 0 1001 | 999 0 1002 | 1000 1 1003 | 1001 1 1004 | 1002 0 1005 | 1003 1 1006 | 1004 0 1007 | 1005 1 1008 | 1006 1 1009 | 1007 0 1010 | 1008 0 1011 | 1009 0 1012 | 1010 0 1013 | 1011 1 1014 | 1012 1 1015 | 1013 1 1016 | 1014 1 1017 | 1015 1 1018 | 1016 1 1019 | 1017 1 1020 | 1018 1 1021 | 1019 0 1022 | 1020 1 1023 | 1021 1 1024 | 1022 1 1025 | 1023 1 1026 | 1024 1 1027 | 1025 1 1028 | 1026 1 1029 | 1027 1 1030 | 1028 1 1031 | 1029 1 1032 | 1030 0 1033 | 1031 1 1034 | 1032 0 1035 | 1033 1 1036 | 1034 1 1037 | 1035 0 1038 | 1036 0 1039 | 1037 0 1040 | 1038 0 1041 | 1039 1 1042 | 1040 0 1043 | 1041 1 1044 | 1042 1 1045 | 1043 1 1046 | 1044 1 1047 | 1045 1 1048 | 1046 1 1049 | 1047 1 1050 | 1048 1 1051 | 1049 1 1052 | 1050 0 1053 | 1051 0 1054 | 1052 0 1055 | 1053 0 1056 | 1054 0 1057 | 1055 0 1058 | 1056 1 1059 | 1057 1 1060 | 1058 1 1061 | 1059 0 1062 | 1060 1 1063 | 1061 1 1064 | 1062 1 1065 | -------------------------------------------------------------------------------- /examples/GLUE-submission/matbert-base-cased/MRPC.tsv: -------------------------------------------------------------------------------- 1 | index prediction 2 | 0 1 3 | 1 1 4 | 2 1 5 | 3 1 6 | 4 1 7 | 5 1 8 | 6 0 9 | 7 0 10 | 8 1 11 | 9 0 12 | 10 1 13 | 11 1 14 | 12 1 15 | 13 0 16 | 14 1 17 | 15 0 18 | 16 1 19 | 17 1 20 | 18 1 21 | 19 1 22 | 20 0 23 | 21 1 24 | 22 1 25 | 23 1 26 | 24 0 27 | 25 1 28 | 26 1 29 | 27 1 30 | 28 0 31 | 29 0 32 | 30 1 33 | 31 0 34 | 32 0 35 | 33 0 36 | 34 1 37 | 35 0 38 | 36 1 39 | 37 1 40 | 38 0 41 | 39 1 42 | 40 1 43 | 41 1 44 | 42 0 45 | 43 1 46 | 44 1 47 | 45 0 48 | 46 0 49 | 47 0 50 | 48 1 51 | 49 1 52 | 50 1 53 | 51 1 54 | 52 1 55 | 53 1 56 | 54 0 57 | 55 1 58 | 56 0 59 | 57 1 60 | 58 0 61 | 59 1 62 | 60 1 63 | 61 1 64 | 62 1 65 | 63 1 66 | 64 1 67 | 65 1 68 | 66 1 69 | 67 0 70 | 68 1 71 | 69 1 72 | 70 1 73 | 71 0 74 | 72 1 75 | 73 0 76 | 74 1 77 | 75 0 78 | 76 1 79 | 77 1 80 | 78 0 81 | 79 1 82 | 80 1 83 | 81 0 84 | 82 1 85 | 83 0 86 | 84 0 87 | 85 0 88 | 86 1 89 | 87 1 90 | 88 0 91 | 89 1 92 | 90 0 93 | 91 0 94 | 92 1 95 | 93 0 96 | 94 1 97 | 95 0 98 | 96 0 99 | 97 1 100 | 98 0 101 | 99 1 102 | 100 0 103 | 101 1 104 | 102 1 105 | 103 1 106 | 104 1 107 | 105 0 108 | 106 0 109 | 107 1 110 | 108 1 111 | 109 1 112 | 110 1 113 | 111 0 114 | 112 0 115 | 113 1 116 | 114 1 117 | 115 0 118 | 116 1 119 | 117 0 120 | 118 1 121 | 119 1 122 | 120 0 123 | 121 1 124 | 122 1 125 | 123 1 126 | 124 1 127 | 125 0 128 | 126 1 129 | 127 1 130 | 128 0 131 | 129 1 132 | 130 0 133 | 131 1 134 | 132 1 135 | 133 0 136 | 134 1 137 | 135 0 138 | 136 1 139 | 137 1 140 | 138 1 141 | 139 1 142 | 140 1 143 | 141 1 144 | 142 1 145 | 143 0 146 | 144 1 147 | 145 1 148 | 146 0 149 | 147 1 150 | 148 1 151 | 149 0 152 | 150 1 153 | 151 0 154 | 152 0 155 | 153 0 156 | 154 1 157 | 155 1 158 | 156 1 159 | 157 0 160 | 158 0 161 | 159 1 162 | 160 1 163 | 161 1 164 | 162 1 165 | 163 1 166 | 164 0 167 | 165 1 168 | 166 1 169 | 167 0 170 | 168 1 171 | 169 1 172 | 170 1 173 | 171 1 174 | 172 1 175 | 173 0 176 | 174 1 177 | 175 1 178 | 176 0 179 | 177 0 180 | 178 1 181 | 179 1 182 | 180 0 183 | 181 0 184 | 182 1 185 | 183 1 186 | 184 1 187 | 185 0 188 | 186 0 189 | 187 0 190 | 188 1 191 | 189 1 192 | 190 0 193 | 191 0 194 | 192 1 195 | 193 1 196 | 194 0 197 | 195 0 198 | 196 0 199 | 197 1 200 | 198 1 201 | 199 0 202 | 200 0 203 | 201 1 204 | 202 1 205 | 203 1 206 | 204 1 207 | 205 0 208 | 206 1 209 | 207 1 210 | 208 0 211 | 209 1 212 | 210 0 213 | 211 0 214 | 212 1 215 | 213 0 216 | 214 1 217 | 215 0 218 | 216 1 219 | 217 1 220 | 218 0 221 | 219 1 222 | 220 1 223 | 221 0 224 | 222 1 225 | 223 0 226 | 224 0 227 | 225 1 228 | 226 1 229 | 227 0 230 | 228 1 231 | 229 0 232 | 230 1 233 | 231 0 234 | 232 1 235 | 233 1 236 | 234 0 237 | 235 1 238 | 236 1 239 | 237 1 240 | 238 0 241 | 239 1 242 | 240 1 243 | 241 0 244 | 242 0 245 | 243 1 246 | 244 0 247 | 245 1 248 | 246 1 249 | 247 1 250 | 248 1 251 | 249 0 252 | 250 1 253 | 251 0 254 | 252 0 255 | 253 1 256 | 254 1 257 | 255 1 258 | 256 1 259 | 257 1 260 | 258 1 261 | 259 1 262 | 260 1 263 | 261 1 264 | 262 1 265 | 263 1 266 | 264 0 267 | 265 1 268 | 266 1 269 | 267 0 270 | 268 1 271 | 269 1 272 | 270 1 273 | 271 0 274 | 272 1 275 | 273 0 276 | 274 0 277 | 275 0 278 | 276 0 279 | 277 1 280 | 278 0 281 | 279 1 282 | 280 1 283 | 281 0 284 | 282 1 285 | 283 1 286 | 284 1 287 | 285 0 288 | 286 1 289 | 287 0 290 | 288 0 291 | 289 0 292 | 290 1 293 | 291 0 294 | 292 1 295 | 293 1 296 | 294 1 297 | 295 0 298 | 296 0 299 | 297 0 300 | 298 1 301 | 299 0 302 | 300 1 303 | 301 1 304 | 302 0 305 | 303 1 306 | 304 1 307 | 305 0 308 | 306 1 309 | 307 0 310 | 308 0 311 | 309 1 312 | 310 1 313 | 311 1 314 | 312 0 315 | 313 1 316 | 314 1 317 | 315 1 318 | 316 1 319 | 317 1 320 | 318 1 321 | 319 0 322 | 320 1 323 | 321 1 324 | 322 1 325 | 323 1 326 | 324 0 327 | 325 1 328 | 326 1 329 | 327 0 330 | 328 1 331 | 329 0 332 | 330 0 333 | 331 1 334 | 332 1 335 | 333 1 336 | 334 1 337 | 335 0 338 | 336 1 339 | 337 1 340 | 338 1 341 | 339 1 342 | 340 1 343 | 341 1 344 | 342 1 345 | 343 1 346 | 344 1 347 | 345 1 348 | 346 1 349 | 347 0 350 | 348 1 351 | 349 1 352 | 350 1 353 | 351 1 354 | 352 0 355 | 353 1 356 | 354 1 357 | 355 0 358 | 356 0 359 | 357 0 360 | 358 1 361 | 359 0 362 | 360 0 363 | 361 0 364 | 362 1 365 | 363 1 366 | 364 1 367 | 365 0 368 | 366 0 369 | 367 0 370 | 368 0 371 | 369 1 372 | 370 1 373 | 371 1 374 | 372 1 375 | 373 1 376 | 374 1 377 | 375 0 378 | 376 0 379 | 377 1 380 | 378 0 381 | 379 0 382 | 380 0 383 | 381 1 384 | 382 1 385 | 383 1 386 | 384 1 387 | 385 0 388 | 386 1 389 | 387 0 390 | 388 1 391 | 389 1 392 | 390 1 393 | 391 0 394 | 392 0 395 | 393 1 396 | 394 0 397 | 395 1 398 | 396 1 399 | 397 1 400 | 398 0 401 | 399 0 402 | 400 1 403 | 401 0 404 | 402 1 405 | 403 1 406 | 404 0 407 | 405 0 408 | 406 0 409 | 407 1 410 | 408 0 411 | 409 0 412 | 410 0 413 | 411 0 414 | 412 1 415 | 413 1 416 | 414 1 417 | 415 0 418 | 416 1 419 | 417 1 420 | 418 1 421 | 419 1 422 | 420 1 423 | 421 1 424 | 422 1 425 | 423 1 426 | 424 0 427 | 425 0 428 | 426 0 429 | 427 1 430 | 428 0 431 | 429 1 432 | 430 0 433 | 431 1 434 | 432 1 435 | 433 1 436 | 434 1 437 | 435 1 438 | 436 1 439 | 437 1 440 | 438 1 441 | 439 0 442 | 440 0 443 | 441 0 444 | 442 1 445 | 443 1 446 | 444 0 447 | 445 1 448 | 446 1 449 | 447 1 450 | 448 1 451 | 449 1 452 | 450 0 453 | 451 1 454 | 452 1 455 | 453 0 456 | 454 0 457 | 455 1 458 | 456 1 459 | 457 0 460 | 458 0 461 | 459 1 462 | 460 0 463 | 461 1 464 | 462 0 465 | 463 0 466 | 464 0 467 | 465 0 468 | 466 1 469 | 467 1 470 | 468 1 471 | 469 1 472 | 470 0 473 | 471 1 474 | 472 1 475 | 473 1 476 | 474 1 477 | 475 1 478 | 476 1 479 | 477 1 480 | 478 0 481 | 479 1 482 | 480 0 483 | 481 1 484 | 482 0 485 | 483 1 486 | 484 0 487 | 485 1 488 | 486 1 489 | 487 1 490 | 488 1 491 | 489 1 492 | 490 1 493 | 491 0 494 | 492 1 495 | 493 0 496 | 494 0 497 | 495 1 498 | 496 0 499 | 497 1 500 | 498 0 501 | 499 1 502 | 500 1 503 | 501 1 504 | 502 1 505 | 503 1 506 | 504 1 507 | 505 1 508 | 506 0 509 | 507 1 510 | 508 1 511 | 509 0 512 | 510 1 513 | 511 1 514 | 512 0 515 | 513 1 516 | 514 1 517 | 515 0 518 | 516 1 519 | 517 1 520 | 518 1 521 | 519 1 522 | 520 1 523 | 521 0 524 | 522 1 525 | 523 1 526 | 524 1 527 | 525 1 528 | 526 0 529 | 527 1 530 | 528 1 531 | 529 1 532 | 530 1 533 | 531 1 534 | 532 1 535 | 533 0 536 | 534 1 537 | 535 1 538 | 536 1 539 | 537 0 540 | 538 0 541 | 539 1 542 | 540 1 543 | 541 1 544 | 542 0 545 | 543 0 546 | 544 1 547 | 545 1 548 | 546 1 549 | 547 0 550 | 548 0 551 | 549 0 552 | 550 0 553 | 551 1 554 | 552 0 555 | 553 1 556 | 554 0 557 | 555 1 558 | 556 0 559 | 557 1 560 | 558 1 561 | 559 1 562 | 560 1 563 | 561 0 564 | 562 0 565 | 563 0 566 | 564 1 567 | 565 1 568 | 566 1 569 | 567 1 570 | 568 0 571 | 569 0 572 | 570 1 573 | 571 1 574 | 572 1 575 | 573 0 576 | 574 1 577 | 575 1 578 | 576 1 579 | 577 1 580 | 578 1 581 | 579 1 582 | 580 1 583 | 581 0 584 | 582 1 585 | 583 1 586 | 584 1 587 | 585 1 588 | 586 0 589 | 587 1 590 | 588 1 591 | 589 1 592 | 590 1 593 | 591 1 594 | 592 1 595 | 593 0 596 | 594 0 597 | 595 1 598 | 596 1 599 | 597 0 600 | 598 1 601 | 599 0 602 | 600 0 603 | 601 1 604 | 602 1 605 | 603 1 606 | 604 1 607 | 605 1 608 | 606 1 609 | 607 0 610 | 608 0 611 | 609 1 612 | 610 1 613 | 611 1 614 | 612 0 615 | 613 1 616 | 614 1 617 | 615 1 618 | 616 1 619 | 617 0 620 | 618 1 621 | 619 1 622 | 620 1 623 | 621 0 624 | 622 0 625 | 623 0 626 | 624 1 627 | 625 0 628 | 626 1 629 | 627 1 630 | 628 0 631 | 629 1 632 | 630 0 633 | 631 1 634 | 632 1 635 | 633 0 636 | 634 0 637 | 635 1 638 | 636 0 639 | 637 1 640 | 638 1 641 | 639 1 642 | 640 1 643 | 641 1 644 | 642 1 645 | 643 1 646 | 644 1 647 | 645 0 648 | 646 1 649 | 647 1 650 | 648 0 651 | 649 0 652 | 650 1 653 | 651 1 654 | 652 0 655 | 653 1 656 | 654 0 657 | 655 1 658 | 656 1 659 | 657 1 660 | 658 1 661 | 659 1 662 | 660 1 663 | 661 0 664 | 662 1 665 | 663 1 666 | 664 1 667 | 665 0 668 | 666 1 669 | 667 0 670 | 668 0 671 | 669 0 672 | 670 1 673 | 671 1 674 | 672 0 675 | 673 1 676 | 674 1 677 | 675 0 678 | 676 1 679 | 677 0 680 | 678 1 681 | 679 1 682 | 680 0 683 | 681 1 684 | 682 1 685 | 683 1 686 | 684 0 687 | 685 1 688 | 686 1 689 | 687 0 690 | 688 0 691 | 689 1 692 | 690 1 693 | 691 0 694 | 692 0 695 | 693 1 696 | 694 1 697 | 695 0 698 | 696 0 699 | 697 1 700 | 698 0 701 | 699 1 702 | 700 0 703 | 701 1 704 | 702 1 705 | 703 0 706 | 704 1 707 | 705 0 708 | 706 0 709 | 707 0 710 | 708 1 711 | 709 1 712 | 710 0 713 | 711 0 714 | 712 1 715 | 713 0 716 | 714 1 717 | 715 1 718 | 716 1 719 | 717 0 720 | 718 1 721 | 719 1 722 | 720 1 723 | 721 1 724 | 722 0 725 | 723 0 726 | 724 1 727 | 725 1 728 | 726 1 729 | 727 1 730 | 728 0 731 | 729 0 732 | 730 1 733 | 731 1 734 | 732 1 735 | 733 0 736 | 734 1 737 | 735 1 738 | 736 1 739 | 737 1 740 | 738 1 741 | 739 0 742 | 740 1 743 | 741 1 744 | 742 1 745 | 743 1 746 | 744 1 747 | 745 0 748 | 746 1 749 | 747 0 750 | 748 1 751 | 749 0 752 | 750 0 753 | 751 1 754 | 752 1 755 | 753 0 756 | 754 1 757 | 755 0 758 | 756 1 759 | 757 0 760 | 758 1 761 | 759 1 762 | 760 1 763 | 761 1 764 | 762 0 765 | 763 1 766 | 764 1 767 | 765 1 768 | 766 1 769 | 767 1 770 | 768 1 771 | 769 0 772 | 770 1 773 | 771 0 774 | 772 1 775 | 773 1 776 | 774 1 777 | 775 1 778 | 776 1 779 | 777 1 780 | 778 0 781 | 779 0 782 | 780 1 783 | 781 0 784 | 782 1 785 | 783 0 786 | 784 1 787 | 785 1 788 | 786 1 789 | 787 0 790 | 788 1 791 | 789 1 792 | 790 1 793 | 791 1 794 | 792 0 795 | 793 1 796 | 794 0 797 | 795 1 798 | 796 0 799 | 797 0 800 | 798 0 801 | 799 1 802 | 800 0 803 | 801 0 804 | 802 1 805 | 803 1 806 | 804 0 807 | 805 1 808 | 806 0 809 | 807 0 810 | 808 1 811 | 809 0 812 | 810 1 813 | 811 1 814 | 812 0 815 | 813 1 816 | 814 0 817 | 815 1 818 | 816 1 819 | 817 0 820 | 818 0 821 | 819 1 822 | 820 0 823 | 821 0 824 | 822 1 825 | 823 1 826 | 824 0 827 | 825 1 828 | 826 0 829 | 827 1 830 | 828 0 831 | 829 1 832 | 830 1 833 | 831 1 834 | 832 1 835 | 833 0 836 | 834 1 837 | 835 0 838 | 836 1 839 | 837 1 840 | 838 0 841 | 839 0 842 | 840 0 843 | 841 0 844 | 842 1 845 | 843 1 846 | 844 0 847 | 845 1 848 | 846 1 849 | 847 1 850 | 848 0 851 | 849 0 852 | 850 0 853 | 851 0 854 | 852 1 855 | 853 1 856 | 854 1 857 | 855 0 858 | 856 1 859 | 857 1 860 | 858 0 861 | 859 1 862 | 860 1 863 | 861 1 864 | 862 0 865 | 863 1 866 | 864 0 867 | 865 0 868 | 866 0 869 | 867 1 870 | 868 0 871 | 869 1 872 | 870 0 873 | 871 1 874 | 872 1 875 | 873 1 876 | 874 1 877 | 875 0 878 | 876 1 879 | 877 0 880 | 878 0 881 | 879 1 882 | 880 0 883 | 881 0 884 | 882 1 885 | 883 0 886 | 884 1 887 | 885 1 888 | 886 1 889 | 887 1 890 | 888 0 891 | 889 1 892 | 890 1 893 | 891 1 894 | 892 0 895 | 893 1 896 | 894 0 897 | 895 0 898 | 896 0 899 | 897 1 900 | 898 0 901 | 899 1 902 | 900 1 903 | 901 1 904 | 902 0 905 | 903 1 906 | 904 1 907 | 905 1 908 | 906 1 909 | 907 1 910 | 908 1 911 | 909 1 912 | 910 0 913 | 911 1 914 | 912 0 915 | 913 0 916 | 914 1 917 | 915 1 918 | 916 1 919 | 917 1 920 | 918 0 921 | 919 1 922 | 920 0 923 | 921 1 924 | 922 1 925 | 923 1 926 | 924 1 927 | 925 1 928 | 926 1 929 | 927 0 930 | 928 0 931 | 929 1 932 | 930 1 933 | 931 0 934 | 932 0 935 | 933 0 936 | 934 0 937 | 935 1 938 | 936 0 939 | 937 0 940 | 938 0 941 | 939 1 942 | 940 1 943 | 941 1 944 | 942 0 945 | 943 1 946 | 944 1 947 | 945 1 948 | 946 1 949 | 947 1 950 | 948 0 951 | 949 1 952 | 950 0 953 | 951 1 954 | 952 0 955 | 953 1 956 | 954 1 957 | 955 1 958 | 956 1 959 | 957 1 960 | 958 1 961 | 959 1 962 | 960 1 963 | 961 1 964 | 962 1 965 | 963 0 966 | 964 0 967 | 965 1 968 | 966 1 969 | 967 1 970 | 968 1 971 | 969 1 972 | 970 1 973 | 971 0 974 | 972 0 975 | 973 1 976 | 974 1 977 | 975 0 978 | 976 1 979 | 977 1 980 | 978 1 981 | 979 1 982 | 980 1 983 | 981 0 984 | 982 0 985 | 983 1 986 | 984 0 987 | 985 1 988 | 986 1 989 | 987 1 990 | 988 1 991 | 989 0 992 | 990 1 993 | 991 1 994 | 992 1 995 | 993 1 996 | 994 0 997 | 995 1 998 | 996 1 999 | 997 1 1000 | 998 1 1001 | 999 1 1002 | 1000 0 1003 | 1001 1 1004 | 1002 0 1005 | 1003 0 1006 | 1004 1 1007 | 1005 0 1008 | 1006 1 1009 | 1007 1 1010 | 1008 1 1011 | 1009 0 1012 | 1010 1 1013 | 1011 0 1014 | 1012 1 1015 | 1013 1 1016 | 1014 1 1017 | 1015 1 1018 | 1016 0 1019 | 1017 1 1020 | 1018 0 1021 | 1019 0 1022 | 1020 1 1023 | 1021 0 1024 | 1022 1 1025 | 1023 0 1026 | 1024 1 1027 | 1025 0 1028 | 1026 0 1029 | 1027 1 1030 | 1028 0 1031 | 1029 1 1032 | 1030 1 1033 | 1031 1 1034 | 1032 1 1035 | 1033 0 1036 | 1034 1 1037 | 1035 0 1038 | 1036 0 1039 | 1037 1 1040 | 1038 1 1041 | 1039 0 1042 | 1040 0 1043 | 1041 1 1044 | 1042 0 1045 | 1043 0 1046 | 1044 1 1047 | 1045 1 1048 | 1046 0 1049 | 1047 1 1050 | 1048 1 1051 | 1049 1 1052 | 1050 1 1053 | 1051 0 1054 | 1052 0 1055 | 1053 1 1056 | 1054 0 1057 | 1055 1 1058 | 1056 1 1059 | 1057 1 1060 | 1058 1 1061 | 1059 1 1062 | 1060 0 1063 | 1061 1 1064 | 1062 1 1065 | 1063 1 1066 | 1064 1 1067 | 1065 1 1068 | 1066 1 1069 | 1067 1 1070 | 1068 0 1071 | 1069 0 1072 | 1070 1 1073 | 1071 1 1074 | 1072 1 1075 | 1073 1 1076 | 1074 1 1077 | 1075 1 1078 | 1076 1 1079 | 1077 1 1080 | 1078 0 1081 | 1079 1 1082 | 1080 0 1083 | 1081 0 1084 | 1082 1 1085 | 1083 1 1086 | 1084 1 1087 | 1085 1 1088 | 1086 1 1089 | 1087 1 1090 | 1088 1 1091 | 1089 1 1092 | 1090 1 1093 | 1091 0 1094 | 1092 1 1095 | 1093 0 1096 | 1094 1 1097 | 1095 1 1098 | 1096 1 1099 | 1097 1 1100 | 1098 1 1101 | 1099 0 1102 | 1100 1 1103 | 1101 1 1104 | 1102 0 1105 | 1103 1 1106 | 1104 1 1107 | 1105 0 1108 | 1106 1 1109 | 1107 0 1110 | 1108 0 1111 | 1109 1 1112 | 1110 1 1113 | 1111 1 1114 | 1112 0 1115 | 1113 0 1116 | 1114 1 1117 | 1115 1 1118 | 1116 0 1119 | 1117 1 1120 | 1118 1 1121 | 1119 1 1122 | 1120 0 1123 | 1121 0 1124 | 1122 1 1125 | 1123 1 1126 | 1124 0 1127 | 1125 1 1128 | 1126 0 1129 | 1127 0 1130 | 1128 1 1131 | 1129 1 1132 | 1130 1 1133 | 1131 1 1134 | 1132 0 1135 | 1133 0 1136 | 1134 0 1137 | 1135 1 1138 | 1136 0 1139 | 1137 1 1140 | 1138 0 1141 | 1139 0 1142 | 1140 0 1143 | 1141 1 1144 | 1142 1 1145 | 1143 1 1146 | 1144 1 1147 | 1145 1 1148 | 1146 1 1149 | 1147 0 1150 | 1148 0 1151 | 1149 1 1152 | 1150 0 1153 | 1151 1 1154 | 1152 0 1155 | 1153 1 1156 | 1154 0 1157 | 1155 1 1158 | 1156 1 1159 | 1157 0 1160 | 1158 0 1161 | 1159 1 1162 | 1160 0 1163 | 1161 1 1164 | 1162 0 1165 | 1163 0 1166 | 1164 1 1167 | 1165 0 1168 | 1166 0 1169 | 1167 1 1170 | 1168 1 1171 | 1169 1 1172 | 1170 1 1173 | 1171 1 1174 | 1172 1 1175 | 1173 1 1176 | 1174 1 1177 | 1175 0 1178 | 1176 1 1179 | 1177 0 1180 | 1178 1 1181 | 1179 1 1182 | 1180 0 1183 | 1181 1 1184 | 1182 1 1185 | 1183 1 1186 | 1184 0 1187 | 1185 1 1188 | 1186 0 1189 | 1187 1 1190 | 1188 1 1191 | 1189 0 1192 | 1190 1 1193 | 1191 0 1194 | 1192 1 1195 | 1193 1 1196 | 1194 1 1197 | 1195 1 1198 | 1196 0 1199 | 1197 1 1200 | 1198 1 1201 | 1199 1 1202 | 1200 0 1203 | 1201 1 1204 | 1202 1 1205 | 1203 1 1206 | 1204 0 1207 | 1205 1 1208 | 1206 0 1209 | 1207 0 1210 | 1208 1 1211 | 1209 1 1212 | 1210 1 1213 | 1211 1 1214 | 1212 1 1215 | 1213 1 1216 | 1214 1 1217 | 1215 1 1218 | 1216 1 1219 | 1217 1 1220 | 1218 1 1221 | 1219 1 1222 | 1220 1 1223 | 1221 1 1224 | 1222 0 1225 | 1223 1 1226 | 1224 0 1227 | 1225 1 1228 | 1226 0 1229 | 1227 1 1230 | 1228 1 1231 | 1229 1 1232 | 1230 1 1233 | 1231 0 1234 | 1232 1 1235 | 1233 1 1236 | 1234 1 1237 | 1235 1 1238 | 1236 1 1239 | 1237 1 1240 | 1238 1 1241 | 1239 1 1242 | 1240 1 1243 | 1241 1 1244 | 1242 1 1245 | 1243 1 1246 | 1244 1 1247 | 1245 1 1248 | 1246 0 1249 | 1247 1 1250 | 1248 1 1251 | 1249 1 1252 | 1250 1 1253 | 1251 1 1254 | 1252 1 1255 | 1253 1 1256 | 1254 0 1257 | 1255 1 1258 | 1256 1 1259 | 1257 1 1260 | 1258 0 1261 | 1259 1 1262 | 1260 1 1263 | 1261 1 1264 | 1262 0 1265 | 1263 1 1266 | 1264 1 1267 | 1265 1 1268 | 1266 0 1269 | 1267 1 1270 | 1268 1 1271 | 1269 1 1272 | 1270 0 1273 | 1271 1 1274 | 1272 1 1275 | 1273 0 1276 | 1274 0 1277 | 1275 1 1278 | 1276 0 1279 | 1277 0 1280 | 1278 1 1281 | 1279 0 1282 | 1280 0 1283 | 1281 1 1284 | 1282 0 1285 | 1283 0 1286 | 1284 1 1287 | 1285 1 1288 | 1286 1 1289 | 1287 0 1290 | 1288 1 1291 | 1289 1 1292 | 1290 1 1293 | 1291 1 1294 | 1292 0 1295 | 1293 0 1296 | 1294 0 1297 | 1295 1 1298 | 1296 1 1299 | 1297 1 1300 | 1298 1 1301 | 1299 1 1302 | 1300 0 1303 | 1301 1 1304 | 1302 1 1305 | 1303 1 1306 | 1304 0 1307 | 1305 0 1308 | 1306 0 1309 | 1307 1 1310 | 1308 1 1311 | 1309 0 1312 | 1310 1 1313 | 1311 1 1314 | 1312 0 1315 | 1313 1 1316 | 1314 1 1317 | 1315 1 1318 | 1316 1 1319 | 1317 1 1320 | 1318 0 1321 | 1319 1 1322 | 1320 0 1323 | 1321 1 1324 | 1322 1 1325 | 1323 1 1326 | 1324 1 1327 | 1325 1 1328 | 1326 0 1329 | 1327 1 1330 | 1328 1 1331 | 1329 0 1332 | 1330 0 1333 | 1331 1 1334 | 1332 1 1335 | 1333 0 1336 | 1334 1 1337 | 1335 0 1338 | 1336 0 1339 | 1337 0 1340 | 1338 0 1341 | 1339 0 1342 | 1340 0 1343 | 1341 0 1344 | 1342 0 1345 | 1343 0 1346 | 1344 1 1347 | 1345 0 1348 | 1346 1 1349 | 1347 1 1350 | 1348 1 1351 | 1349 1 1352 | 1350 0 1353 | 1351 1 1354 | 1352 1 1355 | 1353 0 1356 | 1354 0 1357 | 1355 0 1358 | 1356 1 1359 | 1357 1 1360 | 1358 0 1361 | 1359 1 1362 | 1360 1 1363 | 1361 1 1364 | 1362 1 1365 | 1363 0 1366 | 1364 1 1367 | 1365 0 1368 | 1366 0 1369 | 1367 0 1370 | 1368 1 1371 | 1369 1 1372 | 1370 0 1373 | 1371 0 1374 | 1372 1 1375 | 1373 1 1376 | 1374 0 1377 | 1375 1 1378 | 1376 0 1379 | 1377 1 1380 | 1378 1 1381 | 1379 0 1382 | 1380 1 1383 | 1381 1 1384 | 1382 1 1385 | 1383 0 1386 | 1384 1 1387 | 1385 1 1388 | 1386 0 1389 | 1387 1 1390 | 1388 1 1391 | 1389 1 1392 | 1390 0 1393 | 1391 0 1394 | 1392 0 1395 | 1393 1 1396 | 1394 1 1397 | 1395 0 1398 | 1396 0 1399 | 1397 1 1400 | 1398 1 1401 | 1399 0 1402 | 1400 0 1403 | 1401 0 1404 | 1402 1 1405 | 1403 0 1406 | 1404 1 1407 | 1405 0 1408 | 1406 1 1409 | 1407 1 1410 | 1408 0 1411 | 1409 1 1412 | 1410 0 1413 | 1411 1 1414 | 1412 1 1415 | 1413 1 1416 | 1414 0 1417 | 1415 1 1418 | 1416 1 1419 | 1417 0 1420 | 1418 1 1421 | 1419 0 1422 | 1420 0 1423 | 1421 1 1424 | 1422 0 1425 | 1423 0 1426 | 1424 0 1427 | 1425 1 1428 | 1426 0 1429 | 1427 1 1430 | 1428 1 1431 | 1429 0 1432 | 1430 1 1433 | 1431 1 1434 | 1432 1 1435 | 1433 0 1436 | 1434 0 1437 | 1435 0 1438 | 1436 1 1439 | 1437 1 1440 | 1438 1 1441 | 1439 1 1442 | 1440 1 1443 | 1441 1 1444 | 1442 0 1445 | 1443 0 1446 | 1444 1 1447 | 1445 1 1448 | 1446 1 1449 | 1447 1 1450 | 1448 0 1451 | 1449 1 1452 | 1450 1 1453 | 1451 0 1454 | 1452 1 1455 | 1453 1 1456 | 1454 1 1457 | 1455 1 1458 | 1456 1 1459 | 1457 1 1460 | 1458 1 1461 | 1459 1 1462 | 1460 1 1463 | 1461 1 1464 | 1462 1 1465 | 1463 0 1466 | 1464 1 1467 | 1465 1 1468 | 1466 1 1469 | 1467 0 1470 | 1468 1 1471 | 1469 0 1472 | 1470 0 1473 | 1471 1 1474 | 1472 0 1475 | 1473 0 1476 | 1474 1 1477 | 1475 0 1478 | 1476 0 1479 | 1477 1 1480 | 1478 0 1481 | 1479 0 1482 | 1480 1 1483 | 1481 0 1484 | 1482 1 1485 | 1483 0 1486 | 1484 0 1487 | 1485 1 1488 | 1486 0 1489 | 1487 1 1490 | 1488 1 1491 | 1489 1 1492 | 1490 0 1493 | 1491 1 1494 | 1492 1 1495 | 1493 0 1496 | 1494 0 1497 | 1495 1 1498 | 1496 1 1499 | 1497 0 1500 | 1498 0 1501 | 1499 0 1502 | 1500 1 1503 | 1501 1 1504 | 1502 0 1505 | 1503 0 1506 | 1504 1 1507 | 1505 1 1508 | 1506 1 1509 | 1507 1 1510 | 1508 0 1511 | 1509 0 1512 | 1510 1 1513 | 1511 1 1514 | 1512 0 1515 | 1513 1 1516 | 1514 0 1517 | 1515 1 1518 | 1516 0 1519 | 1517 1 1520 | 1518 0 1521 | 1519 0 1522 | 1520 0 1523 | 1521 0 1524 | 1522 0 1525 | 1523 0 1526 | 1524 1 1527 | 1525 1 1528 | 1526 1 1529 | 1527 1 1530 | 1528 0 1531 | 1529 1 1532 | 1530 1 1533 | 1531 1 1534 | 1532 1 1535 | 1533 1 1536 | 1534 0 1537 | 1535 0 1538 | 1536 1 1539 | 1537 0 1540 | 1538 1 1541 | 1539 0 1542 | 1540 0 1543 | 1541 1 1544 | 1542 0 1545 | 1543 1 1546 | 1544 0 1547 | 1545 1 1548 | 1546 1 1549 | 1547 0 1550 | 1548 1 1551 | 1549 1 1552 | 1550 1 1553 | 1551 0 1554 | 1552 0 1555 | 1553 0 1556 | 1554 1 1557 | 1555 0 1558 | 1556 0 1559 | 1557 1 1560 | 1558 0 1561 | 1559 1 1562 | 1560 1 1563 | 1561 1 1564 | 1562 1 1565 | 1563 1 1566 | 1564 0 1567 | 1565 0 1568 | 1566 1 1569 | 1567 1 1570 | 1568 1 1571 | 1569 1 1572 | 1570 1 1573 | 1571 1 1574 | 1572 1 1575 | 1573 1 1576 | 1574 1 1577 | 1575 1 1578 | 1576 1 1579 | 1577 0 1580 | 1578 0 1581 | 1579 1 1582 | 1580 1 1583 | 1581 0 1584 | 1582 0 1585 | 1583 1 1586 | 1584 0 1587 | 1585 1 1588 | 1586 1 1589 | 1587 1 1590 | 1588 1 1591 | 1589 0 1592 | 1590 0 1593 | 1591 1 1594 | 1592 1 1595 | 1593 0 1596 | 1594 1 1597 | 1595 1 1598 | 1596 1 1599 | 1597 0 1600 | 1598 1 1601 | 1599 1 1602 | 1600 0 1603 | 1601 0 1604 | 1602 1 1605 | 1603 0 1606 | 1604 1 1607 | 1605 0 1608 | 1606 1 1609 | 1607 1 1610 | 1608 0 1611 | 1609 1 1612 | 1610 0 1613 | 1611 1 1614 | 1612 0 1615 | 1613 1 1616 | 1614 1 1617 | 1615 1 1618 | 1616 1 1619 | 1617 0 1620 | 1618 0 1621 | 1619 1 1622 | 1620 0 1623 | 1621 0 1624 | 1622 1 1625 | 1623 1 1626 | 1624 1 1627 | 1625 0 1628 | 1626 1 1629 | 1627 1 1630 | 1628 1 1631 | 1629 0 1632 | 1630 0 1633 | 1631 0 1634 | 1632 1 1635 | 1633 1 1636 | 1634 1 1637 | 1635 1 1638 | 1636 1 1639 | 1637 1 1640 | 1638 0 1641 | 1639 0 1642 | 1640 0 1643 | 1641 1 1644 | 1642 1 1645 | 1643 1 1646 | 1644 1 1647 | 1645 1 1648 | 1646 1 1649 | 1647 0 1650 | 1648 1 1651 | 1649 1 1652 | 1650 0 1653 | 1651 0 1654 | 1652 0 1655 | 1653 1 1656 | 1654 0 1657 | 1655 1 1658 | 1656 0 1659 | 1657 0 1660 | 1658 1 1661 | 1659 1 1662 | 1660 0 1663 | 1661 0 1664 | 1662 0 1665 | 1663 1 1666 | 1664 1 1667 | 1665 1 1668 | 1666 1 1669 | 1667 1 1670 | 1668 0 1671 | 1669 0 1672 | 1670 1 1673 | 1671 1 1674 | 1672 1 1675 | 1673 1 1676 | 1674 1 1677 | 1675 0 1678 | 1676 0 1679 | 1677 1 1680 | 1678 1 1681 | 1679 1 1682 | 1680 0 1683 | 1681 1 1684 | 1682 1 1685 | 1683 0 1686 | 1684 1 1687 | 1685 1 1688 | 1686 0 1689 | 1687 0 1690 | 1688 1 1691 | 1689 1 1692 | 1690 1 1693 | 1691 0 1694 | 1692 1 1695 | 1693 1 1696 | 1694 0 1697 | 1695 1 1698 | 1696 1 1699 | 1697 0 1700 | 1698 0 1701 | 1699 1 1702 | 1700 0 1703 | 1701 1 1704 | 1702 1 1705 | 1703 1 1706 | 1704 0 1707 | 1705 1 1708 | 1706 1 1709 | 1707 0 1710 | 1708 1 1711 | 1709 1 1712 | 1710 1 1713 | 1711 1 1714 | 1712 1 1715 | 1713 0 1716 | 1714 1 1717 | 1715 1 1718 | 1716 1 1719 | 1717 1 1720 | 1718 0 1721 | 1719 1 1722 | 1720 0 1723 | 1721 0 1724 | 1722 1 1725 | 1723 1 1726 | 1724 1 1727 | -------------------------------------------------------------------------------- /examples/GLUE-submission/matbert-base-cased/STS-B.tsv: -------------------------------------------------------------------------------- 1 | index prediction 2 | 0 3.431 3 | 1 4.168 4 | 2 4.996 5 | 3 5.000 6 | 4 2.015 7 | 5 1.792 8 | 6 4.490 9 | 7 1.462 10 | 8 2.713 11 | 9 2.040 12 | 10 2.040 13 | 11 5.000 14 | 12 1.120 15 | 13 4.754 16 | 14 2.451 17 | 15 2.154 18 | 16 4.466 19 | 17 3.191 20 | 18 3.588 21 | 19 0.600 22 | 20 1.711 23 | 21 0.757 24 | 22 4.782 25 | 23 4.807 26 | 24 1.105 27 | 25 2.886 28 | 26 0.832 29 | 27 1.704 30 | 28 1.110 31 | 29 3.051 32 | 30 2.909 33 | 31 4.359 34 | 32 0.882 35 | 33 3.735 36 | 34 2.851 37 | 35 0.384 38 | 36 1.269 39 | 37 2.216 40 | 38 4.226 41 | 39 0.679 42 | 40 3.831 43 | 41 2.888 44 | 42 3.482 45 | 43 0.492 46 | 44 2.304 47 | 45 0.489 48 | 46 0.554 49 | 47 1.344 50 | 48 4.322 51 | 49 2.116 52 | 50 0.597 53 | 51 1.444 54 | 52 0.658 55 | 53 0.460 56 | 54 1.462 57 | 55 0.064 58 | 56 3.690 59 | 57 0.664 60 | 58 2.932 61 | 59 1.998 62 | 60 5.000 63 | 61 0.202 64 | 62 3.399 65 | 63 1.049 66 | 64 3.531 67 | 65 0.466 68 | 66 0.221 69 | 67 3.434 70 | 68 0.050 71 | 69 3.182 72 | 70 3.890 73 | 71 4.171 74 | 72 0.166 75 | 73 3.466 76 | 74 3.348 77 | 75 0.083 78 | 76 0.705 79 | 77 0.086 80 | 78 2.749 81 | 79 3.110 82 | 80 0.080 83 | 81 0.378 84 | 82 2.702 85 | 83 1.800 86 | 84 0.711 87 | 85 0.772 88 | 86 2.164 89 | 87 4.396 90 | 88 3.546 91 | 89 1.207 92 | 90 1.414 93 | 91 3.433 94 | 92 0.167 95 | 93 3.384 96 | 94 1.271 97 | 95 2.986 98 | 96 3.582 99 | 97 1.255 100 | 98 3.297 101 | 99 3.333 102 | 100 0.038 103 | 101 0.392 104 | 102 3.411 105 | 103 2.863 106 | 104 3.200 107 | 105 3.637 108 | 106 0.000 109 | 107 3.197 110 | 108 2.363 111 | 109 3.606 112 | 110 2.594 113 | 111 2.701 114 | 112 2.743 115 | 113 3.478 116 | 114 0.000 117 | 115 0.298 118 | 116 4.411 119 | 117 1.601 120 | 118 0.157 121 | 119 0.604 122 | 120 1.583 123 | 121 0.478 124 | 122 0.223 125 | 123 2.255 126 | 124 0.011 127 | 125 5.000 128 | 126 0.659 129 | 127 1.970 130 | 128 5.000 131 | 129 4.771 132 | 130 4.407 133 | 131 2.186 134 | 132 3.547 135 | 133 4.684 136 | 134 3.560 137 | 135 4.585 138 | 136 5.000 139 | 137 1.780 140 | 138 1.998 141 | 139 4.903 142 | 140 5.000 143 | 141 3.629 144 | 142 3.021 145 | 143 4.371 146 | 144 1.663 147 | 145 1.224 148 | 146 4.164 149 | 147 3.359 150 | 148 1.929 151 | 149 0.723 152 | 150 1.700 153 | 151 1.344 154 | 152 3.355 155 | 153 3.226 156 | 154 4.277 157 | 155 3.640 158 | 156 0.363 159 | 157 0.963 160 | 158 4.010 161 | 159 0.692 162 | 160 5.000 163 | 161 2.519 164 | 162 1.595 165 | 163 3.247 166 | 164 3.807 167 | 165 4.181 168 | 166 4.865 169 | 167 3.298 170 | 168 4.635 171 | 169 4.041 172 | 170 1.263 173 | 171 0.243 174 | 172 4.504 175 | 173 3.817 176 | 174 3.556 177 | 175 3.209 178 | 176 4.429 179 | 177 4.201 180 | 178 0.501 181 | 179 0.179 182 | 180 0.284 183 | 181 2.489 184 | 182 2.986 185 | 183 0.420 186 | 184 1.435 187 | 185 0.144 188 | 186 0.105 189 | 187 0.425 190 | 188 0.956 191 | 189 0.318 192 | 190 4.475 193 | 191 3.302 194 | 192 1.599 195 | 193 3.499 196 | 194 3.564 197 | 195 4.059 198 | 196 3.161 199 | 197 0.655 200 | 198 0.036 201 | 199 3.344 202 | 200 0.786 203 | 201 0.943 204 | 202 0.469 205 | 203 1.244 206 | 204 0.803 207 | 205 0.940 208 | 206 3.997 209 | 207 0.021 210 | 208 2.024 211 | 209 1.960 212 | 210 0.985 213 | 211 0.547 214 | 212 0.735 215 | 213 0.625 216 | 214 2.260 217 | 215 1.430 218 | 216 1.844 219 | 217 4.180 220 | 218 3.775 221 | 219 0.814 222 | 220 0.535 223 | 221 0.046 224 | 222 0.036 225 | 223 1.918 226 | 224 0.033 227 | 225 0.063 228 | 226 2.398 229 | 227 0.244 230 | 228 0.173 231 | 229 0.358 232 | 230 0.038 233 | 231 0.079 234 | 232 3.632 235 | 233 2.483 236 | 234 3.280 237 | 235 3.634 238 | 236 0.665 239 | 237 0.174 240 | 238 0.149 241 | 239 0.083 242 | 240 1.104 243 | 241 0.260 244 | 242 0.067 245 | 243 0.114 246 | 244 1.210 247 | 245 0.807 248 | 246 3.126 249 | 247 3.452 250 | 248 0.033 251 | 249 0.241 252 | 250 3.435 253 | 251 4.642 254 | 252 2.956 255 | 253 2.620 256 | 254 0.000 257 | 255 3.975 258 | 256 0.430 259 | 257 3.857 260 | 258 3.277 261 | 259 0.084 262 | 260 4.691 263 | 261 3.580 264 | 262 3.612 265 | 263 4.054 266 | 264 2.486 267 | 265 1.887 268 | 266 2.036 269 | 267 3.457 270 | 268 0.469 271 | 269 5.000 272 | 270 3.530 273 | 271 1.453 274 | 272 2.264 275 | 273 3.883 276 | 274 4.352 277 | 275 5.000 278 | 276 0.782 279 | 277 0.000 280 | 278 3.518 281 | 279 2.752 282 | 280 0.203 283 | 281 2.499 284 | 282 3.429 285 | 283 3.086 286 | 284 3.576 287 | 285 4.539 288 | 286 2.193 289 | 287 3.515 290 | 288 3.668 291 | 289 0.383 292 | 290 4.538 293 | 291 2.002 294 | 292 3.335 295 | 293 3.898 296 | 294 0.072 297 | 295 0.826 298 | 296 2.086 299 | 297 3.539 300 | 298 2.525 301 | 299 3.450 302 | 300 5.000 303 | 301 3.000 304 | 302 0.894 305 | 303 4.448 306 | 304 3.160 307 | 305 4.988 308 | 306 4.267 309 | 307 2.568 310 | 308 3.862 311 | 309 3.275 312 | 310 1.555 313 | 311 0.446 314 | 312 0.794 315 | 313 2.748 316 | 314 3.163 317 | 315 4.896 318 | 316 2.911 319 | 317 4.222 320 | 318 0.000 321 | 319 0.046 322 | 320 3.977 323 | 321 3.901 324 | 322 3.874 325 | 323 3.314 326 | 324 3.017 327 | 325 0.165 328 | 326 3.103 329 | 327 4.433 330 | 328 3.542 331 | 329 3.104 332 | 330 4.450 333 | 331 2.699 334 | 332 4.956 335 | 333 2.705 336 | 334 3.189 337 | 335 0.815 338 | 336 4.044 339 | 337 3.198 340 | 338 0.171 341 | 339 0.707 342 | 340 3.630 343 | 341 5.000 344 | 342 4.619 345 | 343 2.934 346 | 344 3.199 347 | 345 0.687 348 | 346 2.749 349 | 347 0.505 350 | 348 1.597 351 | 349 1.251 352 | 350 3.034 353 | 351 4.059 354 | 352 0.913 355 | 353 3.570 356 | 354 2.396 357 | 355 0.049 358 | 356 2.458 359 | 357 4.314 360 | 358 0.322 361 | 359 3.721 362 | 360 4.612 363 | 361 4.291 364 | 362 2.342 365 | 363 2.705 366 | 364 5.000 367 | 365 0.178 368 | 366 0.708 369 | 367 3.292 370 | 368 3.961 371 | 369 4.566 372 | 370 3.379 373 | 371 4.375 374 | 372 0.504 375 | 373 0.840 376 | 374 1.588 377 | 375 2.752 378 | 376 1.907 379 | 377 0.000 380 | 378 1.049 381 | 379 1.167 382 | 380 3.571 383 | 381 3.949 384 | 382 2.520 385 | 383 0.685 386 | 384 3.523 387 | 385 2.257 388 | 386 4.768 389 | 387 1.352 390 | 388 3.159 391 | 389 2.232 392 | 390 4.923 393 | 391 2.756 394 | 392 0.141 395 | 393 1.784 396 | 394 2.417 397 | 395 0.329 398 | 396 1.129 399 | 397 5.000 400 | 398 3.230 401 | 399 2.259 402 | 400 2.829 403 | 401 0.604 404 | 402 2.023 405 | 403 3.461 406 | 404 2.857 407 | 405 4.045 408 | 406 0.164 409 | 407 3.340 410 | 408 2.290 411 | 409 2.815 412 | 410 4.637 413 | 411 4.797 414 | 412 1.446 415 | 413 4.150 416 | 414 0.878 417 | 415 2.117 418 | 416 0.000 419 | 417 4.409 420 | 418 2.233 421 | 419 3.600 422 | 420 4.237 423 | 421 1.565 424 | 422 5.000 425 | 423 1.493 426 | 424 2.194 427 | 425 2.771 428 | 426 0.000 429 | 427 5.000 430 | 428 2.586 431 | 429 4.712 432 | 430 1.611 433 | 431 5.000 434 | 432 0.397 435 | 433 3.548 436 | 434 0.736 437 | 435 2.574 438 | 436 2.346 439 | 437 5.000 440 | 438 4.658 441 | 439 4.475 442 | 440 2.624 443 | 441 1.877 444 | 442 0.032 445 | 443 3.831 446 | 444 2.730 447 | 445 0.205 448 | 446 1.162 449 | 447 4.026 450 | 448 1.915 451 | 449 1.469 452 | 450 3.270 453 | 451 3.477 454 | 452 0.760 455 | 453 0.148 456 | 454 5.000 457 | 455 3.036 458 | 456 1.650 459 | 457 4.413 460 | 458 2.323 461 | 459 3.117 462 | 460 0.266 463 | 461 0.185 464 | 462 0.000 465 | 463 2.291 466 | 464 3.603 467 | 465 3.574 468 | 466 2.010 469 | 467 4.501 470 | 468 3.987 471 | 469 0.120 472 | 470 1.590 473 | 471 3.939 474 | 472 1.897 475 | 473 4.855 476 | 474 3.042 477 | 475 2.384 478 | 476 0.729 479 | 477 0.532 480 | 478 3.408 481 | 479 0.350 482 | 480 4.454 483 | 481 0.883 484 | 482 3.851 485 | 483 0.011 486 | 484 4.645 487 | 485 2.276 488 | 486 3.358 489 | 487 4.295 490 | 488 3.236 491 | 489 4.966 492 | 490 4.216 493 | 491 3.907 494 | 492 4.072 495 | 493 2.782 496 | 494 3.207 497 | 495 2.193 498 | 496 0.803 499 | 497 4.172 500 | 498 0.617 501 | 499 5.000 502 | 500 2.733 503 | 501 2.581 504 | 502 1.291 505 | 503 3.036 506 | 504 1.923 507 | 505 3.710 508 | 506 2.974 509 | 507 1.050 510 | 508 3.477 511 | 509 0.007 512 | 510 1.903 513 | 511 2.062 514 | 512 1.873 515 | 513 0.291 516 | 514 2.074 517 | 515 0.277 518 | 516 1.982 519 | 517 1.761 520 | 518 2.138 521 | 519 2.848 522 | 520 0.868 523 | 521 3.154 524 | 522 4.385 525 | 523 2.972 526 | 524 3.177 527 | 525 3.321 528 | 526 1.589 529 | 527 0.896 530 | 528 3.105 531 | 529 0.344 532 | 530 0.446 533 | 531 0.000 534 | 532 1.056 535 | 533 1.109 536 | 534 0.958 537 | 535 0.607 538 | 536 2.007 539 | 537 0.263 540 | 538 1.628 541 | 539 2.526 542 | 540 2.457 543 | 541 1.530 544 | 542 2.943 545 | 543 4.340 546 | 544 4.649 547 | 545 5.000 548 | 546 0.002 549 | 547 2.241 550 | 548 2.475 551 | 549 4.003 552 | 550 2.104 553 | 551 1.805 554 | 552 3.984 555 | 553 2.178 556 | 554 1.681 557 | 555 3.737 558 | 556 3.332 559 | 557 2.232 560 | 558 4.825 561 | 559 1.932 562 | 560 3.309 563 | 561 0.556 564 | 562 1.561 565 | 563 1.223 566 | 564 1.385 567 | 565 2.171 568 | 566 4.673 569 | 567 3.307 570 | 568 1.910 571 | 569 2.370 572 | 570 2.137 573 | 571 1.530 574 | 572 1.300 575 | 573 3.570 576 | 574 3.489 577 | 575 2.664 578 | 576 4.172 579 | 577 3.237 580 | 578 1.803 581 | 579 1.516 582 | 580 1.811 583 | 581 2.685 584 | 582 3.166 585 | 583 2.099 586 | 584 2.248 587 | 585 3.491 588 | 586 0.674 589 | 587 5.000 590 | 588 2.567 591 | 589 1.935 592 | 590 2.733 593 | 591 2.329 594 | 592 4.174 595 | 593 3.052 596 | 594 0.186 597 | 595 3.619 598 | 596 3.268 599 | 597 0.185 600 | 598 3.251 601 | 599 4.318 602 | 600 2.235 603 | 601 0.222 604 | 602 3.338 605 | 603 1.549 606 | 604 0.799 607 | 605 3.536 608 | 606 1.996 609 | 607 2.466 610 | 608 3.063 611 | 609 2.888 612 | 610 5.000 613 | 611 3.333 614 | 612 2.186 615 | 613 1.119 616 | 614 3.966 617 | 615 2.122 618 | 616 2.790 619 | 617 0.705 620 | 618 3.041 621 | 619 4.056 622 | 620 1.160 623 | 621 2.598 624 | 622 0.643 625 | 623 5.000 626 | 624 2.231 627 | 625 2.982 628 | 626 2.744 629 | 627 0.443 630 | 628 1.599 631 | 629 1.614 632 | 630 1.830 633 | 631 3.031 634 | 632 4.300 635 | 633 2.801 636 | 634 2.059 637 | 635 2.379 638 | 636 2.837 639 | 637 4.476 640 | 638 2.707 641 | 639 3.211 642 | 640 1.615 643 | 641 1.891 644 | 642 2.846 645 | 643 2.592 646 | 644 2.352 647 | 645 4.438 648 | 646 3.054 649 | 647 2.093 650 | 648 3.720 651 | 649 2.509 652 | 650 2.373 653 | 651 4.026 654 | 652 1.219 655 | 653 0.851 656 | 654 2.731 657 | 655 3.334 658 | 656 3.699 659 | 657 1.489 660 | 658 2.389 661 | 659 1.758 662 | 660 1.881 663 | 661 3.601 664 | 662 2.155 665 | 663 2.795 666 | 664 4.750 667 | 665 1.821 668 | 666 3.042 669 | 667 2.285 670 | 668 4.738 671 | 669 2.809 672 | 670 2.580 673 | 671 4.466 674 | 672 2.627 675 | 673 4.905 676 | 674 3.366 677 | 675 1.789 678 | 676 1.909 679 | 677 3.165 680 | 678 1.254 681 | 679 0.796 682 | 680 2.626 683 | 681 1.706 684 | 682 3.521 685 | 683 2.314 686 | 684 2.543 687 | 685 2.038 688 | 686 2.152 689 | 687 2.432 690 | 688 1.731 691 | 689 3.823 692 | 690 3.263 693 | 691 2.481 694 | 692 4.767 695 | 693 2.328 696 | 694 3.166 697 | 695 1.658 698 | 696 2.658 699 | 697 2.092 700 | 698 2.160 701 | 699 2.887 702 | 700 3.378 703 | 701 2.605 704 | 702 2.757 705 | 703 4.441 706 | 704 3.688 707 | 705 2.160 708 | 706 2.748 709 | 707 1.092 710 | 708 3.610 711 | 709 2.891 712 | 710 4.349 713 | 711 2.131 714 | 712 1.771 715 | 713 1.686 716 | 714 2.698 717 | 715 2.140 718 | 716 3.016 719 | 717 3.444 720 | 718 1.651 721 | 719 1.836 722 | 720 2.762 723 | 721 2.158 724 | 722 2.381 725 | 723 2.356 726 | 724 3.411 727 | 725 1.093 728 | 726 3.104 729 | 727 3.680 730 | 728 1.570 731 | 729 3.430 732 | 730 1.931 733 | 731 1.722 734 | 732 2.856 735 | 733 1.324 736 | 734 2.773 737 | 735 2.103 738 | 736 1.935 739 | 737 3.752 740 | 738 2.032 741 | 739 3.636 742 | 740 3.818 743 | 741 2.479 744 | 742 1.855 745 | 743 2.763 746 | 744 1.908 747 | 745 2.211 748 | 746 2.047 749 | 747 2.394 750 | 748 3.321 751 | 749 2.390 752 | 750 2.709 753 | 751 2.286 754 | 752 2.997 755 | 753 3.511 756 | 754 2.053 757 | 755 4.239 758 | 756 2.249 759 | 757 1.776 760 | 758 1.853 761 | 759 4.273 762 | 760 4.042 763 | 761 0.685 764 | 762 2.039 765 | 763 2.384 766 | 764 3.243 767 | 765 3.020 768 | 766 3.575 769 | 767 1.988 770 | 768 3.013 771 | 769 2.045 772 | 770 2.650 773 | 771 2.086 774 | 772 1.979 775 | 773 2.756 776 | 774 2.732 777 | 775 2.223 778 | 776 2.106 779 | 777 1.603 780 | 778 2.125 781 | 779 2.369 782 | 780 2.154 783 | 781 3.085 784 | 782 3.832 785 | 783 3.882 786 | 784 1.380 787 | 785 2.955 788 | 786 2.486 789 | 787 3.089 790 | 788 3.352 791 | 789 1.719 792 | 790 2.269 793 | 791 2.311 794 | 792 1.234 795 | 793 2.124 796 | 794 1.996 797 | 795 3.088 798 | 796 0.987 799 | 797 2.906 800 | 798 0.461 801 | 799 2.800 802 | 800 1.563 803 | 801 1.741 804 | 802 1.440 805 | 803 0.678 806 | 804 2.876 807 | 805 4.804 808 | 806 1.964 809 | 807 2.509 810 | 808 3.029 811 | 809 1.599 812 | 810 2.627 813 | 811 1.863 814 | 812 2.485 815 | 813 3.115 816 | 814 2.755 817 | 815 2.208 818 | 816 2.478 819 | 817 2.453 820 | 818 3.786 821 | 819 2.477 822 | 820 1.980 823 | 821 2.695 824 | 822 2.321 825 | 823 2.262 826 | 824 2.795 827 | 825 1.804 828 | 826 3.315 829 | 827 0.932 830 | 828 3.743 831 | 829 4.232 832 | 830 2.396 833 | 831 1.712 834 | 832 2.326 835 | 833 3.807 836 | 834 3.302 837 | 835 0.884 838 | 836 4.157 839 | 837 1.759 840 | 838 1.206 841 | 839 2.475 842 | 840 1.766 843 | 841 3.582 844 | 842 3.090 845 | 843 1.968 846 | 844 3.843 847 | 845 2.573 848 | 846 3.287 849 | 847 2.657 850 | 848 2.183 851 | 849 3.330 852 | 850 3.794 853 | 851 2.575 854 | 852 4.947 855 | 853 2.497 856 | 854 5.000 857 | 855 3.127 858 | 856 1.976 859 | 857 1.996 860 | 858 4.115 861 | 859 2.981 862 | 860 3.140 863 | 861 4.636 864 | 862 4.030 865 | 863 1.762 866 | 864 2.000 867 | 865 1.882 868 | 866 2.042 869 | 867 1.495 870 | 868 1.663 871 | 869 2.575 872 | 870 2.696 873 | 871 2.028 874 | 872 1.904 875 | 873 2.953 876 | 874 2.209 877 | 875 4.584 878 | 876 2.702 879 | 877 3.510 880 | 878 2.528 881 | 879 1.812 882 | 880 3.467 883 | 881 2.842 884 | 882 3.581 885 | 883 4.517 886 | 884 1.815 887 | 885 3.460 888 | 886 3.251 889 | 887 3.469 890 | 888 3.103 891 | 889 2.897 892 | 890 5.000 893 | 891 3.844 894 | 892 1.215 895 | 893 3.800 896 | 894 3.810 897 | 895 4.209 898 | 896 3.429 899 | 897 3.025 900 | 898 1.459 901 | 899 3.253 902 | 900 3.904 903 | 901 4.676 904 | 902 2.826 905 | 903 4.062 906 | 904 4.784 907 | 905 3.238 908 | 906 2.463 909 | 907 3.947 910 | 908 3.882 911 | 909 2.646 912 | 910 2.983 913 | 911 3.373 914 | 912 3.073 915 | 913 4.127 916 | 914 3.315 917 | 915 4.296 918 | 916 2.860 919 | 917 3.428 920 | 918 4.123 921 | 919 4.091 922 | 920 3.323 923 | 921 2.797 924 | 922 4.464 925 | 923 3.796 926 | 924 4.104 927 | 925 2.211 928 | 926 3.922 929 | 927 3.265 930 | 928 3.629 931 | 929 3.154 932 | 930 3.453 933 | 931 3.786 934 | 932 4.123 935 | 933 3.055 936 | 934 1.996 937 | 935 1.442 938 | 936 2.781 939 | 937 2.886 940 | 938 3.647 941 | 939 2.933 942 | 940 3.831 943 | 941 3.647 944 | 942 3.273 945 | 943 3.038 946 | 944 3.568 947 | 945 4.256 948 | 946 4.331 949 | 947 3.280 950 | 948 3.399 951 | 949 2.953 952 | 950 3.987 953 | 951 4.131 954 | 952 3.923 955 | 953 2.565 956 | 954 2.652 957 | 955 1.498 958 | 956 2.747 959 | 957 1.813 960 | 958 3.532 961 | 959 3.279 962 | 960 2.526 963 | 961 3.205 964 | 962 4.201 965 | 963 2.552 966 | 964 3.660 967 | 965 3.607 968 | 966 4.107 969 | 967 3.947 970 | 968 3.282 971 | 969 2.765 972 | 970 2.656 973 | 971 2.868 974 | 972 3.936 975 | 973 3.142 976 | 974 4.122 977 | 975 2.887 978 | 976 3.115 979 | 977 4.628 980 | 978 4.015 981 | 979 4.391 982 | 980 2.938 983 | 981 4.386 984 | 982 2.452 985 | 983 1.584 986 | 984 3.333 987 | 985 3.513 988 | 986 4.445 989 | 987 2.947 990 | 988 1.899 991 | 989 2.521 992 | 990 4.139 993 | 991 2.505 994 | 992 2.976 995 | 993 3.862 996 | 994 1.809 997 | 995 2.857 998 | 996 1.977 999 | 997 3.383 1000 | 998 3.878 1001 | 999 2.787 1002 | 1000 3.294 1003 | 1001 4.528 1004 | 1002 3.276 1005 | 1003 3.008 1006 | 1004 2.632 1007 | 1005 3.124 1008 | 1006 3.503 1009 | 1007 3.250 1010 | 1008 2.388 1011 | 1009 2.968 1012 | 1010 3.501 1013 | 1011 3.271 1014 | 1012 4.617 1015 | 1013 2.983 1016 | 1014 3.597 1017 | 1015 3.270 1018 | 1016 4.036 1019 | 1017 2.832 1020 | 1018 4.073 1021 | 1019 3.284 1022 | 1020 3.278 1023 | 1021 3.415 1024 | 1022 3.336 1025 | 1023 2.948 1026 | 1024 4.197 1027 | 1025 3.041 1028 | 1026 2.747 1029 | 1027 2.647 1030 | 1028 3.701 1031 | 1029 1.608 1032 | 1030 4.492 1033 | 1031 5.000 1034 | 1032 3.739 1035 | 1033 2.734 1036 | 1034 4.150 1037 | 1035 2.804 1038 | 1036 3.035 1039 | 1037 3.558 1040 | 1038 4.214 1041 | 1039 5.000 1042 | 1040 3.332 1043 | 1041 3.022 1044 | 1042 3.913 1045 | 1043 2.739 1046 | 1044 3.611 1047 | 1045 2.322 1048 | 1046 2.980 1049 | 1047 3.715 1050 | 1048 4.237 1051 | 1049 4.282 1052 | 1050 3.852 1053 | 1051 3.864 1054 | 1052 4.032 1055 | 1053 3.333 1056 | 1054 3.145 1057 | 1055 4.249 1058 | 1056 4.276 1059 | 1057 3.624 1060 | 1058 3.353 1061 | 1059 4.327 1062 | 1060 3.264 1063 | 1061 2.965 1064 | 1062 2.551 1065 | 1063 3.564 1066 | 1064 2.106 1067 | 1065 2.479 1068 | 1066 3.328 1069 | 1067 2.246 1070 | 1068 3.457 1071 | 1069 3.617 1072 | 1070 5.000 1073 | 1071 2.915 1074 | 1072 2.606 1075 | 1073 1.536 1076 | 1074 2.832 1077 | 1075 2.775 1078 | 1076 3.677 1079 | 1077 4.325 1080 | 1078 3.542 1081 | 1079 2.054 1082 | 1080 3.836 1083 | 1081 4.085 1084 | 1082 1.916 1085 | 1083 0.653 1086 | 1084 3.368 1087 | 1085 3.634 1088 | 1086 2.848 1089 | 1087 3.161 1090 | 1088 1.221 1091 | 1089 3.919 1092 | 1090 3.335 1093 | 1091 4.756 1094 | 1092 3.750 1095 | 1093 3.593 1096 | 1094 3.204 1097 | 1095 3.079 1098 | 1096 3.346 1099 | 1097 2.510 1100 | 1098 3.596 1101 | 1099 2.186 1102 | 1100 3.291 1103 | 1101 3.248 1104 | 1102 3.812 1105 | 1103 3.651 1106 | 1104 4.051 1107 | 1105 2.839 1108 | 1106 3.476 1109 | 1107 1.573 1110 | 1108 3.326 1111 | 1109 3.153 1112 | 1110 1.063 1113 | 1111 4.761 1114 | 1112 4.057 1115 | 1113 3.907 1116 | 1114 3.432 1117 | 1115 3.056 1118 | 1116 3.049 1119 | 1117 4.112 1120 | 1118 3.418 1121 | 1119 3.308 1122 | 1120 3.112 1123 | 1121 0.662 1124 | 1122 4.429 1125 | 1123 3.807 1126 | 1124 2.404 1127 | 1125 4.325 1128 | 1126 2.898 1129 | 1127 4.381 1130 | 1128 3.891 1131 | 1129 1.922 1132 | 1130 1.268 1133 | 1131 4.056 1134 | 1132 4.411 1135 | 1133 0.580 1136 | 1134 2.598 1137 | 1135 2.894 1138 | 1136 2.276 1139 | 1137 3.233 1140 | 1138 0.708 1141 | 1139 5.000 1142 | 1140 2.202 1143 | 1141 0.851 1144 | 1142 1.670 1145 | 1143 4.706 1146 | 1144 3.638 1147 | 1145 1.722 1148 | 1146 1.413 1149 | 1147 4.260 1150 | 1148 3.426 1151 | 1149 4.705 1152 | 1150 0.142 1153 | 1151 0.000 1154 | 1152 3.910 1155 | 1153 3.974 1156 | 1154 4.680 1157 | 1155 4.006 1158 | 1156 3.551 1159 | 1157 0.099 1160 | 1158 3.682 1161 | 1159 1.419 1162 | 1160 2.694 1163 | 1161 3.781 1164 | 1162 3.007 1165 | 1163 4.652 1166 | 1164 4.002 1167 | 1165 1.053 1168 | 1166 2.233 1169 | 1167 1.522 1170 | 1168 3.115 1171 | 1169 0.613 1172 | 1170 1.804 1173 | 1171 3.841 1174 | 1172 2.521 1175 | 1173 4.334 1176 | 1174 1.590 1177 | 1175 0.000 1178 | 1176 5.000 1179 | 1177 2.219 1180 | 1178 3.430 1181 | 1179 1.148 1182 | 1180 1.827 1183 | 1181 2.500 1184 | 1182 4.346 1185 | 1183 2.472 1186 | 1184 2.084 1187 | 1185 3.597 1188 | 1186 5.000 1189 | 1187 3.001 1190 | 1188 3.288 1191 | 1189 4.221 1192 | 1190 3.587 1193 | 1191 3.802 1194 | 1192 2.293 1195 | 1193 1.645 1196 | 1194 1.109 1197 | 1195 2.773 1198 | 1196 4.841 1199 | 1197 4.364 1200 | 1198 3.600 1201 | 1199 2.995 1202 | 1200 1.088 1203 | 1201 3.001 1204 | 1202 4.171 1205 | 1203 4.531 1206 | 1204 2.817 1207 | 1205 3.198 1208 | 1206 4.604 1209 | 1207 2.853 1210 | 1208 2.475 1211 | 1209 2.767 1212 | 1210 2.697 1213 | 1211 2.956 1214 | 1212 5.000 1215 | 1213 2.852 1216 | 1214 5.000 1217 | 1215 3.113 1218 | 1216 2.532 1219 | 1217 2.778 1220 | 1218 2.586 1221 | 1219 1.062 1222 | 1220 2.194 1223 | 1221 1.630 1224 | 1222 4.888 1225 | 1223 2.619 1226 | 1224 2.944 1227 | 1225 2.620 1228 | 1226 2.661 1229 | 1227 4.124 1230 | 1228 4.852 1231 | 1229 2.043 1232 | 1230 4.360 1233 | 1231 1.586 1234 | 1232 3.203 1235 | 1233 4.114 1236 | 1234 3.807 1237 | 1235 3.176 1238 | 1236 4.010 1239 | 1237 2.730 1240 | 1238 1.628 1241 | 1239 1.975 1242 | 1240 1.085 1243 | 1241 2.849 1244 | 1242 4.029 1245 | 1243 2.892 1246 | 1244 0.268 1247 | 1245 5.000 1248 | 1246 4.357 1249 | 1247 1.630 1250 | 1248 0.079 1251 | 1249 4.334 1252 | 1250 3.668 1253 | 1251 3.647 1254 | 1252 2.937 1255 | 1253 3.229 1256 | 1254 0.442 1257 | 1255 4.531 1258 | 1256 2.858 1259 | 1257 3.708 1260 | 1258 3.083 1261 | 1259 3.050 1262 | 1260 2.479 1263 | 1261 2.102 1264 | 1262 3.897 1265 | 1263 2.924 1266 | 1264 4.240 1267 | 1265 4.542 1268 | 1266 3.784 1269 | 1267 3.685 1270 | 1268 1.002 1271 | 1269 1.683 1272 | 1270 2.331 1273 | 1271 3.244 1274 | 1272 1.302 1275 | 1273 2.218 1276 | 1274 2.327 1277 | 1275 2.109 1278 | 1276 1.445 1279 | 1277 3.808 1280 | 1278 1.338 1281 | 1279 4.535 1282 | 1280 0.145 1283 | 1281 1.387 1284 | 1282 2.711 1285 | 1283 3.386 1286 | 1284 1.370 1287 | 1285 3.933 1288 | 1286 4.388 1289 | 1287 1.869 1290 | 1288 0.154 1291 | 1289 1.761 1292 | 1290 3.704 1293 | 1291 1.072 1294 | 1292 2.676 1295 | 1293 3.169 1296 | 1294 2.998 1297 | 1295 0.634 1298 | 1296 4.454 1299 | 1297 1.852 1300 | 1298 0.706 1301 | 1299 4.071 1302 | 1300 1.308 1303 | 1301 4.145 1304 | 1302 2.898 1305 | 1303 2.031 1306 | 1304 2.289 1307 | 1305 1.948 1308 | 1306 1.899 1309 | 1307 1.018 1310 | 1308 1.026 1311 | 1309 1.909 1312 | 1310 3.749 1313 | 1311 4.136 1314 | 1312 1.889 1315 | 1313 2.636 1316 | 1314 2.359 1317 | 1315 3.818 1318 | 1316 5.000 1319 | 1317 4.677 1320 | 1318 1.959 1321 | 1319 0.325 1322 | 1320 4.842 1323 | 1321 2.144 1324 | 1322 4.505 1325 | 1323 2.848 1326 | 1324 4.875 1327 | 1325 4.653 1328 | 1326 4.754 1329 | 1327 1.817 1330 | 1328 0.606 1331 | 1329 4.088 1332 | 1330 3.852 1333 | 1331 2.487 1334 | 1332 2.157 1335 | 1333 3.810 1336 | 1334 3.385 1337 | 1335 3.862 1338 | 1336 3.668 1339 | 1337 3.401 1340 | 1338 0.754 1341 | 1339 2.964 1342 | 1340 3.411 1343 | 1341 4.841 1344 | 1342 3.193 1345 | 1343 0.641 1346 | 1344 1.075 1347 | 1345 2.813 1348 | 1346 4.253 1349 | 1347 4.279 1350 | 1348 5.000 1351 | 1349 1.896 1352 | 1350 3.893 1353 | 1351 5.000 1354 | 1352 4.340 1355 | 1353 3.774 1356 | 1354 4.402 1357 | 1355 1.296 1358 | 1356 2.589 1359 | 1357 4.850 1360 | 1358 3.173 1361 | 1359 2.768 1362 | 1360 0.997 1363 | 1361 2.942 1364 | 1362 0.606 1365 | 1363 1.878 1366 | 1364 4.478 1367 | 1365 2.180 1368 | 1366 1.390 1369 | 1367 1.886 1370 | 1368 0.865 1371 | 1369 1.703 1372 | 1370 3.330 1373 | 1371 1.641 1374 | 1372 2.853 1375 | 1373 1.045 1376 | 1374 1.175 1377 | 1375 1.958 1378 | 1376 1.821 1379 | 1377 0.832 1380 | 1378 2.817 1381 | -------------------------------------------------------------------------------- /examples/GLUE-submission/matbert-base-cased/WNLI.tsv: -------------------------------------------------------------------------------- 1 | index prediction 2 | 0 0 3 | 1 0 4 | 2 0 5 | 3 0 6 | 4 0 7 | 5 0 8 | 6 0 9 | 7 0 10 | 8 0 11 | 9 0 12 | 10 0 13 | 11 0 14 | 12 0 15 | 13 0 16 | 14 0 17 | 15 0 18 | 16 0 19 | 17 0 20 | 18 0 21 | 19 0 22 | 20 0 23 | 21 0 24 | 22 0 25 | 23 0 26 | 24 0 27 | 25 0 28 | 26 0 29 | 27 0 30 | 28 0 31 | 29 0 32 | 30 0 33 | 31 0 34 | 32 0 35 | 33 0 36 | 34 1 37 | 35 1 38 | 36 1 39 | 37 1 40 | 38 1 41 | 39 0 42 | 40 0 43 | 41 0 44 | 42 0 45 | 43 0 46 | 44 0 47 | 45 0 48 | 46 0 49 | 47 0 50 | 48 0 51 | 49 0 52 | 50 1 53 | 51 1 54 | 52 1 55 | 53 1 56 | 54 0 57 | 55 0 58 | 56 0 59 | 57 0 60 | 58 0 61 | 59 0 62 | 60 0 63 | 61 0 64 | 62 0 65 | 63 0 66 | 64 0 67 | 65 0 68 | 66 0 69 | 67 0 70 | 68 0 71 | 69 1 72 | 70 1 73 | 71 1 74 | 72 1 75 | 73 1 76 | 74 0 77 | 75 0 78 | 76 0 79 | 77 0 80 | 78 0 81 | 79 0 82 | 80 0 83 | 81 0 84 | 82 0 85 | 83 0 86 | 84 0 87 | 85 0 88 | 86 0 89 | 87 0 90 | 88 1 91 | 89 1 92 | 90 1 93 | 91 0 94 | 92 0 95 | 93 0 96 | 94 0 97 | 95 0 98 | 96 0 99 | 97 0 100 | 98 0 101 | 99 0 102 | 100 0 103 | 101 0 104 | 102 0 105 | 103 0 106 | 104 0 107 | 105 0 108 | 106 0 109 | 107 0 110 | 108 0 111 | 109 0 112 | 110 0 113 | 111 0 114 | 112 0 115 | 113 1 116 | 114 1 117 | 115 0 118 | 116 0 119 | 117 0 120 | 118 0 121 | 119 0 122 | 120 0 123 | 121 0 124 | 122 0 125 | 123 0 126 | 124 0 127 | 125 0 128 | 126 0 129 | 127 0 130 | 128 0 131 | 129 0 132 | 130 1 133 | 131 1 134 | 132 1 135 | 133 1 136 | 134 0 137 | 135 0 138 | 136 0 139 | 137 0 140 | 138 0 141 | 139 0 142 | 140 0 143 | 141 0 144 | 142 0 145 | 143 0 146 | 144 0 147 | 145 0 148 | -------------------------------------------------------------------------------- /examples/GLUE-submission/matbert-base-uncased.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lbnlp/MatBERT/31592cf5a26c9730dd39585c38f470b9eba6f4ba/examples/GLUE-submission/matbert-base-uncased.zip -------------------------------------------------------------------------------- /examples/GLUE-submission/matbert-base-uncased/CoLA.tsv: -------------------------------------------------------------------------------- 1 | index prediction 2 | 0 1 3 | 1 1 4 | 2 0 5 | 3 0 6 | 4 0 7 | 5 1 8 | 6 0 9 | 7 0 10 | 8 1 11 | 9 1 12 | 10 1 13 | 11 0 14 | 12 0 15 | 13 0 16 | 14 1 17 | 15 1 18 | 16 0 19 | 17 1 20 | 18 1 21 | 19 1 22 | 20 1 23 | 21 1 24 | 22 1 25 | 23 1 26 | 24 1 27 | 25 1 28 | 26 1 29 | 27 0 30 | 28 0 31 | 29 1 32 | 30 0 33 | 31 1 34 | 32 1 35 | 33 0 36 | 34 1 37 | 35 1 38 | 36 1 39 | 37 1 40 | 38 0 41 | 39 1 42 | 40 0 43 | 41 0 44 | 42 0 45 | 43 1 46 | 44 1 47 | 45 1 48 | 46 1 49 | 47 0 50 | 48 1 51 | 49 1 52 | 50 1 53 | 51 1 54 | 52 1 55 | 53 1 56 | 54 1 57 | 55 1 58 | 56 1 59 | 57 1 60 | 58 1 61 | 59 1 62 | 60 0 63 | 61 1 64 | 62 1 65 | 63 1 66 | 64 1 67 | 65 1 68 | 66 1 69 | 67 1 70 | 68 1 71 | 69 0 72 | 70 1 73 | 71 0 74 | 72 0 75 | 73 1 76 | 74 1 77 | 75 1 78 | 76 1 79 | 77 1 80 | 78 1 81 | 79 1 82 | 80 1 83 | 81 0 84 | 82 0 85 | 83 1 86 | 84 1 87 | 85 1 88 | 86 1 89 | 87 1 90 | 88 1 91 | 89 1 92 | 90 1 93 | 91 1 94 | 92 1 95 | 93 1 96 | 94 1 97 | 95 1 98 | 96 1 99 | 97 1 100 | 98 1 101 | 99 1 102 | 100 0 103 | 101 1 104 | 102 1 105 | 103 0 106 | 104 1 107 | 105 0 108 | 106 1 109 | 107 1 110 | 108 1 111 | 109 1 112 | 110 1 113 | 111 1 114 | 112 1 115 | 113 1 116 | 114 1 117 | 115 0 118 | 116 1 119 | 117 0 120 | 118 0 121 | 119 1 122 | 120 0 123 | 121 0 124 | 122 1 125 | 123 0 126 | 124 1 127 | 125 1 128 | 126 0 129 | 127 1 130 | 128 1 131 | 129 1 132 | 130 0 133 | 131 1 134 | 132 1 135 | 133 1 136 | 134 1 137 | 135 1 138 | 136 1 139 | 137 1 140 | 138 1 141 | 139 0 142 | 140 0 143 | 141 1 144 | 142 1 145 | 143 1 146 | 144 1 147 | 145 1 148 | 146 1 149 | 147 1 150 | 148 1 151 | 149 1 152 | 150 1 153 | 151 1 154 | 152 0 155 | 153 1 156 | 154 1 157 | 155 1 158 | 156 0 159 | 157 1 160 | 158 0 161 | 159 1 162 | 160 1 163 | 161 1 164 | 162 1 165 | 163 1 166 | 164 1 167 | 165 0 168 | 166 0 169 | 167 1 170 | 168 1 171 | 169 1 172 | 170 0 173 | 171 1 174 | 172 1 175 | 173 0 176 | 174 1 177 | 175 0 178 | 176 1 179 | 177 1 180 | 178 1 181 | 179 1 182 | 180 1 183 | 181 1 184 | 182 0 185 | 183 1 186 | 184 1 187 | 185 1 188 | 186 1 189 | 187 1 190 | 188 0 191 | 189 1 192 | 190 1 193 | 191 1 194 | 192 1 195 | 193 1 196 | 194 0 197 | 195 1 198 | 196 1 199 | 197 1 200 | 198 1 201 | 199 1 202 | 200 1 203 | 201 0 204 | 202 1 205 | 203 0 206 | 204 1 207 | 205 1 208 | 206 1 209 | 207 1 210 | 208 1 211 | 209 1 212 | 210 1 213 | 211 1 214 | 212 1 215 | 213 0 216 | 214 1 217 | 215 0 218 | 216 0 219 | 217 1 220 | 218 1 221 | 219 1 222 | 220 1 223 | 221 0 224 | 222 1 225 | 223 1 226 | 224 1 227 | 225 1 228 | 226 1 229 | 227 1 230 | 228 1 231 | 229 1 232 | 230 1 233 | 231 1 234 | 232 1 235 | 233 1 236 | 234 1 237 | 235 1 238 | 236 0 239 | 237 1 240 | 238 1 241 | 239 1 242 | 240 1 243 | 241 1 244 | 242 0 245 | 243 1 246 | 244 1 247 | 245 1 248 | 246 1 249 | 247 1 250 | 248 1 251 | 249 1 252 | 250 0 253 | 251 1 254 | 252 1 255 | 253 1 256 | 254 1 257 | 255 1 258 | 256 1 259 | 257 1 260 | 258 0 261 | 259 0 262 | 260 1 263 | 261 1 264 | 262 1 265 | 263 1 266 | 264 1 267 | 265 1 268 | 266 1 269 | 267 0 270 | 268 1 271 | 269 1 272 | 270 1 273 | 271 0 274 | 272 1 275 | 273 1 276 | 274 1 277 | 275 1 278 | 276 1 279 | 277 1 280 | 278 1 281 | 279 0 282 | 280 1 283 | 281 1 284 | 282 1 285 | 283 1 286 | 284 0 287 | 285 0 288 | 286 0 289 | 287 1 290 | 288 1 291 | 289 1 292 | 290 1 293 | 291 1 294 | 292 1 295 | 293 0 296 | 294 1 297 | 295 1 298 | 296 1 299 | 297 1 300 | 298 1 301 | 299 1 302 | 300 0 303 | 301 1 304 | 302 1 305 | 303 1 306 | 304 1 307 | 305 1 308 | 306 1 309 | 307 1 310 | 308 1 311 | 309 1 312 | 310 1 313 | 311 1 314 | 312 1 315 | 313 1 316 | 314 0 317 | 315 0 318 | 316 0 319 | 317 0 320 | 318 1 321 | 319 0 322 | 320 1 323 | 321 1 324 | 322 1 325 | 323 0 326 | 324 1 327 | 325 1 328 | 326 1 329 | 327 1 330 | 328 1 331 | 329 1 332 | 330 1 333 | 331 1 334 | 332 1 335 | 333 1 336 | 334 1 337 | 335 1 338 | 336 1 339 | 337 1 340 | 338 1 341 | 339 1 342 | 340 1 343 | 341 1 344 | 342 1 345 | 343 1 346 | 344 0 347 | 345 1 348 | 346 1 349 | 347 0 350 | 348 1 351 | 349 1 352 | 350 1 353 | 351 0 354 | 352 1 355 | 353 1 356 | 354 1 357 | 355 1 358 | 356 0 359 | 357 1 360 | 358 1 361 | 359 1 362 | 360 1 363 | 361 0 364 | 362 1 365 | 363 1 366 | 364 1 367 | 365 1 368 | 366 1 369 | 367 1 370 | 368 1 371 | 369 1 372 | 370 1 373 | 371 0 374 | 372 1 375 | 373 1 376 | 374 1 377 | 375 1 378 | 376 1 379 | 377 0 380 | 378 1 381 | 379 1 382 | 380 1 383 | 381 1 384 | 382 1 385 | 383 1 386 | 384 1 387 | 385 1 388 | 386 1 389 | 387 1 390 | 388 1 391 | 389 1 392 | 390 1 393 | 391 1 394 | 392 0 395 | 393 1 396 | 394 1 397 | 395 0 398 | 396 1 399 | 397 1 400 | 398 1 401 | 399 1 402 | 400 1 403 | 401 1 404 | 402 0 405 | 403 1 406 | 404 0 407 | 405 1 408 | 406 1 409 | 407 1 410 | 408 1 411 | 409 1 412 | 410 1 413 | 411 1 414 | 412 0 415 | 413 1 416 | 414 1 417 | 415 0 418 | 416 1 419 | 417 1 420 | 418 0 421 | 419 0 422 | 420 1 423 | 421 1 424 | 422 1 425 | 423 1 426 | 424 0 427 | 425 1 428 | 426 1 429 | 427 1 430 | 428 1 431 | 429 1 432 | 430 1 433 | 431 1 434 | 432 1 435 | 433 1 436 | 434 1 437 | 435 1 438 | 436 1 439 | 437 1 440 | 438 1 441 | 439 1 442 | 440 1 443 | 441 1 444 | 442 1 445 | 443 1 446 | 444 0 447 | 445 0 448 | 446 1 449 | 447 0 450 | 448 0 451 | 449 0 452 | 450 1 453 | 451 1 454 | 452 1 455 | 453 1 456 | 454 1 457 | 455 1 458 | 456 1 459 | 457 1 460 | 458 1 461 | 459 1 462 | 460 1 463 | 461 1 464 | 462 1 465 | 463 1 466 | 464 1 467 | 465 1 468 | 466 1 469 | 467 1 470 | 468 1 471 | 469 1 472 | 470 1 473 | 471 1 474 | 472 1 475 | 473 0 476 | 474 1 477 | 475 0 478 | 476 1 479 | 477 0 480 | 478 1 481 | 479 1 482 | 480 1 483 | 481 1 484 | 482 0 485 | 483 0 486 | 484 1 487 | 485 1 488 | 486 1 489 | 487 1 490 | 488 0 491 | 489 1 492 | 490 1 493 | 491 1 494 | 492 1 495 | 493 1 496 | 494 1 497 | 495 0 498 | 496 1 499 | 497 1 500 | 498 1 501 | 499 1 502 | 500 1 503 | 501 1 504 | 502 1 505 | 503 1 506 | 504 0 507 | 505 1 508 | 506 1 509 | 507 1 510 | 508 1 511 | 509 1 512 | 510 1 513 | 511 1 514 | 512 1 515 | 513 1 516 | 514 1 517 | 515 1 518 | 516 1 519 | 517 1 520 | 518 1 521 | 519 1 522 | 520 1 523 | 521 0 524 | 522 1 525 | 523 1 526 | 524 0 527 | 525 1 528 | 526 1 529 | 527 1 530 | 528 1 531 | 529 1 532 | 530 1 533 | 531 0 534 | 532 1 535 | 533 1 536 | 534 1 537 | 535 1 538 | 536 1 539 | 537 1 540 | 538 1 541 | 539 0 542 | 540 1 543 | 541 1 544 | 542 1 545 | 543 1 546 | 544 1 547 | 545 0 548 | 546 1 549 | 547 1 550 | 548 1 551 | 549 0 552 | 550 1 553 | 551 1 554 | 552 1 555 | 553 1 556 | 554 1 557 | 555 1 558 | 556 1 559 | 557 1 560 | 558 1 561 | 559 1 562 | 560 1 563 | 561 1 564 | 562 1 565 | 563 0 566 | 564 0 567 | 565 1 568 | 566 1 569 | 567 1 570 | 568 1 571 | 569 1 572 | 570 1 573 | 571 0 574 | 572 1 575 | 573 1 576 | 574 1 577 | 575 1 578 | 576 1 579 | 577 1 580 | 578 1 581 | 579 1 582 | 580 0 583 | 581 1 584 | 582 1 585 | 583 1 586 | 584 1 587 | 585 1 588 | 586 1 589 | 587 1 590 | 588 1 591 | 589 1 592 | 590 1 593 | 591 1 594 | 592 0 595 | 593 1 596 | 594 1 597 | 595 1 598 | 596 1 599 | 597 1 600 | 598 1 601 | 599 1 602 | 600 1 603 | 601 0 604 | 602 1 605 | 603 1 606 | 604 0 607 | 605 1 608 | 606 1 609 | 607 1 610 | 608 1 611 | 609 1 612 | 610 1 613 | 611 1 614 | 612 1 615 | 613 1 616 | 614 0 617 | 615 1 618 | 616 1 619 | 617 0 620 | 618 1 621 | 619 1 622 | 620 1 623 | 621 1 624 | 622 1 625 | 623 1 626 | 624 1 627 | 625 1 628 | 626 1 629 | 627 1 630 | 628 1 631 | 629 0 632 | 630 1 633 | 631 1 634 | 632 1 635 | 633 1 636 | 634 1 637 | 635 1 638 | 636 0 639 | 637 1 640 | 638 1 641 | 639 1 642 | 640 1 643 | 641 0 644 | 642 0 645 | 643 1 646 | 644 1 647 | 645 1 648 | 646 1 649 | 647 1 650 | 648 1 651 | 649 1 652 | 650 1 653 | 651 1 654 | 652 1 655 | 653 1 656 | 654 1 657 | 655 1 658 | 656 0 659 | 657 1 660 | 658 0 661 | 659 0 662 | 660 0 663 | 661 1 664 | 662 1 665 | 663 1 666 | 664 1 667 | 665 1 668 | 666 1 669 | 667 1 670 | 668 1 671 | 669 0 672 | 670 1 673 | 671 1 674 | 672 1 675 | 673 1 676 | 674 1 677 | 675 1 678 | 676 1 679 | 677 1 680 | 678 1 681 | 679 1 682 | 680 1 683 | 681 1 684 | 682 1 685 | 683 1 686 | 684 1 687 | 685 1 688 | 686 0 689 | 687 1 690 | 688 1 691 | 689 0 692 | 690 1 693 | 691 1 694 | 692 0 695 | 693 0 696 | 694 0 697 | 695 1 698 | 696 1 699 | 697 1 700 | 698 1 701 | 699 1 702 | 700 0 703 | 701 0 704 | 702 1 705 | 703 1 706 | 704 1 707 | 705 0 708 | 706 0 709 | 707 0 710 | 708 1 711 | 709 1 712 | 710 0 713 | 711 1 714 | 712 1 715 | 713 1 716 | 714 1 717 | 715 0 718 | 716 0 719 | 717 0 720 | 718 0 721 | 719 0 722 | 720 1 723 | 721 0 724 | 722 0 725 | 723 0 726 | 724 1 727 | 725 1 728 | 726 1 729 | 727 0 730 | 728 1 731 | 729 0 732 | 730 0 733 | 731 1 734 | 732 1 735 | 733 1 736 | 734 1 737 | 735 1 738 | 736 1 739 | 737 1 740 | 738 1 741 | 739 1 742 | 740 0 743 | 741 0 744 | 742 1 745 | 743 0 746 | 744 0 747 | 745 1 748 | 746 1 749 | 747 1 750 | 748 1 751 | 749 1 752 | 750 1 753 | 751 1 754 | 752 1 755 | 753 1 756 | 754 1 757 | 755 0 758 | 756 0 759 | 757 1 760 | 758 1 761 | 759 1 762 | 760 1 763 | 761 1 764 | 762 1 765 | 763 1 766 | 764 1 767 | 765 1 768 | 766 1 769 | 767 1 770 | 768 1 771 | 769 1 772 | 770 1 773 | 771 1 774 | 772 1 775 | 773 1 776 | 774 1 777 | 775 0 778 | 776 1 779 | 777 1 780 | 778 1 781 | 779 1 782 | 780 1 783 | 781 1 784 | 782 1 785 | 783 0 786 | 784 1 787 | 785 1 788 | 786 1 789 | 787 1 790 | 788 1 791 | 789 1 792 | 790 1 793 | 791 1 794 | 792 0 795 | 793 1 796 | 794 1 797 | 795 1 798 | 796 0 799 | 797 1 800 | 798 1 801 | 799 0 802 | 800 1 803 | 801 1 804 | 802 1 805 | 803 1 806 | 804 1 807 | 805 1 808 | 806 1 809 | 807 0 810 | 808 1 811 | 809 1 812 | 810 1 813 | 811 1 814 | 812 1 815 | 813 1 816 | 814 1 817 | 815 1 818 | 816 1 819 | 817 1 820 | 818 1 821 | 819 1 822 | 820 1 823 | 821 1 824 | 822 1 825 | 823 0 826 | 824 1 827 | 825 0 828 | 826 1 829 | 827 1 830 | 828 1 831 | 829 1 832 | 830 1 833 | 831 1 834 | 832 1 835 | 833 1 836 | 834 1 837 | 835 0 838 | 836 0 839 | 837 1 840 | 838 1 841 | 839 1 842 | 840 1 843 | 841 1 844 | 842 1 845 | 843 1 846 | 844 1 847 | 845 1 848 | 846 0 849 | 847 1 850 | 848 1 851 | 849 1 852 | 850 1 853 | 851 0 854 | 852 1 855 | 853 0 856 | 854 1 857 | 855 1 858 | 856 1 859 | 857 1 860 | 858 1 861 | 859 1 862 | 860 1 863 | 861 1 864 | 862 1 865 | 863 1 866 | 864 1 867 | 865 0 868 | 866 1 869 | 867 1 870 | 868 0 871 | 869 1 872 | 870 1 873 | 871 1 874 | 872 0 875 | 873 1 876 | 874 1 877 | 875 1 878 | 876 1 879 | 877 0 880 | 878 1 881 | 879 1 882 | 880 1 883 | 881 1 884 | 882 1 885 | 883 1 886 | 884 1 887 | 885 1 888 | 886 1 889 | 887 0 890 | 888 0 891 | 889 1 892 | 890 0 893 | 891 0 894 | 892 1 895 | 893 1 896 | 894 1 897 | 895 0 898 | 896 1 899 | 897 1 900 | 898 1 901 | 899 1 902 | 900 1 903 | 901 1 904 | 902 1 905 | 903 1 906 | 904 1 907 | 905 1 908 | 906 1 909 | 907 1 910 | 908 1 911 | 909 1 912 | 910 0 913 | 911 1 914 | 912 1 915 | 913 1 916 | 914 0 917 | 915 1 918 | 916 1 919 | 917 1 920 | 918 1 921 | 919 1 922 | 920 1 923 | 921 1 924 | 922 1 925 | 923 1 926 | 924 1 927 | 925 1 928 | 926 1 929 | 927 1 930 | 928 1 931 | 929 0 932 | 930 0 933 | 931 0 934 | 932 1 935 | 933 0 936 | 934 1 937 | 935 1 938 | 936 1 939 | 937 1 940 | 938 1 941 | 939 1 942 | 940 1 943 | 941 1 944 | 942 1 945 | 943 1 946 | 944 1 947 | 945 1 948 | 946 1 949 | 947 1 950 | 948 0 951 | 949 1 952 | 950 1 953 | 951 1 954 | 952 1 955 | 953 1 956 | 954 1 957 | 955 1 958 | 956 1 959 | 957 1 960 | 958 1 961 | 959 1 962 | 960 1 963 | 961 1 964 | 962 1 965 | 963 0 966 | 964 0 967 | 965 1 968 | 966 1 969 | 967 0 970 | 968 1 971 | 969 1 972 | 970 0 973 | 971 1 974 | 972 1 975 | 973 1 976 | 974 1 977 | 975 1 978 | 976 1 979 | 977 1 980 | 978 1 981 | 979 1 982 | 980 1 983 | 981 1 984 | 982 0 985 | 983 0 986 | 984 1 987 | 985 0 988 | 986 0 989 | 987 1 990 | 988 1 991 | 989 1 992 | 990 1 993 | 991 0 994 | 992 1 995 | 993 1 996 | 994 1 997 | 995 1 998 | 996 0 999 | 997 1 1000 | 998 1 1001 | 999 0 1002 | 1000 1 1003 | 1001 1 1004 | 1002 1 1005 | 1003 1 1006 | 1004 1 1007 | 1005 1 1008 | 1006 1 1009 | 1007 1 1010 | 1008 0 1011 | 1009 1 1012 | 1010 1 1013 | 1011 1 1014 | 1012 1 1015 | 1013 1 1016 | 1014 1 1017 | 1015 1 1018 | 1016 1 1019 | 1017 1 1020 | 1018 1 1021 | 1019 1 1022 | 1020 1 1023 | 1021 0 1024 | 1022 0 1025 | 1023 1 1026 | 1024 1 1027 | 1025 1 1028 | 1026 1 1029 | 1027 1 1030 | 1028 1 1031 | 1029 1 1032 | 1030 1 1033 | 1031 1 1034 | 1032 1 1035 | 1033 1 1036 | 1034 1 1037 | 1035 1 1038 | 1036 0 1039 | 1037 1 1040 | 1038 0 1041 | 1039 1 1042 | 1040 0 1043 | 1041 1 1044 | 1042 1 1045 | 1043 1 1046 | 1044 1 1047 | 1045 1 1048 | 1046 1 1049 | 1047 1 1050 | 1048 1 1051 | 1049 0 1052 | 1050 1 1053 | 1051 0 1054 | 1052 0 1055 | 1053 0 1056 | 1054 1 1057 | 1055 0 1058 | 1056 1 1059 | 1057 1 1060 | 1058 1 1061 | 1059 0 1062 | 1060 1 1063 | 1061 1 1064 | 1062 1 1065 | -------------------------------------------------------------------------------- /examples/GLUE-submission/matbert-base-uncased/MRPC.tsv: -------------------------------------------------------------------------------- 1 | index prediction 2 | 0 1 3 | 1 1 4 | 2 1 5 | 3 1 6 | 4 0 7 | 5 1 8 | 6 0 9 | 7 0 10 | 8 1 11 | 9 0 12 | 10 1 13 | 11 1 14 | 12 1 15 | 13 0 16 | 14 1 17 | 15 1 18 | 16 1 19 | 17 1 20 | 18 1 21 | 19 1 22 | 20 1 23 | 21 1 24 | 22 1 25 | 23 1 26 | 24 0 27 | 25 1 28 | 26 1 29 | 27 1 30 | 28 1 31 | 29 1 32 | 30 1 33 | 31 0 34 | 32 0 35 | 33 0 36 | 34 0 37 | 35 0 38 | 36 1 39 | 37 1 40 | 38 0 41 | 39 1 42 | 40 1 43 | 41 1 44 | 42 0 45 | 43 1 46 | 44 1 47 | 45 0 48 | 46 1 49 | 47 0 50 | 48 1 51 | 49 1 52 | 50 1 53 | 51 1 54 | 52 1 55 | 53 1 56 | 54 0 57 | 55 1 58 | 56 1 59 | 57 1 60 | 58 0 61 | 59 1 62 | 60 1 63 | 61 1 64 | 62 1 65 | 63 1 66 | 64 1 67 | 65 1 68 | 66 1 69 | 67 1 70 | 68 1 71 | 69 1 72 | 70 1 73 | 71 1 74 | 72 1 75 | 73 0 76 | 74 1 77 | 75 0 78 | 76 1 79 | 77 1 80 | 78 0 81 | 79 0 82 | 80 1 83 | 81 0 84 | 82 1 85 | 83 0 86 | 84 0 87 | 85 0 88 | 86 1 89 | 87 1 90 | 88 1 91 | 89 1 92 | 90 0 93 | 91 1 94 | 92 1 95 | 93 0 96 | 94 1 97 | 95 1 98 | 96 0 99 | 97 1 100 | 98 0 101 | 99 1 102 | 100 1 103 | 101 0 104 | 102 1 105 | 103 1 106 | 104 1 107 | 105 0 108 | 106 1 109 | 107 1 110 | 108 1 111 | 109 1 112 | 110 1 113 | 111 0 114 | 112 0 115 | 113 1 116 | 114 1 117 | 115 0 118 | 116 0 119 | 117 0 120 | 118 1 121 | 119 1 122 | 120 1 123 | 121 0 124 | 122 1 125 | 123 0 126 | 124 1 127 | 125 1 128 | 126 1 129 | 127 1 130 | 128 0 131 | 129 1 132 | 130 1 133 | 131 1 134 | 132 1 135 | 133 1 136 | 134 1 137 | 135 0 138 | 136 1 139 | 137 1 140 | 138 1 141 | 139 1 142 | 140 1 143 | 141 1 144 | 142 1 145 | 143 1 146 | 144 1 147 | 145 1 148 | 146 0 149 | 147 1 150 | 148 1 151 | 149 1 152 | 150 0 153 | 151 0 154 | 152 0 155 | 153 0 156 | 154 1 157 | 155 1 158 | 156 1 159 | 157 1 160 | 158 0 161 | 159 1 162 | 160 1 163 | 161 1 164 | 162 1 165 | 163 1 166 | 164 0 167 | 165 1 168 | 166 1 169 | 167 0 170 | 168 1 171 | 169 1 172 | 170 1 173 | 171 1 174 | 172 1 175 | 173 0 176 | 174 1 177 | 175 1 178 | 176 1 179 | 177 0 180 | 178 1 181 | 179 1 182 | 180 0 183 | 181 0 184 | 182 1 185 | 183 1 186 | 184 1 187 | 185 1 188 | 186 0 189 | 187 0 190 | 188 1 191 | 189 1 192 | 190 0 193 | 191 0 194 | 192 1 195 | 193 1 196 | 194 0 197 | 195 1 198 | 196 0 199 | 197 1 200 | 198 1 201 | 199 0 202 | 200 0 203 | 201 1 204 | 202 1 205 | 203 1 206 | 204 1 207 | 205 0 208 | 206 1 209 | 207 1 210 | 208 0 211 | 209 1 212 | 210 1 213 | 211 0 214 | 212 0 215 | 213 1 216 | 214 1 217 | 215 0 218 | 216 1 219 | 217 1 220 | 218 0 221 | 219 1 222 | 220 1 223 | 221 1 224 | 222 1 225 | 223 0 226 | 224 0 227 | 225 1 228 | 226 0 229 | 227 0 230 | 228 1 231 | 229 1 232 | 230 1 233 | 231 1 234 | 232 1 235 | 233 1 236 | 234 0 237 | 235 1 238 | 236 1 239 | 237 1 240 | 238 0 241 | 239 1 242 | 240 1 243 | 241 1 244 | 242 1 245 | 243 1 246 | 244 0 247 | 245 1 248 | 246 1 249 | 247 1 250 | 248 1 251 | 249 0 252 | 250 1 253 | 251 1 254 | 252 0 255 | 253 1 256 | 254 1 257 | 255 1 258 | 256 1 259 | 257 1 260 | 258 1 261 | 259 1 262 | 260 0 263 | 261 0 264 | 262 1 265 | 263 1 266 | 264 1 267 | 265 1 268 | 266 1 269 | 267 0 270 | 268 1 271 | 269 1 272 | 270 1 273 | 271 0 274 | 272 1 275 | 273 0 276 | 274 0 277 | 275 0 278 | 276 1 279 | 277 1 280 | 278 1 281 | 279 1 282 | 280 1 283 | 281 0 284 | 282 1 285 | 283 1 286 | 284 1 287 | 285 0 288 | 286 1 289 | 287 0 290 | 288 0 291 | 289 0 292 | 290 1 293 | 291 0 294 | 292 1 295 | 293 1 296 | 294 1 297 | 295 0 298 | 296 0 299 | 297 0 300 | 298 1 301 | 299 0 302 | 300 1 303 | 301 1 304 | 302 0 305 | 303 1 306 | 304 1 307 | 305 0 308 | 306 1 309 | 307 1 310 | 308 0 311 | 309 1 312 | 310 1 313 | 311 1 314 | 312 1 315 | 313 1 316 | 314 1 317 | 315 1 318 | 316 1 319 | 317 1 320 | 318 1 321 | 319 0 322 | 320 1 323 | 321 1 324 | 322 1 325 | 323 1 326 | 324 0 327 | 325 1 328 | 326 1 329 | 327 0 330 | 328 0 331 | 329 0 332 | 330 1 333 | 331 1 334 | 332 1 335 | 333 1 336 | 334 1 337 | 335 0 338 | 336 1 339 | 337 1 340 | 338 1 341 | 339 1 342 | 340 1 343 | 341 1 344 | 342 1 345 | 343 1 346 | 344 0 347 | 345 1 348 | 346 1 349 | 347 0 350 | 348 1 351 | 349 1 352 | 350 1 353 | 351 1 354 | 352 0 355 | 353 1 356 | 354 1 357 | 355 1 358 | 356 1 359 | 357 1 360 | 358 1 361 | 359 0 362 | 360 0 363 | 361 0 364 | 362 1 365 | 363 1 366 | 364 1 367 | 365 1 368 | 366 0 369 | 367 0 370 | 368 0 371 | 369 1 372 | 370 1 373 | 371 1 374 | 372 1 375 | 373 1 376 | 374 1 377 | 375 1 378 | 376 0 379 | 377 1 380 | 378 0 381 | 379 0 382 | 380 0 383 | 381 1 384 | 382 0 385 | 383 1 386 | 384 0 387 | 385 0 388 | 386 1 389 | 387 0 390 | 388 1 391 | 389 1 392 | 390 1 393 | 391 0 394 | 392 0 395 | 393 1 396 | 394 0 397 | 395 1 398 | 396 1 399 | 397 1 400 | 398 0 401 | 399 1 402 | 400 1 403 | 401 1 404 | 402 1 405 | 403 1 406 | 404 1 407 | 405 1 408 | 406 0 409 | 407 1 410 | 408 0 411 | 409 0 412 | 410 0 413 | 411 0 414 | 412 1 415 | 413 1 416 | 414 1 417 | 415 1 418 | 416 1 419 | 417 1 420 | 418 1 421 | 419 1 422 | 420 1 423 | 421 1 424 | 422 1 425 | 423 1 426 | 424 0 427 | 425 0 428 | 426 0 429 | 427 0 430 | 428 0 431 | 429 1 432 | 430 0 433 | 431 1 434 | 432 1 435 | 433 1 436 | 434 1 437 | 435 1 438 | 436 1 439 | 437 0 440 | 438 1 441 | 439 0 442 | 440 0 443 | 441 0 444 | 442 1 445 | 443 1 446 | 444 1 447 | 445 1 448 | 446 1 449 | 447 1 450 | 448 1 451 | 449 1 452 | 450 0 453 | 451 1 454 | 452 1 455 | 453 0 456 | 454 1 457 | 455 1 458 | 456 1 459 | 457 1 460 | 458 0 461 | 459 1 462 | 460 1 463 | 461 1 464 | 462 1 465 | 463 0 466 | 464 0 467 | 465 0 468 | 466 1 469 | 467 1 470 | 468 1 471 | 469 1 472 | 470 0 473 | 471 1 474 | 472 1 475 | 473 1 476 | 474 1 477 | 475 1 478 | 476 1 479 | 477 1 480 | 478 1 481 | 479 0 482 | 480 0 483 | 481 1 484 | 482 1 485 | 483 1 486 | 484 0 487 | 485 1 488 | 486 1 489 | 487 1 490 | 488 1 491 | 489 1 492 | 490 1 493 | 491 1 494 | 492 1 495 | 493 1 496 | 494 0 497 | 495 1 498 | 496 1 499 | 497 1 500 | 498 0 501 | 499 1 502 | 500 1 503 | 501 1 504 | 502 1 505 | 503 1 506 | 504 1 507 | 505 1 508 | 506 0 509 | 507 1 510 | 508 1 511 | 509 0 512 | 510 1 513 | 511 1 514 | 512 0 515 | 513 0 516 | 514 1 517 | 515 1 518 | 516 1 519 | 517 0 520 | 518 1 521 | 519 1 522 | 520 1 523 | 521 0 524 | 522 1 525 | 523 1 526 | 524 1 527 | 525 1 528 | 526 0 529 | 527 1 530 | 528 1 531 | 529 1 532 | 530 1 533 | 531 1 534 | 532 1 535 | 533 1 536 | 534 1 537 | 535 1 538 | 536 1 539 | 537 0 540 | 538 0 541 | 539 1 542 | 540 1 543 | 541 1 544 | 542 1 545 | 543 1 546 | 544 1 547 | 545 1 548 | 546 1 549 | 547 0 550 | 548 0 551 | 549 0 552 | 550 1 553 | 551 1 554 | 552 0 555 | 553 1 556 | 554 0 557 | 555 1 558 | 556 1 559 | 557 1 560 | 558 1 561 | 559 1 562 | 560 1 563 | 561 0 564 | 562 1 565 | 563 0 566 | 564 1 567 | 565 1 568 | 566 1 569 | 567 1 570 | 568 1 571 | 569 0 572 | 570 1 573 | 571 1 574 | 572 1 575 | 573 0 576 | 574 1 577 | 575 1 578 | 576 1 579 | 577 0 580 | 578 1 581 | 579 1 582 | 580 1 583 | 581 1 584 | 582 1 585 | 583 1 586 | 584 1 587 | 585 1 588 | 586 0 589 | 587 1 590 | 588 1 591 | 589 1 592 | 590 1 593 | 591 1 594 | 592 1 595 | 593 0 596 | 594 0 597 | 595 1 598 | 596 1 599 | 597 1 600 | 598 1 601 | 599 0 602 | 600 0 603 | 601 1 604 | 602 1 605 | 603 1 606 | 604 1 607 | 605 1 608 | 606 1 609 | 607 0 610 | 608 0 611 | 609 1 612 | 610 1 613 | 611 1 614 | 612 0 615 | 613 1 616 | 614 1 617 | 615 1 618 | 616 1 619 | 617 0 620 | 618 1 621 | 619 1 622 | 620 1 623 | 621 0 624 | 622 1 625 | 623 0 626 | 624 1 627 | 625 0 628 | 626 1 629 | 627 1 630 | 628 1 631 | 629 1 632 | 630 0 633 | 631 1 634 | 632 1 635 | 633 0 636 | 634 0 637 | 635 1 638 | 636 1 639 | 637 1 640 | 638 1 641 | 639 1 642 | 640 1 643 | 641 1 644 | 642 0 645 | 643 1 646 | 644 1 647 | 645 1 648 | 646 1 649 | 647 1 650 | 648 1 651 | 649 0 652 | 650 1 653 | 651 0 654 | 652 0 655 | 653 1 656 | 654 0 657 | 655 1 658 | 656 1 659 | 657 1 660 | 658 1 661 | 659 1 662 | 660 1 663 | 661 1 664 | 662 1 665 | 663 1 666 | 664 1 667 | 665 0 668 | 666 1 669 | 667 0 670 | 668 0 671 | 669 0 672 | 670 1 673 | 671 1 674 | 672 0 675 | 673 1 676 | 674 1 677 | 675 0 678 | 676 1 679 | 677 1 680 | 678 1 681 | 679 1 682 | 680 1 683 | 681 1 684 | 682 1 685 | 683 1 686 | 684 1 687 | 685 1 688 | 686 1 689 | 687 0 690 | 688 1 691 | 689 1 692 | 690 1 693 | 691 0 694 | 692 0 695 | 693 1 696 | 694 1 697 | 695 0 698 | 696 0 699 | 697 1 700 | 698 0 701 | 699 1 702 | 700 0 703 | 701 1 704 | 702 1 705 | 703 0 706 | 704 1 707 | 705 0 708 | 706 0 709 | 707 0 710 | 708 1 711 | 709 1 712 | 710 0 713 | 711 0 714 | 712 1 715 | 713 1 716 | 714 1 717 | 715 1 718 | 716 1 719 | 717 0 720 | 718 1 721 | 719 1 722 | 720 1 723 | 721 1 724 | 722 0 725 | 723 0 726 | 724 1 727 | 725 0 728 | 726 1 729 | 727 1 730 | 728 0 731 | 729 1 732 | 730 1 733 | 731 1 734 | 732 1 735 | 733 0 736 | 734 1 737 | 735 1 738 | 736 1 739 | 737 1 740 | 738 1 741 | 739 1 742 | 740 1 743 | 741 1 744 | 742 1 745 | 743 1 746 | 744 1 747 | 745 0 748 | 746 1 749 | 747 0 750 | 748 1 751 | 749 0 752 | 750 1 753 | 751 1 754 | 752 1 755 | 753 0 756 | 754 1 757 | 755 1 758 | 756 1 759 | 757 1 760 | 758 1 761 | 759 1 762 | 760 1 763 | 761 1 764 | 762 1 765 | 763 1 766 | 764 1 767 | 765 1 768 | 766 1 769 | 767 1 770 | 768 1 771 | 769 0 772 | 770 1 773 | 771 1 774 | 772 1 775 | 773 1 776 | 774 1 777 | 775 1 778 | 776 1 779 | 777 1 780 | 778 0 781 | 779 1 782 | 780 1 783 | 781 1 784 | 782 1 785 | 783 0 786 | 784 0 787 | 785 1 788 | 786 1 789 | 787 0 790 | 788 1 791 | 789 1 792 | 790 1 793 | 791 1 794 | 792 0 795 | 793 0 796 | 794 1 797 | 795 1 798 | 796 0 799 | 797 1 800 | 798 0 801 | 799 0 802 | 800 0 803 | 801 0 804 | 802 1 805 | 803 1 806 | 804 0 807 | 805 1 808 | 806 1 809 | 807 1 810 | 808 1 811 | 809 1 812 | 810 1 813 | 811 1 814 | 812 0 815 | 813 1 816 | 814 0 817 | 815 1 818 | 816 0 819 | 817 0 820 | 818 0 821 | 819 1 822 | 820 0 823 | 821 1 824 | 822 1 825 | 823 1 826 | 824 1 827 | 825 1 828 | 826 0 829 | 827 0 830 | 828 0 831 | 829 1 832 | 830 1 833 | 831 1 834 | 832 1 835 | 833 0 836 | 834 1 837 | 835 0 838 | 836 1 839 | 837 1 840 | 838 1 841 | 839 0 842 | 840 1 843 | 841 1 844 | 842 1 845 | 843 1 846 | 844 0 847 | 845 1 848 | 846 1 849 | 847 1 850 | 848 0 851 | 849 1 852 | 850 0 853 | 851 0 854 | 852 1 855 | 853 1 856 | 854 1 857 | 855 0 858 | 856 1 859 | 857 0 860 | 858 0 861 | 859 1 862 | 860 1 863 | 861 1 864 | 862 1 865 | 863 1 866 | 864 0 867 | 865 1 868 | 866 1 869 | 867 1 870 | 868 1 871 | 869 1 872 | 870 0 873 | 871 1 874 | 872 1 875 | 873 1 876 | 874 1 877 | 875 0 878 | 876 1 879 | 877 0 880 | 878 0 881 | 879 1 882 | 880 0 883 | 881 1 884 | 882 1 885 | 883 1 886 | 884 1 887 | 885 1 888 | 886 1 889 | 887 1 890 | 888 0 891 | 889 1 892 | 890 1 893 | 891 1 894 | 892 0 895 | 893 1 896 | 894 1 897 | 895 0 898 | 896 0 899 | 897 1 900 | 898 1 901 | 899 1 902 | 900 1 903 | 901 1 904 | 902 0 905 | 903 1 906 | 904 1 907 | 905 1 908 | 906 1 909 | 907 1 910 | 908 1 911 | 909 1 912 | 910 1 913 | 911 1 914 | 912 0 915 | 913 0 916 | 914 0 917 | 915 1 918 | 916 1 919 | 917 1 920 | 918 0 921 | 919 1 922 | 920 1 923 | 921 1 924 | 922 1 925 | 923 1 926 | 924 1 927 | 925 1 928 | 926 1 929 | 927 0 930 | 928 1 931 | 929 1 932 | 930 1 933 | 931 0 934 | 932 0 935 | 933 0 936 | 934 0 937 | 935 1 938 | 936 0 939 | 937 0 940 | 938 1 941 | 939 1 942 | 940 1 943 | 941 1 944 | 942 0 945 | 943 1 946 | 944 1 947 | 945 1 948 | 946 1 949 | 947 1 950 | 948 1 951 | 949 1 952 | 950 0 953 | 951 1 954 | 952 0 955 | 953 1 956 | 954 1 957 | 955 1 958 | 956 1 959 | 957 1 960 | 958 1 961 | 959 0 962 | 960 1 963 | 961 1 964 | 962 1 965 | 963 0 966 | 964 0 967 | 965 1 968 | 966 1 969 | 967 1 970 | 968 1 971 | 969 1 972 | 970 1 973 | 971 0 974 | 972 0 975 | 973 1 976 | 974 1 977 | 975 0 978 | 976 1 979 | 977 1 980 | 978 1 981 | 979 1 982 | 980 1 983 | 981 0 984 | 982 0 985 | 983 1 986 | 984 0 987 | 985 0 988 | 986 1 989 | 987 1 990 | 988 1 991 | 989 0 992 | 990 1 993 | 991 0 994 | 992 1 995 | 993 1 996 | 994 1 997 | 995 1 998 | 996 1 999 | 997 1 1000 | 998 1 1001 | 999 1 1002 | 1000 0 1003 | 1001 0 1004 | 1002 0 1005 | 1003 0 1006 | 1004 1 1007 | 1005 0 1008 | 1006 1 1009 | 1007 1 1010 | 1008 1 1011 | 1009 0 1012 | 1010 1 1013 | 1011 0 1014 | 1012 1 1015 | 1013 1 1016 | 1014 1 1017 | 1015 1 1018 | 1016 1 1019 | 1017 1 1020 | 1018 0 1021 | 1019 1 1022 | 1020 1 1023 | 1021 1 1024 | 1022 1 1025 | 1023 1 1026 | 1024 1 1027 | 1025 0 1028 | 1026 0 1029 | 1027 1 1030 | 1028 0 1031 | 1029 1 1032 | 1030 1 1033 | 1031 1 1034 | 1032 1 1035 | 1033 1 1036 | 1034 1 1037 | 1035 0 1038 | 1036 0 1039 | 1037 1 1040 | 1038 1 1041 | 1039 0 1042 | 1040 0 1043 | 1041 1 1044 | 1042 0 1045 | 1043 0 1046 | 1044 1 1047 | 1045 0 1048 | 1046 1 1049 | 1047 1 1050 | 1048 1 1051 | 1049 1 1052 | 1050 1 1053 | 1051 1 1054 | 1052 0 1055 | 1053 1 1056 | 1054 0 1057 | 1055 1 1058 | 1056 1 1059 | 1057 1 1060 | 1058 1 1061 | 1059 1 1062 | 1060 1 1063 | 1061 1 1064 | 1062 1 1065 | 1063 1 1066 | 1064 1 1067 | 1065 1 1068 | 1066 1 1069 | 1067 1 1070 | 1068 1 1071 | 1069 1 1072 | 1070 1 1073 | 1071 1 1074 | 1072 1 1075 | 1073 1 1076 | 1074 1 1077 | 1075 1 1078 | 1076 1 1079 | 1077 1 1080 | 1078 1 1081 | 1079 1 1082 | 1080 0 1083 | 1081 0 1084 | 1082 1 1085 | 1083 1 1086 | 1084 1 1087 | 1085 1 1088 | 1086 1 1089 | 1087 1 1090 | 1088 1 1091 | 1089 1 1092 | 1090 1 1093 | 1091 0 1094 | 1092 1 1095 | 1093 1 1096 | 1094 1 1097 | 1095 1 1098 | 1096 1 1099 | 1097 0 1100 | 1098 1 1101 | 1099 1 1102 | 1100 1 1103 | 1101 1 1104 | 1102 0 1105 | 1103 1 1106 | 1104 1 1107 | 1105 0 1108 | 1106 1 1109 | 1107 0 1110 | 1108 1 1111 | 1109 1 1112 | 1110 1 1113 | 1111 1 1114 | 1112 0 1115 | 1113 0 1116 | 1114 1 1117 | 1115 1 1118 | 1116 0 1119 | 1117 1 1120 | 1118 1 1121 | 1119 1 1122 | 1120 0 1123 | 1121 1 1124 | 1122 1 1125 | 1123 1 1126 | 1124 0 1127 | 1125 1 1128 | 1126 0 1129 | 1127 0 1130 | 1128 1 1131 | 1129 1 1132 | 1130 1 1133 | 1131 1 1134 | 1132 1 1135 | 1133 1 1136 | 1134 0 1137 | 1135 1 1138 | 1136 1 1139 | 1137 1 1140 | 1138 0 1141 | 1139 0 1142 | 1140 0 1143 | 1141 1 1144 | 1142 1 1145 | 1143 1 1146 | 1144 0 1147 | 1145 1 1148 | 1146 1 1149 | 1147 0 1150 | 1148 0 1151 | 1149 1 1152 | 1150 0 1153 | 1151 1 1154 | 1152 1 1155 | 1153 0 1156 | 1154 1 1157 | 1155 1 1158 | 1156 1 1159 | 1157 0 1160 | 1158 0 1161 | 1159 1 1162 | 1160 0 1163 | 1161 1 1164 | 1162 1 1165 | 1163 1 1166 | 1164 1 1167 | 1165 0 1168 | 1166 0 1169 | 1167 1 1170 | 1168 1 1171 | 1169 1 1172 | 1170 1 1173 | 1171 1 1174 | 1172 1 1175 | 1173 1 1176 | 1174 1 1177 | 1175 0 1178 | 1176 1 1179 | 1177 1 1180 | 1178 1 1181 | 1179 1 1182 | 1180 0 1183 | 1181 0 1184 | 1182 1 1185 | 1183 1 1186 | 1184 0 1187 | 1185 1 1188 | 1186 1 1189 | 1187 1 1190 | 1188 1 1191 | 1189 0 1192 | 1190 1 1193 | 1191 0 1194 | 1192 1 1195 | 1193 1 1196 | 1194 1 1197 | 1195 1 1198 | 1196 0 1199 | 1197 1 1200 | 1198 1 1201 | 1199 1 1202 | 1200 0 1203 | 1201 1 1204 | 1202 0 1205 | 1203 1 1206 | 1204 0 1207 | 1205 1 1208 | 1206 0 1209 | 1207 1 1210 | 1208 1 1211 | 1209 1 1212 | 1210 1 1213 | 1211 1 1214 | 1212 1 1215 | 1213 1 1216 | 1214 1 1217 | 1215 1 1218 | 1216 1 1219 | 1217 1 1220 | 1218 1 1221 | 1219 1 1222 | 1220 1 1223 | 1221 1 1224 | 1222 1 1225 | 1223 1 1226 | 1224 0 1227 | 1225 1 1228 | 1226 0 1229 | 1227 1 1230 | 1228 1 1231 | 1229 1 1232 | 1230 1 1233 | 1231 0 1234 | 1232 1 1235 | 1233 1 1236 | 1234 1 1237 | 1235 1 1238 | 1236 1 1239 | 1237 1 1240 | 1238 1 1241 | 1239 1 1242 | 1240 1 1243 | 1241 0 1244 | 1242 1 1245 | 1243 1 1246 | 1244 1 1247 | 1245 1 1248 | 1246 1 1249 | 1247 0 1250 | 1248 1 1251 | 1249 1 1252 | 1250 1 1253 | 1251 0 1254 | 1252 1 1255 | 1253 1 1256 | 1254 1 1257 | 1255 1 1258 | 1256 1 1259 | 1257 1 1260 | 1258 0 1261 | 1259 1 1262 | 1260 1 1263 | 1261 1 1264 | 1262 0 1265 | 1263 1 1266 | 1264 1 1267 | 1265 1 1268 | 1266 0 1269 | 1267 1 1270 | 1268 1 1271 | 1269 1 1272 | 1270 1 1273 | 1271 1 1274 | 1272 1 1275 | 1273 0 1276 | 1274 0 1277 | 1275 1 1278 | 1276 0 1279 | 1277 1 1280 | 1278 1 1281 | 1279 0 1282 | 1280 0 1283 | 1281 1 1284 | 1282 0 1285 | 1283 0 1286 | 1284 1 1287 | 1285 1 1288 | 1286 1 1289 | 1287 0 1290 | 1288 1 1291 | 1289 1 1292 | 1290 1 1293 | 1291 1 1294 | 1292 0 1295 | 1293 0 1296 | 1294 0 1297 | 1295 1 1298 | 1296 1 1299 | 1297 1 1300 | 1298 1 1301 | 1299 1 1302 | 1300 1 1303 | 1301 1 1304 | 1302 1 1305 | 1303 1 1306 | 1304 0 1307 | 1305 0 1308 | 1306 0 1309 | 1307 1 1310 | 1308 1 1311 | 1309 0 1312 | 1310 1 1313 | 1311 1 1314 | 1312 0 1315 | 1313 1 1316 | 1314 1 1317 | 1315 1 1318 | 1316 1 1319 | 1317 1 1320 | 1318 0 1321 | 1319 1 1322 | 1320 1 1323 | 1321 0 1324 | 1322 1 1325 | 1323 0 1326 | 1324 1 1327 | 1325 0 1328 | 1326 1 1329 | 1327 1 1330 | 1328 1 1331 | 1329 1 1332 | 1330 1 1333 | 1331 1 1334 | 1332 1 1335 | 1333 0 1336 | 1334 1 1337 | 1335 0 1338 | 1336 1 1339 | 1337 0 1340 | 1338 0 1341 | 1339 1 1342 | 1340 0 1343 | 1341 1 1344 | 1342 0 1345 | 1343 0 1346 | 1344 1 1347 | 1345 1 1348 | 1346 1 1349 | 1347 1 1350 | 1348 1 1351 | 1349 1 1352 | 1350 0 1353 | 1351 1 1354 | 1352 1 1355 | 1353 0 1356 | 1354 0 1357 | 1355 0 1358 | 1356 1 1359 | 1357 0 1360 | 1358 0 1361 | 1359 1 1362 | 1360 1 1363 | 1361 1 1364 | 1362 1 1365 | 1363 0 1366 | 1364 1 1367 | 1365 0 1368 | 1366 0 1369 | 1367 0 1370 | 1368 1 1371 | 1369 1 1372 | 1370 0 1373 | 1371 0 1374 | 1372 1 1375 | 1373 1 1376 | 1374 0 1377 | 1375 1 1378 | 1376 0 1379 | 1377 1 1380 | 1378 1 1381 | 1379 0 1382 | 1380 1 1383 | 1381 1 1384 | 1382 1 1385 | 1383 1 1386 | 1384 1 1387 | 1385 1 1388 | 1386 0 1389 | 1387 0 1390 | 1388 1 1391 | 1389 1 1392 | 1390 0 1393 | 1391 0 1394 | 1392 0 1395 | 1393 1 1396 | 1394 1 1397 | 1395 0 1398 | 1396 0 1399 | 1397 1 1400 | 1398 1 1401 | 1399 0 1402 | 1400 0 1403 | 1401 0 1404 | 1402 1 1405 | 1403 0 1406 | 1404 1 1407 | 1405 0 1408 | 1406 1 1409 | 1407 1 1410 | 1408 0 1411 | 1409 1 1412 | 1410 0 1413 | 1411 1 1414 | 1412 1 1415 | 1413 1 1416 | 1414 0 1417 | 1415 1 1418 | 1416 1 1419 | 1417 0 1420 | 1418 1 1421 | 1419 1 1422 | 1420 1 1423 | 1421 1 1424 | 1422 0 1425 | 1423 0 1426 | 1424 0 1427 | 1425 1 1428 | 1426 0 1429 | 1427 1 1430 | 1428 0 1431 | 1429 0 1432 | 1430 1 1433 | 1431 1 1434 | 1432 1 1435 | 1433 1 1436 | 1434 1 1437 | 1435 0 1438 | 1436 1 1439 | 1437 1 1440 | 1438 1 1441 | 1439 1 1442 | 1440 1 1443 | 1441 1 1444 | 1442 1 1445 | 1443 0 1446 | 1444 1 1447 | 1445 1 1448 | 1446 1 1449 | 1447 1 1450 | 1448 0 1451 | 1449 1 1452 | 1450 1 1453 | 1451 1 1454 | 1452 1 1455 | 1453 1 1456 | 1454 1 1457 | 1455 1 1458 | 1456 1 1459 | 1457 1 1460 | 1458 1 1461 | 1459 1 1462 | 1460 1 1463 | 1461 1 1464 | 1462 1 1465 | 1463 1 1466 | 1464 1 1467 | 1465 1 1468 | 1466 1 1469 | 1467 0 1470 | 1468 1 1471 | 1469 0 1472 | 1470 1 1473 | 1471 1 1474 | 1472 1 1475 | 1473 0 1476 | 1474 1 1477 | 1475 0 1478 | 1476 0 1479 | 1477 1 1480 | 1478 0 1481 | 1479 0 1482 | 1480 1 1483 | 1481 0 1484 | 1482 1 1485 | 1483 0 1486 | 1484 0 1487 | 1485 1 1488 | 1486 0 1489 | 1487 1 1490 | 1488 1 1491 | 1489 0 1492 | 1490 0 1493 | 1491 1 1494 | 1492 1 1495 | 1493 1 1496 | 1494 1 1497 | 1495 1 1498 | 1496 1 1499 | 1497 0 1500 | 1498 1 1501 | 1499 0 1502 | 1500 1 1503 | 1501 1 1504 | 1502 0 1505 | 1503 0 1506 | 1504 1 1507 | 1505 1 1508 | 1506 1 1509 | 1507 1 1510 | 1508 0 1511 | 1509 0 1512 | 1510 1 1513 | 1511 1 1514 | 1512 1 1515 | 1513 1 1516 | 1514 0 1517 | 1515 1 1518 | 1516 0 1519 | 1517 1 1520 | 1518 0 1521 | 1519 0 1522 | 1520 0 1523 | 1521 0 1524 | 1522 1 1525 | 1523 1 1526 | 1524 1 1527 | 1525 1 1528 | 1526 1 1529 | 1527 1 1530 | 1528 0 1531 | 1529 1 1532 | 1530 1 1533 | 1531 1 1534 | 1532 1 1535 | 1533 1 1536 | 1534 1 1537 | 1535 0 1538 | 1536 1 1539 | 1537 0 1540 | 1538 1 1541 | 1539 0 1542 | 1540 0 1543 | 1541 1 1544 | 1542 0 1545 | 1543 1 1546 | 1544 1 1547 | 1545 1 1548 | 1546 1 1549 | 1547 0 1550 | 1548 1 1551 | 1549 1 1552 | 1550 1 1553 | 1551 0 1554 | 1552 0 1555 | 1553 0 1556 | 1554 1 1557 | 1555 1 1558 | 1556 1 1559 | 1557 1 1560 | 1558 0 1561 | 1559 1 1562 | 1560 1 1563 | 1561 1 1564 | 1562 1 1565 | 1563 1 1566 | 1564 0 1567 | 1565 0 1568 | 1566 1 1569 | 1567 1 1570 | 1568 1 1571 | 1569 1 1572 | 1570 0 1573 | 1571 1 1574 | 1572 0 1575 | 1573 1 1576 | 1574 1 1577 | 1575 1 1578 | 1576 1 1579 | 1577 1 1580 | 1578 1 1581 | 1579 1 1582 | 1580 1 1583 | 1581 0 1584 | 1582 0 1585 | 1583 1 1586 | 1584 0 1587 | 1585 1 1588 | 1586 1 1589 | 1587 0 1590 | 1588 1 1591 | 1589 1 1592 | 1590 1 1593 | 1591 1 1594 | 1592 1 1595 | 1593 1 1596 | 1594 1 1597 | 1595 1 1598 | 1596 1 1599 | 1597 1 1600 | 1598 1 1601 | 1599 1 1602 | 1600 0 1603 | 1601 0 1604 | 1602 1 1605 | 1603 1 1606 | 1604 1 1607 | 1605 0 1608 | 1606 1 1609 | 1607 1 1610 | 1608 0 1611 | 1609 1 1612 | 1610 0 1613 | 1611 1 1614 | 1612 1 1615 | 1613 1 1616 | 1614 1 1617 | 1615 1 1618 | 1616 1 1619 | 1617 0 1620 | 1618 1 1621 | 1619 1 1622 | 1620 0 1623 | 1621 0 1624 | 1622 1 1625 | 1623 1 1626 | 1624 0 1627 | 1625 1 1628 | 1626 1 1629 | 1627 1 1630 | 1628 1 1631 | 1629 0 1632 | 1630 0 1633 | 1631 1 1634 | 1632 1 1635 | 1633 1 1636 | 1634 1 1637 | 1635 1 1638 | 1636 1 1639 | 1637 1 1640 | 1638 0 1641 | 1639 0 1642 | 1640 0 1643 | 1641 1 1644 | 1642 1 1645 | 1643 1 1646 | 1644 1 1647 | 1645 1 1648 | 1646 1 1649 | 1647 1 1650 | 1648 1 1651 | 1649 1 1652 | 1650 0 1653 | 1651 0 1654 | 1652 0 1655 | 1653 1 1656 | 1654 1 1657 | 1655 1 1658 | 1656 0 1659 | 1657 0 1660 | 1658 1 1661 | 1659 0 1662 | 1660 0 1663 | 1661 0 1664 | 1662 0 1665 | 1663 1 1666 | 1664 1 1667 | 1665 0 1668 | 1666 1 1669 | 1667 1 1670 | 1668 0 1671 | 1669 0 1672 | 1670 1 1673 | 1671 0 1674 | 1672 1 1675 | 1673 1 1676 | 1674 1 1677 | 1675 0 1678 | 1676 0 1679 | 1677 1 1680 | 1678 1 1681 | 1679 1 1682 | 1680 0 1683 | 1681 1 1684 | 1682 1 1685 | 1683 0 1686 | 1684 1 1687 | 1685 1 1688 | 1686 0 1689 | 1687 0 1690 | 1688 1 1691 | 1689 1 1692 | 1690 1 1693 | 1691 0 1694 | 1692 1 1695 | 1693 1 1696 | 1694 0 1697 | 1695 1 1698 | 1696 1 1699 | 1697 0 1700 | 1698 1 1701 | 1699 1 1702 | 1700 1 1703 | 1701 1 1704 | 1702 1 1705 | 1703 1 1706 | 1704 0 1707 | 1705 1 1708 | 1706 1 1709 | 1707 0 1710 | 1708 1 1711 | 1709 1 1712 | 1710 1 1713 | 1711 1 1714 | 1712 1 1715 | 1713 0 1716 | 1714 1 1717 | 1715 1 1718 | 1716 1 1719 | 1717 1 1720 | 1718 1 1721 | 1719 1 1722 | 1720 0 1723 | 1721 0 1724 | 1722 1 1725 | 1723 1 1726 | 1724 1 1727 | -------------------------------------------------------------------------------- /examples/GLUE-submission/matbert-base-uncased/STS-B.tsv: -------------------------------------------------------------------------------- 1 | index prediction 2 | 0 3.479 3 | 1 3.845 4 | 2 4.988 5 | 3 4.370 6 | 4 1.863 7 | 5 2.402 8 | 6 4.117 9 | 7 1.835 10 | 8 2.629 11 | 9 1.947 12 | 10 1.947 13 | 11 5.000 14 | 12 1.323 15 | 13 4.225 16 | 14 2.460 17 | 15 1.757 18 | 16 4.537 19 | 17 2.956 20 | 18 3.452 21 | 19 0.871 22 | 20 2.533 23 | 21 0.839 24 | 22 4.925 25 | 23 3.977 26 | 24 1.110 27 | 25 3.164 28 | 26 0.940 29 | 27 1.974 30 | 28 0.889 31 | 29 3.265 32 | 30 2.826 33 | 31 4.390 34 | 32 0.649 35 | 33 3.700 36 | 34 2.661 37 | 35 0.242 38 | 36 0.549 39 | 37 2.339 40 | 38 4.196 41 | 39 0.612 42 | 40 3.711 43 | 41 3.211 44 | 42 3.222 45 | 43 0.409 46 | 44 1.579 47 | 45 1.039 48 | 46 0.546 49 | 47 1.180 50 | 48 4.295 51 | 49 2.441 52 | 50 0.620 53 | 51 1.026 54 | 52 0.824 55 | 53 0.360 56 | 54 1.430 57 | 55 0.000 58 | 56 3.851 59 | 57 1.499 60 | 58 4.380 61 | 59 1.944 62 | 60 4.973 63 | 61 0.021 64 | 62 3.357 65 | 63 1.044 66 | 64 4.156 67 | 65 0.726 68 | 66 0.017 69 | 67 3.117 70 | 68 0.000 71 | 69 2.678 72 | 70 3.737 73 | 71 4.126 74 | 72 0.399 75 | 73 3.538 76 | 74 2.798 77 | 75 0.000 78 | 76 0.980 79 | 77 0.056 80 | 78 2.865 81 | 79 3.186 82 | 80 0.000 83 | 81 0.772 84 | 82 2.568 85 | 83 1.544 86 | 84 1.009 87 | 85 0.828 88 | 86 2.439 89 | 87 4.517 90 | 88 3.754 91 | 89 0.829 92 | 90 2.238 93 | 91 3.955 94 | 92 0.000 95 | 93 3.470 96 | 94 0.903 97 | 95 3.046 98 | 96 3.614 99 | 97 1.584 100 | 98 3.156 101 | 99 3.271 102 | 100 0.000 103 | 101 0.332 104 | 102 3.656 105 | 103 2.755 106 | 104 3.218 107 | 105 3.441 108 | 106 0.000 109 | 107 3.490 110 | 108 2.041 111 | 109 3.606 112 | 110 2.282 113 | 111 3.322 114 | 112 2.466 115 | 113 3.244 116 | 114 0.013 117 | 115 0.110 118 | 116 4.287 119 | 117 1.358 120 | 118 0.107 121 | 119 0.316 122 | 120 1.454 123 | 121 0.398 124 | 122 0.247 125 | 123 0.766 126 | 124 0.021 127 | 125 5.000 128 | 126 0.809 129 | 127 1.822 130 | 128 5.000 131 | 129 4.831 132 | 130 3.638 133 | 131 2.209 134 | 132 3.980 135 | 133 4.288 136 | 134 4.091 137 | 135 4.900 138 | 136 5.000 139 | 137 1.800 140 | 138 1.791 141 | 139 4.994 142 | 140 5.000 143 | 141 4.103 144 | 142 2.864 145 | 143 4.042 146 | 144 2.757 147 | 145 1.452 148 | 146 3.927 149 | 147 3.871 150 | 148 1.929 151 | 149 0.653 152 | 150 1.584 153 | 151 1.914 154 | 152 3.749 155 | 153 3.400 156 | 154 3.432 157 | 155 3.959 158 | 156 0.292 159 | 157 0.801 160 | 158 4.099 161 | 159 0.742 162 | 160 4.964 163 | 161 2.112 164 | 162 2.039 165 | 163 3.239 166 | 164 3.889 167 | 165 4.733 168 | 166 4.413 169 | 167 3.669 170 | 168 4.713 171 | 169 3.540 172 | 170 1.647 173 | 171 0.395 174 | 172 4.086 175 | 173 3.182 176 | 174 3.943 177 | 175 3.581 178 | 176 4.092 179 | 177 4.135 180 | 178 0.311 181 | 179 0.133 182 | 180 0.175 183 | 181 2.975 184 | 182 2.803 185 | 183 0.423 186 | 184 1.003 187 | 185 0.000 188 | 186 0.089 189 | 187 0.486 190 | 188 0.642 191 | 189 0.435 192 | 190 4.635 193 | 191 3.534 194 | 192 1.016 195 | 193 2.848 196 | 194 3.222 197 | 195 4.375 198 | 196 3.616 199 | 197 0.703 200 | 198 0.000 201 | 199 3.327 202 | 200 1.526 203 | 201 1.200 204 | 202 0.625 205 | 203 0.943 206 | 204 0.510 207 | 205 0.733 208 | 206 3.863 209 | 207 0.000 210 | 208 1.530 211 | 209 1.711 212 | 210 0.899 213 | 211 0.679 214 | 212 1.224 215 | 213 0.622 216 | 214 1.110 217 | 215 0.801 218 | 216 1.525 219 | 217 3.361 220 | 218 3.509 221 | 219 1.464 222 | 220 0.369 223 | 221 0.000 224 | 222 0.043 225 | 223 2.945 226 | 224 0.000 227 | 225 0.157 228 | 226 3.140 229 | 227 1.818 230 | 228 0.132 231 | 229 0.289 232 | 230 0.000 233 | 231 0.042 234 | 232 3.092 235 | 233 2.634 236 | 234 3.326 237 | 235 2.677 238 | 236 1.232 239 | 237 0.027 240 | 238 0.000 241 | 239 0.041 242 | 240 1.188 243 | 241 0.135 244 | 242 0.274 245 | 243 0.000 246 | 244 2.411 247 | 245 0.515 248 | 246 3.289 249 | 247 3.096 250 | 248 0.260 251 | 249 0.196 252 | 250 3.456 253 | 251 4.646 254 | 252 1.462 255 | 253 2.972 256 | 254 0.000 257 | 255 3.716 258 | 256 0.346 259 | 257 3.767 260 | 258 2.813 261 | 259 0.000 262 | 260 4.340 263 | 261 4.058 264 | 262 3.325 265 | 263 3.969 266 | 264 2.614 267 | 265 2.295 268 | 266 1.080 269 | 267 3.586 270 | 268 0.632 271 | 269 4.988 272 | 270 4.080 273 | 271 0.827 274 | 272 1.274 275 | 273 3.916 276 | 274 4.446 277 | 275 4.998 278 | 276 0.661 279 | 277 0.000 280 | 278 2.993 281 | 279 3.171 282 | 280 0.191 283 | 281 1.745 284 | 282 3.526 285 | 283 3.554 286 | 284 3.946 287 | 285 4.140 288 | 286 2.499 289 | 287 3.556 290 | 288 3.240 291 | 289 0.172 292 | 290 4.161 293 | 291 2.075 294 | 292 3.018 295 | 293 4.860 296 | 294 0.000 297 | 295 0.733 298 | 296 1.521 299 | 297 3.469 300 | 298 2.834 301 | 299 2.403 302 | 300 5.000 303 | 301 2.773 304 | 302 1.456 305 | 303 4.668 306 | 304 2.268 307 | 305 4.856 308 | 306 4.746 309 | 307 2.343 310 | 308 3.981 311 | 309 3.157 312 | 310 1.404 313 | 311 0.561 314 | 312 1.021 315 | 313 2.980 316 | 314 3.326 317 | 315 4.847 318 | 316 3.268 319 | 317 3.980 320 | 318 0.000 321 | 319 0.000 322 | 320 3.617 323 | 321 3.352 324 | 322 3.462 325 | 323 3.016 326 | 324 3.491 327 | 325 0.073 328 | 326 3.278 329 | 327 3.767 330 | 328 4.078 331 | 329 3.253 332 | 330 4.995 333 | 331 2.972 334 | 332 4.004 335 | 333 2.319 336 | 334 2.903 337 | 335 0.481 338 | 336 3.816 339 | 337 2.919 340 | 338 0.180 341 | 339 0.137 342 | 340 3.710 343 | 341 4.911 344 | 342 4.675 345 | 343 1.420 346 | 344 3.364 347 | 345 0.560 348 | 346 2.606 349 | 347 0.702 350 | 348 2.389 351 | 349 1.737 352 | 350 3.313 353 | 351 3.580 354 | 352 0.980 355 | 353 3.449 356 | 354 2.048 357 | 355 0.000 358 | 356 1.661 359 | 357 3.935 360 | 358 0.580 361 | 359 4.004 362 | 360 4.318 363 | 361 3.759 364 | 362 2.337 365 | 363 3.059 366 | 364 4.678 367 | 365 0.107 368 | 366 0.918 369 | 367 2.654 370 | 368 4.102 371 | 369 4.552 372 | 370 2.926 373 | 371 4.245 374 | 372 0.353 375 | 373 1.025 376 | 374 1.869 377 | 375 2.651 378 | 376 2.453 379 | 377 0.000 380 | 378 1.499 381 | 379 1.288 382 | 380 3.530 383 | 381 3.571 384 | 382 2.966 385 | 383 0.939 386 | 384 3.149 387 | 385 2.007 388 | 386 4.433 389 | 387 1.643 390 | 388 3.206 391 | 389 2.007 392 | 390 3.506 393 | 391 2.939 394 | 392 0.173 395 | 393 1.423 396 | 394 2.602 397 | 395 0.073 398 | 396 0.912 399 | 397 4.982 400 | 398 3.390 401 | 399 2.568 402 | 400 2.232 403 | 401 0.438 404 | 402 1.874 405 | 403 3.736 406 | 404 2.751 407 | 405 3.800 408 | 406 0.000 409 | 407 3.243 410 | 408 2.191 411 | 409 3.016 412 | 410 4.100 413 | 411 5.000 414 | 412 1.589 415 | 413 4.100 416 | 414 0.524 417 | 415 2.003 418 | 416 0.000 419 | 417 4.969 420 | 418 2.748 421 | 419 4.072 422 | 420 3.944 423 | 421 1.142 424 | 422 4.990 425 | 423 1.000 426 | 424 1.813 427 | 425 2.117 428 | 426 0.000 429 | 427 4.987 430 | 428 2.483 431 | 429 4.945 432 | 430 1.478 433 | 431 4.969 434 | 432 0.243 435 | 433 3.388 436 | 434 1.358 437 | 435 2.325 438 | 436 2.703 439 | 437 4.985 440 | 438 4.503 441 | 439 4.413 442 | 440 2.001 443 | 441 2.022 444 | 442 0.000 445 | 443 4.099 446 | 444 2.845 447 | 445 0.118 448 | 446 1.024 449 | 447 3.564 450 | 448 2.147 451 | 449 1.653 452 | 450 3.617 453 | 451 4.049 454 | 452 0.912 455 | 453 0.192 456 | 454 4.953 457 | 455 1.756 458 | 456 1.275 459 | 457 4.649 460 | 458 1.952 461 | 459 3.339 462 | 460 0.420 463 | 461 0.037 464 | 462 0.000 465 | 463 1.322 466 | 464 3.623 467 | 465 3.627 468 | 466 1.993 469 | 467 4.667 470 | 468 4.293 471 | 469 0.240 472 | 470 2.105 473 | 471 4.361 474 | 472 1.738 475 | 473 5.000 476 | 474 2.686 477 | 475 3.250 478 | 476 0.986 479 | 477 0.489 480 | 478 3.640 481 | 479 0.200 482 | 480 4.511 483 | 481 1.815 484 | 482 3.493 485 | 483 0.000 486 | 484 4.236 487 | 485 1.077 488 | 486 3.969 489 | 487 4.605 490 | 488 3.215 491 | 489 4.989 492 | 490 4.138 493 | 491 3.888 494 | 492 4.082 495 | 493 3.207 496 | 494 2.669 497 | 495 2.526 498 | 496 0.715 499 | 497 4.272 500 | 498 0.541 501 | 499 4.990 502 | 500 2.529 503 | 501 2.616 504 | 502 1.539 505 | 503 3.035 506 | 504 1.567 507 | 505 3.623 508 | 506 3.282 509 | 507 0.954 510 | 508 2.688 511 | 509 0.014 512 | 510 2.042 513 | 511 2.718 514 | 512 2.293 515 | 513 0.139 516 | 514 1.897 517 | 515 0.242 518 | 516 2.036 519 | 517 2.297 520 | 518 2.243 521 | 519 3.728 522 | 520 0.644 523 | 521 2.798 524 | 522 4.240 525 | 523 2.653 526 | 524 3.730 527 | 525 3.365 528 | 526 1.288 529 | 527 1.319 530 | 528 2.277 531 | 529 0.207 532 | 530 0.426 533 | 531 0.000 534 | 532 0.869 535 | 533 1.383 536 | 534 0.826 537 | 535 0.624 538 | 536 2.915 539 | 537 0.300 540 | 538 1.692 541 | 539 2.957 542 | 540 2.383 543 | 541 1.197 544 | 542 2.946 545 | 543 3.868 546 | 544 4.993 547 | 545 4.956 548 | 546 0.000 549 | 547 2.615 550 | 548 3.706 551 | 549 4.118 552 | 550 3.135 553 | 551 1.253 554 | 552 4.018 555 | 553 2.431 556 | 554 2.148 557 | 555 3.310 558 | 556 3.283 559 | 557 2.188 560 | 558 4.771 561 | 559 2.030 562 | 560 3.706 563 | 561 0.949 564 | 562 1.880 565 | 563 1.836 566 | 564 1.785 567 | 565 2.786 568 | 566 4.770 569 | 567 3.645 570 | 568 0.869 571 | 569 2.644 572 | 570 1.640 573 | 571 1.259 574 | 572 1.438 575 | 573 3.637 576 | 574 3.996 577 | 575 2.659 578 | 576 4.742 579 | 577 3.076 580 | 578 3.365 581 | 579 2.085 582 | 580 1.575 583 | 581 3.093 584 | 582 3.537 585 | 583 1.999 586 | 584 1.874 587 | 585 2.944 588 | 586 1.392 589 | 587 4.996 590 | 588 2.456 591 | 589 1.181 592 | 590 2.031 593 | 591 2.470 594 | 592 3.755 595 | 593 3.163 596 | 594 0.061 597 | 595 3.666 598 | 596 2.891 599 | 597 1.000 600 | 598 3.074 601 | 599 4.777 602 | 600 2.815 603 | 601 0.313 604 | 602 2.603 605 | 603 1.274 606 | 604 1.263 607 | 605 3.827 608 | 606 2.149 609 | 607 2.063 610 | 608 3.236 611 | 609 3.113 612 | 610 4.961 613 | 611 3.038 614 | 612 2.342 615 | 613 1.643 616 | 614 3.464 617 | 615 2.268 618 | 616 2.888 619 | 617 0.540 620 | 618 2.856 621 | 619 3.437 622 | 620 0.767 623 | 621 2.744 624 | 622 1.571 625 | 623 4.991 626 | 624 2.491 627 | 625 2.872 628 | 626 3.279 629 | 627 0.909 630 | 628 1.095 631 | 629 1.629 632 | 630 1.594 633 | 631 1.884 634 | 632 3.116 635 | 633 2.991 636 | 634 1.840 637 | 635 2.378 638 | 636 3.331 639 | 637 3.053 640 | 638 2.721 641 | 639 3.601 642 | 640 0.889 643 | 641 1.713 644 | 642 2.230 645 | 643 3.947 646 | 644 2.627 647 | 645 4.452 648 | 646 4.299 649 | 647 2.370 650 | 648 4.225 651 | 649 2.172 652 | 650 2.515 653 | 651 4.423 654 | 652 1.486 655 | 653 1.157 656 | 654 2.820 657 | 655 2.850 658 | 656 2.848 659 | 657 0.771 660 | 658 2.210 661 | 659 3.195 662 | 660 1.286 663 | 661 3.716 664 | 662 1.486 665 | 663 3.034 666 | 664 4.817 667 | 665 2.088 668 | 666 2.855 669 | 667 2.798 670 | 668 4.608 671 | 669 2.438 672 | 670 2.142 673 | 671 4.906 674 | 672 2.677 675 | 673 4.907 676 | 674 4.473 677 | 675 1.210 678 | 676 2.299 679 | 677 2.569 680 | 678 1.581 681 | 679 2.231 682 | 680 2.955 683 | 681 2.529 684 | 682 2.749 685 | 683 1.923 686 | 684 2.902 687 | 685 2.358 688 | 686 2.633 689 | 687 2.410 690 | 688 1.996 691 | 689 4.115 692 | 690 2.919 693 | 691 2.646 694 | 692 4.806 695 | 693 2.274 696 | 694 3.213 697 | 695 2.160 698 | 696 2.560 699 | 697 0.927 700 | 698 1.043 701 | 699 3.354 702 | 700 2.592 703 | 701 3.077 704 | 702 3.255 705 | 703 4.199 706 | 704 3.010 707 | 705 2.316 708 | 706 2.403 709 | 707 1.086 710 | 708 3.293 711 | 709 2.742 712 | 710 4.189 713 | 711 2.875 714 | 712 1.284 715 | 713 1.515 716 | 714 2.332 717 | 715 1.803 718 | 716 3.136 719 | 717 2.902 720 | 718 2.274 721 | 719 4.208 722 | 720 2.043 723 | 721 2.083 724 | 722 2.729 725 | 723 2.286 726 | 724 3.340 727 | 725 1.381 728 | 726 2.651 729 | 727 4.316 730 | 728 2.330 731 | 729 3.610 732 | 730 2.588 733 | 731 2.175 734 | 732 2.804 735 | 733 1.011 736 | 734 2.710 737 | 735 1.441 738 | 736 1.793 739 | 737 3.578 740 | 738 2.312 741 | 739 3.269 742 | 740 3.474 743 | 741 2.744 744 | 742 1.700 745 | 743 2.459 746 | 744 1.675 747 | 745 1.854 748 | 746 2.515 749 | 747 1.665 750 | 748 4.047 751 | 749 2.878 752 | 750 2.041 753 | 751 1.898 754 | 752 3.075 755 | 753 1.838 756 | 754 2.048 757 | 755 4.065 758 | 756 2.859 759 | 757 2.614 760 | 758 2.586 761 | 759 3.996 762 | 760 4.392 763 | 761 0.631 764 | 762 1.627 765 | 763 2.375 766 | 764 3.646 767 | 765 2.031 768 | 766 3.179 769 | 767 2.453 770 | 768 2.367 771 | 769 1.225 772 | 770 2.521 773 | 771 1.922 774 | 772 2.637 775 | 773 4.390 776 | 774 3.529 777 | 775 2.498 778 | 776 2.269 779 | 777 1.520 780 | 778 2.608 781 | 779 2.252 782 | 780 2.609 783 | 781 2.188 784 | 782 4.297 785 | 783 3.803 786 | 784 1.787 787 | 785 2.338 788 | 786 2.370 789 | 787 3.329 790 | 788 3.471 791 | 789 2.417 792 | 790 2.845 793 | 791 2.461 794 | 792 2.506 795 | 793 2.464 796 | 794 2.657 797 | 795 3.591 798 | 796 0.962 799 | 797 3.746 800 | 798 1.405 801 | 799 3.054 802 | 800 2.639 803 | 801 2.249 804 | 802 1.340 805 | 803 1.165 806 | 804 3.382 807 | 805 4.468 808 | 806 2.684 809 | 807 2.893 810 | 808 3.871 811 | 809 1.766 812 | 810 3.919 813 | 811 2.079 814 | 812 1.786 815 | 813 2.858 816 | 814 2.602 817 | 815 1.325 818 | 816 2.685 819 | 817 2.202 820 | 818 3.462 821 | 819 2.363 822 | 820 2.411 823 | 821 2.699 824 | 822 2.131 825 | 823 2.666 826 | 824 2.907 827 | 825 2.422 828 | 826 3.090 829 | 827 0.743 830 | 828 3.104 831 | 829 4.476 832 | 830 2.646 833 | 831 2.274 834 | 832 2.556 835 | 833 3.169 836 | 834 3.308 837 | 835 1.112 838 | 836 4.395 839 | 837 1.125 840 | 838 2.039 841 | 839 2.680 842 | 840 2.411 843 | 841 3.206 844 | 842 2.327 845 | 843 2.566 846 | 844 4.836 847 | 845 2.174 848 | 846 3.653 849 | 847 2.684 850 | 848 2.478 851 | 849 4.396 852 | 850 3.505 853 | 851 2.550 854 | 852 4.865 855 | 853 4.571 856 | 854 4.726 857 | 855 3.001 858 | 856 2.941 859 | 857 1.202 860 | 858 3.905 861 | 859 3.183 862 | 860 2.650 863 | 861 4.915 864 | 862 3.464 865 | 863 1.106 866 | 864 1.976 867 | 865 0.801 868 | 866 2.363 869 | 867 1.921 870 | 868 1.056 871 | 869 2.767 872 | 870 2.485 873 | 871 1.708 874 | 872 2.308 875 | 873 2.731 876 | 874 2.134 877 | 875 3.933 878 | 876 2.459 879 | 877 3.237 880 | 878 2.116 881 | 879 1.808 882 | 880 3.517 883 | 881 2.243 884 | 882 3.284 885 | 883 4.304 886 | 884 2.049 887 | 885 3.417 888 | 886 2.397 889 | 887 3.508 890 | 888 2.974 891 | 889 2.701 892 | 890 4.989 893 | 891 3.873 894 | 892 1.196 895 | 893 3.812 896 | 894 3.457 897 | 895 3.785 898 | 896 3.817 899 | 897 3.047 900 | 898 1.584 901 | 899 3.410 902 | 900 4.218 903 | 901 4.489 904 | 902 2.727 905 | 903 3.483 906 | 904 4.564 907 | 905 2.822 908 | 906 2.829 909 | 907 4.213 910 | 908 3.681 911 | 909 2.667 912 | 910 2.392 913 | 911 3.045 914 | 912 2.489 915 | 913 3.515 916 | 914 3.645 917 | 915 4.709 918 | 916 2.702 919 | 917 3.298 920 | 918 4.432 921 | 919 3.760 922 | 920 3.404 923 | 921 3.118 924 | 922 4.303 925 | 923 3.437 926 | 924 4.152 927 | 925 2.566 928 | 926 3.769 929 | 927 3.020 930 | 928 4.404 931 | 929 3.313 932 | 930 3.368 933 | 931 3.335 934 | 932 3.986 935 | 933 3.001 936 | 934 2.037 937 | 935 1.552 938 | 936 2.614 939 | 937 2.848 940 | 938 3.312 941 | 939 2.853 942 | 940 3.675 943 | 941 3.766 944 | 942 3.161 945 | 943 3.045 946 | 944 2.972 947 | 945 3.830 948 | 946 4.236 949 | 947 3.050 950 | 948 3.200 951 | 949 2.716 952 | 950 3.926 953 | 951 3.977 954 | 952 3.859 955 | 953 2.540 956 | 954 2.646 957 | 955 1.696 958 | 956 2.980 959 | 957 1.923 960 | 958 4.147 961 | 959 3.179 962 | 960 3.500 963 | 961 3.054 964 | 962 4.191 965 | 963 2.173 966 | 964 3.368 967 | 965 2.906 968 | 966 4.114 969 | 967 3.928 970 | 968 2.837 971 | 969 2.340 972 | 970 2.923 973 | 971 2.716 974 | 972 3.753 975 | 973 3.282 976 | 974 3.249 977 | 975 3.309 978 | 976 2.691 979 | 977 4.462 980 | 978 3.571 981 | 979 3.848 982 | 980 2.536 983 | 981 3.899 984 | 982 2.821 985 | 983 2.109 986 | 984 3.078 987 | 985 3.316 988 | 986 3.867 989 | 987 2.627 990 | 988 1.530 991 | 989 2.463 992 | 990 4.037 993 | 991 2.787 994 | 992 2.661 995 | 993 3.196 996 | 994 1.785 997 | 995 3.084 998 | 996 1.761 999 | 997 3.702 1000 | 998 3.878 1001 | 999 2.699 1002 | 1000 3.361 1003 | 1001 4.771 1004 | 1002 3.542 1005 | 1003 3.145 1006 | 1004 2.002 1007 | 1005 2.632 1008 | 1006 3.844 1009 | 1007 3.151 1010 | 1008 2.191 1011 | 1009 3.402 1012 | 1010 3.675 1013 | 1011 3.380 1014 | 1012 4.751 1015 | 1013 2.634 1016 | 1014 3.344 1017 | 1015 3.005 1018 | 1016 4.137 1019 | 1017 2.876 1020 | 1018 4.204 1021 | 1019 2.880 1022 | 1020 2.994 1023 | 1021 2.972 1024 | 1022 2.475 1025 | 1023 2.948 1026 | 1024 3.774 1027 | 1025 3.195 1028 | 1026 3.239 1029 | 1027 2.349 1030 | 1028 3.400 1031 | 1029 1.818 1032 | 1030 4.010 1033 | 1031 4.797 1034 | 1032 3.713 1035 | 1033 2.835 1036 | 1034 3.732 1037 | 1035 3.138 1038 | 1036 2.958 1039 | 1037 3.396 1040 | 1038 4.215 1041 | 1039 4.984 1042 | 1040 3.283 1043 | 1041 3.243 1044 | 1042 4.034 1045 | 1043 3.005 1046 | 1044 3.490 1047 | 1045 2.565 1048 | 1046 3.592 1049 | 1047 3.336 1050 | 1048 4.538 1051 | 1049 3.740 1052 | 1050 3.457 1053 | 1051 3.841 1054 | 1052 4.096 1055 | 1053 2.465 1056 | 1054 3.411 1057 | 1055 3.633 1058 | 1056 3.783 1059 | 1057 3.398 1060 | 1058 3.244 1061 | 1059 3.421 1062 | 1060 2.963 1063 | 1061 2.217 1064 | 1062 2.711 1065 | 1063 3.028 1066 | 1064 3.345 1067 | 1065 2.705 1068 | 1066 3.789 1069 | 1067 2.710 1070 | 1068 3.013 1071 | 1069 2.961 1072 | 1070 4.968 1073 | 1071 2.834 1074 | 1072 2.669 1075 | 1073 1.148 1076 | 1074 2.285 1077 | 1075 3.287 1078 | 1076 3.777 1079 | 1077 3.664 1080 | 1078 3.890 1081 | 1079 2.015 1082 | 1080 3.954 1083 | 1081 3.666 1084 | 1082 2.451 1085 | 1083 0.614 1086 | 1084 3.434 1087 | 1085 3.580 1088 | 1086 3.249 1089 | 1087 3.362 1090 | 1088 1.879 1091 | 1089 3.671 1092 | 1090 3.129 1093 | 1091 4.432 1094 | 1092 3.906 1095 | 1093 3.499 1096 | 1094 3.081 1097 | 1095 2.887 1098 | 1096 3.112 1099 | 1097 2.332 1100 | 1098 3.343 1101 | 1099 2.025 1102 | 1100 2.623 1103 | 1101 3.414 1104 | 1102 2.721 1105 | 1103 3.421 1106 | 1104 4.623 1107 | 1105 2.253 1108 | 1106 3.089 1109 | 1107 1.567 1110 | 1108 3.396 1111 | 1109 3.029 1112 | 1110 1.230 1113 | 1111 4.457 1114 | 1112 3.736 1115 | 1113 3.967 1116 | 1114 2.957 1117 | 1115 2.257 1118 | 1116 3.390 1119 | 1117 4.159 1120 | 1118 3.077 1121 | 1119 3.066 1122 | 1120 2.955 1123 | 1121 0.894 1124 | 1122 4.160 1125 | 1123 3.326 1126 | 1124 4.046 1127 | 1125 3.773 1128 | 1126 3.266 1129 | 1127 4.231 1130 | 1128 4.007 1131 | 1129 2.721 1132 | 1130 0.918 1133 | 1131 4.325 1134 | 1132 4.808 1135 | 1133 0.730 1136 | 1134 2.280 1137 | 1135 2.489 1138 | 1136 2.293 1139 | 1137 3.739 1140 | 1138 0.824 1141 | 1139 4.590 1142 | 1140 1.589 1143 | 1141 0.759 1144 | 1142 2.364 1145 | 1143 3.296 1146 | 1144 3.417 1147 | 1145 1.668 1148 | 1146 1.644 1149 | 1147 4.464 1150 | 1148 4.160 1151 | 1149 4.607 1152 | 1150 0.296 1153 | 1151 0.000 1154 | 1152 4.952 1155 | 1153 4.792 1156 | 1154 4.961 1157 | 1155 3.897 1158 | 1156 3.540 1159 | 1157 0.167 1160 | 1158 3.977 1161 | 1159 1.479 1162 | 1160 2.728 1163 | 1161 3.211 1164 | 1162 3.413 1165 | 1163 4.779 1166 | 1164 3.633 1167 | 1165 0.984 1168 | 1166 2.653 1169 | 1167 1.182 1170 | 1168 3.143 1171 | 1169 0.133 1172 | 1170 2.009 1173 | 1171 4.225 1174 | 1172 2.024 1175 | 1173 4.415 1176 | 1174 1.152 1177 | 1175 0.000 1178 | 1176 4.672 1179 | 1177 2.586 1180 | 1178 3.800 1181 | 1179 1.585 1182 | 1180 1.018 1183 | 1181 2.377 1184 | 1182 3.784 1185 | 1183 2.263 1186 | 1184 1.273 1187 | 1185 2.954 1188 | 1186 4.256 1189 | 1187 2.709 1190 | 1188 3.068 1191 | 1189 4.661 1192 | 1190 2.491 1193 | 1191 3.951 1194 | 1192 2.443 1195 | 1193 1.427 1196 | 1194 1.210 1197 | 1195 4.221 1198 | 1196 4.748 1199 | 1197 4.008 1200 | 1198 3.227 1201 | 1199 3.566 1202 | 1200 1.558 1203 | 1201 2.457 1204 | 1202 3.957 1205 | 1203 4.326 1206 | 1204 1.960 1207 | 1205 3.124 1208 | 1206 4.918 1209 | 1207 3.169 1210 | 1208 2.217 1211 | 1209 2.916 1212 | 1210 2.231 1213 | 1211 3.114 1214 | 1212 4.907 1215 | 1213 2.886 1216 | 1214 4.886 1217 | 1215 4.066 1218 | 1216 2.199 1219 | 1217 2.647 1220 | 1218 3.050 1221 | 1219 1.965 1222 | 1220 1.824 1223 | 1221 3.115 1224 | 1222 4.816 1225 | 1223 3.283 1226 | 1224 2.940 1227 | 1225 3.284 1228 | 1226 3.654 1229 | 1227 4.179 1230 | 1228 4.710 1231 | 1229 2.032 1232 | 1230 4.095 1233 | 1231 1.739 1234 | 1232 3.028 1235 | 1233 4.103 1236 | 1234 3.837 1237 | 1235 1.907 1238 | 1236 3.020 1239 | 1237 1.818 1240 | 1238 1.279 1241 | 1239 2.683 1242 | 1240 1.517 1243 | 1241 3.344 1244 | 1242 3.903 1245 | 1243 2.849 1246 | 1244 0.350 1247 | 1245 4.998 1248 | 1246 3.864 1249 | 1247 2.085 1250 | 1248 0.000 1251 | 1249 4.796 1252 | 1250 3.467 1253 | 1251 3.574 1254 | 1252 3.279 1255 | 1253 3.336 1256 | 1254 0.612 1257 | 1255 4.549 1258 | 1256 2.961 1259 | 1257 3.880 1260 | 1258 2.512 1261 | 1259 4.144 1262 | 1260 2.121 1263 | 1261 2.045 1264 | 1262 3.785 1265 | 1263 2.030 1266 | 1264 3.942 1267 | 1265 3.820 1268 | 1266 3.931 1269 | 1267 3.564 1270 | 1268 1.400 1271 | 1269 1.461 1272 | 1270 2.976 1273 | 1271 2.976 1274 | 1272 1.199 1275 | 1273 1.980 1276 | 1274 2.385 1277 | 1275 1.978 1278 | 1276 0.611 1279 | 1277 3.855 1280 | 1278 1.116 1281 | 1279 4.912 1282 | 1280 0.059 1283 | 1281 0.452 1284 | 1282 2.641 1285 | 1283 4.850 1286 | 1284 1.132 1287 | 1285 3.269 1288 | 1286 3.818 1289 | 1287 1.909 1290 | 1288 0.511 1291 | 1289 1.486 1292 | 1290 3.659 1293 | 1291 1.014 1294 | 1292 3.794 1295 | 1293 2.694 1296 | 1294 2.227 1297 | 1295 0.286 1298 | 1296 4.772 1299 | 1297 0.453 1300 | 1298 0.623 1301 | 1299 4.827 1302 | 1300 1.559 1303 | 1301 4.261 1304 | 1302 2.547 1305 | 1303 2.150 1306 | 1304 2.532 1307 | 1305 1.377 1308 | 1306 1.931 1309 | 1307 1.119 1310 | 1308 1.292 1311 | 1309 2.039 1312 | 1310 2.911 1313 | 1311 4.364 1314 | 1312 1.958 1315 | 1313 2.775 1316 | 1314 2.596 1317 | 1315 4.999 1318 | 1316 4.992 1319 | 1317 4.491 1320 | 1318 1.911 1321 | 1319 0.196 1322 | 1320 4.948 1323 | 1321 2.007 1324 | 1322 3.977 1325 | 1323 2.104 1326 | 1324 4.990 1327 | 1325 4.914 1328 | 1326 4.402 1329 | 1327 2.647 1330 | 1328 0.824 1331 | 1329 4.112 1332 | 1330 4.315 1333 | 1331 2.326 1334 | 1332 1.986 1335 | 1333 3.449 1336 | 1334 2.801 1337 | 1335 3.799 1338 | 1336 4.439 1339 | 1337 3.723 1340 | 1338 0.616 1341 | 1339 2.666 1342 | 1340 3.284 1343 | 1341 4.679 1344 | 1342 3.118 1345 | 1343 0.522 1346 | 1344 1.154 1347 | 1345 2.777 1348 | 1346 4.884 1349 | 1347 4.349 1350 | 1348 4.977 1351 | 1349 2.079 1352 | 1350 5.000 1353 | 1351 4.988 1354 | 1352 4.725 1355 | 1353 4.449 1356 | 1354 4.673 1357 | 1355 1.141 1358 | 1356 2.623 1359 | 1357 5.000 1360 | 1358 3.543 1361 | 1359 2.922 1362 | 1360 0.932 1363 | 1361 3.851 1364 | 1362 0.588 1365 | 1363 2.047 1366 | 1364 4.942 1367 | 1365 3.129 1368 | 1366 1.360 1369 | 1367 2.100 1370 | 1368 0.986 1371 | 1369 1.061 1372 | 1370 1.071 1373 | 1371 1.115 1374 | 1372 2.670 1375 | 1373 2.137 1376 | 1374 1.306 1377 | 1375 1.247 1378 | 1376 1.342 1379 | 1377 0.373 1380 | 1378 1.356 1381 | -------------------------------------------------------------------------------- /examples/GLUE-submission/matbert-base-uncased/WNLI.tsv: -------------------------------------------------------------------------------- 1 | index prediction 2 | 0 0 3 | 1 0 4 | 2 0 5 | 3 0 6 | 4 0 7 | 5 0 8 | 6 0 9 | 7 0 10 | 8 0 11 | 9 0 12 | 10 0 13 | 11 0 14 | 12 0 15 | 13 0 16 | 14 0 17 | 15 0 18 | 16 0 19 | 17 0 20 | 18 0 21 | 19 0 22 | 20 0 23 | 21 0 24 | 22 0 25 | 23 0 26 | 24 0 27 | 25 0 28 | 26 0 29 | 27 0 30 | 28 0 31 | 29 0 32 | 30 0 33 | 31 0 34 | 32 0 35 | 33 0 36 | 34 0 37 | 35 0 38 | 36 0 39 | 37 0 40 | 38 0 41 | 39 0 42 | 40 0 43 | 41 0 44 | 42 0 45 | 43 0 46 | 44 0 47 | 45 0 48 | 46 0 49 | 47 0 50 | 48 0 51 | 49 0 52 | 50 0 53 | 51 0 54 | 52 0 55 | 53 0 56 | 54 0 57 | 55 0 58 | 56 0 59 | 57 0 60 | 58 0 61 | 59 0 62 | 60 0 63 | 61 0 64 | 62 0 65 | 63 0 66 | 64 0 67 | 65 0 68 | 66 0 69 | 67 0 70 | 68 0 71 | 69 0 72 | 70 0 73 | 71 0 74 | 72 0 75 | 73 0 76 | 74 0 77 | 75 0 78 | 76 0 79 | 77 0 80 | 78 0 81 | 79 0 82 | 80 0 83 | 81 0 84 | 82 0 85 | 83 0 86 | 84 0 87 | 85 0 88 | 86 0 89 | 87 0 90 | 88 0 91 | 89 0 92 | 90 0 93 | 91 0 94 | 92 0 95 | 93 0 96 | 94 0 97 | 95 0 98 | 96 0 99 | 97 0 100 | 98 0 101 | 99 0 102 | 100 0 103 | 101 0 104 | 102 0 105 | 103 0 106 | 104 0 107 | 105 0 108 | 106 0 109 | 107 0 110 | 108 0 111 | 109 0 112 | 110 0 113 | 111 0 114 | 112 0 115 | 113 0 116 | 114 0 117 | 115 0 118 | 116 0 119 | 117 0 120 | 118 0 121 | 119 0 122 | 120 0 123 | 121 0 124 | 122 0 125 | 123 0 126 | 124 0 127 | 125 0 128 | 126 0 129 | 127 0 130 | 128 0 131 | 129 0 132 | 130 0 133 | 131 0 134 | 132 0 135 | 133 0 136 | 134 0 137 | 135 0 138 | 136 0 139 | 137 0 140 | 138 0 141 | 139 0 142 | 140 0 143 | 141 0 144 | 142 0 145 | 143 0 146 | 144 0 147 | 145 0 148 | -------------------------------------------------------------------------------- /examples/evaluate_glue.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2020 Ceder Group. 3 | # Copyright 2020 The HuggingFace Inc. team. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | """ Finetuning the library models for sequence classification on GLUE.""" 17 | # You can also adapt this script on your own text classification task. Pointers for this are left as comments. 18 | 19 | import logging 20 | import os 21 | import random 22 | import sys 23 | import time 24 | from dataclasses import dataclass, field 25 | from typing import Optional 26 | 27 | import numpy as np 28 | import torch 29 | from datasets import load_dataset, load_metric 30 | 31 | import transformers 32 | from torch.distributed import barrier 33 | from transformers import ( 34 | AutoConfig, 35 | AutoModelForSequenceClassification, 36 | AutoTokenizer, 37 | EvalPrediction, 38 | HfArgumentParser, 39 | PretrainedConfig, 40 | Trainer, 41 | TrainingArguments, 42 | default_data_collator, 43 | set_seed, PreTrainedModel, WEIGHTS_NAME, 44 | ) 45 | from transformers.trainer_utils import is_main_process 46 | 47 | task_to_keys = { 48 | "cola": ("sentence", None), 49 | "mnli": ("premise", "hypothesis"), 50 | "mrpc": ("sentence1", "sentence2"), 51 | "qnli": ("question", "sentence"), 52 | "qqp": ("question1", "question2"), 53 | "rte": ("sentence1", "sentence2"), 54 | "sst2": ("sentence", None), 55 | "stsb": ("sentence1", "sentence2"), 56 | "wnli": ("sentence1", "sentence2"), 57 | } 58 | 59 | logger = logging.getLogger(__name__) 60 | 61 | 62 | @dataclass 63 | class DataTrainingArguments: 64 | """ 65 | Arguments pertaining to what data we are going to input our model for training and eval. 66 | 67 | Using `HfArgumentParser` we can turn this class 68 | into argparse arguments to be able to specify them on 69 | the command line. 70 | """ 71 | 72 | task_name: Optional[str] = field( 73 | default=None, 74 | metadata={"help": "The name of the task to train on: " + ", ".join(task_to_keys.keys())}, 75 | ) 76 | max_seq_length: int = field( 77 | default=128, 78 | metadata={ 79 | "help": "The maximum total input sequence length after tokenization. Sequences longer " 80 | "than this will be truncated, sequences shorter will be padded." 81 | }, 82 | ) 83 | overwrite_cache: bool = field( 84 | default=False, metadata={"help": "Overwrite the cached preprocessed datasets or not."} 85 | ) 86 | pad_to_max_length: bool = field( 87 | default=True, 88 | metadata={ 89 | "help": "Whether to pad all samples to `max_seq_length`. " 90 | "If False, will pad the samples dynamically when batching to the maximum length in the batch." 91 | }, 92 | ) 93 | train_file: Optional[str] = field( 94 | default=None, metadata={"help": "A csv or a json file containing the training data."} 95 | ) 96 | validation_file: Optional[str] = field( 97 | default=None, metadata={"help": "A csv or a json file containing the validation data."} 98 | ) 99 | 100 | def __post_init__(self): 101 | if self.task_name is not None: 102 | self.task_name = self.task_name.lower() 103 | if self.task_name not in task_to_keys.keys(): 104 | raise ValueError("Unknown task, you should pick one in " + ",".join(task_to_keys.keys())) 105 | elif self.train_file is None or self.validation_file is None: 106 | raise ValueError("Need either a GLUE task or a training/validation file.") 107 | else: 108 | extension = self.train_file.split(".")[-1] 109 | assert extension in ["csv", "json"], "`train_file` should be a csv or a json file." 110 | extension = self.validation_file.split(".")[-1] 111 | assert extension in ["csv", "json"], "`validation_file` should be a csv or a json file." 112 | 113 | 114 | @dataclass 115 | class ModelArguments: 116 | """ 117 | Arguments pertaining to which model/config/tokenizer we are going to fine-tune from. 118 | """ 119 | 120 | model_name_or_path: str = field( 121 | metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"} 122 | ) 123 | config_name: Optional[str] = field( 124 | default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"} 125 | ) 126 | do_lower_case: bool = field( 127 | default=True, metadata={"help": "Whether to perform case conversion in tokenizer"} 128 | ) 129 | tokenizer_name: Optional[str] = field( 130 | default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"} 131 | ) 132 | cache_dir: Optional[str] = field( 133 | default=None, 134 | metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"}, 135 | ) 136 | use_fast_tokenizer: bool = field( 137 | default=True, 138 | metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."}, 139 | ) 140 | 141 | 142 | def wrap_save_training(trainer): 143 | original = trainer.save_model 144 | # original_save = trainer._save 145 | 146 | def _func(*args, **kwargs): 147 | result = original(*args, **kwargs) 148 | barrier() 149 | return result 150 | 151 | def _new_save(self, output_dir: Optional[str] = None): 152 | output_dir = output_dir if output_dir is not None else self.args.output_dir 153 | os.makedirs(output_dir, exist_ok=True) 154 | logger.info("Saving model checkpoint to %s", output_dir) 155 | # Save a trained model and configuration using `save_pretrained()`. 156 | # They can then be reloaded using `from_pretrained()` 157 | if not isinstance(self.model, PreTrainedModel): 158 | logger.info("Trainer.model is not a `PreTrainedModel`, only saving its state dict.") 159 | state_dict = {k: v.cpu() for k, v in self.model.state_dict().items()} 160 | torch.save(state_dict, os.path.join(output_dir, WEIGHTS_NAME)) 161 | else: 162 | self.model.save_pretrained(output_dir) 163 | if self.tokenizer is not None and self.is_world_process_zero(): 164 | self.tokenizer.save_pretrained(output_dir) 165 | 166 | # Good practice: save your training arguments together with the trained model 167 | torch.save(self.args, os.path.join(output_dir, "training_args.bin")) 168 | 169 | trainer.save_model = _func 170 | trainer._save = _new_save.__get__(trainer, Trainer) 171 | 172 | 173 | def main(): 174 | # See all possible arguments in src/transformers/training_args.py 175 | # or by passing the --help flag to this script. 176 | # We now keep distinct sets of args, for a cleaner separation of concerns. 177 | 178 | parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)) 179 | if len(sys.argv) == 2 and sys.argv[1].endswith(".json"): 180 | # If we pass only one argument to the script and it's the path to a json file, 181 | # let's parse it to get our arguments. 182 | model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1])) 183 | else: 184 | model_args, data_args, training_args = parser.parse_args_into_dataclasses() 185 | 186 | if ( 187 | os.path.exists(training_args.output_dir) 188 | and os.listdir(training_args.output_dir) 189 | and training_args.do_train 190 | and not training_args.overwrite_output_dir 191 | ): 192 | raise ValueError( 193 | f"Output directory ({training_args.output_dir}) already exists and is not empty. " 194 | "Use --overwrite_output_dir to overcome." 195 | ) 196 | 197 | # Setup logging 198 | logging.basicConfig( 199 | format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", 200 | datefmt="%m/%d/%Y %H:%M:%S", 201 | level=logging.INFO if is_main_process(training_args.local_rank) else logging.WARN, 202 | ) 203 | 204 | # Log on each process the small summary: 205 | logger.warning( 206 | f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}" 207 | + f"distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16}" 208 | ) 209 | # Set the verbosity to info of the Transformers logger (on main process only): 210 | if is_main_process(training_args.local_rank): 211 | transformers.utils.logging.set_verbosity_info() 212 | transformers.utils.logging.enable_default_handler() 213 | transformers.utils.logging.enable_explicit_format() 214 | logger.info(f"Training/evaluation parameters {training_args}") 215 | 216 | # Set seed before initializing model. 217 | set_seed(training_args.seed) 218 | 219 | # Get the datasets: you can either provide your own CSV/JSON training and evaluation files (see below) 220 | # or specify a GLUE benchmark task (the dataset will be downloaded automatically from the datasets Hub). 221 | # 222 | # For CSV/JSON files, this script will use as labels the column called 'label' and as pair of sentences the 223 | # sentences in columns called 'sentence1' and 'sentence2' if such column exists or the first two columns not named 224 | # label if at least two columns are provided. 225 | # 226 | # If the CSVs/JSONs contain only one non-label column, the script does single sentence classification on this 227 | # single column. You can easily tweak this behavior (see below) 228 | # 229 | # In distributed training, the load_dataset function guarantee that only one local process can concurrently 230 | # download the dataset. 231 | if data_args.task_name is not None: 232 | # Downloading and loading a dataset from the hub. 233 | datasets = load_dataset("glue", data_args.task_name) 234 | if data_args.task_name == 'mnli': 235 | ax = load_dataset("glue", "ax") 236 | datasets["test_ax"] = ax["test"] 237 | elif data_args.train_file.endswith(".csv"): 238 | # Loading a dataset from local csv files 239 | datasets = load_dataset( 240 | "csv", data_files={"train": data_args.train_file, "validation": data_args.validation_file} 241 | ) 242 | else: 243 | # Loading a dataset from local json files 244 | datasets = load_dataset( 245 | "json", data_files={"train": data_args.train_file, "validation": data_args.validation_file} 246 | ) 247 | # See more about loading any type of standard or custom dataset at 248 | # https://huggingface.co/docs/datasets/loading_datasets.html. 249 | 250 | # Labels 251 | if data_args.task_name is not None: 252 | is_regression = data_args.task_name == "stsb" 253 | if not is_regression: 254 | label_list = datasets["train"].features["label"].names 255 | num_labels = len(label_list) 256 | else: 257 | num_labels = 1 258 | else: 259 | # Trying to have good defaults here, don't hesitate to tweak to your needs. 260 | is_regression = datasets["train"].features["label"].dtype in ["float32", "float64"] 261 | if is_regression: 262 | num_labels = 1 263 | else: 264 | # A useful fast method: 265 | # https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.unique 266 | label_list = datasets["train"].unique("label") 267 | label_list.sort() # Let's sort it for determinism 268 | num_labels = len(label_list) 269 | 270 | # Load pretrained model and tokenizer 271 | # 272 | # In distributed training, the .from_pretrained methods guarantee that only one local process can concurrently 273 | # download model & vocab. 274 | config = AutoConfig.from_pretrained( 275 | model_args.config_name if model_args.config_name else model_args.model_name_or_path, 276 | num_labels=num_labels, 277 | finetuning_task=data_args.task_name, 278 | cache_dir=model_args.cache_dir, 279 | ) 280 | tokenizer = AutoTokenizer.from_pretrained( 281 | model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path, 282 | do_lower_case=model_args.do_lower_case, 283 | cache_dir=model_args.cache_dir, 284 | use_fast=model_args.use_fast_tokenizer, 285 | ) 286 | model = AutoModelForSequenceClassification.from_pretrained( 287 | model_args.model_name_or_path, 288 | from_tf=bool(".ckpt" in model_args.model_name_or_path), 289 | config=config, 290 | cache_dir=model_args.cache_dir, 291 | ) 292 | 293 | # Preprocessing the datasets 294 | if data_args.task_name is not None: 295 | sentence1_key, sentence2_key = task_to_keys[data_args.task_name] 296 | else: 297 | # Again, we try to have some nice defaults but don't hesitate to tweak to your use case. 298 | non_label_column_names = [name for name in datasets["train"].column_names if name != "label"] 299 | if "sentence1" in non_label_column_names and "sentence2" in non_label_column_names: 300 | sentence1_key, sentence2_key = "sentence1", "sentence2" 301 | else: 302 | if len(non_label_column_names) >= 2: 303 | sentence1_key, sentence2_key = non_label_column_names[:2] 304 | else: 305 | sentence1_key, sentence2_key = non_label_column_names[0], None 306 | 307 | # Padding strategy 308 | if data_args.pad_to_max_length: 309 | padding = "max_length" 310 | max_length = data_args.max_seq_length 311 | else: 312 | # We will pad later, dynamically at batch creation, to the max sequence length in each batch 313 | padding = False 314 | max_length = None 315 | 316 | # Some models have set the order of the labels to use, so let's make sure we do use it. 317 | label_to_id = None 318 | if ( 319 | model.config.label2id != PretrainedConfig(num_labels=num_labels).label2id 320 | and data_args.task_name is not None 321 | and is_regression 322 | ): 323 | # Some have all caps in their config, some don't. 324 | label_name_to_id = {k.lower(): v for k, v in model.config.label2id.items()} 325 | if list(sorted(label_name_to_id.keys())) == list(sorted(label_list)): 326 | label_to_id = {i: label_name_to_id[label_list[i]] for i in range(num_labels)} 327 | else: 328 | logger.warn( 329 | "Your model seems to have been trained with labels, but they don't match the dataset: ", 330 | f"model labels: {list(sorted(label_name_to_id.keys()))}, dataset labels: {list(sorted(label_list))}." 331 | "\nIgnoring the model labels as a result.", 332 | ) 333 | elif data_args.task_name is None: 334 | label_to_id = {v: i for i, v in enumerate(label_list)} 335 | 336 | def preprocess_function(examples): 337 | # Tokenize the texts 338 | args = ( 339 | (examples[sentence1_key],) if sentence2_key is None else (examples[sentence1_key], examples[sentence2_key]) 340 | ) 341 | result = tokenizer(*args, padding=padding, max_length=max_length, truncation=True) 342 | 343 | # Map labels to IDs (not necessary for GLUE tasks) 344 | if label_to_id is not None and "label" in examples: 345 | result["label"] = [label_to_id[l] for l in examples["label"]] 346 | return result 347 | 348 | datasets = datasets.map(preprocess_function, batched=True, load_from_cache_file=not data_args.overwrite_cache) 349 | 350 | train_dataset = datasets["train"] 351 | eval_dataset = datasets["validation_matched" if data_args.task_name == "mnli" else "validation"] 352 | if data_args.task_name is not None: 353 | test_dataset = datasets["test_matched" if data_args.task_name == "mnli" else "test"] 354 | 355 | # Log a few random samples from the training set: 356 | for index in random.sample(range(len(train_dataset)), 3): 357 | logger.info(f"Sample {index} of the training set: {train_dataset[index]}.") 358 | 359 | # Get the metric function 360 | exp_id = f'exp-{time.time()}' 361 | if data_args.task_name is not None: 362 | metric = load_metric("glue", data_args.task_name, experiment_id=exp_id) 363 | 364 | # TODO: When datasets metrics include regular accuracy, make an else here and remove special branch from 365 | # compute_metrics 366 | 367 | # You can define your custom compute_metrics function. It takes an `EvalPrediction` object (a namedtuple with a 368 | # predictions and label_ids field) and has to return a dictionary string to float. 369 | def compute_metrics(p: EvalPrediction): 370 | preds = p.predictions[0] if isinstance(p.predictions, tuple) else p.predictions 371 | preds = np.squeeze(preds) if is_regression else np.argmax(preds, axis=1) 372 | if data_args.task_name is not None: 373 | result = metric.compute(predictions=preds, references=p.label_ids) 374 | if len(result) > 1: 375 | result["combined_score"] = np.mean(list(result.values())).item() 376 | return result 377 | elif is_regression: 378 | return {"mse": ((preds - p.label_ids) ** 2).mean().item()} 379 | else: 380 | return {"accuracy": (preds == p.label_ids).astype(np.float32).mean().item()} 381 | 382 | # Initialize our Trainer 383 | trainer = Trainer( 384 | model=model, 385 | args=training_args, 386 | train_dataset=train_dataset, 387 | eval_dataset=eval_dataset if training_args.do_eval else None, 388 | compute_metrics=compute_metrics, 389 | tokenizer=tokenizer, 390 | # Data collator will default to DataCollatorWithPadding, so we change it if we already did the padding. 391 | data_collator=default_data_collator if data_args.pad_to_max_length else None, 392 | ) 393 | 394 | # Training 395 | wrap_save_training(trainer) 396 | if training_args.do_train: 397 | trainer.train( 398 | model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path) else None 399 | ) 400 | trainer.save_model() # Saves the tokenizer too for easy upload 401 | 402 | # Evaluation 403 | eval_results = {} 404 | if training_args.do_eval: 405 | logger.info("*** Evaluate ***") 406 | 407 | # Loop to handle MNLI double evaluation (matched, mis-matched) 408 | tasks = [data_args.task_name] 409 | eval_datasets = [eval_dataset] 410 | if data_args.task_name == "mnli": 411 | tasks.append("mnli-mm") 412 | eval_datasets.append(datasets["validation_mismatched"]) 413 | 414 | for eval_dataset, task in zip(eval_datasets, tasks): 415 | eval_result = trainer.evaluate(eval_dataset=eval_dataset) 416 | 417 | output_eval_file = os.path.join(training_args.output_dir, f"eval_results_{task}.txt") 418 | if trainer.is_world_process_zero(): 419 | with open(output_eval_file, "w") as writer: 420 | logger.info(f"***** Eval results {task} *****") 421 | for key, value in eval_result.items(): 422 | logger.info(f" {key} = {value}") 423 | writer.write(f"{key} = {value}\n") 424 | 425 | eval_results.update(eval_result) 426 | 427 | if training_args.do_predict: 428 | logger.info("*** Test ***") 429 | 430 | # Loop to handle MNLI double evaluation (matched, mis-matched) 431 | tasks = [data_args.task_name] 432 | test_datasets = [test_dataset] 433 | if data_args.task_name == "mnli": 434 | tasks.append("mnli-mm") 435 | tasks.append("mnli-ax") 436 | test_datasets.append(datasets["test_mismatched"]) 437 | test_datasets.append(datasets["test_ax"]) 438 | 439 | for test_dataset, task in zip(test_datasets, tasks): 440 | # Removing the `label` columns because it contains -1 and Trainer won't like that. 441 | test_dataset.remove_columns_("label") 442 | predictions = trainer.predict(test_dataset=test_dataset).predictions 443 | predictions = np.squeeze(predictions) if is_regression else np.argmax(predictions, axis=1) 444 | 445 | output_test_file = os.path.join(training_args.output_dir, f"test_results_{task}.txt") 446 | if trainer.is_world_process_zero(): 447 | with open(output_test_file, "w") as writer: 448 | logger.info(f"***** Test results {task} *****") 449 | writer.write("index\tprediction\n") 450 | for index, item in enumerate(predictions): 451 | if is_regression: 452 | writer.write(f"{index}\t{item:3.3f}\n") 453 | else: 454 | item = label_list[item] 455 | writer.write(f"{index}\t{item}\n") 456 | return eval_results 457 | 458 | 459 | def _mp_fn(index): 460 | # For xla_spawn (TPUs) 461 | main() 462 | 463 | 464 | if __name__ == "__main__": 465 | main() 466 | -------------------------------------------------------------------------------- /matbert/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lbnlp/MatBERT/31592cf5a26c9730dd39585c38f470b9eba6f4ba/matbert/__init__.py -------------------------------------------------------------------------------- /matbert/training/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lbnlp/MatBERT/31592cf5a26c9730dd39585c38f470b9eba6f4ba/matbert/training/__init__.py -------------------------------------------------------------------------------- /matbert/training/configs/bert-base-cased-wd.json: -------------------------------------------------------------------------------- 1 | { 2 | "tokenized_lmdb_path": "data_2Mpapers_tokenized_cased_30522.lmdb", 3 | "tokenizer_path": "tokenizer_cased_30522", 4 | "cased": true, 5 | "output_dir": "model_2Mpapers_cased_30522", 6 | "bert_config": { 7 | "vocab_size": 30522, 8 | "hidden_size": 768, 9 | "num_hidden_layers": 12, 10 | "num_attention_heads": 12, 11 | "intermediate_size": 3072, 12 | "hidden_act": "gelu", 13 | "hidden_dropout_prob": 0.1, 14 | "attention_probs_dropout_prob": 0.1, 15 | "max_position_embeddings": 512, 16 | "type_vocab_size": 2, 17 | "initializer_range": 0.02, 18 | "layer_norm_eps": 1e-12, 19 | "pad_token_id": 0 20 | }, 21 | "load_checkpoint": true, 22 | "mlm_probability": 0.15, 23 | "per_device_train_batch_size": 12, 24 | "gradient_accumulation_steps": 2, 25 | "weight_decay": 0.01, 26 | "dataloader_drop_last": true, 27 | "num_train_epochs": 5, 28 | "save_total_limit": 4, 29 | "save_steps": 3500, 30 | "fp16": true, 31 | "fp16_opt_level": "O2", 32 | "seed": 42 33 | } -------------------------------------------------------------------------------- /matbert/training/configs/bert-base-cased.json: -------------------------------------------------------------------------------- 1 | { 2 | "tokenized_lmdb_path": "data_2Mpapers_tokenized_cased_30522.lmdb", 3 | "tokenizer_path": "tokenizer_cased_30522", 4 | "cased": true, 5 | "output_dir": "model_2Mpapers_cased_30522", 6 | "bert_config": { 7 | "vocab_size": 30522, 8 | "hidden_size": 768, 9 | "num_hidden_layers": 12, 10 | "num_attention_heads": 12, 11 | "intermediate_size": 3072, 12 | "hidden_act": "gelu", 13 | "hidden_dropout_prob": 0.1, 14 | "attention_probs_dropout_prob": 0.1, 15 | "max_position_embeddings": 512, 16 | "type_vocab_size": 2, 17 | "initializer_range": 0.02, 18 | "layer_norm_eps": 1e-12, 19 | "pad_token_id": 0 20 | }, 21 | "load_checkpoint": true, 22 | "mlm_probability": 0.15, 23 | "per_device_train_batch_size": 12, 24 | "gradient_accumulation_steps": 2, 25 | "weight_decay": 0.0, 26 | "dataloader_drop_last": true, 27 | "num_train_epochs": 5, 28 | "save_total_limit": 4, 29 | "save_steps": 3500, 30 | "fp16": true, 31 | "fp16_opt_level": "O2", 32 | "seed": 42 33 | } -------------------------------------------------------------------------------- /matbert/training/configs/bert-base-uncased-wd.json: -------------------------------------------------------------------------------- 1 | { 2 | "tokenized_lmdb_path": "data_2Mpapers_tokenized_uncased_30522.lmdb", 3 | "tokenizer_path": "tokenizer_uncased_30522", 4 | "cased": false, 5 | "output_dir": "model_2Mpapers_uncased_30522", 6 | "bert_config": { 7 | "vocab_size": 30522, 8 | "hidden_size": 768, 9 | "num_hidden_layers": 12, 10 | "num_attention_heads": 12, 11 | "intermediate_size": 3072, 12 | "hidden_act": "gelu", 13 | "hidden_dropout_prob": 0.1, 14 | "attention_probs_dropout_prob": 0.1, 15 | "max_position_embeddings": 512, 16 | "type_vocab_size": 2, 17 | "initializer_range": 0.02, 18 | "layer_norm_eps": 1e-12, 19 | "pad_token_id": 0 20 | }, 21 | "load_checkpoint": true, 22 | "mlm_probability": 0.15, 23 | "per_device_train_batch_size": 12, 24 | "gradient_accumulation_steps": 2, 25 | "weight_decay": 0.01, 26 | "dataloader_drop_last": true, 27 | "num_train_epochs": 5, 28 | "save_total_limit": 4, 29 | "save_steps": 3500, 30 | "fp16": true, 31 | "fp16_opt_level": "O2", 32 | "seed": 42 33 | } -------------------------------------------------------------------------------- /matbert/training/configs/bert-base-uncased.json: -------------------------------------------------------------------------------- 1 | { 2 | "tokenized_lmdb_path": "data_2Mpapers_tokenized_uncased_30522.lmdb", 3 | "tokenizer_path": "tokenizer_uncased_30522", 4 | "cased": false, 5 | "output_dir": "model_2Mpapers_uncased_30522", 6 | "bert_config": { 7 | "vocab_size": 30522, 8 | "hidden_size": 768, 9 | "num_hidden_layers": 12, 10 | "num_attention_heads": 12, 11 | "intermediate_size": 3072, 12 | "hidden_act": "gelu", 13 | "hidden_dropout_prob": 0.1, 14 | "attention_probs_dropout_prob": 0.1, 15 | "max_position_embeddings": 512, 16 | "type_vocab_size": 2, 17 | "initializer_range": 0.02, 18 | "layer_norm_eps": 1e-12, 19 | "pad_token_id": 0 20 | }, 21 | "load_checkpoint": true, 22 | "mlm_probability": 0.15, 23 | "per_device_train_batch_size": 12, 24 | "gradient_accumulation_steps": 2, 25 | "weight_decay": 0.0, 26 | "dataloader_drop_last": true, 27 | "num_train_epochs": 5, 28 | "save_total_limit": 4, 29 | "save_steps": 3500, 30 | "fp16": true, 31 | "fp16_opt_level": "O2", 32 | "seed": 42 33 | } -------------------------------------------------------------------------------- /matbert/training/dataset.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | from typing import List, Tuple 4 | 5 | import lmdb 6 | import numpy 7 | import torch 8 | from torch.utils.data import Dataset 9 | 10 | __author__ = 'Haoyan Huo' 11 | __email__ = 'haoyan.huo@lbl.gov' 12 | __maintainer__ = 'Haoyan Huo' 13 | 14 | __all__ = ['SynthesisParagraphsDataset'] 15 | 16 | 17 | class SynthesisParagraphsDataset(Dataset): 18 | def __init__(self, 19 | training_lmdb: str, 20 | skip: int = 0, 21 | min_tokens: int = 22, 22 | max_tokens: int = 512): 23 | """ 24 | Constructor. 25 | 26 | :param training_lmdb: Path to a tokenized paragraphs database. 27 | :param skip: How many samples to skip. 28 | :param min_tokens: Minimal number of tokens. 29 | :param max_tokens: Maximal number of tokens to keep. 30 | """ 31 | 32 | self.db_env = lmdb.open( 33 | training_lmdb, readonly=True, readahead=True, lock=False) 34 | self.db_txn = self.db_env.begin(buffers=True) 35 | 36 | self._skip = 0 37 | self.skip = skip 38 | self.min_tokens = min_tokens 39 | self.max_tokens = max_tokens 40 | 41 | meta_fn = os.path.join(training_lmdb, 'meta.txt') 42 | dois, token_counts = self._load_token_counts(meta_fn) 43 | self.dois: List[bytes] = dois 44 | self.token_counts: List[Tuple[int, bytes, int]] = token_counts 45 | 46 | dtype_fn = os.path.join(training_lmdb, 'dtype.txt') 47 | with open(dtype_fn) as f: 48 | self.dtype = numpy.dtype(f.read().strip()) 49 | 50 | @property 51 | def skip(self): 52 | return self._skip 53 | 54 | @skip.setter 55 | def skip(self, s: int): 56 | self._skip = s 57 | if self._skip: 58 | logging.info('Skipping %d items in the current epoch', self.skip) 59 | 60 | def _load_token_counts(self, meta_fn: str): 61 | """ 62 | Load token count stats from meta file. 63 | The meta file will have a format like this: 64 | 65 | 10.1000/some-doi (single TAB) 0:446,1:306,2:0,3:118,4:103,5:120,6:253,7:76,8:76 66 | """ 67 | dois = [] 68 | token_counts = [] 69 | with open(meta_fn, 'rb') as f: 70 | for i, line in enumerate(f): 71 | if not line.strip(): 72 | continue 73 | 74 | _doi, _token_counts = line.split(b'\t') 75 | 76 | # Re-order i 77 | i = len(dois) 78 | dois.append(_doi) 79 | 80 | for _token_count in _token_counts.split(b','): 81 | ip, count = _token_count.split(b':') 82 | count = int(count) 83 | if count >= self.min_tokens: 84 | token_counts.append((i, ip, count)) 85 | return dois, token_counts 86 | 87 | def __len__(self): 88 | return len(self.token_counts) 89 | 90 | zero_tensor = torch.tensor([0], dtype=torch.long) 91 | 92 | def __getitem__(self, i) -> torch.Tensor: 93 | if self._skip > 0: 94 | self._skip -= 1 95 | return self.zero_tensor 96 | 97 | doi_i, ip, count = self.token_counts[i] 98 | doi = self.dois[doi_i] 99 | paragraph_id = doi + b' ' + ip 100 | paragraph_tokens = self.db_txn.get(paragraph_id) 101 | 102 | paragraph_tokens_array = numpy.frombuffer(paragraph_tokens, dtype=self.dtype).astype(numpy.long) 103 | if paragraph_tokens_array.size > self.max_tokens: 104 | # Keep the last element because we need [SEP] 105 | paragraph_tokens_array = numpy.concatenate( 106 | (paragraph_tokens_array[:self.max_tokens - 1], paragraph_tokens_array[-1:]) 107 | ) 108 | return torch.tensor(paragraph_tokens_array, dtype=torch.long) 109 | -------------------------------------------------------------------------------- /matbert/training/options.py: -------------------------------------------------------------------------------- 1 | import json 2 | from dataclasses import dataclass, field 3 | from typing import Optional 4 | 5 | from transformers import HfArgumentParser 6 | 7 | __author__ = 'Haoyan Huo' 8 | __email__ = 'haoyan.huo@lbl.gov' 9 | __maintainer__ = 'Haoyan Huo' 10 | 11 | __all__ = [ 12 | 'DataOpts', 'ModelOpts', 'TrainingOpts', 'MiscOpts', 13 | 'BertOpts', 'GPT2Opts', 'parse_with_config' 14 | ] 15 | 16 | 17 | @dataclass 18 | class DataOpts: 19 | tokenizer_path: str = field( 20 | default=None, 21 | metadata={"help": "Path to a pretrained BERT tokenizer."} 22 | ) 23 | cased: bool = field( 24 | default=None, 25 | metadata={"help": "Whether the tokenizer is cased."} 26 | ) 27 | tokenized_lmdb_path: str = field( 28 | default=None, 29 | metadata={"help": "The training database containing tokenized paragraphs."} 30 | ) 31 | 32 | 33 | @dataclass 34 | class ModelOpts: 35 | output_dir: str = field( 36 | default=None, 37 | metadata={"help": "Model output dir."} 38 | ) 39 | logging_dir: Optional[str] = field( 40 | default=None, 41 | metadata={"help": "Logging dir, default will be the run-logs in output_dir."} 42 | ) 43 | load_checkpoint: bool = field( 44 | default=True, 45 | metadata={"help": "Load the latest checkpoint in output dir."} 46 | ) 47 | save_total_limit: Optional[int] = field( 48 | default=None, 49 | metadata={"help": "Number of intermediate models to keep. Default is to keep all."} 50 | ) 51 | save_steps: Optional[int] = field( 52 | default=10_000, 53 | metadata={"help": "Steps to save between checkpoints."} 54 | ) 55 | 56 | 57 | @dataclass 58 | class TrainingOpts: 59 | mlm_probability: float = field( 60 | default=0.15, 61 | metadata={"help": "Ratio of tokens to mask for masked language modeling loss."} 62 | ) 63 | 64 | per_device_train_batch_size: int = field( 65 | default=8, 66 | metadata={"help": "Batch size of each device."} 67 | ) 68 | num_train_epochs: int = field( 69 | default=5, 70 | metadata={"help": "Number of training epochs."} 71 | ) 72 | gradient_accumulation_steps: int = field( 73 | default=1, 74 | metadata={"help": "Number of steps between gradient accumulation steps."} 75 | ) 76 | dataloader_drop_last: bool = field( 77 | default=True, 78 | metadata={"help": "Whether to drop the last incomplete batch in training set."} 79 | ) 80 | weight_decay: float = field( 81 | default=0.0, 82 | metadata={"help": "Weight decay of Adam optimizer."} 83 | ) 84 | 85 | 86 | @dataclass 87 | class MiscOpts: 88 | config: Optional[str] = field( 89 | default=None, 90 | metadata={"help": "Filename to a config json file."} 91 | ) 92 | seed: int = field( 93 | default=42, 94 | metadata={"help": "Random seed."} 95 | ) 96 | fp16: bool = field( 97 | default=False, 98 | metadata={"help": "Use fp16 training."}, 99 | ) 100 | fp16_opt_level: str = field( 101 | default='O1', 102 | metadata={"help": "Level of AMP optimization."} 103 | ) 104 | local_rank: int = field( 105 | default=-1, 106 | metadata={"help": "Local rank in a distributed context."} 107 | ) 108 | 109 | 110 | @dataclass 111 | class BertOpts(ModelOpts, DataOpts, TrainingOpts, MiscOpts): 112 | bert_config: dict = field( 113 | default_factory=dict, 114 | metadata={"help": "BERT config. Use a json file to specify a dict."} 115 | ) 116 | 117 | 118 | @dataclass 119 | class GPT2Opts(ModelOpts, DataOpts, TrainingOpts, MiscOpts): 120 | gpt2_config: dict = field( 121 | default_factory=dict, 122 | metadata={"help": "GPT2 config. Use a json file to specify a dict."} 123 | ) 124 | 125 | 126 | def parse_with_config(cls: type = BertOpts): 127 | parser = HfArgumentParser(cls) 128 | opts = vars(parser.parse_args()) 129 | 130 | if opts['config'] is not None: 131 | with open(opts['config']) as f: 132 | config = json.load(f) 133 | for key, value in config.items(): 134 | opts[key] = value 135 | 136 | return cls(**opts) 137 | -------------------------------------------------------------------------------- /matbert/training/script_collect.py: -------------------------------------------------------------------------------- 1 | import html 2 | import os 3 | from argparse import ArgumentParser 4 | 5 | import lmdb 6 | from pymongo import MongoClient 7 | from tqdm import tqdm 8 | 9 | __author__ = 'Haoyan Huo' 10 | __email__ = 'haoyan.huo@lbl.gov' 11 | __maintainer__ = 'Haoyan Huo' 12 | 13 | __all__ = ['collect_paragraphs'] 14 | 15 | 16 | def collect_paragraphs( 17 | db_hostname: str, 18 | db_username: str, 19 | db_password: str, 20 | lmdb_path: str, 21 | num_papers: int, 22 | ): 23 | """ 24 | Collect papers from synthesisproject.lbl.gov and save them into LMDB database 25 | for later training. 26 | 27 | :param db_hostname: Hostname of the database. 28 | :param db_username: Username for authentication. 29 | :param db_password: Password for authentication. 30 | :param lmdb_path: Path to a LMDB database instance. 31 | :param num_papers: Number of papers to collect. 32 | :return: None 33 | """ 34 | assert not os.path.exists(lmdb_path), "Target LMDB path already exists! Delete it first." 35 | 36 | # Open connections to synthesisproject.lbl.gov 37 | # Full text and paragraphs are saved separately. 38 | print('Connecting to database...') 39 | full_text_db = MongoClient(db_hostname).FullText 40 | full_text_db.authenticate(db_username, db_password) 41 | print('Connected to FullText database.') 42 | paper_meta = full_text_db.Paper_Metadata 43 | prod_db = MongoClient(db_hostname).SynPro 44 | prod_db.authenticate(db_username, db_password) 45 | all_paragraphs = prod_db.Paragraphs 46 | print('Connected to SynPro database.') 47 | 48 | # Retrieve 1.5 more papers just in case there are empty papers 49 | num_req_papers = int(num_papers * 1.5) 50 | print(f'Requesting {num_req_papers} paper metadata...') 51 | papers = list(paper_meta.aggregate( 52 | [{'$sample': {'size': num_req_papers}}], allowDiskUse=True 53 | )) 54 | 55 | # For safety, just map 100GB (this is the current upper bound of the size of the 56 | # synthesisproject text database. 57 | env = lmdb.open( 58 | lmdb_path, 59 | readonly=False, map_size=1024 * 1024 * 1024 * 100, lock=True) 60 | txn = env.begin(buffers=True, write=True) 61 | 62 | with tqdm(total=num_papers, desc='Downloading papers') as bar, \ 63 | open(os.path.join(lmdb_path, 'meta.txt'), 'w') as meta_f: 64 | for paper in papers: 65 | # We don't filter bad paragraphs (e.g. equations) since this 66 | # should be preferably done by training programs. 67 | final_paragraphs = [] 68 | for paragraph in all_paragraphs.find({'DOI': paper['DOI']}): 69 | # If multiple paragraphs accidentally are in one string, we treat 70 | # them as a single paragraph. 71 | paragraph = html.unescape(paragraph['text']).replace('\n', ' ').strip() 72 | final_paragraphs.append(paragraph) 73 | 74 | # Only collect papers that have at least one paragraph! 75 | if not len(final_paragraphs): 76 | continue 77 | 78 | # In LMDB, entries are stored as (b"10.0000/test:10", b"Test paragraph...") 79 | for i, paragraph in enumerate(final_paragraphs): 80 | key = f'{paper["DOI"]} {i}' 81 | content = paragraph.encode('utf8', errors='ignore') 82 | txn.put(key.encode('utf8'), content) 83 | 84 | # Keep a reference in the meta file since LMDB is unordered. 85 | meta_f.write('%s\t%d\n' % (paper['DOI'], len(final_paragraphs))) 86 | 87 | bar.update(1) 88 | # We actually collected more than enough papers. Here we should exit 89 | # early when the number of papers is more than or equal to what is desired. 90 | if bar.n >= num_papers: 91 | break 92 | 93 | txn.commit() 94 | env.close() 95 | 96 | 97 | def _main(): 98 | parser = ArgumentParser(description='Collect paragraphs for training MatBERT.') 99 | 100 | parser.add_argument('--db_hostname', '-host', type=str, required=True, 101 | help='Hostname of synthesisproject.lbl.gov') 102 | parser.add_argument('--db_username', '-user', type=str, required=True, 103 | help='Username of your credentials for accessing the database.') 104 | parser.add_argument('--db_password', '-pass', type=str, required=True, 105 | help='Password of your credentials for accessing the database.') 106 | parser.add_argument('--lmdb_path', '-lmdb', type=str, required=True, 107 | help='Destination folder for storing LMDB database.') 108 | parser.add_argument('--num_papers', '-num', type=int, default=2 * 1000 * 1000, 109 | help='How many papers to collect from the database.') 110 | 111 | args = parser.parse_args() 112 | collect_paragraphs( 113 | args.db_hostname, args.db_username, args.db_password, 114 | args.lmdb_path, args.num_papers) 115 | 116 | 117 | if __name__ == '__main__': 118 | _main() 119 | -------------------------------------------------------------------------------- /matbert/training/script_tokenize_lmdb.py: -------------------------------------------------------------------------------- 1 | import os 2 | from argparse import ArgumentParser 3 | from collections import defaultdict 4 | from multiprocessing import Semaphore, cpu_count, Process, Queue 5 | 6 | import lmdb 7 | import numpy 8 | from tqdm import tqdm 9 | from transformers import BertTokenizerFast 10 | 11 | __author__ = 'Haoyan Huo' 12 | __email__ = 'haoyan.huo@lbl.gov' 13 | __maintainer__ = 'Haoyan Huo' 14 | 15 | __all__ = ['tokenize_lmdb'] 16 | 17 | 18 | def _tokenize_subprocess(tokenizer: BertTokenizerFast, semaphore: Semaphore, 19 | documents_queue: Queue, writer_queue: Queue): 20 | """ 21 | Tokenize paragraphs in a subprocess. 22 | 23 | :param tokenizer: A bert tokenizer. 24 | :param semaphore: A semaphore to control the throttle of source documents. 25 | :param documents_queue: A queue from which documents are fetched. 26 | :param writer_queue: A queue to which tokenized documents are written. 27 | :return: 28 | """ 29 | while True: 30 | item = documents_queue.get() 31 | semaphore.release() 32 | if item is None: 33 | break 34 | 35 | key, paragraph = item 36 | tokens = tokenizer.tokenize(paragraph.decode('utf8'), add_special_tokens=True) 37 | token_ids = numpy.array(tokenizer.convert_tokens_to_ids(tokens)) 38 | 39 | writer_queue.put((key, token_ids)) 40 | 41 | 42 | def _write_db_subprocess(tokenized_lmdb_path: str, writer_queue: Queue, dtype: numpy.dtype): 43 | dst_env = lmdb.open( 44 | tokenized_lmdb_path, readonly=False, lock=True, map_size=1024 * 1024 * 1024 * 100) 45 | dst_txn = dst_env.begin(buffers=True, write=True) 46 | 47 | dst_meta = os.path.join(tokenized_lmdb_path, 'meta.txt') 48 | dtype_meta = os.path.join(tokenized_lmdb_path, 'dtype.txt') 49 | 50 | with open(dtype_meta, 'w') as f: 51 | f.write(str(dtype)) 52 | 53 | meta_maps = defaultdict(dict) 54 | 55 | while True: 56 | # Main process sends None to indicate EOF. 57 | item = writer_queue.get() 58 | if item is None: 59 | break 60 | 61 | # Using a consistent dtype long so that trainer can directly use the 62 | # mapped memory to create arrays. 63 | key, token_ids = item 64 | doi, ip = key.split(b' ') 65 | 66 | dst_txn.put(key, token_ids.astype(dtype).tobytes()) 67 | 68 | # Minus 2 since we have [CLS] and [SEP]. 69 | meta_maps[doi][int(ip)] = token_ids.size - 2 70 | 71 | dst_txn.commit() 72 | dst_env.close() 73 | 74 | with open(dst_meta, 'wb') as dst_meta_f: 75 | for doi, token_counts in meta_maps.items(): 76 | token_count_s = b','.join( 77 | f'{ip}:{c}'.encode('utf8') 78 | for ip, c in sorted(token_counts.items())) 79 | dst_meta_f.write(doi) 80 | dst_meta_f.write(b'\t') 81 | dst_meta_f.write(token_count_s) 82 | dst_meta_f.write(b'\n') 83 | 84 | 85 | def tokenize_lmdb( 86 | lmdb_path: str, 87 | tokenized_lmdb_path: str, 88 | bert_tokenizer: str, 89 | cased: bool, 90 | processes: int = cpu_count(), 91 | dtype: str = 'uint16' 92 | ): 93 | """ 94 | Tokenize all paragraphs in a LMDB database. 95 | 96 | :param lmdb_path: Source folder of plain text paragraphs. 97 | :param tokenized_lmdb_path: Destination folder of tokenized paragraphs. 98 | :param bert_tokenizer: Folder that contains a bert tokenizer. 99 | :param cased: Whether the tokenizer is cased. 100 | :param processes: Number of processes to use. 101 | :param dtype: What dtype to use in database (affects database size). 102 | :return: 103 | """ 104 | tokenizer = BertTokenizerFast.from_pretrained( 105 | bert_tokenizer, do_lower_case=not cased) 106 | 107 | dtype = numpy.dtype(dtype) 108 | 109 | assert tokenizer.vocab_size < numpy.iinfo(dtype).max, \ 110 | f"Vocabulary size is greater than the maximum of dtyle {dtype}" 111 | 112 | src_env = lmdb.open( 113 | lmdb_path, readonly=True, lock=False) 114 | src_txn = src_env.begin(buffers=True) 115 | 116 | semaphore = Semaphore(4096) 117 | tokenized_queue = Queue() 118 | 119 | def _paragraph_generator(): 120 | for key, value in iter(src_txn.cursor()): 121 | # If queue insertion is too fast, we get throttled. 122 | semaphore.acquire() 123 | 124 | yield key.tobytes(), value.tobytes() 125 | 126 | # Create database writer. 127 | db_writer = Process( 128 | target=_write_db_subprocess, 129 | args=(tokenized_lmdb_path, tokenized_queue, dtype)) 130 | db_writer.start() 131 | 132 | # Create workers. 133 | document_queues = [Queue() for _ in range(processes)] 134 | workers = [Process( 135 | target=_tokenize_subprocess, 136 | args=(tokenizer, semaphore, document_queues[i], tokenized_queue)) for i in range(processes)] 137 | [i.start() for i in workers] 138 | 139 | # Distribute tasks. 140 | for i, item in enumerate(tqdm(_paragraph_generator(), desc='Tokenizing paragraphs')): 141 | document_queues[i % len(document_queues)].put(item) 142 | 143 | print('Notifying workers EOF...') 144 | for queue in document_queues: 145 | queue.put(None) 146 | 147 | # Wait for workers to finish 148 | for i, worker in enumerate(workers): 149 | print(f'Waiting for worker {i} to finish...') 150 | worker.join() 151 | 152 | print('Notifying database writer EOF...') 153 | tokenized_queue.put(None) 154 | 155 | # Wait for database write to finish 156 | print('Waiting for database writer to finish...') 157 | db_writer.join() 158 | 159 | 160 | def _main(): 161 | parser = ArgumentParser(description='Tokenize paragraphs in a LMDB paragraphs database.') 162 | 163 | parser.add_argument('--lmdb_path', '-input', type=str, required=True, 164 | help='Source folder of the LMDB database.') 165 | parser.add_argument('--tokenized_lmdb_path', '-output', type=str, required=True, 166 | help='Source folder of the LMDB database.') 167 | parser.add_argument('--tokenizer_path', '-tokenizer', type=str, required=True, 168 | help='Folder that contains a BERT tokenizer.') 169 | parser.add_argument('--cased', '-cased', action='store_true', 170 | help='Tokenizer should be case-sensitive.') 171 | parser.add_argument('--processes', '-p', type=int, default=cpu_count(), 172 | help='Tokenizer should be case-sensitive.') 173 | parser.add_argument('--dtype', '-dtype', type=str, default='uint16', 174 | help='Dtype of the stored numpy arrays.') 175 | 176 | args = parser.parse_args() 177 | 178 | assert not os.path.exists(args.tokenized_lmdb_path), f"Output dir {args.tokenized_lmdb_path} already exists!" 179 | 180 | tokenize_lmdb( 181 | lmdb_path=args.lmdb_path, 182 | tokenized_lmdb_path=args.tokenized_lmdb_path, 183 | bert_tokenizer=args.tokenizer_path, 184 | processes=args.processes, 185 | cased=args.cased, 186 | dtype=args.dtype, 187 | ) 188 | 189 | 190 | if __name__ == '__main__': 191 | _main() 192 | -------------------------------------------------------------------------------- /matbert/training/script_train_bert.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import math 3 | import os 4 | from pathlib import Path 5 | from typing import Tuple 6 | 7 | import torch.distributed as dist 8 | from transformers import ( 9 | BertConfig, BertTokenizerFast, BertForMaskedLM, 10 | DataCollatorForLanguageModeling) 11 | from transformers import Trainer, TrainingArguments 12 | 13 | from matbert.training.dataset import SynthesisParagraphsDataset 14 | from matbert.training.options import parse_with_config, BertOpts 15 | 16 | __author__ = 'Haoyan Huo' 17 | __email__ = 'haoyan.huo@lbl.gov' 18 | __maintainer__ = 'Haoyan Huo' 19 | 20 | logger = logging.getLogger(__name__) 21 | logging.basicConfig( 22 | format="%(asctime)s [%(levelname)s][%(name)s]: %(message)s", 23 | datefmt="%m/%d %H:%M:%S", 24 | level=logging.INFO, 25 | ) 26 | 27 | 28 | class MyTrainer(Trainer): 29 | # Hack function used to save models at the end of every epoch. 30 | def evaluate(self, eval_dataset=None): 31 | if hasattr(self, '_last_epoch'): 32 | if math.floor(self.epoch) != math.floor(self._last_epoch): 33 | save_path = os.path.join(self.args.output_dir, 'epoch_save_%d' % int(math.floor(self._last_epoch))) 34 | self.save_model(save_path) 35 | 36 | setattr(self, '_last_epoch', self.epoch) 37 | 38 | 39 | def prepare_model(opts: BertOpts) -> Tuple[BertForMaskedLM, str, int]: 40 | config = BertConfig(**opts.bert_config) 41 | 42 | checkpoints = list(map(str, Path(opts.output_dir).glob("checkpoint-*"))) 43 | 44 | if opts.load_checkpoint and len(checkpoints) > 0: 45 | checkpoint_steps = [int(x.rsplit('-')[1]) for x in checkpoints] 46 | latest_global_step = max(checkpoint_steps) 47 | checkpoint_dir = os.path.join( 48 | opts.output_dir, 'checkpoint-%d' % latest_global_step) 49 | model = BertForMaskedLM.from_pretrained(checkpoint_dir, config=config) 50 | else: 51 | checkpoint_dir = None 52 | latest_global_step = 0 53 | model = BertForMaskedLM(config=config) 54 | 55 | return model, checkpoint_dir, latest_global_step 56 | 57 | 58 | def main(): 59 | opts = parse_with_config() 60 | 61 | assert os.path.exists(opts.tokenizer_path), \ 62 | "Tokenizer (--tokenizer_path) not specified or does not exist!" 63 | assert os.path.exists(opts.tokenized_lmdb_path), \ 64 | "Training data (--tokenized_lmdb_path) not specified or does not exist!" 65 | assert opts.output_dir is not None, \ 66 | "Output dir (--output_dir) is None!" 67 | 68 | tokenizer = BertTokenizerFast.from_pretrained( 69 | opts.tokenizer_path, do_lower_case=not opts.cased) 70 | 71 | trainer_arguments = TrainingArguments( 72 | output_dir=opts.output_dir, 73 | overwrite_output_dir=True, 74 | num_train_epochs=opts.num_train_epochs, 75 | gradient_accumulation_steps=opts.gradient_accumulation_steps, 76 | per_device_train_batch_size=opts.per_device_train_batch_size, 77 | weight_decay=opts.weight_decay, 78 | dataloader_drop_last=opts.dataloader_drop_last, 79 | save_steps=opts.save_steps, 80 | save_total_limit=opts.save_total_limit, 81 | logging_dir=opts.logging_dir or os.path.join(opts.output_dir, 'run-logs'), 82 | dataloader_num_workers=4, 83 | fp16=opts.fp16, 84 | fp16_opt_level=opts.fp16_opt_level, 85 | seed=opts.seed, 86 | local_rank=opts.local_rank, 87 | 88 | # A nice hack for storing models every epoch. See below 89 | evaluate_during_training=True, 90 | eval_steps=1, 91 | ) 92 | 93 | model, checkpoint_dir, latest_global_step = prepare_model(opts) 94 | logger.info('Loaded model from %s', checkpoint_dir) 95 | 96 | dataset = SynthesisParagraphsDataset( 97 | training_lmdb=opts.tokenized_lmdb_path, 98 | max_tokens=model.config.max_position_embeddings 99 | ) 100 | data_collator = DataCollatorForLanguageModeling( 101 | tokenizer=tokenizer, 102 | mlm=True, 103 | mlm_probability=opts.mlm_probability, 104 | ) 105 | 106 | # Recover how many samples to skip 107 | # Need to call trainer_arguments.device to setup pytorch. 108 | _ = trainer_arguments.device 109 | if opts.local_rank > -1: 110 | num_replicas = dist.get_world_size() 111 | local_data_size = int(math.ceil(len(dataset) * 1.0 / num_replicas)) 112 | else: 113 | local_data_size = len(dataset) 114 | local_batches = local_data_size // opts.per_device_train_batch_size 115 | steps_trained_in_current_epoch = latest_global_step % ( 116 | local_batches // opts.gradient_accumulation_steps 117 | ) 118 | dataset.skip = steps_trained_in_current_epoch * \ 119 | opts.gradient_accumulation_steps * \ 120 | opts.per_device_train_batch_size 121 | 122 | trainer = MyTrainer( 123 | model=model, 124 | args=trainer_arguments, 125 | data_collator=data_collator, 126 | train_dataset=dataset, 127 | prediction_loss_only=True, 128 | ) 129 | 130 | trainer.train(checkpoint_dir) 131 | trainer.save_model(opts.output_dir) 132 | 133 | 134 | if __name__ == '__main__': 135 | main() 136 | -------------------------------------------------------------------------------- /matbert/training/script_train_tokenizer_bert.py: -------------------------------------------------------------------------------- 1 | import os 2 | from argparse import ArgumentParser 3 | 4 | import lmdb 5 | from tokenizers import BertWordPieceTokenizer 6 | from tqdm import tqdm 7 | 8 | __author__ = 'Haoyan Huo' 9 | __email__ = 'haoyan.huo@lbl.gov' 10 | __maintainer__ = 'Haoyan Huo' 11 | 12 | __all__ = ['prepare_for_tokenizer'] 13 | 14 | 15 | def prepare_for_tokenizer( 16 | lmdb_path: str, 17 | ): 18 | """ 19 | Convert a LMDB-stored paragraphs set into a plain text for use with tokenizer. 20 | The plain text file will be removed if any exception occurs while creating it. 21 | """ 22 | pt_filename = os.path.join(lmdb_path, 'all_paragraphs.txt') 23 | assert not os.path.exists(pt_filename), f"Plain text file {pt_filename} already exists! Delete it first." 24 | 25 | env = lmdb.open( 26 | lmdb_path, readonly=True, lock=False) 27 | txn = env.begin(buffers=True) 28 | 29 | try: 30 | with tqdm(desc='Dumping paragraphs', unit='B', unit_scale=True) as bar, \ 31 | open(pt_filename, 'wb') as f: 32 | for key, value in iter(txn.cursor()): 33 | f.write(value) 34 | f.write(b'\n') 35 | bar.update(len(value) + 1) 36 | env.close() 37 | except: 38 | if os.path.exists(pt_filename): 39 | os.remove(pt_filename) 40 | raise 41 | 42 | return pt_filename 43 | 44 | 45 | def _main(): 46 | parser = ArgumentParser(description='Train a MatBERT tokenizer.') 47 | 48 | parser.add_argument('--lmdb_path', '-lmdb', type=str, required=True, 49 | help='Source folder of the LMDB database.') 50 | parser.add_argument('--dont_remove_file', '-no_remove', action='store_true', 51 | help='Remove the temporary file containing all paragraphs.') 52 | parser.add_argument('--save_dir', '-save', type=str, required=True, 53 | help='Destination folder of the tokenizer.') 54 | 55 | parser.add_argument('--vocab_size', '-size', type=int, default=30_522, 56 | help='Size of the vocabulary.') 57 | parser.add_argument('--cased', '-cased', action='store_true', 58 | help='Tokenizer should be case-sensitive.') 59 | 60 | args = parser.parse_args() 61 | 62 | assert not os.path.exists(args.save_dir), f"Output dir {args.save_dir} already exists!" 63 | os.makedirs(args.save_dir) 64 | 65 | pt_filename = prepare_for_tokenizer(args.lmdb_path) 66 | try: 67 | print(f'Creating new tokenizer to "{os.path.realpath(args.save_dir)}". ' 68 | f'Configs: cased={args.cased}, vocab_size={args.vocab_size}') 69 | tokenizer = BertWordPieceTokenizer(lowercase=not args.cased) 70 | tokenizer.train(files=pt_filename, vocab_size=args.vocab_size) 71 | tokenizer.save_model(args.save_dir) 72 | finally: 73 | if not args.dont_remove_file: 74 | os.remove(pt_filename) 75 | 76 | 77 | if __name__ == '__main__': 78 | _main() 79 | -------------------------------------------------------------------------------- /matbert/training/script_train_tokenizer_gpt2.py: -------------------------------------------------------------------------------- 1 | import os 2 | from argparse import ArgumentParser 3 | 4 | from tokenizers import ByteLevelBPETokenizer 5 | 6 | __author__ = 'Haoyan Huo' 7 | __email__ = 'haoyan.huo@lbl.gov' 8 | __maintainer__ = 'Haoyan Huo' 9 | 10 | from matbert.training.script_train_tokenizer_bert import prepare_for_tokenizer 11 | 12 | 13 | def _main(): 14 | parser = ArgumentParser(description='Train a MatGPT2 tokenizer.') 15 | 16 | parser.add_argument('--lmdb_path', '-lmdb', type=str, required=True, 17 | help='Source folder of the LMDB database.') 18 | parser.add_argument('--dont_remove_file', '-no_remove', action='store_true', 19 | help='Do not remove the temporary file containing all paragraphs.') 20 | parser.add_argument('--save_dir', '-save', type=str, required=True, 21 | help='Destination folder of the tokenizer.') 22 | 23 | parser.add_argument('--vocab_size', '-size', type=int, default=50_257, 24 | help='Size of the vocabulary.') 25 | parser.add_argument('--cased', '-cased', action='store_true', 26 | help='Tokenizer should be case-sensitive.') 27 | 28 | args = parser.parse_args() 29 | 30 | assert not os.path.exists(args.save_dir), f"Output dir {args.save_dir} already exists!" 31 | os.makedirs(args.save_dir) 32 | 33 | pt_filename = prepare_for_tokenizer(args.lmdb_path) 34 | try: 35 | print(f'Creating new tokenizer to "{os.path.realpath(args.save_dir)}". ' 36 | f'Configs: cased={args.cased}, vocab_size={args.vocab_size}') 37 | tokenizer = ByteLevelBPETokenizer(lowercase=not args.cased) 38 | tokenizer.train(files=pt_filename, vocab_size=args.vocab_size, special_tokens=['<|endoftext|>']) 39 | tokenizer.save_model(args.save_dir) 40 | finally: 41 | if not args.dont_remove_file: 42 | os.remove(pt_filename) 43 | 44 | 45 | if __name__ == '__main__': 46 | _main() 47 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pymongo 2 | torch 3 | tqdm 4 | transformers -------------------------------------------------------------------------------- /requirements_training.txt: -------------------------------------------------------------------------------- 1 | lmdb 2 | numpy 3 | pymongo 4 | torch 5 | tqdm 6 | transformers==3.3.1 -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from setuptools import setup, find_packages 4 | 5 | cur_dir = os.path.realpath(os.path.dirname(__file__)) 6 | 7 | if __name__ == "__main__": 8 | setup( 9 | name='MatBERT', 10 | version="0.0.1", 11 | python_requires='>=3.6', 12 | author="Haoyan Huo", 13 | author_email="haoyan.huo@lbl.gov", 14 | license="MIT License", 15 | packages=find_packages(), 16 | zip_safe=False, 17 | install_requires=open(os.path.join(cur_dir, 'requirements.txt')).readlines() 18 | ) 19 | --------------------------------------------------------------------------------