├── .gitignore ├── LICENSE.TXT ├── README.md ├── docs ├── dataprocess.md ├── images │ ├── bg.png │ ├── code_math.png │ ├── long_context.png │ └── multilingual.png └── trainer.md ├── examples ├── continuous_pretraining.ipynb └── tokenizer │ └── tokenizer.model ├── optimus ├── __init__.py ├── dataprocess │ ├── README.md │ ├── __init__.py │ ├── dataset │ │ ├── IndustryCorpus.py │ │ ├── __init__.py │ │ ├── agentInstruct.py │ │ ├── ayaInstruct.py │ │ ├── codeBagel.py │ │ ├── culturaX.py │ │ ├── finemath.py │ │ ├── fineweb-edu-2.py │ │ ├── fineweb.py │ │ ├── industryCorpus2.py │ │ ├── languageFiltered.py │ │ ├── long_alpaca.py │ │ ├── openPerfectblend.py │ │ ├── orca_agentinstruct.py │ │ ├── parallel.py │ │ ├── proof-pile-2.py │ │ ├── smolTalk.py │ │ ├── the-stack.py │ │ └── wikipedia.py │ ├── inspect_dataset.py │ ├── pack_dataset.py │ ├── subsample_dataset.py │ └── tokenize_dataset.py ├── train.py └── trainer │ ├── __init__.py │ ├── configuration │ ├── __init__.py │ ├── configs.py │ ├── dataset.py │ ├── distributed.py │ ├── model.py │ ├── system.py │ └── train.py │ ├── data.py │ ├── distributed.py │ ├── model │ ├── __init__.py │ ├── encoder │ │ ├── __init__.py │ │ ├── bert.py │ │ └── eurobert.py │ ├── load.py │ ├── model.py │ └── tools.py │ ├── pretrain.py │ ├── profile_model.py │ ├── script │ ├── cache.py │ └── warmup_stable_decay_lr.py │ └── trainer.md └── pyproject.toml /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE.TXT: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/LICENSE.TXT -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/README.md -------------------------------------------------------------------------------- /docs/dataprocess.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/docs/dataprocess.md -------------------------------------------------------------------------------- /docs/images/bg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/docs/images/bg.png -------------------------------------------------------------------------------- /docs/images/code_math.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/docs/images/code_math.png -------------------------------------------------------------------------------- /docs/images/long_context.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/docs/images/long_context.png -------------------------------------------------------------------------------- /docs/images/multilingual.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/docs/images/multilingual.png -------------------------------------------------------------------------------- /docs/trainer.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/docs/trainer.md -------------------------------------------------------------------------------- /examples/continuous_pretraining.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/examples/continuous_pretraining.ipynb -------------------------------------------------------------------------------- /examples/tokenizer/tokenizer.model: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/examples/tokenizer/tokenizer.model -------------------------------------------------------------------------------- /optimus/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /optimus/dataprocess/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/dataprocess/README.md -------------------------------------------------------------------------------- /optimus/dataprocess/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/dataprocess/__init__.py -------------------------------------------------------------------------------- /optimus/dataprocess/dataset/IndustryCorpus.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/dataprocess/dataset/IndustryCorpus.py -------------------------------------------------------------------------------- /optimus/dataprocess/dataset/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /optimus/dataprocess/dataset/agentInstruct.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/dataprocess/dataset/agentInstruct.py -------------------------------------------------------------------------------- /optimus/dataprocess/dataset/ayaInstruct.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/dataprocess/dataset/ayaInstruct.py -------------------------------------------------------------------------------- /optimus/dataprocess/dataset/codeBagel.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/dataprocess/dataset/codeBagel.py -------------------------------------------------------------------------------- /optimus/dataprocess/dataset/culturaX.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/dataprocess/dataset/culturaX.py -------------------------------------------------------------------------------- /optimus/dataprocess/dataset/finemath.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/dataprocess/dataset/finemath.py -------------------------------------------------------------------------------- /optimus/dataprocess/dataset/fineweb-edu-2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/dataprocess/dataset/fineweb-edu-2.py -------------------------------------------------------------------------------- /optimus/dataprocess/dataset/fineweb.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/dataprocess/dataset/fineweb.py -------------------------------------------------------------------------------- /optimus/dataprocess/dataset/industryCorpus2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/dataprocess/dataset/industryCorpus2.py -------------------------------------------------------------------------------- /optimus/dataprocess/dataset/languageFiltered.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/dataprocess/dataset/languageFiltered.py -------------------------------------------------------------------------------- /optimus/dataprocess/dataset/long_alpaca.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/dataprocess/dataset/long_alpaca.py -------------------------------------------------------------------------------- /optimus/dataprocess/dataset/openPerfectblend.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/dataprocess/dataset/openPerfectblend.py -------------------------------------------------------------------------------- /optimus/dataprocess/dataset/orca_agentinstruct.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/dataprocess/dataset/orca_agentinstruct.py -------------------------------------------------------------------------------- /optimus/dataprocess/dataset/parallel.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/dataprocess/dataset/parallel.py -------------------------------------------------------------------------------- /optimus/dataprocess/dataset/proof-pile-2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/dataprocess/dataset/proof-pile-2.py -------------------------------------------------------------------------------- /optimus/dataprocess/dataset/smolTalk.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/dataprocess/dataset/smolTalk.py -------------------------------------------------------------------------------- /optimus/dataprocess/dataset/the-stack.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/dataprocess/dataset/the-stack.py -------------------------------------------------------------------------------- /optimus/dataprocess/dataset/wikipedia.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/dataprocess/dataset/wikipedia.py -------------------------------------------------------------------------------- /optimus/dataprocess/inspect_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/dataprocess/inspect_dataset.py -------------------------------------------------------------------------------- /optimus/dataprocess/pack_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/dataprocess/pack_dataset.py -------------------------------------------------------------------------------- /optimus/dataprocess/subsample_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/dataprocess/subsample_dataset.py -------------------------------------------------------------------------------- /optimus/dataprocess/tokenize_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/dataprocess/tokenize_dataset.py -------------------------------------------------------------------------------- /optimus/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/train.py -------------------------------------------------------------------------------- /optimus/trainer/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /optimus/trainer/configuration/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /optimus/trainer/configuration/configs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/trainer/configuration/configs.py -------------------------------------------------------------------------------- /optimus/trainer/configuration/dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/trainer/configuration/dataset.py -------------------------------------------------------------------------------- /optimus/trainer/configuration/distributed.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/trainer/configuration/distributed.py -------------------------------------------------------------------------------- /optimus/trainer/configuration/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/trainer/configuration/model.py -------------------------------------------------------------------------------- /optimus/trainer/configuration/system.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/trainer/configuration/system.py -------------------------------------------------------------------------------- /optimus/trainer/configuration/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/trainer/configuration/train.py -------------------------------------------------------------------------------- /optimus/trainer/data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/trainer/data.py -------------------------------------------------------------------------------- /optimus/trainer/distributed.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/trainer/distributed.py -------------------------------------------------------------------------------- /optimus/trainer/model/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /optimus/trainer/model/encoder/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /optimus/trainer/model/encoder/bert.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/trainer/model/encoder/bert.py -------------------------------------------------------------------------------- /optimus/trainer/model/encoder/eurobert.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/trainer/model/encoder/eurobert.py -------------------------------------------------------------------------------- /optimus/trainer/model/load.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/trainer/model/load.py -------------------------------------------------------------------------------- /optimus/trainer/model/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/trainer/model/model.py -------------------------------------------------------------------------------- /optimus/trainer/model/tools.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/trainer/model/tools.py -------------------------------------------------------------------------------- /optimus/trainer/pretrain.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/trainer/pretrain.py -------------------------------------------------------------------------------- /optimus/trainer/profile_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/trainer/profile_model.py -------------------------------------------------------------------------------- /optimus/trainer/script/cache.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/trainer/script/cache.py -------------------------------------------------------------------------------- /optimus/trainer/script/warmup_stable_decay_lr.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/trainer/script/warmup_stable_decay_lr.py -------------------------------------------------------------------------------- /optimus/trainer/trainer.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/optimus/trainer/trainer.md -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nicolas-BZRD/EuroBERT/HEAD/pyproject.toml --------------------------------------------------------------------------------