├── .gitignore ├── LICENSE ├── README.md ├── accelerate_configs ├── 1_gpu.yaml ├── 1_node_8_gpus_deepspeed_zero1.yaml ├── 1_node_8_gpus_deepspeed_zero2.yaml ├── 1_node_8_gpus_deepspeed_zero3.yaml ├── 1_node_only.yaml └── 8_node_8_gpus_deepspeed_zero2.yaml ├── configs └── llada_pretraining.yaml ├── lm_chat_validation ├── description.txt └── questions.jsonl ├── models ├── __init__.py ├── common_modules.py ├── configuration_llada.py ├── logging.py ├── lr_schedulers.py ├── misc.py ├── modeling_llada.py ├── modeling_magvitv2.py ├── modeling_utils.py ├── sampling.py └── training_utils.py ├── parquet ├── __init__.py └── my_dataset.py ├── reasoning └── test.py ├── requirements.txt ├── scripts └── train.sh └── training ├── __init__.py ├── data.py ├── optimizer.py ├── prompting_utils.py ├── train_llada.py └── utils.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Auraithm/LLADA_pretraining/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Auraithm/LLADA_pretraining/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Auraithm/LLADA_pretraining/HEAD/README.md -------------------------------------------------------------------------------- /accelerate_configs/1_gpu.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Auraithm/LLADA_pretraining/HEAD/accelerate_configs/1_gpu.yaml -------------------------------------------------------------------------------- /accelerate_configs/1_node_8_gpus_deepspeed_zero1.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Auraithm/LLADA_pretraining/HEAD/accelerate_configs/1_node_8_gpus_deepspeed_zero1.yaml -------------------------------------------------------------------------------- /accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Auraithm/LLADA_pretraining/HEAD/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml -------------------------------------------------------------------------------- /accelerate_configs/1_node_8_gpus_deepspeed_zero3.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Auraithm/LLADA_pretraining/HEAD/accelerate_configs/1_node_8_gpus_deepspeed_zero3.yaml -------------------------------------------------------------------------------- /accelerate_configs/1_node_only.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Auraithm/LLADA_pretraining/HEAD/accelerate_configs/1_node_only.yaml -------------------------------------------------------------------------------- /accelerate_configs/8_node_8_gpus_deepspeed_zero2.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Auraithm/LLADA_pretraining/HEAD/accelerate_configs/8_node_8_gpus_deepspeed_zero2.yaml -------------------------------------------------------------------------------- /configs/llada_pretraining.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Auraithm/LLADA_pretraining/HEAD/configs/llada_pretraining.yaml -------------------------------------------------------------------------------- /lm_chat_validation/description.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Auraithm/LLADA_pretraining/HEAD/lm_chat_validation/description.txt -------------------------------------------------------------------------------- /lm_chat_validation/questions.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Auraithm/LLADA_pretraining/HEAD/lm_chat_validation/questions.jsonl -------------------------------------------------------------------------------- /models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Auraithm/LLADA_pretraining/HEAD/models/__init__.py -------------------------------------------------------------------------------- /models/common_modules.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Auraithm/LLADA_pretraining/HEAD/models/common_modules.py -------------------------------------------------------------------------------- /models/configuration_llada.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Auraithm/LLADA_pretraining/HEAD/models/configuration_llada.py -------------------------------------------------------------------------------- /models/logging.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Auraithm/LLADA_pretraining/HEAD/models/logging.py -------------------------------------------------------------------------------- /models/lr_schedulers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Auraithm/LLADA_pretraining/HEAD/models/lr_schedulers.py -------------------------------------------------------------------------------- /models/misc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Auraithm/LLADA_pretraining/HEAD/models/misc.py -------------------------------------------------------------------------------- /models/modeling_llada.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Auraithm/LLADA_pretraining/HEAD/models/modeling_llada.py -------------------------------------------------------------------------------- /models/modeling_magvitv2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Auraithm/LLADA_pretraining/HEAD/models/modeling_magvitv2.py -------------------------------------------------------------------------------- /models/modeling_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Auraithm/LLADA_pretraining/HEAD/models/modeling_utils.py -------------------------------------------------------------------------------- /models/sampling.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Auraithm/LLADA_pretraining/HEAD/models/sampling.py -------------------------------------------------------------------------------- /models/training_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Auraithm/LLADA_pretraining/HEAD/models/training_utils.py -------------------------------------------------------------------------------- /parquet/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Auraithm/LLADA_pretraining/HEAD/parquet/__init__.py -------------------------------------------------------------------------------- /parquet/my_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Auraithm/LLADA_pretraining/HEAD/parquet/my_dataset.py -------------------------------------------------------------------------------- /reasoning/test.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Auraithm/LLADA_pretraining/HEAD/requirements.txt -------------------------------------------------------------------------------- /scripts/train.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Auraithm/LLADA_pretraining/HEAD/scripts/train.sh -------------------------------------------------------------------------------- /training/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Auraithm/LLADA_pretraining/HEAD/training/__init__.py -------------------------------------------------------------------------------- /training/data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Auraithm/LLADA_pretraining/HEAD/training/data.py -------------------------------------------------------------------------------- /training/optimizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Auraithm/LLADA_pretraining/HEAD/training/optimizer.py -------------------------------------------------------------------------------- /training/prompting_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Auraithm/LLADA_pretraining/HEAD/training/prompting_utils.py -------------------------------------------------------------------------------- /training/train_llada.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Auraithm/LLADA_pretraining/HEAD/training/train_llada.py -------------------------------------------------------------------------------- /training/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Auraithm/LLADA_pretraining/HEAD/training/utils.py --------------------------------------------------------------------------------