├── .gitignore ├── .isort.cfg ├── .pre-commit-config.yaml ├── INSTALL.md ├── README.md ├── assets ├── 1.png ├── Technical_Report.pdf ├── demo.png └── logo.png ├── requirements.txt ├── setup.py ├── unitoken ├── .isort.cfg ├── TRAIN.md ├── configs │ └── data │ │ ├── config_phase1_allava710k_llavadetail559k_sharecap1246k.yaml │ │ └── config_phase2_sftmixed8d6m_ocr3m_mjv10m.yaml ├── data │ ├── __init__.py │ ├── convertsation.py │ └── item_processor.py ├── demos │ ├── demo_freeform.py │ ├── demo_image2image.py │ ├── demo_image_generation.py │ └── inference.py ├── exps │ ├── UniToken-AnyRes-stage1.sh │ ├── UniToken-AnyRes-stage2.sh │ ├── UniToken-base-stage1.sh │ └── UniToken-base-stage2.sh ├── finetune_solver_anyres.py ├── finetune_solver_base.py ├── generate_examples │ └── generate.py ├── inference_solver_anyres.py ├── inference_solver_base.py ├── model │ ├── __init__.py │ ├── chameleon │ │ ├── __init__.py │ │ ├── configuration_chameleon.py │ │ ├── convert_chameleon_weights_to_hf.py │ │ ├── image_processing_chameleon.py │ │ ├── modeling_chameleon.py │ │ └── processing_chameleon.py │ ├── chameleon_vae_ori │ │ ├── __init__.py │ │ ├── image_tokenizer.py │ │ ├── vocab.py │ │ └── vqgan.py │ ├── configuration_xllmx_chameleon.py │ ├── emu3 │ │ ├── __init__.py │ │ ├── configuration_emu3.py │ │ ├── modeling_emu3.py │ │ ├── processing_emu3.py │ │ ├── tokenization_emu3.py │ │ └── utils_emu3.py │ ├── emu3_vae_ori │ │ ├── __init__.py │ │ ├── configuration_emu3visionvq.py │ │ ├── image_processing_emu3visionvq.py │ │ └── modeling_emu3visionvq.py │ ├── model_api.py │ ├── modeling_gen_chameleon.py │ ├── modeling_xllmx_chameleon.py │ ├── modeling_xllmx_chameleon_anyres.py │ └── modeling_xllmx_chameleon_base.py ├── pre_tokenize │ ├── concat_record.py │ └── pre_tokenize.py └── tools │ ├── __init__.py │ └── mm_utils.py └── xllmx ├── __init__.py ├── data ├── __init__.py ├── conversation │ ├── __init__.py │ └── template.py ├── data_reader.py ├── dataset.py ├── item_processor.py └── sampler.py ├── model ├── __init__.py ├── components.py └── tokenizer.py ├── solvers ├── __init__.py └── finetune │ ├── __init__.py │ └── finetune.py └── util ├── __init__.py ├── ckpt.py ├── dist.py ├── lr_sched.py ├── misc.py └── tensor_type.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/.gitignore -------------------------------------------------------------------------------- /.isort.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/.isort.cfg -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/.pre-commit-config.yaml -------------------------------------------------------------------------------- /INSTALL.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/INSTALL.md -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/README.md -------------------------------------------------------------------------------- /assets/1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/assets/1.png -------------------------------------------------------------------------------- /assets/Technical_Report.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/assets/Technical_Report.pdf -------------------------------------------------------------------------------- /assets/demo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/assets/demo.png -------------------------------------------------------------------------------- /assets/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/assets/logo.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/requirements.txt -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/setup.py -------------------------------------------------------------------------------- /unitoken/.isort.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/.isort.cfg -------------------------------------------------------------------------------- /unitoken/TRAIN.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/TRAIN.md -------------------------------------------------------------------------------- /unitoken/configs/data/config_phase1_allava710k_llavadetail559k_sharecap1246k.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/configs/data/config_phase1_allava710k_llavadetail559k_sharecap1246k.yaml -------------------------------------------------------------------------------- /unitoken/configs/data/config_phase2_sftmixed8d6m_ocr3m_mjv10m.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/configs/data/config_phase2_sftmixed8d6m_ocr3m_mjv10m.yaml -------------------------------------------------------------------------------- /unitoken/data/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /unitoken/data/convertsation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/data/convertsation.py -------------------------------------------------------------------------------- /unitoken/data/item_processor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/data/item_processor.py -------------------------------------------------------------------------------- /unitoken/demos/demo_freeform.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/demos/demo_freeform.py -------------------------------------------------------------------------------- /unitoken/demos/demo_image2image.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/demos/demo_image2image.py -------------------------------------------------------------------------------- /unitoken/demos/demo_image_generation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/demos/demo_image_generation.py -------------------------------------------------------------------------------- /unitoken/demos/inference.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/demos/inference.py -------------------------------------------------------------------------------- /unitoken/exps/UniToken-AnyRes-stage1.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/exps/UniToken-AnyRes-stage1.sh -------------------------------------------------------------------------------- /unitoken/exps/UniToken-AnyRes-stage2.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/exps/UniToken-AnyRes-stage2.sh -------------------------------------------------------------------------------- /unitoken/exps/UniToken-base-stage1.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/exps/UniToken-base-stage1.sh -------------------------------------------------------------------------------- /unitoken/exps/UniToken-base-stage2.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/exps/UniToken-base-stage2.sh -------------------------------------------------------------------------------- /unitoken/finetune_solver_anyres.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/finetune_solver_anyres.py -------------------------------------------------------------------------------- /unitoken/finetune_solver_base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/finetune_solver_base.py -------------------------------------------------------------------------------- /unitoken/generate_examples/generate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/generate_examples/generate.py -------------------------------------------------------------------------------- /unitoken/inference_solver_anyres.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/inference_solver_anyres.py -------------------------------------------------------------------------------- /unitoken/inference_solver_base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/inference_solver_base.py -------------------------------------------------------------------------------- /unitoken/model/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/model/__init__.py -------------------------------------------------------------------------------- /unitoken/model/chameleon/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/model/chameleon/__init__.py -------------------------------------------------------------------------------- /unitoken/model/chameleon/configuration_chameleon.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/model/chameleon/configuration_chameleon.py -------------------------------------------------------------------------------- /unitoken/model/chameleon/convert_chameleon_weights_to_hf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/model/chameleon/convert_chameleon_weights_to_hf.py -------------------------------------------------------------------------------- /unitoken/model/chameleon/image_processing_chameleon.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/model/chameleon/image_processing_chameleon.py -------------------------------------------------------------------------------- /unitoken/model/chameleon/modeling_chameleon.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/model/chameleon/modeling_chameleon.py -------------------------------------------------------------------------------- /unitoken/model/chameleon/processing_chameleon.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/model/chameleon/processing_chameleon.py -------------------------------------------------------------------------------- /unitoken/model/chameleon_vae_ori/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/model/chameleon_vae_ori/__init__.py -------------------------------------------------------------------------------- /unitoken/model/chameleon_vae_ori/image_tokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/model/chameleon_vae_ori/image_tokenizer.py -------------------------------------------------------------------------------- /unitoken/model/chameleon_vae_ori/vocab.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/model/chameleon_vae_ori/vocab.py -------------------------------------------------------------------------------- /unitoken/model/chameleon_vae_ori/vqgan.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/model/chameleon_vae_ori/vqgan.py -------------------------------------------------------------------------------- /unitoken/model/configuration_xllmx_chameleon.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/model/configuration_xllmx_chameleon.py -------------------------------------------------------------------------------- /unitoken/model/emu3/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/model/emu3/__init__.py -------------------------------------------------------------------------------- /unitoken/model/emu3/configuration_emu3.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/model/emu3/configuration_emu3.py -------------------------------------------------------------------------------- /unitoken/model/emu3/modeling_emu3.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/model/emu3/modeling_emu3.py -------------------------------------------------------------------------------- /unitoken/model/emu3/processing_emu3.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/model/emu3/processing_emu3.py -------------------------------------------------------------------------------- /unitoken/model/emu3/tokenization_emu3.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/model/emu3/tokenization_emu3.py -------------------------------------------------------------------------------- /unitoken/model/emu3/utils_emu3.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/model/emu3/utils_emu3.py -------------------------------------------------------------------------------- /unitoken/model/emu3_vae_ori/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/model/emu3_vae_ori/__init__.py -------------------------------------------------------------------------------- /unitoken/model/emu3_vae_ori/configuration_emu3visionvq.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/model/emu3_vae_ori/configuration_emu3visionvq.py -------------------------------------------------------------------------------- /unitoken/model/emu3_vae_ori/image_processing_emu3visionvq.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/model/emu3_vae_ori/image_processing_emu3visionvq.py -------------------------------------------------------------------------------- /unitoken/model/emu3_vae_ori/modeling_emu3visionvq.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/model/emu3_vae_ori/modeling_emu3visionvq.py -------------------------------------------------------------------------------- /unitoken/model/model_api.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/model/model_api.py -------------------------------------------------------------------------------- /unitoken/model/modeling_gen_chameleon.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/model/modeling_gen_chameleon.py -------------------------------------------------------------------------------- /unitoken/model/modeling_xllmx_chameleon.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/model/modeling_xllmx_chameleon.py -------------------------------------------------------------------------------- /unitoken/model/modeling_xllmx_chameleon_anyres.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/model/modeling_xllmx_chameleon_anyres.py -------------------------------------------------------------------------------- /unitoken/model/modeling_xllmx_chameleon_base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/model/modeling_xllmx_chameleon_base.py -------------------------------------------------------------------------------- /unitoken/pre_tokenize/concat_record.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/pre_tokenize/concat_record.py -------------------------------------------------------------------------------- /unitoken/pre_tokenize/pre_tokenize.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/pre_tokenize/pre_tokenize.py -------------------------------------------------------------------------------- /unitoken/tools/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/tools/__init__.py -------------------------------------------------------------------------------- /unitoken/tools/mm_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/unitoken/tools/mm_utils.py -------------------------------------------------------------------------------- /xllmx/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /xllmx/data/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /xllmx/data/conversation/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /xllmx/data/conversation/template.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/xllmx/data/conversation/template.py -------------------------------------------------------------------------------- /xllmx/data/data_reader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/xllmx/data/data_reader.py -------------------------------------------------------------------------------- /xllmx/data/dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/xllmx/data/dataset.py -------------------------------------------------------------------------------- /xllmx/data/item_processor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/xllmx/data/item_processor.py -------------------------------------------------------------------------------- /xllmx/data/sampler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/xllmx/data/sampler.py -------------------------------------------------------------------------------- /xllmx/model/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /xllmx/model/components.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/xllmx/model/components.py -------------------------------------------------------------------------------- /xllmx/model/tokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/xllmx/model/tokenizer.py -------------------------------------------------------------------------------- /xllmx/solvers/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /xllmx/solvers/finetune/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/xllmx/solvers/finetune/__init__.py -------------------------------------------------------------------------------- /xllmx/solvers/finetune/finetune.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/xllmx/solvers/finetune/finetune.py -------------------------------------------------------------------------------- /xllmx/util/__init__.py: -------------------------------------------------------------------------------- 1 | from . import ckpt, dist 2 | -------------------------------------------------------------------------------- /xllmx/util/ckpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/xllmx/util/ckpt.py -------------------------------------------------------------------------------- /xllmx/util/dist.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/xllmx/util/dist.py -------------------------------------------------------------------------------- /xllmx/util/lr_sched.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/xllmx/util/lr_sched.py -------------------------------------------------------------------------------- /xllmx/util/misc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/xllmx/util/misc.py -------------------------------------------------------------------------------- /xllmx/util/tensor_type.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SxJyJay/UniToken/HEAD/xllmx/util/tensor_type.py --------------------------------------------------------------------------------