├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── augmentation ├── get_aug.py └── video │ ├── base_video_aug.py │ └── utils │ └── unnorm.py ├── callbacks └── full_callback.py ├── configs ├── base │ └── base_utils.py ├── csldaily │ ├── csldaily_stage1_configs │ │ ├── CSL_example_s1_config.py │ │ └── CSL_example_s1_dyn_config.py │ └── csldaily_stage2_configs │ │ └── CSL_example_s2_config.py ├── phoenix2014t │ ├── phoenix_stage1_configs │ │ └── PHX_example_s1_dyn_config.py │ └── phoenix_stage2_configs │ │ └── PHX_example_s2_dyn_config.py └── standards │ └── standard_meta_model_zero_config.py ├── dataloaders ├── csldaily_video_dataset.py ├── data_utils │ ├── file_utils.py │ ├── lmdb_utils.py │ └── my_concat_dataset.py ├── get_dataset.py └── phoenix_video_dataset.py ├── environment.yml ├── environment_variables.py ├── images └── overview_sign2gpt.png ├── losses ├── base_loss.py ├── get_loss.py └── loss_functions │ ├── bce_loss.py │ └── ce_loss.py ├── main.py ├── metrics ├── accuracy_score.py ├── bleu_score.py ├── class_accuracy_score.py ├── class_f1_score.py ├── f1_score.py ├── rouge_score.py └── utils │ ├── mscoco_rouge.py │ └── sacrebleu.py ├── models ├── dinov2 │ ├── layers │ │ ├── __init__.py │ │ ├── attention.py │ │ ├── block.py │ │ ├── dino_head.py │ │ ├── drop_path.py │ │ ├── layer_scale.py │ │ ├── mlp.py │ │ ├── patch_embed.py │ │ └── swiglu_ffn.py │ └── model │ │ └── vision_transformer.py ├── get_models.py ├── huggingface │ └── modeling_xglm.py ├── metaformer │ ├── downsamplers │ │ └── downsampler.py │ ├── emb │ │ ├── my_no_pos.py │ │ ├── my_sine.py │ │ └── sine_pos.py │ ├── meta_model.py │ ├── net │ │ ├── attentions │ │ │ └── local_mask_attention.py │ │ ├── downsampler_net.py │ │ ├── meta_block.py │ │ ├── multiheaddispatch.py │ │ └── rotary_embeddings.py │ └── post │ │ ├── identity_head.py │ │ └── zero_fasttext_prototype_head.py ├── model_sign_encoder │ └── basic_sign_encoder.py ├── model_utils │ ├── clamp_utils.py │ └── masked_norm.py ├── post_models │ ├── linear_pos_head.py │ └── post_utils │ │ ├── learnable_positional_embedding.py │ │ ├── sine_positional_embedding.py │ │ └── zero_positional_embedding.py ├── spatial_models │ └── frame_models │ │ └── dino_adaptor_model.py └── trial_models │ ├── test_pretraining.py │ └── test_stage2_model.py ├── optimizer └── get_optimizer.py ├── scheduler ├── get_scheduler.py ├── reducelr.py └── warmup_with_cosine.py ├── scripts ├── csldaily │ ├── image_lmdb_creator.py │ ├── pseudo_gloss_zn.py │ ├── tsv_processing.py │ └── video_creator.py └── pseudo_gloss_de.py ├── train_utils ├── checkpoint_helpers.py └── seed_helpers.py └── trainer ├── base └── base_trainer.py ├── complete_translation_trainer.py └── psuedo_gloss_trainer.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/.gitignore -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/Dockerfile -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/README.md -------------------------------------------------------------------------------- /augmentation/get_aug.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/augmentation/get_aug.py -------------------------------------------------------------------------------- /augmentation/video/base_video_aug.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/augmentation/video/base_video_aug.py -------------------------------------------------------------------------------- /augmentation/video/utils/unnorm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/augmentation/video/utils/unnorm.py -------------------------------------------------------------------------------- /callbacks/full_callback.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/callbacks/full_callback.py -------------------------------------------------------------------------------- /configs/base/base_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/configs/base/base_utils.py -------------------------------------------------------------------------------- /configs/csldaily/csldaily_stage1_configs/CSL_example_s1_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/configs/csldaily/csldaily_stage1_configs/CSL_example_s1_config.py -------------------------------------------------------------------------------- /configs/csldaily/csldaily_stage1_configs/CSL_example_s1_dyn_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/configs/csldaily/csldaily_stage1_configs/CSL_example_s1_dyn_config.py -------------------------------------------------------------------------------- /configs/csldaily/csldaily_stage2_configs/CSL_example_s2_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/configs/csldaily/csldaily_stage2_configs/CSL_example_s2_config.py -------------------------------------------------------------------------------- /configs/phoenix2014t/phoenix_stage1_configs/PHX_example_s1_dyn_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/configs/phoenix2014t/phoenix_stage1_configs/PHX_example_s1_dyn_config.py -------------------------------------------------------------------------------- /configs/phoenix2014t/phoenix_stage2_configs/PHX_example_s2_dyn_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/configs/phoenix2014t/phoenix_stage2_configs/PHX_example_s2_dyn_config.py -------------------------------------------------------------------------------- /configs/standards/standard_meta_model_zero_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/configs/standards/standard_meta_model_zero_config.py -------------------------------------------------------------------------------- /dataloaders/csldaily_video_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/dataloaders/csldaily_video_dataset.py -------------------------------------------------------------------------------- /dataloaders/data_utils/file_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/dataloaders/data_utils/file_utils.py -------------------------------------------------------------------------------- /dataloaders/data_utils/lmdb_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/dataloaders/data_utils/lmdb_utils.py -------------------------------------------------------------------------------- /dataloaders/data_utils/my_concat_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/dataloaders/data_utils/my_concat_dataset.py -------------------------------------------------------------------------------- /dataloaders/get_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/dataloaders/get_dataset.py -------------------------------------------------------------------------------- /dataloaders/phoenix_video_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/dataloaders/phoenix_video_dataset.py -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/environment.yml -------------------------------------------------------------------------------- /environment_variables.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/environment_variables.py -------------------------------------------------------------------------------- /images/overview_sign2gpt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/images/overview_sign2gpt.png -------------------------------------------------------------------------------- /losses/base_loss.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/losses/base_loss.py -------------------------------------------------------------------------------- /losses/get_loss.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/losses/get_loss.py -------------------------------------------------------------------------------- /losses/loss_functions/bce_loss.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/losses/loss_functions/bce_loss.py -------------------------------------------------------------------------------- /losses/loss_functions/ce_loss.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/losses/loss_functions/ce_loss.py -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/main.py -------------------------------------------------------------------------------- /metrics/accuracy_score.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/metrics/accuracy_score.py -------------------------------------------------------------------------------- /metrics/bleu_score.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/metrics/bleu_score.py -------------------------------------------------------------------------------- /metrics/class_accuracy_score.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/metrics/class_accuracy_score.py -------------------------------------------------------------------------------- /metrics/class_f1_score.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/metrics/class_f1_score.py -------------------------------------------------------------------------------- /metrics/f1_score.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/metrics/f1_score.py -------------------------------------------------------------------------------- /metrics/rouge_score.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/metrics/rouge_score.py -------------------------------------------------------------------------------- /metrics/utils/mscoco_rouge.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/metrics/utils/mscoco_rouge.py -------------------------------------------------------------------------------- /metrics/utils/sacrebleu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/metrics/utils/sacrebleu.py -------------------------------------------------------------------------------- /models/dinov2/layers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/models/dinov2/layers/__init__.py -------------------------------------------------------------------------------- /models/dinov2/layers/attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/models/dinov2/layers/attention.py -------------------------------------------------------------------------------- /models/dinov2/layers/block.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/models/dinov2/layers/block.py -------------------------------------------------------------------------------- /models/dinov2/layers/dino_head.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/models/dinov2/layers/dino_head.py -------------------------------------------------------------------------------- /models/dinov2/layers/drop_path.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/models/dinov2/layers/drop_path.py -------------------------------------------------------------------------------- /models/dinov2/layers/layer_scale.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/models/dinov2/layers/layer_scale.py -------------------------------------------------------------------------------- /models/dinov2/layers/mlp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/models/dinov2/layers/mlp.py -------------------------------------------------------------------------------- /models/dinov2/layers/patch_embed.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/models/dinov2/layers/patch_embed.py -------------------------------------------------------------------------------- /models/dinov2/layers/swiglu_ffn.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/models/dinov2/layers/swiglu_ffn.py -------------------------------------------------------------------------------- /models/dinov2/model/vision_transformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/models/dinov2/model/vision_transformer.py -------------------------------------------------------------------------------- /models/get_models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/models/get_models.py -------------------------------------------------------------------------------- /models/huggingface/modeling_xglm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/models/huggingface/modeling_xglm.py -------------------------------------------------------------------------------- /models/metaformer/downsamplers/downsampler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/models/metaformer/downsamplers/downsampler.py -------------------------------------------------------------------------------- /models/metaformer/emb/my_no_pos.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/models/metaformer/emb/my_no_pos.py -------------------------------------------------------------------------------- /models/metaformer/emb/my_sine.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/models/metaformer/emb/my_sine.py -------------------------------------------------------------------------------- /models/metaformer/emb/sine_pos.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/models/metaformer/emb/sine_pos.py -------------------------------------------------------------------------------- /models/metaformer/meta_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/models/metaformer/meta_model.py -------------------------------------------------------------------------------- /models/metaformer/net/attentions/local_mask_attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/models/metaformer/net/attentions/local_mask_attention.py -------------------------------------------------------------------------------- /models/metaformer/net/downsampler_net.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/models/metaformer/net/downsampler_net.py -------------------------------------------------------------------------------- /models/metaformer/net/meta_block.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/models/metaformer/net/meta_block.py -------------------------------------------------------------------------------- /models/metaformer/net/multiheaddispatch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/models/metaformer/net/multiheaddispatch.py -------------------------------------------------------------------------------- /models/metaformer/net/rotary_embeddings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/models/metaformer/net/rotary_embeddings.py -------------------------------------------------------------------------------- /models/metaformer/post/identity_head.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/models/metaformer/post/identity_head.py -------------------------------------------------------------------------------- /models/metaformer/post/zero_fasttext_prototype_head.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/models/metaformer/post/zero_fasttext_prototype_head.py -------------------------------------------------------------------------------- /models/model_sign_encoder/basic_sign_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/models/model_sign_encoder/basic_sign_encoder.py -------------------------------------------------------------------------------- /models/model_utils/clamp_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/models/model_utils/clamp_utils.py -------------------------------------------------------------------------------- /models/model_utils/masked_norm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/models/model_utils/masked_norm.py -------------------------------------------------------------------------------- /models/post_models/linear_pos_head.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/models/post_models/linear_pos_head.py -------------------------------------------------------------------------------- /models/post_models/post_utils/learnable_positional_embedding.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/models/post_models/post_utils/learnable_positional_embedding.py -------------------------------------------------------------------------------- /models/post_models/post_utils/sine_positional_embedding.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/models/post_models/post_utils/sine_positional_embedding.py -------------------------------------------------------------------------------- /models/post_models/post_utils/zero_positional_embedding.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/models/post_models/post_utils/zero_positional_embedding.py -------------------------------------------------------------------------------- /models/spatial_models/frame_models/dino_adaptor_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/models/spatial_models/frame_models/dino_adaptor_model.py -------------------------------------------------------------------------------- /models/trial_models/test_pretraining.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/models/trial_models/test_pretraining.py -------------------------------------------------------------------------------- /models/trial_models/test_stage2_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/models/trial_models/test_stage2_model.py -------------------------------------------------------------------------------- /optimizer/get_optimizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/optimizer/get_optimizer.py -------------------------------------------------------------------------------- /scheduler/get_scheduler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/scheduler/get_scheduler.py -------------------------------------------------------------------------------- /scheduler/reducelr.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/scheduler/reducelr.py -------------------------------------------------------------------------------- /scheduler/warmup_with_cosine.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/scheduler/warmup_with_cosine.py -------------------------------------------------------------------------------- /scripts/csldaily/image_lmdb_creator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/scripts/csldaily/image_lmdb_creator.py -------------------------------------------------------------------------------- /scripts/csldaily/pseudo_gloss_zn.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/scripts/csldaily/pseudo_gloss_zn.py -------------------------------------------------------------------------------- /scripts/csldaily/tsv_processing.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/scripts/csldaily/tsv_processing.py -------------------------------------------------------------------------------- /scripts/csldaily/video_creator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/scripts/csldaily/video_creator.py -------------------------------------------------------------------------------- /scripts/pseudo_gloss_de.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/scripts/pseudo_gloss_de.py -------------------------------------------------------------------------------- /train_utils/checkpoint_helpers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/train_utils/checkpoint_helpers.py -------------------------------------------------------------------------------- /train_utils/seed_helpers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/train_utils/seed_helpers.py -------------------------------------------------------------------------------- /trainer/base/base_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/trainer/base/base_trainer.py -------------------------------------------------------------------------------- /trainer/complete_translation_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/trainer/complete_translation_trainer.py -------------------------------------------------------------------------------- /trainer/psuedo_gloss_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwongsa/Sign2GPT/HEAD/trainer/psuedo_gloss_trainer.py --------------------------------------------------------------------------------