├── .github └── workflows │ ├── code-quality.yml │ └── pytest.yml ├── .gitignore ├── .pylintrc ├── DPF ├── __init__.py ├── configs │ ├── __init__.py │ ├── dataset_config.py │ ├── files_config.py │ ├── sharded_config.py │ ├── sharded_files_config.py │ └── shards_config.py ├── connectors │ ├── __init__.py │ ├── connector.py │ ├── errors.py │ ├── local_connector.py │ └── s3_connector.py ├── dataloaders │ ├── __init__.py │ ├── dataloader_utils.py │ ├── files_dataset.py │ └── shards_dataset.py ├── dataset_reader.py ├── dataset_reader_utils.py ├── datatypes.py ├── filters │ ├── __init__.py │ ├── audios │ │ ├── audio_filter.py │ │ └── info_filter.py │ ├── column_filter.py │ ├── data_filter.py │ ├── images │ │ ├── __init__.py │ │ ├── aesthetic_filter.py │ │ ├── aesthetic_improved_filter.py │ │ ├── blip_captioning_filter.py │ │ ├── cliplabels_filter.py │ │ ├── dummy_gpu_filter.py │ │ ├── hash_filters.py │ │ ├── img_filter.py │ │ ├── info_filter.py │ │ ├── llava34b_captioning_filter.py │ │ ├── llava_captioning_filter.py │ │ ├── nsfw_filter.py │ │ ├── ocr_filter.py │ │ ├── ocr_model │ │ │ ├── __init__.py │ │ │ ├── dataset.py │ │ │ ├── model.py │ │ │ ├── modules │ │ │ │ ├── feature_extraction.py │ │ │ │ ├── prediction.py │ │ │ │ ├── sequence_modeling.py │ │ │ │ └── transformation.py │ │ │ └── utils.py │ │ ├── text_detection_filter.py │ │ └── watermarks_filter.py │ ├── multigpu_filter.py │ ├── text2image │ │ ├── __init__.py │ │ ├── blip_filter.py │ │ ├── clip_filter.py │ │ ├── ruclip_filter.py │ │ └── t2i_filter.py │ ├── texts │ │ ├── __init__.py │ │ ├── google_translate_filter.py │ │ ├── lang_filter.py │ │ ├── regex_filter.py │ │ └── regexs.py │ ├── utils │ │ ├── __init__.py │ │ └── fp16_module.py │ └── videos │ │ ├── __init__.py │ │ ├── farneback_filter.py │ │ ├── grounding_gpt │ │ ├── lego │ │ │ ├── LEGO.py │ │ │ ├── __init__.py │ │ │ ├── constants.py │ │ │ ├── conversation.py │ │ │ ├── mm_utils.py │ │ │ ├── model │ │ │ │ ├── builder.py │ │ │ │ └── utils.py │ │ │ └── utils.py │ │ └── video_llama │ │ │ ├── __init__.py │ │ │ ├── common │ │ │ ├── __init__.py │ │ │ ├── config.py │ │ │ ├── dist_utils.py │ │ │ ├── gradcam.py │ │ │ ├── logger.py │ │ │ ├── optims.py │ │ │ ├── registry.py │ │ │ └── utils.py │ │ │ ├── configs │ │ │ ├── datasets │ │ │ │ ├── cc_sbu │ │ │ │ │ ├── align.yaml │ │ │ │ │ └── defaults.yaml │ │ │ │ ├── instruct │ │ │ │ │ ├── llava_instruct.yaml │ │ │ │ │ └── webvid_instruct.yaml │ │ │ │ ├── laion │ │ │ │ │ └── defaults.yaml │ │ │ │ └── webvid │ │ │ │ │ └── defaults.yaml │ │ │ ├── default.yaml │ │ │ └── models │ │ │ │ ├── minigpt4.yaml │ │ │ │ └── video_llama.yaml │ │ │ ├── conversation │ │ │ ├── __init__.py │ │ │ └── conversation_video.py │ │ │ ├── datasets │ │ │ ├── __init__.py │ │ │ ├── builders │ │ │ │ ├── __init__.py │ │ │ │ ├── base_dataset_builder.py │ │ │ │ ├── image_text_pair_builder.py │ │ │ │ ├── instruct_builder.py │ │ │ │ └── video_caption_builder.py │ │ │ ├── data_utils.py │ │ │ └── datasets │ │ │ │ ├── __init__.py │ │ │ │ ├── base_dataset.py │ │ │ │ ├── caption_datasets.py │ │ │ │ ├── cc_sbu_dataset.py │ │ │ │ ├── dataloader_utils.py │ │ │ │ ├── laion_dataset.py │ │ │ │ ├── llava_instruct_dataset.py │ │ │ │ ├── video_instruct_dataset.py │ │ │ │ └── webvid_datasets.py │ │ │ ├── models │ │ │ ├── ImageBind │ │ │ │ ├── CODE_OF_CONDUCT.md │ │ │ │ ├── CONTRIBUTING.md │ │ │ │ ├── LICENSE │ │ │ │ ├── README.md │ │ │ │ ├── bpe │ │ │ │ │ └── bpe_simple_vocab_16e6.txt.gz │ │ │ │ ├── data.py │ │ │ │ ├── model_card.md │ │ │ │ ├── models │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── helpers.py │ │ │ │ │ ├── imagebind_model.py │ │ │ │ │ ├── multimodal_preprocessors.py │ │ │ │ │ └── transformer.py │ │ │ │ └── requirements.txt │ │ │ ├── Qformer.py │ │ │ ├── __init__.py │ │ │ ├── base_model.py │ │ │ ├── blip2.py │ │ │ ├── blip2_outputs.py │ │ │ ├── eva_vit.py │ │ │ ├── modeling_llama.py │ │ │ └── video_llama.py │ │ │ ├── processors │ │ │ ├── __init__.py │ │ │ ├── base_processor.py │ │ │ ├── blip_processors.py │ │ │ ├── functional_video.py │ │ │ ├── randaugment.py │ │ │ ├── transforms_video.py │ │ │ └── video_processor.py │ │ │ ├── runners │ │ │ ├── __init__.py │ │ │ ├── runner_base.py │ │ │ └── test.py │ │ │ └── tasks │ │ │ ├── __init__.py │ │ │ ├── base_task.py │ │ │ ├── image_text_pretrain.py │ │ │ └── video_text_pretrain.py │ │ ├── grounding_gpt_filter.py │ │ ├── image_filter_adapter.py │ │ ├── info_filter.py │ │ ├── lita_filter.py │ │ ├── raft_core │ │ ├── corr.py │ │ ├── download.py │ │ ├── extractor.py │ │ ├── model.py │ │ ├── update.py │ │ └── utils.py │ │ ├── raft_filter.py │ │ ├── video_filter.py │ │ └── video_llava_filter.py ├── modalities.py ├── pipelines │ ├── __init__.py │ ├── filter_pipeline.py │ ├── pipeline_stages.py │ └── types.py ├── processors │ ├── __init__.py │ ├── files_processor.py │ ├── helpers │ │ ├── __init__.py │ │ └── dataframe_helper.py │ ├── processor.py │ ├── processor_mixins.py │ ├── sharded_files_processor.py │ ├── sharded_processor.py │ ├── shards_processor.py │ └── writers │ │ ├── __init__.py │ │ ├── filewriter.py │ │ ├── sharded_files_writer.py │ │ ├── shards_writer.py │ │ └── utils.py ├── transforms │ ├── __init__.py │ ├── base_file_transforms.py │ ├── image_resize_transforms.py │ ├── resizer.py │ └── video_ffmpeg_transforms.py ├── types.py ├── utils │ ├── __init__.py │ ├── constants.py │ ├── image_utils.py │ ├── logger.py │ └── utils.py └── validators │ ├── __init__.py │ ├── errors.py │ ├── format_validators │ ├── __init__.py │ ├── files_validator.py │ ├── sharded_files_validator.py │ ├── sharded_validator.py │ └── shards_validator.py │ ├── utils.py │ └── validator.py ├── LICENSE ├── README.md ├── docs ├── filters.md ├── formats.md ├── pipelines.md ├── processor.md └── transforms.md ├── examples ├── example_dataset │ ├── 0.csv │ ├── 0.tar │ ├── 1.csv │ ├── 1.tar │ ├── 2.csv │ └── 2.tar ├── example_video_dataset │ ├── 0 │ │ ├── 0.mp4 │ │ ├── 1.mp4 │ │ ├── 2.mp4 │ │ ├── 3.mp4 │ │ └── 4.mp4 │ └── 0.csv ├── image_filters_example.ipynb ├── pipeline_video_example.ipynb ├── process_dataset_example.ipynb ├── text_filters_example.ipynb └── video_filters_example.ipynb ├── pyproject.toml ├── scripts └── multi_gpu_filter.py ├── simple_example.py └── tests ├── datasets ├── files_bad_image │ ├── data.csv │ └── imgs │ │ └── test │ │ ├── 0.jpg │ │ ├── 1.jpg │ │ └── 2.jpg ├── files_correct │ ├── data.csv │ └── imgs │ │ └── test │ │ ├── 0.jpg │ │ └── 1.jpg ├── sharded_files_correct │ ├── 0 │ │ ├── 0.jpg │ │ └── 1.jpg │ └── 0.csv ├── sharded_files_wrong_columns │ ├── 0 │ │ ├── 0.jpg │ │ └── 1.jpg │ ├── 1 │ │ ├── 0.jpg │ │ └── 1.jpg │ ├── 0.csv │ └── 1.csv ├── sharded_files_wrong_folder │ ├── 0 │ │ └── 1.jpg │ └── 0.csv ├── shards_bad_image │ ├── 0.csv │ └── 0.tar ├── shards_correct │ ├── 0.csv │ └── 0.tar ├── shards_wrong_columns │ ├── 0.csv │ ├── 0.tar │ ├── 1.csv │ └── 1.tar └── shards_wrong_tar │ ├── 0.csv │ └── 0.tar ├── test_column_filters.py ├── test_configs.py ├── test_filesystem.py ├── test_image_filters.py ├── test_pipelines.py ├── test_readers.py ├── test_resizer.py ├── test_validators.py └── test_writers.py /.github/workflows/code-quality.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/.github/workflows/code-quality.yml -------------------------------------------------------------------------------- /.github/workflows/pytest.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/.github/workflows/pytest.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/.gitignore -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/.pylintrc -------------------------------------------------------------------------------- /DPF/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/__init__.py -------------------------------------------------------------------------------- /DPF/configs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/configs/__init__.py -------------------------------------------------------------------------------- /DPF/configs/dataset_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/configs/dataset_config.py -------------------------------------------------------------------------------- /DPF/configs/files_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/configs/files_config.py -------------------------------------------------------------------------------- /DPF/configs/sharded_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/configs/sharded_config.py -------------------------------------------------------------------------------- /DPF/configs/sharded_files_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/configs/sharded_files_config.py -------------------------------------------------------------------------------- /DPF/configs/shards_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/configs/shards_config.py -------------------------------------------------------------------------------- /DPF/connectors/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/connectors/__init__.py -------------------------------------------------------------------------------- /DPF/connectors/connector.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/connectors/connector.py -------------------------------------------------------------------------------- /DPF/connectors/errors.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/connectors/errors.py -------------------------------------------------------------------------------- /DPF/connectors/local_connector.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/connectors/local_connector.py -------------------------------------------------------------------------------- /DPF/connectors/s3_connector.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/connectors/s3_connector.py -------------------------------------------------------------------------------- /DPF/dataloaders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/dataloaders/__init__.py -------------------------------------------------------------------------------- /DPF/dataloaders/dataloader_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/dataloaders/dataloader_utils.py -------------------------------------------------------------------------------- /DPF/dataloaders/files_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/dataloaders/files_dataset.py -------------------------------------------------------------------------------- /DPF/dataloaders/shards_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/dataloaders/shards_dataset.py -------------------------------------------------------------------------------- /DPF/dataset_reader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/dataset_reader.py -------------------------------------------------------------------------------- /DPF/dataset_reader_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/dataset_reader_utils.py -------------------------------------------------------------------------------- /DPF/datatypes.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/datatypes.py -------------------------------------------------------------------------------- /DPF/filters/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/__init__.py -------------------------------------------------------------------------------- /DPF/filters/audios/audio_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/audios/audio_filter.py -------------------------------------------------------------------------------- /DPF/filters/audios/info_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/audios/info_filter.py -------------------------------------------------------------------------------- /DPF/filters/column_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/column_filter.py -------------------------------------------------------------------------------- /DPF/filters/data_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/data_filter.py -------------------------------------------------------------------------------- /DPF/filters/images/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /DPF/filters/images/aesthetic_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/images/aesthetic_filter.py -------------------------------------------------------------------------------- /DPF/filters/images/aesthetic_improved_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/images/aesthetic_improved_filter.py -------------------------------------------------------------------------------- /DPF/filters/images/blip_captioning_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/images/blip_captioning_filter.py -------------------------------------------------------------------------------- /DPF/filters/images/cliplabels_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/images/cliplabels_filter.py -------------------------------------------------------------------------------- /DPF/filters/images/dummy_gpu_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/images/dummy_gpu_filter.py -------------------------------------------------------------------------------- /DPF/filters/images/hash_filters.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/images/hash_filters.py -------------------------------------------------------------------------------- /DPF/filters/images/img_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/images/img_filter.py -------------------------------------------------------------------------------- /DPF/filters/images/info_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/images/info_filter.py -------------------------------------------------------------------------------- /DPF/filters/images/llava34b_captioning_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/images/llava34b_captioning_filter.py -------------------------------------------------------------------------------- /DPF/filters/images/llava_captioning_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/images/llava_captioning_filter.py -------------------------------------------------------------------------------- /DPF/filters/images/nsfw_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/images/nsfw_filter.py -------------------------------------------------------------------------------- /DPF/filters/images/ocr_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/images/ocr_filter.py -------------------------------------------------------------------------------- /DPF/filters/images/ocr_model/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /DPF/filters/images/ocr_model/dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/images/ocr_model/dataset.py -------------------------------------------------------------------------------- /DPF/filters/images/ocr_model/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/images/ocr_model/model.py -------------------------------------------------------------------------------- /DPF/filters/images/ocr_model/modules/feature_extraction.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/images/ocr_model/modules/feature_extraction.py -------------------------------------------------------------------------------- /DPF/filters/images/ocr_model/modules/prediction.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/images/ocr_model/modules/prediction.py -------------------------------------------------------------------------------- /DPF/filters/images/ocr_model/modules/sequence_modeling.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/images/ocr_model/modules/sequence_modeling.py -------------------------------------------------------------------------------- /DPF/filters/images/ocr_model/modules/transformation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/images/ocr_model/modules/transformation.py -------------------------------------------------------------------------------- /DPF/filters/images/ocr_model/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/images/ocr_model/utils.py -------------------------------------------------------------------------------- /DPF/filters/images/text_detection_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/images/text_detection_filter.py -------------------------------------------------------------------------------- /DPF/filters/images/watermarks_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/images/watermarks_filter.py -------------------------------------------------------------------------------- /DPF/filters/multigpu_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/multigpu_filter.py -------------------------------------------------------------------------------- /DPF/filters/text2image/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /DPF/filters/text2image/blip_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/text2image/blip_filter.py -------------------------------------------------------------------------------- /DPF/filters/text2image/clip_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/text2image/clip_filter.py -------------------------------------------------------------------------------- /DPF/filters/text2image/ruclip_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/text2image/ruclip_filter.py -------------------------------------------------------------------------------- /DPF/filters/text2image/t2i_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/text2image/t2i_filter.py -------------------------------------------------------------------------------- /DPF/filters/texts/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /DPF/filters/texts/google_translate_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/texts/google_translate_filter.py -------------------------------------------------------------------------------- /DPF/filters/texts/lang_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/texts/lang_filter.py -------------------------------------------------------------------------------- /DPF/filters/texts/regex_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/texts/regex_filter.py -------------------------------------------------------------------------------- /DPF/filters/texts/regexs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/texts/regexs.py -------------------------------------------------------------------------------- /DPF/filters/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/utils/__init__.py -------------------------------------------------------------------------------- /DPF/filters/utils/fp16_module.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/utils/fp16_module.py -------------------------------------------------------------------------------- /DPF/filters/videos/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /DPF/filters/videos/farneback_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/farneback_filter.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/lego/LEGO.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/lego/LEGO.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/lego/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/lego/__init__.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/lego/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/lego/constants.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/lego/conversation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/lego/conversation.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/lego/mm_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/lego/mm_utils.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/lego/model/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/lego/model/builder.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/lego/model/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/lego/model/utils.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/lego/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/lego/utils.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/__init__.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/common/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/common/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/common/config.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/common/dist_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/common/dist_utils.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/common/gradcam.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/common/gradcam.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/common/logger.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/common/logger.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/common/optims.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/common/optims.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/common/registry.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/common/registry.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/common/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/common/utils.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/configs/datasets/cc_sbu/align.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/configs/datasets/cc_sbu/align.yaml -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/configs/datasets/cc_sbu/defaults.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/configs/datasets/cc_sbu/defaults.yaml -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/configs/datasets/instruct/llava_instruct.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/configs/datasets/instruct/llava_instruct.yaml -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/configs/datasets/instruct/webvid_instruct.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/configs/datasets/instruct/webvid_instruct.yaml -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/configs/datasets/laion/defaults.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/configs/datasets/laion/defaults.yaml -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/configs/datasets/webvid/defaults.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/configs/datasets/webvid/defaults.yaml -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/configs/default.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/configs/default.yaml -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/configs/models/minigpt4.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/configs/models/minigpt4.yaml -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/configs/models/video_llama.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/configs/models/video_llama.yaml -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/conversation/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/conversation/conversation_video.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/conversation/conversation_video.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/datasets/builders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/datasets/builders/__init__.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/datasets/builders/base_dataset_builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/datasets/builders/base_dataset_builder.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/datasets/builders/image_text_pair_builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/datasets/builders/image_text_pair_builder.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/datasets/builders/instruct_builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/datasets/builders/instruct_builder.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/datasets/builders/video_caption_builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/datasets/builders/video_caption_builder.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/datasets/data_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/datasets/data_utils.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/datasets/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/datasets/datasets/base_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/datasets/datasets/base_dataset.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/datasets/datasets/caption_datasets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/datasets/datasets/caption_datasets.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/datasets/datasets/cc_sbu_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/datasets/datasets/cc_sbu_dataset.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/datasets/datasets/dataloader_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/datasets/datasets/dataloader_utils.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/datasets/datasets/laion_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/datasets/datasets/laion_dataset.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/datasets/datasets/llava_instruct_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/datasets/datasets/llava_instruct_dataset.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/datasets/datasets/video_instruct_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/datasets/datasets/video_instruct_dataset.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/datasets/datasets/webvid_datasets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/datasets/datasets/webvid_datasets.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/models/ImageBind/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/models/ImageBind/CODE_OF_CONDUCT.md -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/models/ImageBind/CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/models/ImageBind/CONTRIBUTING.md -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/models/ImageBind/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/models/ImageBind/LICENSE -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/models/ImageBind/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/models/ImageBind/README.md -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/models/ImageBind/bpe/bpe_simple_vocab_16e6.txt.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/models/ImageBind/bpe/bpe_simple_vocab_16e6.txt.gz -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/models/ImageBind/data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/models/ImageBind/data.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/models/ImageBind/model_card.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/models/ImageBind/model_card.md -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/models/ImageBind/models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/models/ImageBind/models/helpers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/models/ImageBind/models/helpers.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/models/ImageBind/models/imagebind_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/models/ImageBind/models/imagebind_model.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/models/ImageBind/models/multimodal_preprocessors.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/models/ImageBind/models/multimodal_preprocessors.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/models/ImageBind/models/transformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/models/ImageBind/models/transformer.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/models/ImageBind/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/models/ImageBind/requirements.txt -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/models/Qformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/models/Qformer.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/models/__init__.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/models/base_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/models/base_model.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/models/blip2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/models/blip2.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/models/blip2_outputs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/models/blip2_outputs.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/models/eva_vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/models/eva_vit.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/models/modeling_llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/models/modeling_llama.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/models/video_llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/models/video_llama.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/processors/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/processors/__init__.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/processors/base_processor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/processors/base_processor.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/processors/blip_processors.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/processors/blip_processors.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/processors/functional_video.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/processors/functional_video.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/processors/randaugment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/processors/randaugment.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/processors/transforms_video.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/processors/transforms_video.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/processors/video_processor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/processors/video_processor.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/runners/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/runners/__init__.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/runners/runner_base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/runners/runner_base.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/runners/test.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/tasks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/tasks/__init__.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/tasks/base_task.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/tasks/base_task.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/tasks/image_text_pretrain.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/tasks/image_text_pretrain.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt/video_llama/tasks/video_text_pretrain.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt/video_llama/tasks/video_text_pretrain.py -------------------------------------------------------------------------------- /DPF/filters/videos/grounding_gpt_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/grounding_gpt_filter.py -------------------------------------------------------------------------------- /DPF/filters/videos/image_filter_adapter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/image_filter_adapter.py -------------------------------------------------------------------------------- /DPF/filters/videos/info_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/info_filter.py -------------------------------------------------------------------------------- /DPF/filters/videos/lita_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/lita_filter.py -------------------------------------------------------------------------------- /DPF/filters/videos/raft_core/corr.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/raft_core/corr.py -------------------------------------------------------------------------------- /DPF/filters/videos/raft_core/download.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/raft_core/download.py -------------------------------------------------------------------------------- /DPF/filters/videos/raft_core/extractor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/raft_core/extractor.py -------------------------------------------------------------------------------- /DPF/filters/videos/raft_core/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/raft_core/model.py -------------------------------------------------------------------------------- /DPF/filters/videos/raft_core/update.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/raft_core/update.py -------------------------------------------------------------------------------- /DPF/filters/videos/raft_core/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/raft_core/utils.py -------------------------------------------------------------------------------- /DPF/filters/videos/raft_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/raft_filter.py -------------------------------------------------------------------------------- /DPF/filters/videos/video_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/video_filter.py -------------------------------------------------------------------------------- /DPF/filters/videos/video_llava_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/filters/videos/video_llava_filter.py -------------------------------------------------------------------------------- /DPF/modalities.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/modalities.py -------------------------------------------------------------------------------- /DPF/pipelines/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/pipelines/__init__.py -------------------------------------------------------------------------------- /DPF/pipelines/filter_pipeline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/pipelines/filter_pipeline.py -------------------------------------------------------------------------------- /DPF/pipelines/pipeline_stages.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/pipelines/pipeline_stages.py -------------------------------------------------------------------------------- /DPF/pipelines/types.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/pipelines/types.py -------------------------------------------------------------------------------- /DPF/processors/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/processors/__init__.py -------------------------------------------------------------------------------- /DPF/processors/files_processor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/processors/files_processor.py -------------------------------------------------------------------------------- /DPF/processors/helpers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/processors/helpers/__init__.py -------------------------------------------------------------------------------- /DPF/processors/helpers/dataframe_helper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/processors/helpers/dataframe_helper.py -------------------------------------------------------------------------------- /DPF/processors/processor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/processors/processor.py -------------------------------------------------------------------------------- /DPF/processors/processor_mixins.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/processors/processor_mixins.py -------------------------------------------------------------------------------- /DPF/processors/sharded_files_processor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/processors/sharded_files_processor.py -------------------------------------------------------------------------------- /DPF/processors/sharded_processor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/processors/sharded_processor.py -------------------------------------------------------------------------------- /DPF/processors/shards_processor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/processors/shards_processor.py -------------------------------------------------------------------------------- /DPF/processors/writers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/processors/writers/__init__.py -------------------------------------------------------------------------------- /DPF/processors/writers/filewriter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/processors/writers/filewriter.py -------------------------------------------------------------------------------- /DPF/processors/writers/sharded_files_writer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/processors/writers/sharded_files_writer.py -------------------------------------------------------------------------------- /DPF/processors/writers/shards_writer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/processors/writers/shards_writer.py -------------------------------------------------------------------------------- /DPF/processors/writers/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/processors/writers/utils.py -------------------------------------------------------------------------------- /DPF/transforms/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/transforms/__init__.py -------------------------------------------------------------------------------- /DPF/transforms/base_file_transforms.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/transforms/base_file_transforms.py -------------------------------------------------------------------------------- /DPF/transforms/image_resize_transforms.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/transforms/image_resize_transforms.py -------------------------------------------------------------------------------- /DPF/transforms/resizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/transforms/resizer.py -------------------------------------------------------------------------------- /DPF/transforms/video_ffmpeg_transforms.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/transforms/video_ffmpeg_transforms.py -------------------------------------------------------------------------------- /DPF/types.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/types.py -------------------------------------------------------------------------------- /DPF/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/utils/__init__.py -------------------------------------------------------------------------------- /DPF/utils/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/utils/constants.py -------------------------------------------------------------------------------- /DPF/utils/image_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/utils/image_utils.py -------------------------------------------------------------------------------- /DPF/utils/logger.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/utils/logger.py -------------------------------------------------------------------------------- /DPF/utils/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/utils/utils.py -------------------------------------------------------------------------------- /DPF/validators/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/validators/__init__.py -------------------------------------------------------------------------------- /DPF/validators/errors.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/validators/errors.py -------------------------------------------------------------------------------- /DPF/validators/format_validators/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/validators/format_validators/__init__.py -------------------------------------------------------------------------------- /DPF/validators/format_validators/files_validator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/validators/format_validators/files_validator.py -------------------------------------------------------------------------------- /DPF/validators/format_validators/sharded_files_validator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/validators/format_validators/sharded_files_validator.py -------------------------------------------------------------------------------- /DPF/validators/format_validators/sharded_validator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/validators/format_validators/sharded_validator.py -------------------------------------------------------------------------------- /DPF/validators/format_validators/shards_validator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/validators/format_validators/shards_validator.py -------------------------------------------------------------------------------- /DPF/validators/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/validators/utils.py -------------------------------------------------------------------------------- /DPF/validators/validator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/DPF/validators/validator.py -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/README.md -------------------------------------------------------------------------------- /docs/filters.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/docs/filters.md -------------------------------------------------------------------------------- /docs/formats.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/docs/formats.md -------------------------------------------------------------------------------- /docs/pipelines.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/docs/pipelines.md -------------------------------------------------------------------------------- /docs/processor.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/docs/processor.md -------------------------------------------------------------------------------- /docs/transforms.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/docs/transforms.md -------------------------------------------------------------------------------- /examples/example_dataset/0.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/examples/example_dataset/0.csv -------------------------------------------------------------------------------- /examples/example_dataset/0.tar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/examples/example_dataset/0.tar -------------------------------------------------------------------------------- /examples/example_dataset/1.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/examples/example_dataset/1.csv -------------------------------------------------------------------------------- /examples/example_dataset/1.tar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/examples/example_dataset/1.tar -------------------------------------------------------------------------------- /examples/example_dataset/2.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/examples/example_dataset/2.csv -------------------------------------------------------------------------------- /examples/example_dataset/2.tar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/examples/example_dataset/2.tar -------------------------------------------------------------------------------- /examples/example_video_dataset/0.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/examples/example_video_dataset/0.csv -------------------------------------------------------------------------------- /examples/example_video_dataset/0/0.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/examples/example_video_dataset/0/0.mp4 -------------------------------------------------------------------------------- /examples/example_video_dataset/0/1.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/examples/example_video_dataset/0/1.mp4 -------------------------------------------------------------------------------- /examples/example_video_dataset/0/2.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/examples/example_video_dataset/0/2.mp4 -------------------------------------------------------------------------------- /examples/example_video_dataset/0/3.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/examples/example_video_dataset/0/3.mp4 -------------------------------------------------------------------------------- /examples/example_video_dataset/0/4.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/examples/example_video_dataset/0/4.mp4 -------------------------------------------------------------------------------- /examples/image_filters_example.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/examples/image_filters_example.ipynb -------------------------------------------------------------------------------- /examples/pipeline_video_example.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/examples/pipeline_video_example.ipynb -------------------------------------------------------------------------------- /examples/process_dataset_example.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/examples/process_dataset_example.ipynb -------------------------------------------------------------------------------- /examples/text_filters_example.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/examples/text_filters_example.ipynb -------------------------------------------------------------------------------- /examples/video_filters_example.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/examples/video_filters_example.ipynb -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/pyproject.toml -------------------------------------------------------------------------------- /scripts/multi_gpu_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/scripts/multi_gpu_filter.py -------------------------------------------------------------------------------- /simple_example.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/simple_example.py -------------------------------------------------------------------------------- /tests/datasets/files_bad_image/data.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/tests/datasets/files_bad_image/data.csv -------------------------------------------------------------------------------- /tests/datasets/files_bad_image/imgs/test/0.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/tests/datasets/files_bad_image/imgs/test/0.jpg -------------------------------------------------------------------------------- /tests/datasets/files_bad_image/imgs/test/1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/tests/datasets/files_bad_image/imgs/test/1.jpg -------------------------------------------------------------------------------- /tests/datasets/files_bad_image/imgs/test/2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/tests/datasets/files_bad_image/imgs/test/2.jpg -------------------------------------------------------------------------------- /tests/datasets/files_correct/data.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/tests/datasets/files_correct/data.csv -------------------------------------------------------------------------------- /tests/datasets/files_correct/imgs/test/0.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/tests/datasets/files_correct/imgs/test/0.jpg -------------------------------------------------------------------------------- /tests/datasets/files_correct/imgs/test/1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/tests/datasets/files_correct/imgs/test/1.jpg -------------------------------------------------------------------------------- /tests/datasets/sharded_files_correct/0.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/tests/datasets/sharded_files_correct/0.csv -------------------------------------------------------------------------------- /tests/datasets/sharded_files_correct/0/0.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/tests/datasets/sharded_files_correct/0/0.jpg -------------------------------------------------------------------------------- /tests/datasets/sharded_files_correct/0/1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/tests/datasets/sharded_files_correct/0/1.jpg -------------------------------------------------------------------------------- /tests/datasets/sharded_files_wrong_columns/0.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/tests/datasets/sharded_files_wrong_columns/0.csv -------------------------------------------------------------------------------- /tests/datasets/sharded_files_wrong_columns/0/0.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/tests/datasets/sharded_files_wrong_columns/0/0.jpg -------------------------------------------------------------------------------- /tests/datasets/sharded_files_wrong_columns/0/1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/tests/datasets/sharded_files_wrong_columns/0/1.jpg -------------------------------------------------------------------------------- /tests/datasets/sharded_files_wrong_columns/1.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/tests/datasets/sharded_files_wrong_columns/1.csv -------------------------------------------------------------------------------- /tests/datasets/sharded_files_wrong_columns/1/0.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/tests/datasets/sharded_files_wrong_columns/1/0.jpg -------------------------------------------------------------------------------- /tests/datasets/sharded_files_wrong_columns/1/1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/tests/datasets/sharded_files_wrong_columns/1/1.jpg -------------------------------------------------------------------------------- /tests/datasets/sharded_files_wrong_folder/0.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/tests/datasets/sharded_files_wrong_folder/0.csv -------------------------------------------------------------------------------- /tests/datasets/sharded_files_wrong_folder/0/1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/tests/datasets/sharded_files_wrong_folder/0/1.jpg -------------------------------------------------------------------------------- /tests/datasets/shards_bad_image/0.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/tests/datasets/shards_bad_image/0.csv -------------------------------------------------------------------------------- /tests/datasets/shards_bad_image/0.tar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/tests/datasets/shards_bad_image/0.tar -------------------------------------------------------------------------------- /tests/datasets/shards_correct/0.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/tests/datasets/shards_correct/0.csv -------------------------------------------------------------------------------- /tests/datasets/shards_correct/0.tar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/tests/datasets/shards_correct/0.tar -------------------------------------------------------------------------------- /tests/datasets/shards_wrong_columns/0.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/tests/datasets/shards_wrong_columns/0.csv -------------------------------------------------------------------------------- /tests/datasets/shards_wrong_columns/0.tar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/tests/datasets/shards_wrong_columns/0.tar -------------------------------------------------------------------------------- /tests/datasets/shards_wrong_columns/1.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/tests/datasets/shards_wrong_columns/1.csv -------------------------------------------------------------------------------- /tests/datasets/shards_wrong_columns/1.tar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/tests/datasets/shards_wrong_columns/1.tar -------------------------------------------------------------------------------- /tests/datasets/shards_wrong_tar/0.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/tests/datasets/shards_wrong_tar/0.csv -------------------------------------------------------------------------------- /tests/datasets/shards_wrong_tar/0.tar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/tests/datasets/shards_wrong_tar/0.tar -------------------------------------------------------------------------------- /tests/test_column_filters.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/tests/test_column_filters.py -------------------------------------------------------------------------------- /tests/test_configs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/tests/test_configs.py -------------------------------------------------------------------------------- /tests/test_filesystem.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/tests/test_filesystem.py -------------------------------------------------------------------------------- /tests/test_image_filters.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/tests/test_image_filters.py -------------------------------------------------------------------------------- /tests/test_pipelines.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/tests/test_pipelines.py -------------------------------------------------------------------------------- /tests/test_readers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/tests/test_readers.py -------------------------------------------------------------------------------- /tests/test_resizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/tests/test_resizer.py -------------------------------------------------------------------------------- /tests/test_validators.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/tests/test_validators.py -------------------------------------------------------------------------------- /tests/test_writers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/DataProcessingFramework/HEAD/tests/test_writers.py --------------------------------------------------------------------------------