├── .gitignore ├── LICENSE ├── README.md ├── VIDEOPHY2 ├── README.md ├── data_utils │ ├── __init__.py │ ├── processors │ │ ├── __init__.py │ │ ├── builder.py │ │ ├── caption_processor.py │ │ └── default_processor.py │ ├── randaugment.py │ ├── registry.py │ └── xgpt3_dataset.py ├── examples │ ├── A_bobsled_team_pushes_their_sled_off_the_starting_line,_accelerating_down_a_snowy_track.mp4 │ ├── A_canoeist_uses_a_single-bladed_paddle_to_propel_their_canoe_across_a_lake,_the_paddle's_movement_visible_against_the_still_water_1.mp4 │ ├── A_carpenter_pulls_apart_two_glued_pieces_of_wood_using_a_chisel,_resulting_in_two_separate_boards.mp4 │ ├── A_construction_worker_uses_a_sandblasting_machine_to_clean_the_brick_facade_of_a_building,_removing_layers_of_grime.mp4 │ ├── A_hedge_trimmer_is_used_to_shape_a_tall,_dense_privet_hedge,_the_blades_visibly_cutting_through_the_leaves_and_stems_1.mp4 │ ├── A_jetski_moves_quickly_through_a_narrow_channel_between_two_rocky_outcrops_0.mp4 │ ├── A_large,_heavy_pot_is_pushed_on_a_slightly_inclined_kitchen_counter;_it_nearly_slides_off_but_remains_in_place.mp4 │ ├── A_leaf_blower_is_pointed_at_a_patch_of_leaves_on_a_lawn;_the_leaves_are_forcefully_displaced_in_a_specific_direction_0.mp4 │ ├── A_paint_roller_applies_a_coat_of_beige_paint_to_a_textured_wall,_showing_the_paint_filling_in_the_textures_0.mp4 │ ├── A_person_uses_a_leaf_blower_to_clear_leaves_from_a_paved_driveway,_the_leaves_propelled_into_a_nearby_flower_bed.mp4 │ ├── A_player_lunges_to_reach_a_low_shot,_reaching_the_shuttlecock_with_their_racquet.mp4 │ ├── A_potter_wedges_clay,_forcefully_pushing_and_folding_the_clay_to_remove_air_bubbles.mp4 │ ├── A_seamstress_pulls_the_fabric_taut,_guiding_the_needle_with_their_fingers_as_they_sew_a_straight_seam_2.mp4 │ ├── A_streamer,_cut_from_paper,_descends_from_a_high_point,_twisting_and_turning_gently_0.mp4 │ ├── A_surfer_performs_a_sharp_turn_on_their_board,_creating_a_spray_of_water_0.mp4 │ ├── Beer_is_poured_from_a_keg_into_a_plastic_cup,_the_liquid_splashing_slightly_on_the_rim.mp4 │ ├── Hands_fold_a_map,_showing_the_creases_forming_as_sections_are_brought_together_0.mp4 │ ├── Nunchucks_are_used_to_break_a_single_brick_from_a_wall,_creating_visible_cracks_in_the_surrounding_bricks.mp4 │ ├── Syrup_is_poured_onto_pancakes_from_a_bottle,_the_syrup_coating_the_surface_1.mp4 │ ├── Two_knives_are_thrown_in_a_criss-cross_pattern,_nearly_colliding_in_mid-air_before_striking_a_stack_of_wooden_blocks.mp4 │ ├── output_pc.csv │ ├── output_rule.csv │ ├── output_sa.csv │ ├── rule.csv │ └── sa_pc.csv ├── inference.py ├── main_graph.png ├── mplug_owl_video │ ├── __init__.py │ ├── configuration_mplug_owl.py │ ├── modeling_mplug_owl.py │ ├── processing_mplug_owl.py │ └── tokenization_mplug_owl.py ├── paper.pdf ├── template.py ├── utils.py └── videophy2.png ├── asset ├── .DS_Store ├── Lavie_The_net_catches_the_fast-moving_soccer_ball._1.gif ├── Lavie_Water_drips_from_a_leaky_tap_into_a_sink._1.gif ├── Pika_A_whisk_churns_heavy_cream_into_whipped_cream._0.gif ├── Pika_Wooden_swing_dangles_over_the_sand_in_the_sandpit._0.gif ├── VideoPhysics.png ├── leaderboard.png ├── main_graph.png ├── physics_verylong.png ├── vc2_Plastic_frisbee_lands_on_a_lush_grass_lawn_0.gif └── vc2_Water_pouring_from_a_watering_can_onto_plants_0.gif ├── examples ├── A_car_brush_lathers_shampoo_onto_a_dirty_vehicle_1.mp4 ├── A_spatula_scoops_up_cake_batter_into_a_baking_pan._0.mp4 ├── A_wooden_spoon_stirs_the_hot_soup_in_the_pot._1.mp4 ├── An_apple_falls_and_bounces_on_the_hard_ground._1.mp4 ├── Coffee_accepting_a_gentle_pour_of_milk_1.mp4 ├── Pushpin_skates_across_a_glossy_desk_surface._1.mp4 ├── Shoe_squishes_a_bug._0.mp4 ├── example.csv ├── physics_testing.csv ├── sa_testing.csv ├── train_example.csv ├── videocon_format_train.csv ├── videocon_format_val.csv ├── videocon_physics_pc_testing.csv └── videocon_physics_sa_testing.csv ├── preprint.pdf ├── requirements.txt ├── utils ├── constants.py ├── prepare_data.py └── prepare_train_data.py └── videocon └── training ├── configs └── video.yaml ├── pipeline_video ├── __init__.py ├── data_utils │ ├── __init__.py │ ├── processors │ │ ├── __init__.py │ │ ├── builder.py │ │ ├── caption_processor.py │ │ └── default_processor.py │ ├── randaugment.py │ ├── registry.py │ └── xgpt3_dataset.py ├── entailment_inference.py ├── model_unload_merge.py ├── mplug_owl_video │ ├── __init__.py │ ├── configuration_mplug_owl.py │ ├── modeling_mplug_owl.py │ ├── processing_mplug_owl.py │ └── tokenization_mplug_owl.py ├── train.py └── utils.py ├── scripts └── train_it.sh └── wandb ├── debug-internal.log ├── debug.log ├── latest-run └── run-20240609_214112-olqxxhne ├── files ├── code │ └── videocon │ │ └── training │ │ └── pipeline_video │ │ └── train.py ├── conda-environment.yaml ├── config.yaml ├── diff.patch ├── output.log ├── requirements.txt ├── wandb-metadata.json └── wandb-summary.json ├── logs ├── debug-internal.log └── debug.log └── run-olqxxhne.wandb /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/README.md -------------------------------------------------------------------------------- /VIDEOPHY2/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/README.md -------------------------------------------------------------------------------- /VIDEOPHY2/data_utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/data_utils/__init__.py -------------------------------------------------------------------------------- /VIDEOPHY2/data_utils/processors/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/data_utils/processors/__init__.py -------------------------------------------------------------------------------- /VIDEOPHY2/data_utils/processors/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/data_utils/processors/builder.py -------------------------------------------------------------------------------- /VIDEOPHY2/data_utils/processors/caption_processor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/data_utils/processors/caption_processor.py -------------------------------------------------------------------------------- /VIDEOPHY2/data_utils/processors/default_processor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/data_utils/processors/default_processor.py -------------------------------------------------------------------------------- /VIDEOPHY2/data_utils/randaugment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/data_utils/randaugment.py -------------------------------------------------------------------------------- /VIDEOPHY2/data_utils/registry.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/data_utils/registry.py -------------------------------------------------------------------------------- /VIDEOPHY2/data_utils/xgpt3_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/data_utils/xgpt3_dataset.py -------------------------------------------------------------------------------- /VIDEOPHY2/examples/A_bobsled_team_pushes_their_sled_off_the_starting_line,_accelerating_down_a_snowy_track.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/examples/A_bobsled_team_pushes_their_sled_off_the_starting_line,_accelerating_down_a_snowy_track.mp4 -------------------------------------------------------------------------------- /VIDEOPHY2/examples/A_canoeist_uses_a_single-bladed_paddle_to_propel_their_canoe_across_a_lake,_the_paddle's_movement_visible_against_the_still_water_1.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/examples/A_canoeist_uses_a_single-bladed_paddle_to_propel_their_canoe_across_a_lake,_the_paddle's_movement_visible_against_the_still_water_1.mp4 -------------------------------------------------------------------------------- /VIDEOPHY2/examples/A_carpenter_pulls_apart_two_glued_pieces_of_wood_using_a_chisel,_resulting_in_two_separate_boards.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/examples/A_carpenter_pulls_apart_two_glued_pieces_of_wood_using_a_chisel,_resulting_in_two_separate_boards.mp4 -------------------------------------------------------------------------------- /VIDEOPHY2/examples/A_construction_worker_uses_a_sandblasting_machine_to_clean_the_brick_facade_of_a_building,_removing_layers_of_grime.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/examples/A_construction_worker_uses_a_sandblasting_machine_to_clean_the_brick_facade_of_a_building,_removing_layers_of_grime.mp4 -------------------------------------------------------------------------------- /VIDEOPHY2/examples/A_hedge_trimmer_is_used_to_shape_a_tall,_dense_privet_hedge,_the_blades_visibly_cutting_through_the_leaves_and_stems_1.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/examples/A_hedge_trimmer_is_used_to_shape_a_tall,_dense_privet_hedge,_the_blades_visibly_cutting_through_the_leaves_and_stems_1.mp4 -------------------------------------------------------------------------------- /VIDEOPHY2/examples/A_jetski_moves_quickly_through_a_narrow_channel_between_two_rocky_outcrops_0.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/examples/A_jetski_moves_quickly_through_a_narrow_channel_between_two_rocky_outcrops_0.mp4 -------------------------------------------------------------------------------- /VIDEOPHY2/examples/A_large,_heavy_pot_is_pushed_on_a_slightly_inclined_kitchen_counter;_it_nearly_slides_off_but_remains_in_place.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/examples/A_large,_heavy_pot_is_pushed_on_a_slightly_inclined_kitchen_counter;_it_nearly_slides_off_but_remains_in_place.mp4 -------------------------------------------------------------------------------- /VIDEOPHY2/examples/A_leaf_blower_is_pointed_at_a_patch_of_leaves_on_a_lawn;_the_leaves_are_forcefully_displaced_in_a_specific_direction_0.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/examples/A_leaf_blower_is_pointed_at_a_patch_of_leaves_on_a_lawn;_the_leaves_are_forcefully_displaced_in_a_specific_direction_0.mp4 -------------------------------------------------------------------------------- /VIDEOPHY2/examples/A_paint_roller_applies_a_coat_of_beige_paint_to_a_textured_wall,_showing_the_paint_filling_in_the_textures_0.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/examples/A_paint_roller_applies_a_coat_of_beige_paint_to_a_textured_wall,_showing_the_paint_filling_in_the_textures_0.mp4 -------------------------------------------------------------------------------- /VIDEOPHY2/examples/A_person_uses_a_leaf_blower_to_clear_leaves_from_a_paved_driveway,_the_leaves_propelled_into_a_nearby_flower_bed.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/examples/A_person_uses_a_leaf_blower_to_clear_leaves_from_a_paved_driveway,_the_leaves_propelled_into_a_nearby_flower_bed.mp4 -------------------------------------------------------------------------------- /VIDEOPHY2/examples/A_player_lunges_to_reach_a_low_shot,_reaching_the_shuttlecock_with_their_racquet.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/examples/A_player_lunges_to_reach_a_low_shot,_reaching_the_shuttlecock_with_their_racquet.mp4 -------------------------------------------------------------------------------- /VIDEOPHY2/examples/A_potter_wedges_clay,_forcefully_pushing_and_folding_the_clay_to_remove_air_bubbles.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/examples/A_potter_wedges_clay,_forcefully_pushing_and_folding_the_clay_to_remove_air_bubbles.mp4 -------------------------------------------------------------------------------- /VIDEOPHY2/examples/A_seamstress_pulls_the_fabric_taut,_guiding_the_needle_with_their_fingers_as_they_sew_a_straight_seam_2.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/examples/A_seamstress_pulls_the_fabric_taut,_guiding_the_needle_with_their_fingers_as_they_sew_a_straight_seam_2.mp4 -------------------------------------------------------------------------------- /VIDEOPHY2/examples/A_streamer,_cut_from_paper,_descends_from_a_high_point,_twisting_and_turning_gently_0.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/examples/A_streamer,_cut_from_paper,_descends_from_a_high_point,_twisting_and_turning_gently_0.mp4 -------------------------------------------------------------------------------- /VIDEOPHY2/examples/A_surfer_performs_a_sharp_turn_on_their_board,_creating_a_spray_of_water_0.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/examples/A_surfer_performs_a_sharp_turn_on_their_board,_creating_a_spray_of_water_0.mp4 -------------------------------------------------------------------------------- /VIDEOPHY2/examples/Beer_is_poured_from_a_keg_into_a_plastic_cup,_the_liquid_splashing_slightly_on_the_rim.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/examples/Beer_is_poured_from_a_keg_into_a_plastic_cup,_the_liquid_splashing_slightly_on_the_rim.mp4 -------------------------------------------------------------------------------- /VIDEOPHY2/examples/Hands_fold_a_map,_showing_the_creases_forming_as_sections_are_brought_together_0.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/examples/Hands_fold_a_map,_showing_the_creases_forming_as_sections_are_brought_together_0.mp4 -------------------------------------------------------------------------------- /VIDEOPHY2/examples/Nunchucks_are_used_to_break_a_single_brick_from_a_wall,_creating_visible_cracks_in_the_surrounding_bricks.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/examples/Nunchucks_are_used_to_break_a_single_brick_from_a_wall,_creating_visible_cracks_in_the_surrounding_bricks.mp4 -------------------------------------------------------------------------------- /VIDEOPHY2/examples/Syrup_is_poured_onto_pancakes_from_a_bottle,_the_syrup_coating_the_surface_1.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/examples/Syrup_is_poured_onto_pancakes_from_a_bottle,_the_syrup_coating_the_surface_1.mp4 -------------------------------------------------------------------------------- /VIDEOPHY2/examples/Two_knives_are_thrown_in_a_criss-cross_pattern,_nearly_colliding_in_mid-air_before_striking_a_stack_of_wooden_blocks.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/examples/Two_knives_are_thrown_in_a_criss-cross_pattern,_nearly_colliding_in_mid-air_before_striking_a_stack_of_wooden_blocks.mp4 -------------------------------------------------------------------------------- /VIDEOPHY2/examples/output_pc.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/examples/output_pc.csv -------------------------------------------------------------------------------- /VIDEOPHY2/examples/output_rule.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/examples/output_rule.csv -------------------------------------------------------------------------------- /VIDEOPHY2/examples/output_sa.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/examples/output_sa.csv -------------------------------------------------------------------------------- /VIDEOPHY2/examples/rule.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/examples/rule.csv -------------------------------------------------------------------------------- /VIDEOPHY2/examples/sa_pc.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/examples/sa_pc.csv -------------------------------------------------------------------------------- /VIDEOPHY2/inference.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/inference.py -------------------------------------------------------------------------------- /VIDEOPHY2/main_graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/main_graph.png -------------------------------------------------------------------------------- /VIDEOPHY2/mplug_owl_video/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/mplug_owl_video/__init__.py -------------------------------------------------------------------------------- /VIDEOPHY2/mplug_owl_video/configuration_mplug_owl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/mplug_owl_video/configuration_mplug_owl.py -------------------------------------------------------------------------------- /VIDEOPHY2/mplug_owl_video/modeling_mplug_owl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/mplug_owl_video/modeling_mplug_owl.py -------------------------------------------------------------------------------- /VIDEOPHY2/mplug_owl_video/processing_mplug_owl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/mplug_owl_video/processing_mplug_owl.py -------------------------------------------------------------------------------- /VIDEOPHY2/mplug_owl_video/tokenization_mplug_owl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/mplug_owl_video/tokenization_mplug_owl.py -------------------------------------------------------------------------------- /VIDEOPHY2/paper.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/paper.pdf -------------------------------------------------------------------------------- /VIDEOPHY2/template.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/template.py -------------------------------------------------------------------------------- /VIDEOPHY2/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/utils.py -------------------------------------------------------------------------------- /VIDEOPHY2/videophy2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/VIDEOPHY2/videophy2.png -------------------------------------------------------------------------------- /asset/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/asset/.DS_Store -------------------------------------------------------------------------------- /asset/Lavie_The_net_catches_the_fast-moving_soccer_ball._1.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/asset/Lavie_The_net_catches_the_fast-moving_soccer_ball._1.gif -------------------------------------------------------------------------------- /asset/Lavie_Water_drips_from_a_leaky_tap_into_a_sink._1.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/asset/Lavie_Water_drips_from_a_leaky_tap_into_a_sink._1.gif -------------------------------------------------------------------------------- /asset/Pika_A_whisk_churns_heavy_cream_into_whipped_cream._0.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/asset/Pika_A_whisk_churns_heavy_cream_into_whipped_cream._0.gif -------------------------------------------------------------------------------- /asset/Pika_Wooden_swing_dangles_over_the_sand_in_the_sandpit._0.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/asset/Pika_Wooden_swing_dangles_over_the_sand_in_the_sandpit._0.gif -------------------------------------------------------------------------------- /asset/VideoPhysics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/asset/VideoPhysics.png -------------------------------------------------------------------------------- /asset/leaderboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/asset/leaderboard.png -------------------------------------------------------------------------------- /asset/main_graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/asset/main_graph.png -------------------------------------------------------------------------------- /asset/physics_verylong.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/asset/physics_verylong.png -------------------------------------------------------------------------------- /asset/vc2_Plastic_frisbee_lands_on_a_lush_grass_lawn_0.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/asset/vc2_Plastic_frisbee_lands_on_a_lush_grass_lawn_0.gif -------------------------------------------------------------------------------- /asset/vc2_Water_pouring_from_a_watering_can_onto_plants_0.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/asset/vc2_Water_pouring_from_a_watering_can_onto_plants_0.gif -------------------------------------------------------------------------------- /examples/A_car_brush_lathers_shampoo_onto_a_dirty_vehicle_1.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/examples/A_car_brush_lathers_shampoo_onto_a_dirty_vehicle_1.mp4 -------------------------------------------------------------------------------- /examples/A_spatula_scoops_up_cake_batter_into_a_baking_pan._0.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/examples/A_spatula_scoops_up_cake_batter_into_a_baking_pan._0.mp4 -------------------------------------------------------------------------------- /examples/A_wooden_spoon_stirs_the_hot_soup_in_the_pot._1.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/examples/A_wooden_spoon_stirs_the_hot_soup_in_the_pot._1.mp4 -------------------------------------------------------------------------------- /examples/An_apple_falls_and_bounces_on_the_hard_ground._1.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/examples/An_apple_falls_and_bounces_on_the_hard_ground._1.mp4 -------------------------------------------------------------------------------- /examples/Coffee_accepting_a_gentle_pour_of_milk_1.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/examples/Coffee_accepting_a_gentle_pour_of_milk_1.mp4 -------------------------------------------------------------------------------- /examples/Pushpin_skates_across_a_glossy_desk_surface._1.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/examples/Pushpin_skates_across_a_glossy_desk_surface._1.mp4 -------------------------------------------------------------------------------- /examples/Shoe_squishes_a_bug._0.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/examples/Shoe_squishes_a_bug._0.mp4 -------------------------------------------------------------------------------- /examples/example.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/examples/example.csv -------------------------------------------------------------------------------- /examples/physics_testing.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/examples/physics_testing.csv -------------------------------------------------------------------------------- /examples/sa_testing.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/examples/sa_testing.csv -------------------------------------------------------------------------------- /examples/train_example.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/examples/train_example.csv -------------------------------------------------------------------------------- /examples/videocon_format_train.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/examples/videocon_format_train.csv -------------------------------------------------------------------------------- /examples/videocon_format_val.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/examples/videocon_format_val.csv -------------------------------------------------------------------------------- /examples/videocon_physics_pc_testing.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/examples/videocon_physics_pc_testing.csv -------------------------------------------------------------------------------- /examples/videocon_physics_sa_testing.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/examples/videocon_physics_sa_testing.csv -------------------------------------------------------------------------------- /preprint.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/preprint.pdf -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/requirements.txt -------------------------------------------------------------------------------- /utils/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/utils/constants.py -------------------------------------------------------------------------------- /utils/prepare_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/utils/prepare_data.py -------------------------------------------------------------------------------- /utils/prepare_train_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/utils/prepare_train_data.py -------------------------------------------------------------------------------- /videocon/training/configs/video.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/videocon/training/configs/video.yaml -------------------------------------------------------------------------------- /videocon/training/pipeline_video/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /videocon/training/pipeline_video/data_utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/videocon/training/pipeline_video/data_utils/__init__.py -------------------------------------------------------------------------------- /videocon/training/pipeline_video/data_utils/processors/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/videocon/training/pipeline_video/data_utils/processors/__init__.py -------------------------------------------------------------------------------- /videocon/training/pipeline_video/data_utils/processors/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/videocon/training/pipeline_video/data_utils/processors/builder.py -------------------------------------------------------------------------------- /videocon/training/pipeline_video/data_utils/processors/caption_processor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/videocon/training/pipeline_video/data_utils/processors/caption_processor.py -------------------------------------------------------------------------------- /videocon/training/pipeline_video/data_utils/processors/default_processor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/videocon/training/pipeline_video/data_utils/processors/default_processor.py -------------------------------------------------------------------------------- /videocon/training/pipeline_video/data_utils/randaugment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/videocon/training/pipeline_video/data_utils/randaugment.py -------------------------------------------------------------------------------- /videocon/training/pipeline_video/data_utils/registry.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/videocon/training/pipeline_video/data_utils/registry.py -------------------------------------------------------------------------------- /videocon/training/pipeline_video/data_utils/xgpt3_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/videocon/training/pipeline_video/data_utils/xgpt3_dataset.py -------------------------------------------------------------------------------- /videocon/training/pipeline_video/entailment_inference.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/videocon/training/pipeline_video/entailment_inference.py -------------------------------------------------------------------------------- /videocon/training/pipeline_video/model_unload_merge.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/videocon/training/pipeline_video/model_unload_merge.py -------------------------------------------------------------------------------- /videocon/training/pipeline_video/mplug_owl_video/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/videocon/training/pipeline_video/mplug_owl_video/__init__.py -------------------------------------------------------------------------------- /videocon/training/pipeline_video/mplug_owl_video/configuration_mplug_owl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/videocon/training/pipeline_video/mplug_owl_video/configuration_mplug_owl.py -------------------------------------------------------------------------------- /videocon/training/pipeline_video/mplug_owl_video/modeling_mplug_owl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/videocon/training/pipeline_video/mplug_owl_video/modeling_mplug_owl.py -------------------------------------------------------------------------------- /videocon/training/pipeline_video/mplug_owl_video/processing_mplug_owl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/videocon/training/pipeline_video/mplug_owl_video/processing_mplug_owl.py -------------------------------------------------------------------------------- /videocon/training/pipeline_video/mplug_owl_video/tokenization_mplug_owl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/videocon/training/pipeline_video/mplug_owl_video/tokenization_mplug_owl.py -------------------------------------------------------------------------------- /videocon/training/pipeline_video/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/videocon/training/pipeline_video/train.py -------------------------------------------------------------------------------- /videocon/training/pipeline_video/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/videocon/training/pipeline_video/utils.py -------------------------------------------------------------------------------- /videocon/training/scripts/train_it.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/videocon/training/scripts/train_it.sh -------------------------------------------------------------------------------- /videocon/training/wandb/debug-internal.log: -------------------------------------------------------------------------------- 1 | run-20240609_214112-olqxxhne/logs/debug-internal.log -------------------------------------------------------------------------------- /videocon/training/wandb/debug.log: -------------------------------------------------------------------------------- 1 | run-20240609_214112-olqxxhne/logs/debug.log -------------------------------------------------------------------------------- /videocon/training/wandb/latest-run: -------------------------------------------------------------------------------- 1 | run-20240609_214112-olqxxhne -------------------------------------------------------------------------------- /videocon/training/wandb/run-20240609_214112-olqxxhne/files/code/videocon/training/pipeline_video/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/videocon/training/wandb/run-20240609_214112-olqxxhne/files/code/videocon/training/pipeline_video/train.py -------------------------------------------------------------------------------- /videocon/training/wandb/run-20240609_214112-olqxxhne/files/conda-environment.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/videocon/training/wandb/run-20240609_214112-olqxxhne/files/conda-environment.yaml -------------------------------------------------------------------------------- /videocon/training/wandb/run-20240609_214112-olqxxhne/files/config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/videocon/training/wandb/run-20240609_214112-olqxxhne/files/config.yaml -------------------------------------------------------------------------------- /videocon/training/wandb/run-20240609_214112-olqxxhne/files/diff.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/videocon/training/wandb/run-20240609_214112-olqxxhne/files/diff.patch -------------------------------------------------------------------------------- /videocon/training/wandb/run-20240609_214112-olqxxhne/files/output.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/videocon/training/wandb/run-20240609_214112-olqxxhne/files/output.log -------------------------------------------------------------------------------- /videocon/training/wandb/run-20240609_214112-olqxxhne/files/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/videocon/training/wandb/run-20240609_214112-olqxxhne/files/requirements.txt -------------------------------------------------------------------------------- /videocon/training/wandb/run-20240609_214112-olqxxhne/files/wandb-metadata.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/videocon/training/wandb/run-20240609_214112-olqxxhne/files/wandb-metadata.json -------------------------------------------------------------------------------- /videocon/training/wandb/run-20240609_214112-olqxxhne/files/wandb-summary.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/videocon/training/wandb/run-20240609_214112-olqxxhne/files/wandb-summary.json -------------------------------------------------------------------------------- /videocon/training/wandb/run-20240609_214112-olqxxhne/logs/debug-internal.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/videocon/training/wandb/run-20240609_214112-olqxxhne/logs/debug-internal.log -------------------------------------------------------------------------------- /videocon/training/wandb/run-20240609_214112-olqxxhne/logs/debug.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/videocon/training/wandb/run-20240609_214112-olqxxhne/logs/debug.log -------------------------------------------------------------------------------- /videocon/training/wandb/run-20240609_214112-olqxxhne/run-olqxxhne.wandb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hritikbansal/videophy/HEAD/videocon/training/wandb/run-20240609_214112-olqxxhne/run-olqxxhne.wandb --------------------------------------------------------------------------------