├── .gitattributes ├── .gitignore ├── .gitmodules ├── LICENSE ├── README.md ├── assets └── mmsearch_r1_overview.png └── mmsearch_r1 ├── data ├── data_example.ipynb ├── mini_data.pq └── mmsearch_r1_infoseek_sub_2k.parquet ├── monkey_patch └── monkey_patch.py ├── prompts ├── after_image_search_prompt_qwenvl.pkl ├── after_text_search_prompt_qwenvl.pkl └── round_1_user_prompt_qwenvl.pkl ├── scripts ├── inference_torch_demo.py └── run_mmsearch_r1_grpo.sh ├── trainer └── multimodal │ ├── config │ └── ppo_trainer.yaml │ ├── core_algos.py │ ├── main_ppo.py │ └── ray_trainer.py ├── utils ├── dataset │ ├── __init__.py │ ├── mm_rl_dataset.py │ └── rl_dataset.py ├── reward_score_mm │ ├── __init__.py │ ├── geo3k.py │ ├── gsm8k.py │ ├── math.py │ ├── mmsearch_r1_score.py │ ├── prime_code │ │ ├── __init__.py │ │ ├── testing_util.py │ │ └── utils.py │ └── prime_math │ │ ├── __init__.py │ │ ├── grader.py │ │ └── math_normalize.py ├── tools │ ├── image_search.py │ └── text_search.py └── torch_functional.py └── workers └── multimodal ├── actor └── dp_actor.py ├── fsdp_workers.py ├── reward ├── __init__.py ├── mmsearch_r1.py ├── naive.py └── utils.py └── rollout └── vllm_rollout_spmd.py /.gitattributes: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/.gitattributes -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/.gitignore -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/.gitmodules -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/README.md -------------------------------------------------------------------------------- /assets/mmsearch_r1_overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/assets/mmsearch_r1_overview.png -------------------------------------------------------------------------------- /mmsearch_r1/data/data_example.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/mmsearch_r1/data/data_example.ipynb -------------------------------------------------------------------------------- /mmsearch_r1/data/mini_data.pq: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/mmsearch_r1/data/mini_data.pq -------------------------------------------------------------------------------- /mmsearch_r1/data/mmsearch_r1_infoseek_sub_2k.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/mmsearch_r1/data/mmsearch_r1_infoseek_sub_2k.parquet -------------------------------------------------------------------------------- /mmsearch_r1/monkey_patch/monkey_patch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/mmsearch_r1/monkey_patch/monkey_patch.py -------------------------------------------------------------------------------- /mmsearch_r1/prompts/after_image_search_prompt_qwenvl.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/mmsearch_r1/prompts/after_image_search_prompt_qwenvl.pkl -------------------------------------------------------------------------------- /mmsearch_r1/prompts/after_text_search_prompt_qwenvl.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/mmsearch_r1/prompts/after_text_search_prompt_qwenvl.pkl -------------------------------------------------------------------------------- /mmsearch_r1/prompts/round_1_user_prompt_qwenvl.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/mmsearch_r1/prompts/round_1_user_prompt_qwenvl.pkl -------------------------------------------------------------------------------- /mmsearch_r1/scripts/inference_torch_demo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/mmsearch_r1/scripts/inference_torch_demo.py -------------------------------------------------------------------------------- /mmsearch_r1/scripts/run_mmsearch_r1_grpo.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/mmsearch_r1/scripts/run_mmsearch_r1_grpo.sh -------------------------------------------------------------------------------- /mmsearch_r1/trainer/multimodal/config/ppo_trainer.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/mmsearch_r1/trainer/multimodal/config/ppo_trainer.yaml -------------------------------------------------------------------------------- /mmsearch_r1/trainer/multimodal/core_algos.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/mmsearch_r1/trainer/multimodal/core_algos.py -------------------------------------------------------------------------------- /mmsearch_r1/trainer/multimodal/main_ppo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/mmsearch_r1/trainer/multimodal/main_ppo.py -------------------------------------------------------------------------------- /mmsearch_r1/trainer/multimodal/ray_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/mmsearch_r1/trainer/multimodal/ray_trainer.py -------------------------------------------------------------------------------- /mmsearch_r1/utils/dataset/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/mmsearch_r1/utils/dataset/__init__.py -------------------------------------------------------------------------------- /mmsearch_r1/utils/dataset/mm_rl_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/mmsearch_r1/utils/dataset/mm_rl_dataset.py -------------------------------------------------------------------------------- /mmsearch_r1/utils/dataset/rl_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/mmsearch_r1/utils/dataset/rl_dataset.py -------------------------------------------------------------------------------- /mmsearch_r1/utils/reward_score_mm/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/mmsearch_r1/utils/reward_score_mm/__init__.py -------------------------------------------------------------------------------- /mmsearch_r1/utils/reward_score_mm/geo3k.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/mmsearch_r1/utils/reward_score_mm/geo3k.py -------------------------------------------------------------------------------- /mmsearch_r1/utils/reward_score_mm/gsm8k.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/mmsearch_r1/utils/reward_score_mm/gsm8k.py -------------------------------------------------------------------------------- /mmsearch_r1/utils/reward_score_mm/math.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/mmsearch_r1/utils/reward_score_mm/math.py -------------------------------------------------------------------------------- /mmsearch_r1/utils/reward_score_mm/mmsearch_r1_score.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/mmsearch_r1/utils/reward_score_mm/mmsearch_r1_score.py -------------------------------------------------------------------------------- /mmsearch_r1/utils/reward_score_mm/prime_code/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/mmsearch_r1/utils/reward_score_mm/prime_code/__init__.py -------------------------------------------------------------------------------- /mmsearch_r1/utils/reward_score_mm/prime_code/testing_util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/mmsearch_r1/utils/reward_score_mm/prime_code/testing_util.py -------------------------------------------------------------------------------- /mmsearch_r1/utils/reward_score_mm/prime_code/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/mmsearch_r1/utils/reward_score_mm/prime_code/utils.py -------------------------------------------------------------------------------- /mmsearch_r1/utils/reward_score_mm/prime_math/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/mmsearch_r1/utils/reward_score_mm/prime_math/__init__.py -------------------------------------------------------------------------------- /mmsearch_r1/utils/reward_score_mm/prime_math/grader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/mmsearch_r1/utils/reward_score_mm/prime_math/grader.py -------------------------------------------------------------------------------- /mmsearch_r1/utils/reward_score_mm/prime_math/math_normalize.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/mmsearch_r1/utils/reward_score_mm/prime_math/math_normalize.py -------------------------------------------------------------------------------- /mmsearch_r1/utils/tools/image_search.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/mmsearch_r1/utils/tools/image_search.py -------------------------------------------------------------------------------- /mmsearch_r1/utils/tools/text_search.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/mmsearch_r1/utils/tools/text_search.py -------------------------------------------------------------------------------- /mmsearch_r1/utils/torch_functional.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/mmsearch_r1/utils/torch_functional.py -------------------------------------------------------------------------------- /mmsearch_r1/workers/multimodal/actor/dp_actor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/mmsearch_r1/workers/multimodal/actor/dp_actor.py -------------------------------------------------------------------------------- /mmsearch_r1/workers/multimodal/fsdp_workers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/mmsearch_r1/workers/multimodal/fsdp_workers.py -------------------------------------------------------------------------------- /mmsearch_r1/workers/multimodal/reward/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/mmsearch_r1/workers/multimodal/reward/__init__.py -------------------------------------------------------------------------------- /mmsearch_r1/workers/multimodal/reward/mmsearch_r1.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/mmsearch_r1/workers/multimodal/reward/mmsearch_r1.py -------------------------------------------------------------------------------- /mmsearch_r1/workers/multimodal/reward/naive.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/mmsearch_r1/workers/multimodal/reward/naive.py -------------------------------------------------------------------------------- /mmsearch_r1/workers/multimodal/reward/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/mmsearch_r1/workers/multimodal/reward/utils.py -------------------------------------------------------------------------------- /mmsearch_r1/workers/multimodal/rollout/vllm_rollout_spmd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EvolvingLMMs-Lab/multimodal-search-r1/HEAD/mmsearch_r1/workers/multimodal/rollout/vllm_rollout_spmd.py --------------------------------------------------------------------------------