├── .gitignore ├── LICENSE.md ├── MCTS ├── MCTS.py └── __init__.py ├── README.md ├── agents ├── HRL_agent_RL.py ├── HRL_agent_mcts.py ├── MCTS_agent.py ├── __init__.py ├── base_agent.py ├── belief.py └── random_agent.py ├── algos ├── __init__.py ├── a2c.py ├── a2c_mp.py ├── arena.py └── arena_mp2.py ├── arguments.py ├── assets ├── collab_fig.gif ├── cover_fig_final.pdf └── cover_fig_final.png ├── checkpoints └── README.md ├── dataset ├── env_to_pred.json ├── match_demo_test.json ├── object_info.json ├── object_info_small.json ├── test_env_set_help.pik ├── train_env_set_help.pik └── watch_scenes_split.json ├── demo ├── virtualhome_demo.ipynb └── watch_and_help_demo.ipynb ├── envs ├── __init__.py ├── base_environment.py ├── envs.py ├── graph_env.py ├── python_environment.py └── unity_environment.py ├── evaluation ├── calc_metrics_reward.py └── eval_pred.py ├── gen_data ├── __init__.py ├── data │ ├── class_name_size.json │ ├── init_pool.json │ ├── object_info.json │ ├── object_info1.json │ ├── object_info2.json │ ├── object_info3.json │ ├── object_info4.json │ ├── object_info5.json │ ├── object_info6.json │ └── object_info7.json ├── init_goal_setter │ ├── __init__.py │ ├── init_goal_base.py │ └── tasks.py ├── vh_init.py └── vh_init_gen_test.py ├── models ├── __init__.py ├── actor_critic.py ├── actor_critic_hl_mcts.py ├── base_nets.py ├── distributions.py ├── graph_nn.py └── single_policy.py ├── requirements.txt ├── scripts ├── download_ckpts.sh ├── test_watch.sh └── train_watch.sh ├── testing_agents ├── test_hp.py ├── test_hp_pred_goal.py ├── test_hp_random_goal.py ├── test_hrl.py ├── test_hrl_pred.py ├── test_hybrid.py ├── test_hybrid_pred_goal.py ├── test_random_action.py └── test_single_agent.py ├── tests └── test_simspeed.py ├── training_agents └── train_a2c.py ├── utils ├── __init__.py ├── convert_pkl_tojson_pred.py ├── jquery.js ├── kill.py ├── memory.py ├── rewards.py ├── utils.py ├── utils_environment.py ├── utils_goals.py ├── utils_models.py ├── utils_plot.py ├── utils_rl_agent.py └── utils_videos.py └── watch ├── helper.py ├── network ├── encoder_decoder.py └── module_graph.py ├── predicate-train.py └── predicate ├── demo_dataset_graph.py └── utils.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/LICENSE.md -------------------------------------------------------------------------------- /MCTS/MCTS.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/MCTS/MCTS.py -------------------------------------------------------------------------------- /MCTS/__init__.py: -------------------------------------------------------------------------------- 1 | from .MCTS import * -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/README.md -------------------------------------------------------------------------------- /agents/HRL_agent_RL.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/agents/HRL_agent_RL.py -------------------------------------------------------------------------------- /agents/HRL_agent_mcts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/agents/HRL_agent_mcts.py -------------------------------------------------------------------------------- /agents/MCTS_agent.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/agents/MCTS_agent.py -------------------------------------------------------------------------------- /agents/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/agents/__init__.py -------------------------------------------------------------------------------- /agents/base_agent.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/agents/base_agent.py -------------------------------------------------------------------------------- /agents/belief.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/agents/belief.py -------------------------------------------------------------------------------- /agents/random_agent.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/agents/random_agent.py -------------------------------------------------------------------------------- /algos/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/algos/__init__.py -------------------------------------------------------------------------------- /algos/a2c.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/algos/a2c.py -------------------------------------------------------------------------------- /algos/a2c_mp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/algos/a2c_mp.py -------------------------------------------------------------------------------- /algos/arena.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/algos/arena.py -------------------------------------------------------------------------------- /algos/arena_mp2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/algos/arena_mp2.py -------------------------------------------------------------------------------- /arguments.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/arguments.py -------------------------------------------------------------------------------- /assets/collab_fig.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/assets/collab_fig.gif -------------------------------------------------------------------------------- /assets/cover_fig_final.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/assets/cover_fig_final.pdf -------------------------------------------------------------------------------- /assets/cover_fig_final.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/assets/cover_fig_final.png -------------------------------------------------------------------------------- /checkpoints/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/checkpoints/README.md -------------------------------------------------------------------------------- /dataset/env_to_pred.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/dataset/env_to_pred.json -------------------------------------------------------------------------------- /dataset/match_demo_test.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/dataset/match_demo_test.json -------------------------------------------------------------------------------- /dataset/object_info.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/dataset/object_info.json -------------------------------------------------------------------------------- /dataset/object_info_small.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/dataset/object_info_small.json -------------------------------------------------------------------------------- /dataset/test_env_set_help.pik: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/dataset/test_env_set_help.pik -------------------------------------------------------------------------------- /dataset/train_env_set_help.pik: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/dataset/train_env_set_help.pik -------------------------------------------------------------------------------- /dataset/watch_scenes_split.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/dataset/watch_scenes_split.json -------------------------------------------------------------------------------- /demo/virtualhome_demo.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/demo/virtualhome_demo.ipynb -------------------------------------------------------------------------------- /demo/watch_and_help_demo.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/demo/watch_and_help_demo.ipynb -------------------------------------------------------------------------------- /envs/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /envs/base_environment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/envs/base_environment.py -------------------------------------------------------------------------------- /envs/envs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/envs/envs.py -------------------------------------------------------------------------------- /envs/graph_env.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/envs/graph_env.py -------------------------------------------------------------------------------- /envs/python_environment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/envs/python_environment.py -------------------------------------------------------------------------------- /envs/unity_environment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/envs/unity_environment.py -------------------------------------------------------------------------------- /evaluation/calc_metrics_reward.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/evaluation/calc_metrics_reward.py -------------------------------------------------------------------------------- /evaluation/eval_pred.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/evaluation/eval_pred.py -------------------------------------------------------------------------------- /gen_data/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /gen_data/data/class_name_size.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/gen_data/data/class_name_size.json -------------------------------------------------------------------------------- /gen_data/data/init_pool.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/gen_data/data/init_pool.json -------------------------------------------------------------------------------- /gen_data/data/object_info.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/gen_data/data/object_info.json -------------------------------------------------------------------------------- /gen_data/data/object_info1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/gen_data/data/object_info1.json -------------------------------------------------------------------------------- /gen_data/data/object_info2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/gen_data/data/object_info2.json -------------------------------------------------------------------------------- /gen_data/data/object_info3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/gen_data/data/object_info3.json -------------------------------------------------------------------------------- /gen_data/data/object_info4.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/gen_data/data/object_info4.json -------------------------------------------------------------------------------- /gen_data/data/object_info5.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/gen_data/data/object_info5.json -------------------------------------------------------------------------------- /gen_data/data/object_info6.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/gen_data/data/object_info6.json -------------------------------------------------------------------------------- /gen_data/data/object_info7.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/gen_data/data/object_info7.json -------------------------------------------------------------------------------- /gen_data/init_goal_setter/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /gen_data/init_goal_setter/init_goal_base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/gen_data/init_goal_setter/init_goal_base.py -------------------------------------------------------------------------------- /gen_data/init_goal_setter/tasks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/gen_data/init_goal_setter/tasks.py -------------------------------------------------------------------------------- /gen_data/vh_init.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/gen_data/vh_init.py -------------------------------------------------------------------------------- /gen_data/vh_init_gen_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/gen_data/vh_init_gen_test.py -------------------------------------------------------------------------------- /models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/models/__init__.py -------------------------------------------------------------------------------- /models/actor_critic.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/models/actor_critic.py -------------------------------------------------------------------------------- /models/actor_critic_hl_mcts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/models/actor_critic_hl_mcts.py -------------------------------------------------------------------------------- /models/base_nets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/models/base_nets.py -------------------------------------------------------------------------------- /models/distributions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/models/distributions.py -------------------------------------------------------------------------------- /models/graph_nn.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/models/graph_nn.py -------------------------------------------------------------------------------- /models/single_policy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/models/single_policy.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/requirements.txt -------------------------------------------------------------------------------- /scripts/download_ckpts.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/scripts/download_ckpts.sh -------------------------------------------------------------------------------- /scripts/test_watch.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/scripts/test_watch.sh -------------------------------------------------------------------------------- /scripts/train_watch.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/scripts/train_watch.sh -------------------------------------------------------------------------------- /testing_agents/test_hp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/testing_agents/test_hp.py -------------------------------------------------------------------------------- /testing_agents/test_hp_pred_goal.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/testing_agents/test_hp_pred_goal.py -------------------------------------------------------------------------------- /testing_agents/test_hp_random_goal.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/testing_agents/test_hp_random_goal.py -------------------------------------------------------------------------------- /testing_agents/test_hrl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/testing_agents/test_hrl.py -------------------------------------------------------------------------------- /testing_agents/test_hrl_pred.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/testing_agents/test_hrl_pred.py -------------------------------------------------------------------------------- /testing_agents/test_hybrid.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/testing_agents/test_hybrid.py -------------------------------------------------------------------------------- /testing_agents/test_hybrid_pred_goal.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/testing_agents/test_hybrid_pred_goal.py -------------------------------------------------------------------------------- /testing_agents/test_random_action.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/testing_agents/test_random_action.py -------------------------------------------------------------------------------- /testing_agents/test_single_agent.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/testing_agents/test_single_agent.py -------------------------------------------------------------------------------- /tests/test_simspeed.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/tests/test_simspeed.py -------------------------------------------------------------------------------- /training_agents/train_a2c.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/training_agents/train_a2c.py -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /utils/convert_pkl_tojson_pred.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/utils/convert_pkl_tojson_pred.py -------------------------------------------------------------------------------- /utils/jquery.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/utils/jquery.js -------------------------------------------------------------------------------- /utils/kill.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/utils/kill.py -------------------------------------------------------------------------------- /utils/memory.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/utils/memory.py -------------------------------------------------------------------------------- /utils/rewards.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/utils/rewards.py -------------------------------------------------------------------------------- /utils/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/utils/utils.py -------------------------------------------------------------------------------- /utils/utils_environment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/utils/utils_environment.py -------------------------------------------------------------------------------- /utils/utils_goals.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/utils/utils_goals.py -------------------------------------------------------------------------------- /utils/utils_models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/utils/utils_models.py -------------------------------------------------------------------------------- /utils/utils_plot.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/utils/utils_plot.py -------------------------------------------------------------------------------- /utils/utils_rl_agent.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/utils/utils_rl_agent.py -------------------------------------------------------------------------------- /utils/utils_videos.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/utils/utils_videos.py -------------------------------------------------------------------------------- /watch/helper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/watch/helper.py -------------------------------------------------------------------------------- /watch/network/encoder_decoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/watch/network/encoder_decoder.py -------------------------------------------------------------------------------- /watch/network/module_graph.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/watch/network/module_graph.py -------------------------------------------------------------------------------- /watch/predicate-train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/watch/predicate-train.py -------------------------------------------------------------------------------- /watch/predicate/demo_dataset_graph.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/watch/predicate/demo_dataset_graph.py -------------------------------------------------------------------------------- /watch/predicate/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xavierpuigf/watch_and_help/HEAD/watch/predicate/utils.py --------------------------------------------------------------------------------