├── LICENSE ├── LLM ├── .gitignore ├── .python-version ├── CMakeLists.txt ├── LICENSE ├── README.md ├── configs │ ├── archive │ │ └── task_planning │ │ │ ├── context │ │ │ ├── ic1.yaml │ │ │ ├── ic2.yaml │ │ │ └── ic3.yaml │ │ │ ├── serve_apple.yaml │ │ │ ├── shelving_bowls.yaml │ │ │ ├── system │ │ │ └── eRDTransformer.yaml │ │ │ └── unpack_groceries.yaml │ ├── models │ │ └── pretrained │ │ │ ├── embeddings │ │ │ ├── bert-base-uncased.yaml │ │ │ ├── bert-large-uncased.yaml │ │ │ ├── llama2-7b-chat.yaml │ │ │ ├── llama2-7b.yaml │ │ │ ├── mistral-7b.yaml │ │ │ ├── sentence-transformer-mpnet.yaml │ │ │ ├── text_embedding_3_large.yaml │ │ │ ├── text_embedding_3_small.yaml │ │ │ └── text_embedding_ada_002.yaml │ │ │ └── generative │ │ │ ├── gpt_3_5_turbo.yaml │ │ │ ├── gpt_3_5_turbo_cot.yaml │ │ │ ├── gpt_4.yaml │ │ │ ├── gpt_4_cot.yaml │ │ │ └── llama2_7b_chat.yaml │ └── prompts │ │ ├── evaluation │ │ └── p1 │ │ │ ├── memory_1_1.yaml │ │ │ ├── memory_1_2.yaml │ │ │ ├── memory_2_1.yaml │ │ │ ├── memory_2_2.yaml │ │ │ ├── packing_1.yaml │ │ │ ├── packing_2.yaml │ │ │ ├── packing_3.yaml │ │ │ ├── packing_4.yaml │ │ │ ├── retrieval_1_v1.yaml │ │ │ ├── retrieval_1_v2.yaml │ │ │ ├── retrieval_2_v1.yaml │ │ │ ├── retrieval_2_v2.yaml │ │ │ ├── retrieval_3_v1.yaml │ │ │ ├── retrieval_3_v2.yaml │ │ │ ├── retrieval_4_v1.yaml │ │ │ ├── retrieval_4_v2.yaml │ │ │ ├── retrieval_5_v2.yaml │ │ │ └── retrieval_5_v3.yaml │ │ ├── examples │ │ ├── example_1.yaml │ │ ├── example_2.yaml │ │ ├── example_3.yaml │ │ ├── example_4.yaml │ │ ├── example_5.yaml │ │ └── example_6.yaml │ │ └── system │ │ ├── geomtric_reasoning_v1.yaml │ │ ├── goal_prediction_v1.yaml │ │ ├── goal_prediction_v2.yaml │ │ ├── hypothesis_generation_v1.yaml │ │ ├── task_planning_goal_condition_v1.yaml │ │ ├── task_planning_goal_condition_v2.yaml │ │ ├── task_planning_hypothesis_condition_v1.yaml │ │ └── task_planning_v1.yaml ├── fm_planning │ ├── __init__.py │ ├── fm_agents │ │ ├── __init__.py │ │ ├── base.py │ │ └── utils.py │ ├── models │ │ ├── __init__.py │ │ └── pretrained │ │ │ ├── __init__.py │ │ │ ├── base.py │ │ │ ├── embeddings │ │ │ ├── __init__.py │ │ │ ├── hugging_face_emb.py │ │ │ ├── openai_emb.py │ │ │ └── utils.py │ │ │ ├── generative │ │ │ ├── __init__.py │ │ │ ├── llama_gen.py │ │ │ ├── openai_gen.py │ │ │ └── utils.py │ │ │ └── utils.py │ └── utils │ │ ├── __init__.py │ │ ├── configs.py │ │ ├── nest.py │ │ ├── random.py │ │ ├── tensors.py │ │ ├── timing.py │ │ └── typing.py ├── package.xml ├── pyproject.toml ├── scripts │ ├── demos │ │ └── llm_planner.py │ └── llm_planner.py └── setup.cfg ├── README.md ├── conda_env.yml ├── images └── teaser_v1.png └── relational_dynamics ├── .DS_Store ├── __pycache__ ├── __init__.cpython-38.pyc └── base_RD.cpython-38.pyc ├── base_RD.py ├── config ├── __pycache__ │ └── base_config.cpython-38.pyc └── base_config.py ├── dataloader ├── __pycache__ │ ├── __init__.cpython-38.pyc │ ├── dataloader.cpython-38.pyc │ ├── farthest_point_sampling.cpython-38.pyc │ └── real_robot_dataloader.cpython-38.pyc └── dataloader.py ├── main.py ├── model ├── __pycache__ │ ├── GNN_pytorch_geometry.cpython-38.pyc │ ├── __init__.cpython-38.pyc │ ├── contact_model.cpython-38.pyc │ ├── encoder_decoder.cpython-38.pyc │ ├── losses.cpython-38.pyc │ ├── models.cpython-38.pyc │ └── pointconv_util_groupnorm.cpython-38.pyc ├── models.py └── pointconv_util_groupnorm.py └── utils ├── __pycache__ ├── __init__.cpython-38.pyc ├── colors.cpython-38.pyc ├── data_utils.cpython-38.pyc ├── math_util.cpython-38.pyc ├── other_util.cpython-38.pyc ├── parse_util.cpython-38.pyc ├── tensorboardx_logger.cpython-38.pyc ├── torch_util.cpython-38.pyc └── torch_utils.cpython-38.pyc ├── colors.py ├── data_utils.py ├── math_util.py ├── other_util.py ├── parse_util.py ├── torch_util.py └── transformations.py /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Yixuan Huang, Christopher Agia 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /LLM/.gitignore: -------------------------------------------------------------------------------- 1 | # Output directory 2 | data 3 | *.DS_Store 4 | 5 | # Byte-compiled / optimized / DLL files 6 | __pycache__/ 7 | *.py[cod] 8 | *$py.class 9 | 10 | # C extensions 11 | *.so 12 | 13 | # Distribution / packaging 14 | .Python 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | lib/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | wheels/ 27 | share/python-wheels/ 28 | *.egg-info/ 29 | .installed.cfg 30 | *.egg 31 | MANIFEST 32 | 33 | # PyInstaller 34 | # Usually these files are written by a python script from a template 35 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 36 | *.manifest 37 | *.spec 38 | 39 | # Installer logs 40 | pip-log.txt 41 | pip-delete-this-directory.txt 42 | 43 | # Unit test / coverage reports 44 | htmlcov/ 45 | .tox/ 46 | .nox/ 47 | .coverage 48 | .coverage.* 49 | .cache 50 | nosetests.xml 51 | coverage.xml 52 | *.cover 53 | *.py,cover 54 | .hypothesis/ 55 | .pytest_cache/ 56 | cover/ 57 | 58 | # Translations 59 | *.mo 60 | *.pot 61 | 62 | # Django stuff: 63 | *.log 64 | local_settings.py 65 | db.sqlite3 66 | db.sqlite3-journal 67 | 68 | # Flask stuff: 69 | instance/ 70 | .webassets-cache 71 | 72 | # Scrapy stuff: 73 | .scrapy 74 | 75 | # Sphinx documentation 76 | docs/_build/ 77 | 78 | # PyBuilder 79 | .pybuilder/ 80 | target/ 81 | 82 | # Jupyter Notebook 83 | .ipynb_checkpoints 84 | 85 | # IPython 86 | profile_default/ 87 | ipython_config.py 88 | 89 | # pyenv 90 | # For a library or package, you might want to ignore these files since the code is 91 | # intended to run in multiple environments; otherwise, check them in: 92 | # .python-version 93 | 94 | # pipenv 95 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 96 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 97 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 98 | # install all needed dependencies. 99 | #Pipfile.lock 100 | 101 | # poetry 102 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 103 | # This is especially recommended for binary packages to ensure reproducibility, and is more 104 | # commonly ignored for libraries. 105 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 106 | #poetry.lock 107 | 108 | # pdm 109 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 110 | #pdm.lock 111 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 112 | # in version control. 113 | # https://pdm.fming.dev/#use-with-ide 114 | .pdm.toml 115 | 116 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 117 | __pypackages__/ 118 | 119 | # Celery stuff 120 | celerybeat-schedule 121 | celerybeat.pid 122 | 123 | # SageMath parsed files 124 | *.sage.py 125 | 126 | # Environments 127 | .env 128 | .venv 129 | env/ 130 | venv/ 131 | ENV/ 132 | env.bak/ 133 | venv.bak/ 134 | 135 | # Spyder project settings 136 | .spyderproject 137 | .spyproject 138 | 139 | # Rope project settings 140 | .ropeproject 141 | 142 | # mkdocs documentation 143 | /site 144 | 145 | # mypy 146 | .mypy_cache/ 147 | .dmypy.json 148 | dmypy.json 149 | 150 | # Pyre type checker 151 | .pyre/ 152 | 153 | # pytype static type analyzer 154 | .pytype/ 155 | 156 | # Cython debug symbols 157 | cython_debug/ 158 | 159 | # PyCharm 160 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 161 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 162 | # and can be added to the global gitignore or merged into this file. For a more nuclear 163 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 164 | #.idea/ 165 | -------------------------------------------------------------------------------- /LLM/.python-version: -------------------------------------------------------------------------------- 1 | 3.8.16 2 | -------------------------------------------------------------------------------- /LLM/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.0.2) 2 | project(fm_planning) 3 | 4 | ## Compile as C++11, supported in ROS Kinetic and newer 5 | # add_compile_options(-std=c++11) 6 | 7 | ## Find catkin macros and libraries 8 | ## if COMPONENTS list like find_package(catkin REQUIRED COMPONENTS xyz) 9 | ## is used, also find other catkin packages 10 | find_package(catkin REQUIRED COMPONENTS 11 | rospy 12 | std_msgs 13 | message_generation 14 | ) 15 | 16 | ## System dependencies are found with CMake's conventions 17 | # find_package(Boost REQUIRED COMPONENTS system) 18 | 19 | 20 | ## Uncomment this if the package has a setup.py. This macro ensures 21 | ## modules and global scripts declared therein get installed 22 | ## See http://ros.org/doc/api/catkin/html/user_guide/setup_dot_py.html 23 | # catkin_python_setup() 24 | 25 | ################################################ 26 | ## Declare ROS messages, services and actions ## 27 | ################################################ 28 | 29 | ## To declare and build messages, services or actions from within this 30 | ## package, follow these steps: 31 | ## * Let MSG_DEP_SET be the set of packages whose message types you use in 32 | ## your messages/services/actions (e.g. std_msgs, actionlib_msgs, ...). 33 | ## * In the file package.xml: 34 | ## * add a build_depend tag for "message_generation" 35 | ## * add a build_depend and a exec_depend tag for each package in MSG_DEP_SET 36 | ## * If MSG_DEP_SET isn't empty the following dependency has been pulled in 37 | ## but can be declared for certainty nonetheless: 38 | ## * add a exec_depend tag for "message_runtime" 39 | ## * In this file (CMakeLists.txt): 40 | ## * add "message_generation" and every package in MSG_DEP_SET to 41 | ## find_package(catkin REQUIRED COMPONENTS ...) 42 | ## * add "message_runtime" and every package in MSG_DEP_SET to 43 | ## catkin_package(CATKIN_DEPENDS ...) 44 | ## * uncomment the add_*_files sections below as needed 45 | ## and list every .msg/.srv/.action file to be processed 46 | ## * uncomment the generate_messages entry below 47 | ## * add every package in MSG_DEP_SET to generate_messages(DEPENDENCIES ...) 48 | 49 | ## Generate messages in the 'msg' folder 50 | # add_message_files( 51 | # FILES 52 | # Message1.msg 53 | # Message2.msg 54 | # ) 55 | 56 | ## Generate services in the 'srv' folder 57 | add_service_files( 58 | FILES 59 | InitObjects.srv 60 | Goals.srv 61 | TaskPlans.srv 62 | TaskPlansFromGoals.srv 63 | ) 64 | 65 | ## Generate actions in the 'action' folder 66 | # add_action_files( 67 | # FILES 68 | # Action1.action 69 | # Action2.action 70 | # ) 71 | 72 | ## Generate added messages and services with any dependencies listed here 73 | generate_messages( 74 | DEPENDENCIES 75 | std_msgs 76 | ) 77 | 78 | ################################################ 79 | ## Declare ROS dynamic reconfigure parameters ## 80 | ################################################ 81 | 82 | ## To declare and build dynamic reconfigure parameters within this 83 | ## package, follow these steps: 84 | ## * In the file package.xml: 85 | ## * add a build_depend and a exec_depend tag for "dynamic_reconfigure" 86 | ## * In this file (CMakeLists.txt): 87 | ## * add "dynamic_reconfigure" to 88 | ## find_package(catkin REQUIRED COMPONENTS ...) 89 | ## * uncomment the "generate_dynamic_reconfigure_options" section below 90 | ## and list every .cfg file to be processed 91 | 92 | ## Generate dynamic reconfigure parameters in the 'cfg' folder 93 | # generate_dynamic_reconfigure_options( 94 | # cfg/DynReconf1.cfg 95 | # cfg/DynReconf2.cfg 96 | # ) 97 | 98 | ################################### 99 | ## catkin specific configuration ## 100 | ################################### 101 | ## The catkin_package macro generates cmake config files for your package 102 | ## Declare things to be passed to dependent projects 103 | ## INCLUDE_DIRS: uncomment this if your package contains header files 104 | ## LIBRARIES: libraries you create in this project that dependent projects also need 105 | ## CATKIN_DEPENDS: catkin_packages dependent projects also need 106 | ## DEPENDS: system dependencies of this project that dependent projects also need 107 | catkin_package( 108 | # INCLUDE_DIRS include 109 | # LIBRARIES fm_planning 110 | # CATKIN_DEPENDS rospy std_msgs 111 | # DEPENDS system_lib 112 | ) 113 | 114 | ########### 115 | ## Build ## 116 | ########### 117 | 118 | ## Specify additional locations of header files 119 | ## Your package locations should be listed before other locations 120 | include_directories( 121 | # include 122 | ${catkin_INCLUDE_DIRS} 123 | ) 124 | 125 | ## Declare a C++ library 126 | # add_library(${PROJECT_NAME} 127 | # src/${PROJECT_NAME}/fm_planning.cpp 128 | # ) 129 | 130 | ## Add cmake target dependencies of the library 131 | ## as an example, code may need to be generated before libraries 132 | ## either from message generation or dynamic reconfigure 133 | # add_dependencies(${PROJECT_NAME} ${${PROJECT_NAME}_EXPORTED_TARGETS} ${catkin_EXPORTED_TARGETS}) 134 | 135 | ## Declare a C++ executable 136 | ## With catkin_make all packages are built within a single CMake context 137 | ## The recommended prefix ensures that target names across packages don't collide 138 | # add_executable(${PROJECT_NAME}_node src/fm_planning_node.cpp) 139 | 140 | ## Rename C++ executable without prefix 141 | ## The above recommended prefix causes long target names, the following renames the 142 | ## target back to the shorter version for ease of user use 143 | ## e.g. "rosrun someones_pkg node" instead of "rosrun someones_pkg someones_pkg_node" 144 | # set_target_properties(${PROJECT_NAME}_node PROPERTIES OUTPUT_NAME node PREFIX "") 145 | 146 | ## Add cmake target dependencies of the executable 147 | ## same as for the library above 148 | # add_dependencies(${PROJECT_NAME}_node ${${PROJECT_NAME}_EXPORTED_TARGETS} ${catkin_EXPORTED_TARGETS}) 149 | 150 | ## Specify libraries to link a library or executable target against 151 | # target_link_libraries(${PROJECT_NAME}_node 152 | # ${catkin_LIBRARIES} 153 | # ) 154 | 155 | ############# 156 | ## Install ## 157 | ############# 158 | 159 | # all install targets should use catkin DESTINATION variables 160 | # See http://ros.org/doc/api/catkin/html/adv_user_guide/variables.html 161 | 162 | ## Mark executable scripts (Python etc.) for installation 163 | ## in contrast to setup.py, you can choose the destination 164 | catkin_install_python(PROGRAMS 165 | scripts/ros/llm_planning_node.py 166 | DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION} 167 | ) 168 | 169 | ## Mark executables for installation 170 | ## See http://docs.ros.org/melodic/api/catkin/html/howto/format1/building_executables.html 171 | # install(TARGETS ${PROJECT_NAME}_node 172 | # RUNTIME DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION} 173 | # ) 174 | 175 | ## Mark libraries for installation 176 | ## See http://docs.ros.org/melodic/api/catkin/html/howto/format1/building_libraries.html 177 | # install(TARGETS ${PROJECT_NAME} 178 | # ARCHIVE DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION} 179 | # LIBRARY DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION} 180 | # RUNTIME DESTINATION ${CATKIN_GLOBAL_BIN_DESTINATION} 181 | # ) 182 | 183 | ## Mark cpp header files for installation 184 | # install(DIRECTORY include/${PROJECT_NAME}/ 185 | # DESTINATION ${CATKIN_PACKAGE_INCLUDE_DESTINATION} 186 | # FILES_MATCHING PATTERN "*.h" 187 | # PATTERN ".svn" EXCLUDE 188 | # ) 189 | 190 | ## Mark other files for installation (e.g. launch and bag files, etc.) 191 | # install(FILES 192 | # # myfile1 193 | # # myfile2 194 | # DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION} 195 | # ) 196 | 197 | ############# 198 | ## Testing ## 199 | ############# 200 | 201 | ## Add gtest based cpp test target and link libraries 202 | # catkin_add_gtest(${PROJECT_NAME}-test test/test_fm_planning.cpp) 203 | # if(TARGET ${PROJECT_NAME}-test) 204 | # target_link_libraries(${PROJECT_NAME}-test ${PROJECT_NAME}) 205 | # endif() 206 | 207 | ## Add folders to be run by python nosetests 208 | # catkin_add_nosetests(test) 209 | -------------------------------------------------------------------------------- /LLM/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Christopher Agia 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /LLM/README.md: -------------------------------------------------------------------------------- 1 | # fm-planning 2 | Planning with Foundation Models (FMs): Language Models and Vision-Language Models 3 | -------------------------------------------------------------------------------- /LLM/configs/archive/task_planning/context/ic1.yaml: -------------------------------------------------------------------------------- 1 | - role: user 2 | content: 3 | "Objects: [cup, sink, kitchen_table]\n 4 | 5 | Object Relationships: [Above(cup, kitchen_table), Contact(cup, kitchen_table)]\n 6 | 7 | Instruction: Place the cup in the sink.\n 8 | 9 | Goals: [[Inside(cup, sink), Contact(cup, sink)]]\n 10 | 11 | Plans: [[Pick(cup, kitchen_table), Place(cup, sink)]]\n" -------------------------------------------------------------------------------- /LLM/configs/archive/task_planning/context/ic2.yaml: -------------------------------------------------------------------------------- 1 | - role: user 2 | content: 3 | "Objects: [socks, jacket, drawer, closet, bed]\n 4 | 5 | Object Relationships: [Inside(socks, drawer), 6 | Contact(socks, drawer), 7 | Inside(jacket, closet), 8 | Closed(drawer), 9 | Closed(cloest)]\n 10 | 11 | Instruction: Fetch me a pair of socks. I'm in the bedroom. \n 12 | 13 | Goals: [[Above(socks, bed), Contact(socks, bed)]]\n 14 | 15 | Plans: [[Open(drawer), Pick(socks, drawer), Place(socks, bed), Close(drawer)]]\n" -------------------------------------------------------------------------------- /LLM/configs/archive/task_planning/context/ic3.yaml: -------------------------------------------------------------------------------- 1 | - role: user 2 | content: 3 | "Objects: [book_1, book_2, bookshelf, reading_chair, coffee_table]\n 4 | 5 | Object Relationships: [Above(book_1, book_2), Above(book_2, bookshelf), 6 | Contact(book_1, book_2), Contact(book_2, bookshelf)]\n 7 | 8 | Instruction: Bring me book_2.\n 9 | 10 | Goals: [[Above(book_2, reading_chair), Contact(book_2, reading_chair)], 11 | [Above(book_2, coffee_table), Contact(book_2, coffee_table)]]\n 12 | 13 | Plans: [[Pick(book_1, book_2), Place(book_1, bookshelf), Pick(book_2, bookshelf), Place(book_2, reading_chair)], 14 | [Pick(book_1, book_2), Place(book_1, bookshelf), Pick(book_2, bookshelf), Place(book_2, coffee_table)]\n" -------------------------------------------------------------------------------- /LLM/configs/archive/task_planning/serve_apple.yaml: -------------------------------------------------------------------------------- 1 | # System prompt. 2 | - configs/prompts/task_planning/system/eRDTransformer.yaml 3 | 4 | # In-context prompts. 5 | - configs/prompts/task_planning/context/ic1.yaml 6 | - configs/prompts/task_planning/context/ic2.yaml 7 | - configs/prompts/task_planning/context/ic3.yaml 8 | 9 | # Task prompt. 10 | - role: user 11 | content: 12 | "Objects: [apple, onion, grocery_box, fruit_bowl, garbage_bin, kitchen_cabinet]\n 13 | 14 | Object Relationships: [Inside(apple, kitchen_cabinet), 15 | Contact(apple, kitchen_cabinet), 16 | Inside(onion, kitchen_cabinet), 17 | Contact(onion, kitchen_cabinet), 18 | Opened(grocery_box), 19 | Opened(garbage_bin), 20 | Closed(kitchen_cabinet)]\n 21 | 22 | Instruction: Serve the apple in the fruit bowl.\n" -------------------------------------------------------------------------------- /LLM/configs/archive/task_planning/shelving_bowls.yaml: -------------------------------------------------------------------------------- 1 | # System prompt. 2 | - configs/prompts/task_planning/system/eRDTransformer.yaml 3 | 4 | # In-context prompts. 5 | - configs/prompts/task_planning/context/ic1.yaml 6 | - configs/prompts/task_planning/context/ic2.yaml 7 | - configs/prompts/task_planning/context/ic3.yaml 8 | 9 | # Task prompt. 10 | - role: user 11 | content: 12 | "Objects: [small_bowl, large_bowl, medium_bowl, dishwasher, shelf]\n 13 | 14 | Object Relationships: [Inside(small_bowl, dishwasher), 15 | Inside(large_bowl, dishwasher), 16 | Inside(medium_bowl, dishwasher), 17 | Front(small_bowl, large_bowl), 18 | Front(large_bowl, medium_bowl)]\n 19 | 20 | Instruction: Put all the bowls back on the shelf and sort them by increasing size.\n" -------------------------------------------------------------------------------- /LLM/configs/archive/task_planning/system/eRDTransformer.yaml: -------------------------------------------------------------------------------- 1 | - role: system 2 | content: 3 | "I am the task planning system of a mobile manipulator robot operating in a household environment. 4 | My job is to analyze the user's instruction and the state of the environment, then 1) predict goals 5 | that the robot should achieve to fulfill the user's instruction and 2) plan candidate sequences of 6 | actions that the robot should execute to satisfy the predicted goals.\n 7 | 8 | The robot can perceive the following information about the environment:\n 9 | - The objects in the environment\n 10 | - The relationships among objects (e.g., above, below)\n 11 | - The state of objects (e.g., open or closed)\n 12 | 13 | The robot can detect the following relationships among objects:\n 14 | - Left(a, b): Object a is to the left of object b\n 15 | - Right(a, b): Object a is to the right of object b\n 16 | - Front(a, b): Object a is in front of object b\n 17 | - Behind(a, b): Object a is behind object b\n 18 | - Above(a, b): Object a is above object b\n 19 | - Below(a, b): Object a is below object b\n 20 | - Contact(a, b): Object a is in contact with object b\n 21 | - Boundary(a, b): Object a is within the boundary of object b\n 22 | - Inside(a, b): Object a is inside object b\n 23 | - Opened(a): Object a is opened\n 24 | - Closed(a): Object a is closed\n 25 | 26 | The robot can execute the following actions:\n 27 | - Pick(a, b): The robot picks object a from object b. The robot must NOT already be holding an object\n 28 | - Place(a, b): The robot places object a on or in object b\n 29 | - Open(a): The robot opens object a. The robot must NOT already be holding an object\n 30 | - Close(a): The robot closes object a. The robot must NOT already be holding an object\n 31 | 32 | There may be one or multiple ways to fulfill the user's instruction. Therefore, I will consider the 33 | most likely goals and plans, if there are multiple, and output them in the following format:\n 34 | Rules:\n 35 | - I will output goals as a list of lists after 'Goals:'. Each nested list represents one goal\n 36 | - I will output plans as a list of lists after 'Plans:'. Each nested list represents one plan\n 37 | - I will output one plan for each goal. Hence, each goal and its corresponding plan will be located 38 | at the same index in the 'Goals' and 'Plans' lists\n 39 | - I will not output all possible goals and plans, but the most likely ones to fulfill the user's instruction\n 40 | - If there are multiple ways to fulfill the instruction, I will output the simplest goals and plans first\n 41 | - I will consider the feasibility of the plans based on the the defined actions and their preconditions. 42 | For example, the actions Pick(a), Open(a), Close(a) cannot be executed in the robot is already holding an object\n 43 | 44 | Examples:\n" -------------------------------------------------------------------------------- /LLM/configs/archive/task_planning/unpack_groceries.yaml: -------------------------------------------------------------------------------- 1 | # System prompt. 2 | - configs/prompts/task_planning/system/eRDTransformer.yaml 3 | 4 | # In-context prompts. 5 | - configs/prompts/task_planning/context/ic1.yaml 6 | - configs/prompts/task_planning/context/ic2.yaml 7 | - configs/prompts/task_planning/context/ic3.yaml 8 | 9 | # Task prompt. 10 | - role: user 11 | content: 12 | "Objects: [apple, onion, grocery_box, fruit_bowl, garbage_bin, kitchen_cabinet]\n 13 | 14 | Object Relationships: [Inside(apple, grocery_box), 15 | Contact(apple, grocery_box), 16 | Inside(onion, grocery_box), 17 | Contact(onion, grocery_box), 18 | Opened(grocery_box), 19 | Opened(garbage_bin), 20 | Closed(kitchen_cabinet)]\n 21 | 22 | Instruction: Unpack the grocery box into the kitchen cabinet. Start by opening the cabinet.\n" -------------------------------------------------------------------------------- /LLM/configs/models/pretrained/embeddings/bert-base-uncased.yaml: -------------------------------------------------------------------------------- 1 | model: HuggingFaceEmbeddingModel 2 | model_kwargs: 3 | model: "bert-base-uncased" -------------------------------------------------------------------------------- /LLM/configs/models/pretrained/embeddings/bert-large-uncased.yaml: -------------------------------------------------------------------------------- 1 | model: HuggingFaceEmbeddingModel 2 | model_kwargs: 3 | model: "bert-large-uncased" 4 | -------------------------------------------------------------------------------- /LLM/configs/models/pretrained/embeddings/llama2-7b-chat.yaml: -------------------------------------------------------------------------------- 1 | model: HuggingFaceEmbeddingModel 2 | model_kwargs: 3 | model: "meta-llama/Llama-2-7b-chat-hf" 4 | -------------------------------------------------------------------------------- /LLM/configs/models/pretrained/embeddings/llama2-7b.yaml: -------------------------------------------------------------------------------- 1 | model: HuggingFaceEmbeddingModel 2 | model_kwargs: 3 | model: "meta-llama/Llama-2-7b-hf" 4 | -------------------------------------------------------------------------------- /LLM/configs/models/pretrained/embeddings/mistral-7b.yaml: -------------------------------------------------------------------------------- 1 | model: HuggingFaceEmbeddingModel 2 | model_kwargs: 3 | model: "intfloat/e5-mistral-7b-instruct" 4 | -------------------------------------------------------------------------------- /LLM/configs/models/pretrained/embeddings/sentence-transformer-mpnet.yaml: -------------------------------------------------------------------------------- 1 | model: HuggingFaceEmbeddingModel 2 | model_kwargs: 3 | model: "sentence-transformers/all-mpnet-base-v2" 4 | -------------------------------------------------------------------------------- /LLM/configs/models/pretrained/embeddings/text_embedding_3_large.yaml: -------------------------------------------------------------------------------- 1 | model: OpenAIEmbeddingModel 2 | model_kwargs: 3 | model: "text-embedding-3-large" -------------------------------------------------------------------------------- /LLM/configs/models/pretrained/embeddings/text_embedding_3_small.yaml: -------------------------------------------------------------------------------- 1 | model: OpenAIEmbeddingModel 2 | model_kwargs: 3 | model: "text-embedding-3-small" -------------------------------------------------------------------------------- /LLM/configs/models/pretrained/embeddings/text_embedding_ada_002.yaml: -------------------------------------------------------------------------------- 1 | model: OpenAIEmbeddingModel 2 | model_kwargs: 3 | model: "text-embedding-ada-002" -------------------------------------------------------------------------------- /LLM/configs/models/pretrained/generative/gpt_3_5_turbo.yaml: -------------------------------------------------------------------------------- 1 | model: OpenAIGenerativeModel 2 | model_kwargs: 3 | model: "gpt-3.5-turbo" 4 | logprobs: True 5 | max_tokens: 10 6 | temperature: 0.0 -------------------------------------------------------------------------------- /LLM/configs/models/pretrained/generative/gpt_3_5_turbo_cot.yaml: -------------------------------------------------------------------------------- 1 | model: OpenAIGenerativeModel 2 | model_kwargs: 3 | model: "gpt-3.5-turbo" 4 | logprobs: True 5 | temperature: 0.0 -------------------------------------------------------------------------------- /LLM/configs/models/pretrained/generative/gpt_4.yaml: -------------------------------------------------------------------------------- 1 | model: OpenAIGenerativeModel 2 | model_kwargs: 3 | model: "gpt-4-0125-preview" 4 | logprobs: True 5 | max_tokens: 10 6 | temperature: 0.0 -------------------------------------------------------------------------------- /LLM/configs/models/pretrained/generative/gpt_4_cot.yaml: -------------------------------------------------------------------------------- 1 | model: OpenAIGenerativeModel 2 | model_kwargs: 3 | model: "gpt-4-0125-preview" 4 | logprobs: True 5 | temperature: 0.0 -------------------------------------------------------------------------------- /LLM/configs/models/pretrained/generative/llama2_7b_chat.yaml: -------------------------------------------------------------------------------- 1 | model: LlamaGenerativeModel 2 | model_kwargs: 3 | model: "meta-llama/Llama-2-7b-chat-hf" 4 | do_sample: False -------------------------------------------------------------------------------- /LLM/configs/prompts/evaluation/p1/memory_1_1.yaml: -------------------------------------------------------------------------------- 1 | prompt: BehaviorPromptManager 2 | prompt_kwargs: 3 | # Task prompt 4 | task_prompt: 5 | # Task 6 | task: memory_1_1 7 | instruction: "Put object_1 in the low drawer, then close the drawer." 8 | objects: 9 | # - "object_1" 10 | # - "bowl" 11 | # - "low_drawer" 12 | # - "high_drawer" 13 | # - "ground" 14 | - "object_1" 15 | - "object_2" 16 | - "low_drawer" 17 | - "high_drawer" 18 | - "ground" 19 | predicates: 20 | # - "On(object_1, ground)" 21 | # - "On(bowl, ground)" 22 | # - "Closed(low_drawer)" 23 | # - "Closed(high_drawer)" 24 | - "On(object_1, ground)" 25 | - "On(object_2, ground)" 26 | - "Closed(low_drawer)" 27 | - "Closed(high_drawer)" 28 | 29 | # Behaviors 30 | goals: Null 31 | plans: Null 32 | hypotheses: Null 33 | geometric: Null 34 | 35 | # OpenAI 36 | role: user 37 | name: Null 38 | name_query: Null 39 | name_response: Null 40 | 41 | # System prompts 42 | system_prompts: 43 | - LLM/configs/prompts/system/goal_prediction_v2.yaml 44 | - LLM/configs/prompts/system/task_planning_goal_condition_v2.yaml 45 | 46 | # Example prompts 47 | example_prompts: 48 | - LLM/configs/prompts/examples/example_1.yaml 49 | - LLM/configs/prompts/examples/example_2.yaml 50 | - LLM/configs/prompts/examples/example_3.yaml 51 | 52 | # Event prompts 53 | event_prompts: Null -------------------------------------------------------------------------------- /LLM/configs/prompts/evaluation/p1/memory_1_2.yaml: -------------------------------------------------------------------------------- 1 | prompt: BehaviorPromptManager 2 | prompt_kwargs: 3 | # Task prompt 4 | task_prompt: 5 | # Task 6 | task: memory_1_2 7 | instruction: "Place object_1 on object_2. Don't forget to close the drawer." 8 | objects: 9 | - "object_1" 10 | - "object_2" 11 | - "low_drawer" 12 | - "high_drawer" 13 | - "ground" 14 | predicates: 15 | - "Inside(object_1, low_drawer)" 16 | - "On(object_2, ground)" 17 | - "Closed(low_drawer)" 18 | - "Closed(high_drawer)" 19 | 20 | # Behaviors 21 | goals: Null 22 | plans: Null 23 | hypotheses: Null 24 | geometric: Null 25 | 26 | # OpenAI 27 | role: user 28 | name: Null 29 | name_query: Null 30 | name_response: Null 31 | 32 | # System prompts 33 | system_prompts: 34 | - LLM/configs/prompts/system/goal_prediction_v2.yaml 35 | - LLM/configs/prompts/system/task_planning_goal_condition_v2.yaml 36 | 37 | # Example prompts 38 | example_prompts: 39 | - LLM/configs/prompts/examples/example_1.yaml 40 | - LLM/configs/prompts/examples/example_2.yaml 41 | - LLM/configs/prompts/examples/example_3.yaml 42 | 43 | # Event prompts 44 | event_prompts: Null -------------------------------------------------------------------------------- /LLM/configs/prompts/evaluation/p1/memory_2_1.yaml: -------------------------------------------------------------------------------- 1 | prompt: BehaviorPromptManager 2 | prompt_kwargs: 3 | # Task prompt 4 | task_prompt: 5 | # Task 6 | task: memory_2_1 7 | instruction: "Put object_1 and object_2 in the low drawer, then close the drawer." 8 | objects: 9 | - "object_1" 10 | - "object_2" 11 | - "object_3" 12 | - "object_4" 13 | - "low_drawer" 14 | - "high_drawer" 15 | - "ground" 16 | predicates: 17 | - "On(object_1, ground)" 18 | - "On(object_2, ground)" 19 | - "On(object_3, ground)" 20 | - "On(object_4, ground)" 21 | - "Closed(low_drawer)" 22 | - "Closed(high_drawer)" 23 | 24 | # Behaviors 25 | goals: Null 26 | plans: Null 27 | hypotheses: Null 28 | geometric: Null 29 | 30 | # OpenAI 31 | role: user 32 | name: Null 33 | name_query: Null 34 | name_response: Null 35 | 36 | # System prompts 37 | system_prompts: 38 | - LLM/configs/prompts/system/goal_prediction_v2.yaml 39 | - LLM/configs/prompts/system/task_planning_goal_condition_v2.yaml 40 | 41 | # Example prompts 42 | example_prompts: 43 | - LLM/configs/prompts/examples/example_1.yaml 44 | - LLM/configs/prompts/examples/example_2.yaml 45 | - LLM/configs/prompts/examples/example_3.yaml 46 | 47 | # Event prompts 48 | event_prompts: Null -------------------------------------------------------------------------------- /LLM/configs/prompts/evaluation/p1/memory_2_2.yaml: -------------------------------------------------------------------------------- 1 | prompt: BehaviorPromptManager 2 | prompt_kwargs: 3 | # Task prompt 4 | task_prompt: 5 | # Task 6 | task: memory_2_2 7 | # instruction: "Place object_1 on the bowl. Don't forget to close the drawer." 8 | instruction: "Place object_1 on object_3. Don't forget to close the drawer." 9 | objects: 10 | - "object_1" 11 | - "object_2" 12 | - "object_3" 13 | - "object_4" 14 | - "low_drawer" 15 | - "high_drawer" 16 | - "ground" 17 | predicates: 18 | - "Inside(object_1, low_drawer)" 19 | - "Inside(object_2, low_drawer)" 20 | - "On(object_3, ground)" 21 | - "On(object_4, ground)" 22 | - "Closed(low_drawer)" 23 | - "Closed(high_drawer)" 24 | 25 | # Behaviors 26 | goals: Null 27 | plans: Null 28 | hypotheses: Null 29 | geometric: Null 30 | 31 | # OpenAI 32 | role: user 33 | name: Null 34 | name_query: Null 35 | name_response: Null 36 | 37 | # System prompts 38 | system_prompts: 39 | - LLM/configs/prompts/system/goal_prediction_v2.yaml 40 | - LLM/configs/prompts/system/task_planning_goal_condition_v2.yaml 41 | 42 | # Example prompts 43 | example_prompts: 44 | - LLM/configs/prompts/examples/example_1.yaml 45 | - LLM/configs/prompts/examples/example_2.yaml 46 | - LLM/configs/prompts/examples/example_3.yaml 47 | 48 | # Event prompts 49 | event_prompts: Null -------------------------------------------------------------------------------- /LLM/configs/prompts/evaluation/p1/packing_1.yaml: -------------------------------------------------------------------------------- 1 | prompt: BehaviorPromptManager 2 | prompt_kwargs: 3 | # Task prompt 4 | task_prompt: 5 | # Task 6 | task: packing_1 7 | instruction: "Put all the objects on the counter." 8 | objects: 9 | - "object_1" 10 | - "object_2" 11 | - "object_3" 12 | - "object_4" 13 | - "ground" 14 | - "counter" 15 | predicates: 16 | - "On(object_1, ground)" 17 | - "On(object_2, ground)" 18 | - "On(object_3, ground)" 19 | - "On(object_4, ground)" 20 | 21 | # Behaviors 22 | goals: Null 23 | plans: Null 24 | hypotheses: Null 25 | geometric: Null 26 | 27 | # OpenAI 28 | role: user 29 | name: Null 30 | name_query: Null 31 | name_response: Null 32 | 33 | # System prompts 34 | system_prompts: 35 | - LLM/configs/prompts/system/goal_prediction_v1.yaml 36 | - LLM/configs/prompts/system/task_planning_goal_condition_v1.yaml 37 | 38 | # Example prompts 39 | example_prompts: 40 | - LLM/configs/prompts/examples/example_1.yaml 41 | - LLM/configs/prompts/examples/example_2.yaml 42 | - LLM/configs/prompts/examples/example_3.yaml 43 | 44 | # Event prompts 45 | event_prompts: Null -------------------------------------------------------------------------------- /LLM/configs/prompts/evaluation/p1/packing_2.yaml: -------------------------------------------------------------------------------- 1 | prompt: BehaviorPromptManager 2 | prompt_kwargs: 3 | # Task prompt 4 | task_prompt: 5 | # Task 6 | task: packing_2 7 | instruction: "Put all the objects on the shelf." 8 | objects: 9 | - "object_1" 10 | - "object_2" 11 | - "object_3" 12 | - "ground" 13 | - "shelf" 14 | predicates: 15 | - "On(object_1, ground)" 16 | - "On(object_2, ground)" 17 | - "On(object_3, ground)" 18 | 19 | # Behaviors 20 | goals: Null 21 | plans: Null 22 | hypotheses: Null 23 | geometric: Null 24 | 25 | # OpenAI 26 | role: user 27 | name: Null 28 | name_query: Null 29 | name_response: Null 30 | 31 | # System prompts 32 | system_prompts: 33 | - LLM/configs/prompts/system/goal_prediction_v1.yaml 34 | - LLM/configs/prompts/system/task_planning_goal_condition_v1.yaml 35 | 36 | # Example prompts 37 | example_prompts: 38 | - LLM/configs/prompts/examples/example_1.yaml 39 | - LLM/configs/prompts/examples/example_2.yaml 40 | - LLM/configs/prompts/examples/example_3.yaml 41 | 42 | # Event prompts 43 | event_prompts: Null -------------------------------------------------------------------------------- /LLM/configs/prompts/evaluation/p1/packing_3.yaml: -------------------------------------------------------------------------------- 1 | prompt: BehaviorPromptManager 2 | prompt_kwargs: 3 | # Task prompt 4 | task_prompt: 5 | # Task 6 | task: packing_3 7 | instruction: "Put all the objects on the shelf." 8 | objects: 9 | - "object_1" 10 | - "object_2" 11 | - "object_3" 12 | - "object_4" 13 | - "ground" 14 | - "shelf" 15 | predicates: 16 | - "On(object_1, ground)" 17 | - "On(object_2, ground)" 18 | - "On(object_3, ground)" 19 | - "On(object_4, ground)" 20 | 21 | # Behaviors 22 | goals: Null 23 | plans: Null 24 | hypotheses: Null 25 | geometric: Null 26 | 27 | # OpenAI 28 | role: user 29 | name: Null 30 | name_query: Null 31 | name_response: Null 32 | 33 | # System prompts 34 | system_prompts: 35 | - LLM/configs/prompts/system/goal_prediction_v1.yaml 36 | - LLM/configs/prompts/system/task_planning_goal_condition_v1.yaml 37 | 38 | # Example prompts 39 | example_prompts: 40 | - LLM/configs/prompts/examples/example_1.yaml 41 | - LLM/configs/prompts/examples/example_2.yaml 42 | - LLM/configs/prompts/examples/example_3.yaml 43 | 44 | # Event prompts 45 | event_prompts: Null -------------------------------------------------------------------------------- /LLM/configs/prompts/evaluation/p1/packing_4.yaml: -------------------------------------------------------------------------------- 1 | prompt: BehaviorPromptManager 2 | prompt_kwargs: 3 | # Task prompt 4 | task_prompt: 5 | # Task 6 | task: packing_4 7 | instruction: "Put all the objects on the shelf." 8 | objects: 9 | - "object_1" 10 | - "object_2" 11 | - "object_3" 12 | - "object_4" 13 | - "object_5" 14 | - "ground" 15 | - "shelf" 16 | predicates: 17 | - "On(object_1, ground)" 18 | - "On(object_2, ground)" 19 | - "On(object_3, ground)" 20 | - "On(object_4, ground)" 21 | - "On(object_5, ground)" 22 | 23 | # Behaviors 24 | goals: Null 25 | plans: Null 26 | hypotheses: Null 27 | geometric: Null 28 | 29 | # OpenAI 30 | role: user 31 | name: Null 32 | name_query: Null 33 | name_response: Null 34 | 35 | # System prompts 36 | system_prompts: 37 | - LLM/configs/prompts/system/goal_prediction_v1.yaml 38 | - LLM/configs/prompts/system/task_planning_goal_condition_v1.yaml 39 | 40 | # Example prompts 41 | example_prompts: 42 | - LLM/configs/prompts/examples/example_1.yaml 43 | - LLM/configs/prompts/examples/example_2.yaml 44 | - LLM/configs/prompts/examples/example_3.yaml 45 | 46 | # Event prompts 47 | event_prompts: Null -------------------------------------------------------------------------------- /LLM/configs/prompts/evaluation/p1/retrieval_1_v1.yaml: -------------------------------------------------------------------------------- 1 | prompt: BehaviorPromptManager 2 | prompt_kwargs: 3 | # Task prompt 4 | task_prompt: 5 | # Task 6 | task: retrieval_1_v1 7 | instruction: "Retrieve object_1." 8 | objects: 9 | - "object_1" 10 | - "object_2" 11 | - "object_3" 12 | - "shelf" 13 | - "ground" 14 | predicates: 15 | - "On(object_1, shelf)" 16 | - "On(object_2, shelf)" 17 | - "On(object_3, shelf)" 18 | 19 | # Behaviors 20 | goals: Null 21 | plans: Null 22 | hypotheses: Null 23 | geometric: Null 24 | 25 | # OpenAI 26 | role: user 27 | name: Null 28 | name_query: Null 29 | name_response: Null 30 | 31 | # System prompts 32 | system_prompts: 33 | - LLM/configs/prompts/system/goal_prediction_v2.yaml 34 | - LLM/configs/prompts/system/task_planning_goal_condition_v2.yaml 35 | 36 | # Example prompts 37 | example_prompts: 38 | - LLM/configs/prompts/examples/example_4.yaml 39 | - LLM/configs/prompts/examples/example_5.yaml 40 | 41 | # Event prompts 42 | event_prompts: Null -------------------------------------------------------------------------------- /LLM/configs/prompts/evaluation/p1/retrieval_1_v2.yaml: -------------------------------------------------------------------------------- 1 | prompt: BehaviorPromptManager 2 | prompt_kwargs: 3 | # Task prompt 4 | task_prompt: 5 | # Task 6 | task: retrieval_1_v2 7 | instruction: "Retrieve object_1." 8 | objects: 9 | - "object_1" 10 | - "object_2" 11 | - "object_3" 12 | - "shelf" 13 | - "ground" 14 | predicates: 15 | - "On(object_1, shelf)" 16 | - "On(object_2, shelf)" 17 | - "On(object_3, shelf)" 18 | - "Blocking(object_2, object_1)" 19 | - "Blocking(object_3, object_1)" 20 | - "Blocking(object_3, object_2)" 21 | 22 | # Behaviors 23 | goals: [["On(object_1, ground)"]] 24 | plans: Null 25 | hypotheses: Null 26 | geometric: Null 27 | 28 | # OpenAI 29 | role: user 30 | name: Null 31 | name_query: Null 32 | name_response: Null 33 | 34 | # System prompts 35 | system_prompts: 36 | - LLM/configs/prompts/system/goal_prediction_v2.yaml 37 | - LLM/configs/prompts/system/task_planning_goal_condition_v2.yaml 38 | 39 | # Example prompts 40 | example_prompts: 41 | - LLM/configs/prompts/examples/example_4.yaml 42 | - LLM/configs/prompts/examples/example_5.yaml 43 | 44 | # Event prompts 45 | event_prompts: Null -------------------------------------------------------------------------------- /LLM/configs/prompts/evaluation/p1/retrieval_2_v1.yaml: -------------------------------------------------------------------------------- 1 | prompt: BehaviorPromptManager 2 | prompt_kwargs: 3 | # Task prompt 4 | task_prompt: 5 | # Task 6 | task: retrieval_2_v1 7 | instruction: "Retrieve object_1." 8 | objects: 9 | - "object_1" 10 | - "object_2" 11 | - "object_3" 12 | - "object_4" 13 | - "shelf" 14 | - "ground" 15 | predicates: 16 | - "On(object_1, shelf)" 17 | - "On(object_2, shelf)" 18 | - "On(object_3, shelf)" 19 | - "On(object_4, shelf)" 20 | 21 | # Behaviors 22 | goals: Null 23 | plans: Null 24 | hypotheses: Null 25 | geometric: Null 26 | 27 | # OpenAI 28 | role: user 29 | name: Null 30 | name_query: Null 31 | name_response: Null 32 | 33 | # System prompts 34 | system_prompts: 35 | - LLM/configs/prompts/system/goal_prediction_v2.yaml 36 | - LLM/configs/prompts/system/task_planning_goal_condition_v2.yaml 37 | 38 | # Example prompts 39 | example_prompts: 40 | - LLM/configs/prompts/examples/example_4.yaml 41 | - LLM/configs/prompts/examples/example_5.yaml 42 | 43 | # Event prompts 44 | event_prompts: Null -------------------------------------------------------------------------------- /LLM/configs/prompts/evaluation/p1/retrieval_2_v2.yaml: -------------------------------------------------------------------------------- 1 | prompt: BehaviorPromptManager 2 | prompt_kwargs: 3 | # Task prompt 4 | task_prompt: 5 | # Task 6 | task: retrieval_2_v2 7 | instruction: "Retrieve object_1." 8 | objects: 9 | - "object_1" 10 | - "object_2" 11 | - "object_3" 12 | - "object_4" 13 | - "shelf" 14 | - "ground" 15 | predicates: 16 | - "On(object_1, shelf)" 17 | - "On(object_2, shelf)" 18 | - "On(object_3, shelf)" 19 | - "On(object_4, shelf)" 20 | - "Blocking(object_3, object_4)" 21 | - "Blocking(object_3, object_1)" 22 | - "Blocking(object_2, object_1)" 23 | - "Blocking(object_4, object_1)" 24 | 25 | # Behaviors 26 | goals: [["On(object_1, ground)"]] 27 | plans: Null 28 | hypotheses: Null 29 | geometric: Null 30 | 31 | # OpenAI 32 | role: user 33 | name: Null 34 | name_query: Null 35 | name_response: Null 36 | 37 | # System prompts 38 | system_prompts: 39 | - LLM/configs/prompts/system/goal_prediction_v2.yaml 40 | - LLM/configs/prompts/system/task_planning_goal_condition_v2.yaml 41 | 42 | # Example prompts 43 | example_prompts: 44 | - LLM/configs/prompts/examples/example_4.yaml 45 | - LLM/configs/prompts/examples/example_5.yaml 46 | 47 | # Event prompts 48 | event_prompts: Null -------------------------------------------------------------------------------- /LLM/configs/prompts/evaluation/p1/retrieval_3_v1.yaml: -------------------------------------------------------------------------------- 1 | prompt: BehaviorPromptManager 2 | prompt_kwargs: 3 | # Task prompt 4 | task_prompt: 5 | # Task 6 | task: retrieval_3_v1 7 | instruction: "Retrieve object_1." 8 | objects: 9 | - "object_1" 10 | - "object_2" 11 | - "object_3" 12 | - "object_4" 13 | - "object_5" 14 | - "shelf" 15 | - "ground" 16 | predicates: 17 | - "On(object_1, shelf)" 18 | - "On(object_2, shelf)" 19 | - "On(object_3, shelf)" 20 | - "On(object_4, shelf)" 21 | - "On(object_5, shelf)" 22 | 23 | # Behaviors 24 | goals: Null 25 | plans: Null 26 | hypotheses: Null 27 | geometric: Null 28 | 29 | # OpenAI 30 | role: user 31 | name: Null 32 | name_query: Null 33 | name_response: Null 34 | 35 | # System prompts 36 | system_prompts: 37 | - LLM/configs/prompts/system/goal_prediction_v2.yaml 38 | - LLM/configs/prompts/system/task_planning_goal_condition_v2.yaml 39 | 40 | # Example prompts 41 | example_prompts: 42 | - LLM/configs/prompts/examples/example_4.yaml 43 | - LLM/configs/prompts/examples/example_5.yaml 44 | 45 | # Event prompts 46 | event_prompts: Null -------------------------------------------------------------------------------- /LLM/configs/prompts/evaluation/p1/retrieval_3_v2.yaml: -------------------------------------------------------------------------------- 1 | prompt: BehaviorPromptManager 2 | prompt_kwargs: 3 | # Task prompt 4 | task_prompt: 5 | # Task 6 | task: retrieval_3_v2 7 | instruction: | 8 | "Retrieve object_1 and put it on the ground. 9 | 10 | Remember to pay close attention to the 'Blocking' predicates, which indicate 11 | what objects are preventing other objects from being accessible by the robot. 12 | For example, Blocking(object_1, object_2) means that object_1 is blocking object_2; 13 | hence, object_1 can be retrieved directly. However, in the following case, object_1 14 | might be blocked by other objects as well. 15 | 16 | Rule: Restrict your output to the format in the examples." 17 | objects: 18 | - "object_1" 19 | - "object_2" 20 | - "object_3" 21 | - "object_4" 22 | - "object_5" 23 | - "shelf" 24 | - "ground" 25 | predicates: 26 | - "On(object_1, shelf)" 27 | - "On(object_2, shelf)" 28 | - "On(object_3, shelf)" 29 | - "On(object_4, shelf)" 30 | - "On(object_5, shelf)" 31 | - "Blocking(object_3, object_4)" 32 | - "Blocking(object_2, object_5)" 33 | - "Blocking(object_3, object_1)" 34 | - "Blocking(object_2, object_1)" 35 | - "Blocking(object_5, object_1)" 36 | - "Blocking(object_4, object_1)" 37 | 38 | # Behaviors 39 | goals: [["On(object_1, ground)"]] 40 | plans: Null 41 | hypotheses: Null 42 | geometric: Null 43 | 44 | # OpenAI 45 | role: user 46 | name: Null 47 | name_query: Null 48 | name_response: Null 49 | 50 | # System prompts 51 | system_prompts: 52 | - LLM/configs/prompts/system/goal_prediction_v2.yaml 53 | - LLM/configs/prompts/system/task_planning_goal_condition_v2.yaml 54 | 55 | # Example prompts 56 | example_prompts: 57 | - LLM/configs/prompts/examples/example_4.yaml 58 | - LLM/configs/prompts/examples/example_5.yaml 59 | 60 | # Event prompts 61 | event_prompts: Null -------------------------------------------------------------------------------- /LLM/configs/prompts/evaluation/p1/retrieval_4_v1.yaml: -------------------------------------------------------------------------------- 1 | prompt: BehaviorPromptManager 2 | prompt_kwargs: 3 | # Task prompt 4 | task_prompt: 5 | # Task 6 | task: retrieval_4_v1 7 | instruction: "Retrieve object_1 and place it on the shelf." 8 | objects: 9 | - "object_1" 10 | - "object_2" 11 | - "shelf" 12 | - "ground" 13 | predicates: 14 | - "Inside(object_1, object_2)" 15 | - "On(object_2, shelf)" 16 | 17 | # Behaviors 18 | goals: Null 19 | plans: Null 20 | hypotheses: Null 21 | geometric: Null 22 | 23 | # OpenAI 24 | role: user 25 | name: Null 26 | name_query: Null 27 | name_response: Null 28 | 29 | # System prompts 30 | system_prompts: 31 | - LLM/configs/prompts/system/goal_prediction_v2.yaml 32 | - LLM/configs/prompts/system/task_planning_goal_condition_v2.yaml 33 | 34 | # Example prompts 35 | example_prompts: 36 | - LLM/configs/prompts/examples/example_4.yaml 37 | - LLM/configs/prompts/examples/example_5.yaml 38 | - LLM/configs/prompts/examples/example_6.yaml 39 | 40 | # Event prompts 41 | event_prompts: Null -------------------------------------------------------------------------------- /LLM/configs/prompts/evaluation/p1/retrieval_4_v2.yaml: -------------------------------------------------------------------------------- 1 | prompt: BehaviorPromptManager 2 | prompt_kwargs: 3 | # Task prompt 4 | task_prompt: 5 | # Task 6 | task: retrieval_4_v2 7 | instruction: "Retrieve object_1 and place it on the shelf." 8 | objects: 9 | - "object_1" 10 | - "object_2" 11 | - "shelf" 12 | - "ground" 13 | predicates: 14 | - "Inside(object_1, object_2)" 15 | - "On(object_2, shelf)" 16 | - "Blocking(object_2, object_1)" 17 | 18 | # Behaviors 19 | goals: Null 20 | plans: Null 21 | hypotheses: Null 22 | geometric: Null 23 | 24 | # OpenAI 25 | role: user 26 | name: Null 27 | name_query: Null 28 | name_response: Null 29 | 30 | # System prompts 31 | system_prompts: 32 | - LLM/configs/prompts/system/goal_prediction_v2.yaml 33 | - LLM/configs/prompts/system/task_planning_goal_condition_v2.yaml 34 | 35 | # Example prompts 36 | example_prompts: 37 | - LLM/configs/prompts/examples/example_4.yaml 38 | - LLM/configs/prompts/examples/example_5.yaml 39 | - LLM/configs/prompts/examples/example_6.yaml 40 | 41 | # Event prompts 42 | event_prompts: Null -------------------------------------------------------------------------------- /LLM/configs/prompts/evaluation/p1/retrieval_5_v2.yaml: -------------------------------------------------------------------------------- 1 | prompt: BehaviorPromptManager 2 | prompt_kwargs: 3 | # Task prompt 4 | task_prompt: 5 | # Task 6 | task: retrieval_5_v2 7 | instruction: "Retrieve object_1 and place it on the shelf." 8 | objects: 9 | - "object_1" 10 | - "object_2" 11 | - "object_3" 12 | - "shelf" 13 | - "ground" 14 | predicates: 15 | - "Inside(object_1, object_2)" 16 | - "On(object_2, shelf)" 17 | - "Blocking(object_2, object_1)" 18 | - "Blocking(object_3, object_2)" 19 | - "Blocking(object_3, object_1)" 20 | - "On(object_3, shelf)" 21 | 22 | # Behaviors 23 | goals: Null 24 | plans: Null 25 | hypotheses: Null 26 | geometric: Null 27 | 28 | # OpenAI 29 | role: user 30 | name: Null 31 | name_query: Null 32 | name_response: Null 33 | 34 | # System prompts 35 | system_prompts: 36 | - LLM/configs/prompts/system/goal_prediction_v2.yaml 37 | - LLM/configs/prompts/system/task_planning_goal_condition_v2.yaml 38 | 39 | # Example prompts 40 | example_prompts: 41 | - LLM/configs/prompts/examples/example_4.yaml 42 | - LLM/configs/prompts/examples/example_5.yaml 43 | - LLM/configs/prompts/examples/example_6.yaml 44 | 45 | # Event prompts 46 | event_prompts: Null -------------------------------------------------------------------------------- /LLM/configs/prompts/evaluation/p1/retrieval_5_v3.yaml: -------------------------------------------------------------------------------- 1 | prompt: BehaviorPromptManager 2 | prompt_kwargs: 3 | # Task prompt 4 | task_prompt: 5 | # Task 6 | task: retrieval_5_v3 7 | instruction: "Retrieve object_1 and place it on the sofa." 8 | objects: 9 | - "object_1" 10 | - "object_2" 11 | - "object_3" 12 | - "shelf" 13 | - "ground" 14 | - "sofa" 15 | predicates: 16 | - "Inside(object_1, object_2)" 17 | - "On(object_2, shelf)" 18 | - "Blocking(object_2, object_1)" 19 | - "Blocking(object_3, object_2)" 20 | - "Blocking(object_3, object_1)" 21 | - "On(object_3, shelf)" 22 | 23 | # Behaviors 24 | goals: Null 25 | plans: Null 26 | hypotheses: Null 27 | geometric: Null 28 | 29 | # OpenAI 30 | role: user 31 | name: Null 32 | name_query: Null 33 | name_response: Null 34 | 35 | # System prompts 36 | system_prompts: 37 | - LLM/configs/prompts/system/goal_prediction_v2.yaml 38 | - LLM/configs/prompts/system/task_planning_goal_condition_v2.yaml 39 | 40 | # Example prompts 41 | example_prompts: 42 | - LLM/configs/prompts/examples/example_4.yaml 43 | - LLM/configs/prompts/examples/example_5.yaml 44 | - LLM/configs/prompts/examples/example_6.yaml 45 | 46 | # Event prompts 47 | event_prompts: Null -------------------------------------------------------------------------------- /LLM/configs/prompts/examples/example_1.yaml: -------------------------------------------------------------------------------- 1 | # Task 2 | task: example_1 3 | instruction: "Put object_1 on the sink." 4 | objects: 5 | - "object_1" 6 | - "sink" 7 | - "kitchen_table" 8 | predicates: 9 | - "On(object_1, kitchen_table)" 10 | 11 | # Behaviors 12 | goals: [["On(object_1, sink)"]] 13 | plans: [["Pick(object_1, kitchen_table)", "Place(object_1, sink)"]] 14 | hypotheses: Null 15 | geometric: Null 16 | 17 | # OpenAI 18 | role: system 19 | name: Null 20 | name_query: example_user 21 | name_response: example_assistant -------------------------------------------------------------------------------- /LLM/configs/prompts/examples/example_2.yaml: -------------------------------------------------------------------------------- 1 | # Task 2 | task: example_2 3 | instruction: "Get me object_1 from the drawer. I'm in the bedroom. Don't leave the drawer open." 4 | objects: 5 | - "object_1" 6 | - "object_2" 7 | - "drawer" 8 | - "closet" 9 | - "bed" 10 | predicates: 11 | - "Inside(object_1, drawer)" 12 | - "Inside(object_2, closet)" 13 | - "Closed(drawer)" 14 | - "Closed(closet)" 15 | 16 | # Behaviors 17 | goals: [["On(object_1, bed)", "Closed(drawer)"]] 18 | plans: [["Open(drawer)", "Pick(object_1, drawer)", "Place(object_1, bed)", "Close(drawer)"]] 19 | hypotheses: Null 20 | geometric: Null 21 | 22 | # OpenAI 23 | role: system 24 | name: Null 25 | name_query: example_user 26 | name_response: example_assistant -------------------------------------------------------------------------------- /LLM/configs/prompts/examples/example_3.yaml: -------------------------------------------------------------------------------- 1 | # Task 2 | task: example_3 3 | instruction: "Bring me object_2. I'm sitting on the reading_chair by the coffee_table." 4 | objects: 5 | - "object_1" 6 | - "object_2" 7 | - "bookshelf" 8 | - "reading_chair" 9 | - "coffee_table" 10 | predicates: 11 | - "On(object_1, object_2)" 12 | - "On(object_2, bookshelf)" 13 | 14 | # Behaviors 15 | goals: [ 16 | ["On(object_2, coffee_table)"], 17 | ["On(object_2, reading_chair)"] 18 | ] 19 | plans: [ 20 | ["Pick(object_1, object_2)", "Place(object_1, bookshelf)", "Pick(object_2, bookshelf)", "Place(object_2, coffee_table)"], 21 | ["Pick(object_1, object_2)", "Place(object_1, bookshelf)", "Pick(object_2, bookshelf)", "Place(object_2, reading_chair)"] 22 | ] 23 | hypotheses: Null 24 | geometric: Null 25 | 26 | # OpenAI 27 | role: system 28 | name: Null 29 | name_query: example_user 30 | name_response: example_assistant -------------------------------------------------------------------------------- /LLM/configs/prompts/examples/example_4.yaml: -------------------------------------------------------------------------------- 1 | # Task 2 | task: example_4 3 | instruction: "Please retrieve object_1." 4 | objects: 5 | - "object_1" 6 | - "object_2" 7 | - "shelf" 8 | - "ground" 9 | predicates: 10 | - "On(object_1, shelf)" 11 | - "On(object_2, shelf)" 12 | 13 | # Behaviors 14 | goals: [ 15 | ["On(object_1, ground)"], 16 | ["On(object_2, ground)", "On(object_1, ground)"], 17 | ] 18 | plans: [ 19 | ["Pick(object_1, shelf)", "Place(object_1, ground)"], 20 | ["Pick(object_2, shelf)", "Place(object_2, ground)", "Pick(object_1, shelf)", "Place(object_1, ground)"], 21 | ] 22 | hypotheses: Null 23 | geometric: Null 24 | 25 | # OpenAI 26 | role: system 27 | name: Null 28 | name_query: example_user 29 | name_response: example_assistant -------------------------------------------------------------------------------- /LLM/configs/prompts/examples/example_5.yaml: -------------------------------------------------------------------------------- 1 | # Task 2 | task: example_5 3 | instruction: "Please retrieve object_1." 4 | objects: 5 | - "object_1" 6 | - "object_2" 7 | - "shelf" 8 | - "ground" 9 | predicates: 10 | - "On(object_1, shelf)" 11 | - "On(object_2, shelf)" 12 | - "Blocking(object_2, object_1)" 13 | 14 | # Behaviors 15 | goals: [ 16 | ["On(object_2, ground)", "On(object_1, ground)"], 17 | ] 18 | plans: [ 19 | ["Pick(object_2, shelf)", "Place(object_2, ground)", "Pick(object_1, shelf)", "Place(object_1, ground)"], 20 | ] 21 | hypotheses: Null 22 | geometric: Null 23 | 24 | # OpenAI 25 | role: system 26 | name: Null 27 | name_query: example_user 28 | name_response: example_assistant -------------------------------------------------------------------------------- /LLM/configs/prompts/examples/example_6.yaml: -------------------------------------------------------------------------------- 1 | # Task 2 | task: example_6 3 | instruction: "Please retrieve object_1." 4 | objects: 5 | - "object_1" 6 | - "box" 7 | - "ground" 8 | predicates: 9 | - "Inside(object_1, box)" 10 | - "On(box, ground)" 11 | 12 | # Behaviors 13 | goals: [ 14 | ["On(object_1, ground)"] 15 | ] 16 | plans: [ 17 | ["Pick(object_1, box)", "Place(object_1, ground)"] 18 | ] 19 | hypotheses: Null 20 | geometric: Null 21 | 22 | # OpenAI 23 | role: system 24 | name: Null 25 | name_query: example_user 26 | name_response: example_assistant -------------------------------------------------------------------------------- /LLM/configs/prompts/system/geomtric_reasoning_v1.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yixuanhuang98/Points2Plans/78c0b0f0a23d2a352022584d369398741c553772/LLM/configs/prompts/system/geomtric_reasoning_v1.yaml -------------------------------------------------------------------------------- /LLM/configs/prompts/system/goal_prediction_v1.yaml: -------------------------------------------------------------------------------- 1 | behavior: goal_prediction 2 | behavior_kwargs: Null 3 | 4 | role: system 5 | name: Null 6 | content: 7 | "I am the reasoning system of a mobile manipulator robot operating in a household environment. 8 | Given 1) an instruction from a human user and 2) the current symbolic state of the environment, I will 9 | predict a set of possible symbolic goals that the robot could achieve to fulfill the user's instruction.\n 10 | 11 | Definitions:\n 12 | - Symbolic states and symbolic goals are defined as a set of predicates expressed over specific objects.\n 13 | - The term 'predicate' refers to an object state (e.g., Opened(cabinet)) or a relationship among objects (e.g., On(cup, shelf)).\n 14 | 15 | The robot can perceive the following information about the environment:\n 16 | - The objects in the environment\n 17 | - The states of individual objects\n 18 | - The relationships among objects\n 19 | 20 | The robot can detect the following states of individual objects:\n 21 | - Opened(a): Object a is opened\n 22 | - Closed(a): Object a is closed\n 23 | 24 | The robot can detect the following relationships among objects:\n 25 | - On(a, b): Object a is on object b\n 26 | - Inside(a, b): Object a is in object b\n 27 | 28 | There may be multiple symbolic goals that fulfill the user's instruction. 29 | Therefore, I will format my output in the following form:\n 30 | 31 | Goals: List[List[str]]\n 32 | 33 | Rules:\n 34 | - I will output a set of symbolic goals as a list of lists after 'Goals:'. Each nested list represents one goal\n 35 | - I will not output all possible symbolic goals, but the most likely goals that fulfill the user's instruction\n 36 | - If there are multiple symbolic goals that fulfill the instruction, I will output the simplest goals first" -------------------------------------------------------------------------------- /LLM/configs/prompts/system/goal_prediction_v2.yaml: -------------------------------------------------------------------------------- 1 | behavior: goal_prediction 2 | behavior_kwargs: Null 3 | 4 | role: system 5 | name: Null 6 | content: 7 | "I am the reasoning system of a mobile manipulator robot operating in a household environment. 8 | Given 1) an instruction from a human user and 2) the current symbolic state of the environment, I will 9 | predict a set of possible symbolic goals that the robot could achieve to fulfill the user's instruction.\n 10 | 11 | Definitions:\n 12 | - Symbolic states and symbolic goals are defined as a set of predicates expressed over specific objects.\n 13 | - The term 'predicate' refers to an object state (e.g., Opened(cabinet)) or a relationship among objects (e.g., On(cup, shelf)).\n 14 | 15 | The robot can perceive the following information about the environment:\n 16 | - The objects in the environment\n 17 | - The states of individual objects\n 18 | - The relationships among objects\n 19 | 20 | The robot can detect the following states of individual objects:\n 21 | - Opened(a): Object a is opened\n 22 | - Closed(a): Object a is closed\n 23 | 24 | The robot can detect the following relationships among objects:\n 25 | - On(a, b): Object a is on object b\n 26 | - Inside(a, b): Object a is in object b\n 27 | - Blocking(a, b): Object a is blocking object b\n 28 | 29 | There may be multiple symbolic goals that fulfill the user's instruction. 30 | Therefore, I will format my output in the following form:\n 31 | 32 | Goals: List[List[str]]\n 33 | 34 | Rules:\n 35 | - I will output a set of symbolic goals as a list of lists after 'Goals:'. Each nested list represents one goal\n 36 | - I will not output all possible symbolic goals, but the most likely goals that fulfill the user's instruction\n 37 | - If there are multiple symbolic goals that fulfill the instruction, I will output the simplest goals first" -------------------------------------------------------------------------------- /LLM/configs/prompts/system/hypothesis_generation_v1.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yixuanhuang98/Points2Plans/78c0b0f0a23d2a352022584d369398741c553772/LLM/configs/prompts/system/hypothesis_generation_v1.yaml -------------------------------------------------------------------------------- /LLM/configs/prompts/system/task_planning_goal_condition_v1.yaml: -------------------------------------------------------------------------------- 1 | behavior: task_planning 2 | behavior_kwargs: 3 | goal_condition: True 4 | 5 | role: system 6 | name: Null 7 | content: 8 | "I am the task planning system of a mobile manipulator robot operating in a household environment. 9 | Given 1) an instruction from a human user, 2) the current symbolic state of the environment, and 3) 10 | a set of possible symbolic goals that the robot could achieve to fulfill the user's instruction, I will 11 | predict a set of task plans that the robot should execute to satisfy the symbolic goals.\n 12 | 13 | Definitions:\n 14 | - Symbolic states and symbolic goals are defined as a set of predicates expressed over specific objects.\n 15 | - The term 'predicate' refers to an object state (e.g., Opened(cabinet)) or a relationship among objects (e.g., On(cup, shelf)).\n 16 | - A task plan is a sequence of actions that the robot can execute (e.g., Pick(cup, table), Place(cup, shelf))\n 17 | 18 | The robot can perceive the following information about the environment:\n 19 | - The objects in the environment\n 20 | - The states of individual objects\n 21 | - The relationships among objects\n 22 | 23 | The robot can detect the following states of individual objects:\n 24 | - Opened(a): Object a is opened\n 25 | - Closed(a): Object a is closed\n 26 | 27 | The robot can detect the following relationships among objects:\n 28 | - On(a, b): Object a is on object b\n 29 | - Inside(a, b): Object a is in object b\n 30 | 31 | The robot can execute the following actions:\n 32 | - Pick(a, b): The robot picks object a from object b\n 33 | - Place(a, b): The robot places object a on or in object b\n 34 | - Open(a): The robot opens object a\n 35 | - Close(a): The robot closes object a\n 36 | 37 | Action preconditions:\n 38 | - If the robot is already holding an object, it CANNOT Pick, Open, or Close another object\n 39 | - The robot CAN ONLY Place an object that it is already holding\n 40 | 41 | There may be multiple symbolic goals that fulfill the user's instruction. 42 | Therefore, I will format my output in the following form:\n 43 | 44 | Plans: List[List[str]]\n 45 | 46 | Rules:\n 47 | - I will output a set of task plans as a list of lists after 'Plans:'. Each nested list represents one task plan\n 48 | - I will output one task plan for each symbolic goal. Hence, each goal and its corresponding plan will be located 49 | at the same index in the 'Goals' and 'Plans' lists\n 50 | - I will only output task plans that are feasible with respect to the defined action preconditions." 51 | -------------------------------------------------------------------------------- /LLM/configs/prompts/system/task_planning_goal_condition_v2.yaml: -------------------------------------------------------------------------------- 1 | behavior: task_planning 2 | behavior_kwargs: 3 | goal_condition: True 4 | 5 | role: system 6 | name: Null 7 | content: 8 | "I am the task planning system of a mobile manipulator robot operating in a household environment. 9 | Given 1) an instruction from a human user, 2) the current symbolic state of the environment, and 3) 10 | a set of possible symbolic goals that the robot could achieve to fulfill the user's instruction, I will 11 | predict a set of task plans that the robot should execute to satisfy the symbolic goals.\n 12 | 13 | Definitions:\n 14 | - Symbolic states and symbolic goals are defined as a set of predicates expressed over specific objects.\n 15 | - The term 'predicate' refers to an object state (e.g., Opened(cabinet)) or a relationship among objects (e.g., On(cup, shelf)).\n 16 | - A task plan is a sequence of actions that the robot can execute (e.g., Pick(cup, table), Place(cup, shelf))\n 17 | 18 | The robot can perceive the following information about the environment:\n 19 | - The objects in the environment\n 20 | - The states of individual objects\n 21 | - The relationships among objects\n 22 | 23 | The robot can detect the following states of individual objects:\n 24 | - Opened(a): Object a is opened\n 25 | - Closed(a): Object a is closed\n 26 | 27 | The robot can detect the following relationships among objects:\n 28 | - On(a, b): Object a is on object b\n 29 | - Inside(a, b): Object a is in object b\n 30 | - Blocking(a, b): Object a is blocking object b\n 31 | 32 | The robot can execute the following actions:\n 33 | - Pick(a, b): The robot picks object a from object b\n 34 | - Place(a, b): The robot places object a on or in object b\n 35 | - Open(a): The robot opens object a\n 36 | - Close(a): The robot closes object a\n 37 | 38 | Action preconditions:\n 39 | - If the robot is already holding an object, it CANNOT Pick, Open, or Close another object\n 40 | - The robot CAN ONLY Place an object that it is already holding\n 41 | - The robot CANNOT Pick an object if it is blocked by another object\n 42 | 43 | There may be multiple symbolic goals that fulfill the user's instruction. 44 | Therefore, I will format my output in the following form:\n 45 | 46 | Plans: List[List[str]]\n 47 | 48 | Rules:\n 49 | - I will output a set of task plans as a list of lists after 'Plans:'. Each nested list represents one task plan\n 50 | - I will output one task plan for each symbolic goal. Hence, each goal and its corresponding plan will be located 51 | at the same index in the 'Goals' and 'Plans' lists\n 52 | - I will only output task plans that are feasible with respect to the defined action preconditions." 53 | -------------------------------------------------------------------------------- /LLM/configs/prompts/system/task_planning_hypothesis_condition_v1.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yixuanhuang98/Points2Plans/78c0b0f0a23d2a352022584d369398741c553772/LLM/configs/prompts/system/task_planning_hypothesis_condition_v1.yaml -------------------------------------------------------------------------------- /LLM/configs/prompts/system/task_planning_v1.yaml: -------------------------------------------------------------------------------- 1 | behavior: task_planning 2 | behavior_kwargs: Null 3 | 4 | role: system 5 | name: Null 6 | content: 7 | "I am the task planning system of a mobile manipulator robot operating in a household environment. 8 | Given 1) an instruction from a human user and 2) the current symbolic state of the environment, I will 9 | predict a set of possible task plans that the robot could execute to fulfill the user's instruction.\n 10 | 11 | Definitions:\n 12 | - Symbolic states and symbolic goals are defined as a set of predicates expressed over specific objects.\n 13 | - The term 'predicate' refers to an object state (e.g., Opened(cabinet)) or a relationship among objects (e.g., On(cup, shelf)).\n 14 | - A task plan is a sequence of actions that the robot can execute (e.g., Pick(cup, table), Place(cup, shelf))\n 15 | 16 | The robot can perceive the following information about the environment:\n 17 | - The objects in the environment\n 18 | - The states of individual objects\n 19 | - The relationships among objects\n 20 | 21 | The robot can detect the following states of individual objects:\n 22 | - Opened(a): Object a is opened\n 23 | - Closed(a): Object a is closed\n 24 | 25 | The robot can detect the following relationships among objects:\n 26 | - On(a, b): Object a is on object b\n 27 | - Inside(a, b): Object a is in object b\n 28 | 29 | The robot can execute the following actions:\n 30 | - Pick(a, b): The robot picks object a from object b\n 31 | - Place(a, b): The robot places object a on or in object b\n 32 | - Open(a): The robot opens object a\n 33 | - Close(a): The robot closes object a\n 34 | 35 | Action preconditions:\n 36 | - If the robot is already holding an object, it CANNOT Pick, Open, or Close another object\n 37 | - The robot CAN ONLY Place an object that it is already holding\n 38 | 39 | There may be multiple task plans that fulfill the user's instruction. 40 | Therefore, I will format my output in the following form:\n 41 | 42 | Plans: List[List[str]]\n 43 | 44 | Rules:\n 45 | - I will output a set of task plans as a list of lists after 'Plans:'. Each nested list represents one task plan\n 46 | - I will not output all possible task plans, but the most likely plans that fulfill the user's instruction\n 47 | - If there are multiple task plans that fulfill the instruction, I will output the simplest plans first\n 48 | - I will only output task plans that are feasible with respect to the defined action preconditions." 49 | -------------------------------------------------------------------------------- /LLM/fm_planning/__init__.py: -------------------------------------------------------------------------------- 1 | from . import models 2 | from . import utils 3 | -------------------------------------------------------------------------------- /LLM/fm_planning/fm_agents/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yixuanhuang98/Points2Plans/78c0b0f0a23d2a352022584d369398741c553772/LLM/fm_planning/fm_agents/__init__.py -------------------------------------------------------------------------------- /LLM/fm_planning/fm_agents/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yixuanhuang98/Points2Plans/78c0b0f0a23d2a352022584d369398741c553772/LLM/fm_planning/fm_agents/base.py -------------------------------------------------------------------------------- /LLM/fm_planning/fm_agents/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yixuanhuang98/Points2Plans/78c0b0f0a23d2a352022584d369398741c553772/LLM/fm_planning/fm_agents/utils.py -------------------------------------------------------------------------------- /LLM/fm_planning/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .pretrained import * 2 | -------------------------------------------------------------------------------- /LLM/fm_planning/models/pretrained/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import PretrainedModel 2 | from .generative import * 3 | from .embeddings import * 4 | from .utils import * 5 | -------------------------------------------------------------------------------- /LLM/fm_planning/models/pretrained/base.py: -------------------------------------------------------------------------------- 1 | from typing import Generic, Any, TypeVar 2 | 3 | import abc 4 | 5 | from LLM.fm_planning.utils.typing import ModelBatchType 6 | 7 | 8 | class PretrainedModel(abc.ABC, Generic[ModelBatchType]): 9 | """Abstract base class for pretrained models.""" 10 | 11 | @abc.abstractmethod 12 | def forward(self, x: ModelBatchType) -> Any: 13 | """Compute forward pass.""" 14 | pass 15 | 16 | 17 | PretrainedModelType = TypeVar("PretrainedModelType", bound=PretrainedModel) 18 | -------------------------------------------------------------------------------- /LLM/fm_planning/models/pretrained/embeddings/__init__.py: -------------------------------------------------------------------------------- 1 | from .openai_emb import OpenAIEmbeddingModel 2 | -------------------------------------------------------------------------------- /LLM/fm_planning/models/pretrained/embeddings/hugging_face_emb.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | 3 | import torch 4 | import torch.nn.functional as F 5 | 6 | from transformers import AutoTokenizer, AutoModel # type: ignore 7 | 8 | from LLM.fm_planning.utils import typing, tensors 9 | from LLM.fm_planning.models.pretrained.base import PretrainedModel 10 | from .utils import mean_pooling 11 | 12 | 13 | class HuggingFaceEmbeddingModel(PretrainedModel[str]): 14 | def __init__(self, model: str, device: str = "auto", **kwargs: Any): 15 | super().__init__() 16 | self._model_name = model 17 | self._device = tensors.device(device) 18 | self._model = AutoModel.from_pretrained(model).to(self._device) 19 | self._tokenizer = AutoTokenizer.from_pretrained(model, use_fast=True) 20 | 21 | @torch.no_grad() 22 | def forward(self, x: str) -> typing.Tensor: 23 | """Compute embeddings for a prompt.""" 24 | 25 | # BERT models. 26 | if self._model_name in ["bert-base-uncased", "bert-large-uncased"]: 27 | encoded_input = self._tokenizer(x, return_tensors="pt").to(self._device) 28 | output = self._model(**encoded_input) 29 | embedding = output[1] 30 | 31 | # MPNet model. 32 | elif self._model_name == "sentence-transformers/all-mpnet-base-v2": 33 | encoded_input = self._tokenizer( 34 | x, padding=True, truncation=True, return_tensors="pt" 35 | ).to(self._device) 36 | output = self._model(**encoded_input) 37 | embedding = mean_pooling(output, encoded_input["attention_mask"]) 38 | 39 | # Mistral (7B) model. 40 | elif self._model_name == "intfloat/e5-mistral-7b-instruct": 41 | encoded_input = self._tokenizer( 42 | [x], return_attention_mask=False, padding=False, truncation=True 43 | ) 44 | encoded_input["input_ids"][0] += [self._tokenizer.eos_token_id] 45 | encoded_input = self._tokenizer.pad( 46 | encoded_input, 47 | padding=True, 48 | return_attention_mask=True, 49 | return_tensors="pt", 50 | ).to(self._device) 51 | output = self._model(**encoded_input) 52 | embedding = output.last_hidden_state[:, -1, :] 53 | 54 | # Llama 2 (7B) models. 55 | elif self._model_name in [ 56 | "meta-llama/Llama-2-7b-hf", 57 | "meta-llama/Llama-2-7b-chat-hf", 58 | ]: 59 | encoded_input = self._tokenizer(x, return_tensors="pt").to(self._device) 60 | output = self._model(**encoded_input) 61 | embedding = mean_pooling(output, encoded_input["attention_mask"]) 62 | 63 | embedding = embedding.squeeze() 64 | embedding = F.normalize(embedding, dim=0).cpu().numpy() 65 | return embedding 66 | -------------------------------------------------------------------------------- /LLM/fm_planning/models/pretrained/embeddings/openai_emb.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, List, Any 2 | 3 | import os 4 | import time 5 | import openai 6 | from openai.types.embedding import Embedding 7 | 8 | from LLM.fm_planning.models.pretrained.base import PretrainedModel 9 | 10 | 11 | class OpenAIEmbeddingModel(PretrainedModel[List[str]]): 12 | def __init__(self, model: str, api_key: Optional[str] = None, **kwargs: Any): 13 | super().__init__() 14 | self._model = model 15 | self._api_key = ( 16 | api_key if api_key is not None else os.environ.get("OPENAI_API_KEY") 17 | ) 18 | if api_key is None: 19 | raise ValueError("API key must be provided.") 20 | self._api_key = api_key 21 | self._client = openai.OpenAI(api_key=self._api_key) 22 | 23 | def forward(self, x: List[str]) -> List[Embedding]: 24 | """Compute embeddings for a prompt.""" 25 | success = False 26 | while not success: 27 | try: 28 | response = self._client.embeddings.create(input=x, model=self._model) 29 | success = True 30 | except: 31 | time.sleep(1) 32 | continue 33 | 34 | embeddings = [embedding for embedding in response.data] 35 | return embeddings 36 | -------------------------------------------------------------------------------- /LLM/fm_planning/models/pretrained/embeddings/utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | def mean_pooling(model_output, attention_mask: torch.Tensor) -> torch.Tensor: 5 | """Perform mean pooling on the model output. 6 | 7 | Args: 8 | model_output: Output from the HuggingFace model. 9 | attention_mask (torch.Tensor): Attention mask tensor. 10 | 11 | Returns: 12 | torch.Tensor: Mean pooled tensor. 13 | """ 14 | token_embeddings = model_output[0] 15 | input_mask_expanded = ( 16 | attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float() 17 | ) 18 | mean_pool = torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp( 19 | input_mask_expanded.sum(1), min=1e-9 20 | ) 21 | return mean_pool 22 | -------------------------------------------------------------------------------- /LLM/fm_planning/models/pretrained/generative/__init__.py: -------------------------------------------------------------------------------- 1 | from .openai_gen import OpenAIGenerativeModel 2 | from .utils import BehaviorPromptManager 3 | -------------------------------------------------------------------------------- /LLM/fm_planning/models/pretrained/generative/llama_gen.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, Any 2 | 3 | import torch 4 | 5 | from transformers import AutoModelForCausalLM, AutoTokenizer # type: ignore 6 | 7 | from LLM.fm_planning.models.pretrained.base import PretrainedModel 8 | 9 | 10 | class LlamaGenerativeModel(PretrainedModel[str]): 11 | """Llama generative model.""" 12 | 13 | def __init__(self, model: str, device: str = "auto", **kwargs: Any): 14 | """Construct Llama generative model.""" 15 | super().__init__() 16 | self._tokenizer = AutoTokenizer.from_pretrained(model, use_fast=True) 17 | self._model = AutoModelForCausalLM.from_pretrained( 18 | pretrained_model_name_or_path=model, 19 | torch_dtype=torch.float16, 20 | device_map=device, 21 | **kwargs, 22 | ) 23 | self.device: torch.device = self._model.device 24 | 25 | def forward(self, x: str) -> str: 26 | """Compute completion for a prompt.""" 27 | # Send request. 28 | input_ids: torch.Tensor = self._tokenizer.encode( 29 | x, add_special_tokens=True, return_tensors="pt" 30 | ) 31 | response_ids = self._model.generate(input_ids.to(self.device))[0] 32 | response = self._tokenizer.decode( 33 | response_ids[input_ids.shape[-1] :], skip_special_tokens=True 34 | ) 35 | 36 | return response 37 | -------------------------------------------------------------------------------- /LLM/fm_planning/models/pretrained/generative/openai_gen.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Optional, List, Dict 2 | 3 | import os 4 | import time 5 | import openai 6 | from openai.types.chat.chat_completion import ChatCompletion 7 | 8 | from LLM.fm_planning.models.pretrained.base import PretrainedModel 9 | 10 | 11 | class OpenAIGenerativeModel(PretrainedModel[List[Dict[str, Any]]]): 12 | """OpenAI generative model.""" 13 | 14 | def __init__(self, model: str, api_key: Optional[str] = None, **kwargs: Any): 15 | """Construct OpenAI generative model.""" 16 | super().__init__() 17 | self._model = model 18 | self._api_key = ( 19 | api_key if api_key is not None else os.environ.get("OPENAI_API_KEY") 20 | ) 21 | if api_key is None: 22 | raise ValueError("API key must be provided.") 23 | self._api_key = api_key 24 | self._client = openai.OpenAI(api_key=self._api_key) 25 | self._kwargs = kwargs 26 | 27 | def forward(self, x: List[Dict[str, Any]]) -> Dict[str, Any]: 28 | """Compute completion for a prompt.""" 29 | # Send request. 30 | success = False 31 | while not success: 32 | try: 33 | response: ChatCompletion = self._client.chat.completions.create( 34 | messages=x, 35 | model=self._model, 36 | **self._kwargs, 37 | ) 38 | success = True 39 | except: 40 | time.sleep(1) 41 | continue 42 | 43 | response = response.model_dump() 44 | return response 45 | -------------------------------------------------------------------------------- /LLM/fm_planning/models/pretrained/generative/utils.py: -------------------------------------------------------------------------------- 1 | from typing import Union, List, Dict, Optional, Any, Tuple, Set 2 | 3 | import yaml 4 | import pathlib 5 | import dataclasses 6 | 7 | 8 | def format_openai( 9 | role: str, 10 | content: str, 11 | name: Optional[str] = None, 12 | ) -> Dict[str, Any]: 13 | """Format a prompt for OpenAI.""" 14 | prompt = {"role": role, "content": content} 15 | if name is not None: 16 | prompt["name"] = name 17 | return prompt 18 | 19 | 20 | @dataclasses.dataclass 21 | class SystemPrompt: 22 | content: str 23 | # Behavior 24 | behavior: str 25 | behavior_kwargs: Optional[Dict[str, Any]] = None 26 | # OpenAI 27 | role: Optional[str] = None 28 | name: Optional[str] = None 29 | 30 | @classmethod 31 | def from_config(cls, config: Dict[str, Any]) -> "SystemPrompt": 32 | """Create a behavior prompt from a config.""" 33 | return cls(**config) 34 | 35 | @classmethod 36 | def from_yaml(cls, file: Union[str, pathlib.Path]) -> "SystemPrompt": 37 | """Create a behavior prompt from a YAML file.""" 38 | if isinstance(file, str): 39 | file = pathlib.Path(file) 40 | 41 | with file.open("r") as f: 42 | config = yaml.safe_load(f) 43 | 44 | return cls.from_config(config) 45 | 46 | def prompt(self, openai: bool = True) -> Union[str, Dict[str, Any]]: 47 | """Return the prompt.""" 48 | if openai: 49 | assert self.role is not None 50 | prompt = format_openai(role=self.role, content=self.content, name=self.name) 51 | else: 52 | prompt = self.content 53 | 54 | return prompt 55 | 56 | 57 | @dataclasses.dataclass 58 | class BehaviorPrompt: 59 | # Task 60 | task: Optional[str] = None 61 | instruction: Optional[str] = None 62 | objects: Optional[List[str]] = None 63 | predicates: Optional[List[str]] = None 64 | # Behaviors 65 | goals: Optional[List[List[str]]] = None 66 | plans: Optional[List[List[str]]] = None 67 | hypotheses: Optional[List[List[str]]] = None 68 | geometric: Optional[bool] = None 69 | # OpenAI 70 | role: Optional[str] = None 71 | name: Optional[str] = None 72 | name_query: Optional[str] = None 73 | name_response: Optional[str] = None 74 | 75 | @classmethod 76 | def from_config(cls, config: Dict[str, Any]) -> "BehaviorPrompt": 77 | """Create a behavior prompt from a config.""" 78 | return cls(**config) 79 | 80 | @classmethod 81 | def from_yaml(cls, file: Union[str, pathlib.Path]) -> "BehaviorPrompt": 82 | """Create a behavior prompt from a YAML file.""" 83 | if isinstance(file, str): 84 | file = pathlib.Path(file) 85 | 86 | with file.open("r") as f: 87 | config = yaml.safe_load(f) 88 | 89 | return cls.from_config(config) 90 | 91 | def _prefix(self) -> str: 92 | """Return the prefix of the task prompt.""" 93 | assert ( 94 | self.objects is not None 95 | and self.predicates is not None 96 | and self.instruction is not None 97 | ) 98 | prefix = "" 99 | prefix += f"Objects: {self.objects}\n\n" 100 | prefix += f"Object Relationships: {self.predicates}\n\n" 101 | prefix += f"User Instruction: {self.instruction}" 102 | return prefix 103 | 104 | def _goal_prediction(self) -> Tuple[str, Optional[str]]: 105 | """Return the goal prediction prompt.""" 106 | query = self._prefix() + "\n\n" + "Goals: " 107 | response = str(self.goals) if self.goals else None 108 | return query, response 109 | 110 | def _task_planning(self, goal_condition: bool = False) -> Tuple[str, Optional[str]]: 111 | """Return the task planning prompt.""" 112 | if goal_condition: 113 | query, goals = self._goal_prediction() 114 | assert goals is not None 115 | query += goals + "\n\n" + "Plans: " 116 | else: 117 | query = self._prefix() + "\n\n" + "Plans: " 118 | 119 | response = str(self.plans) if self.plans else None 120 | return query, response 121 | 122 | def _hypothesis_generation(self) -> str: 123 | """Return the hypothesis generation prompt.""" 124 | assert self.hypotheses is not None 125 | raise NotImplementedError 126 | 127 | def _geometric_reasoning(self) -> str: 128 | """Return the geometric reasoning prompt.""" 129 | assert self.geometric is not None 130 | raise NotImplementedError 131 | 132 | def prompt( 133 | self, 134 | behavior: str, 135 | behavior_kwargs: Optional[Dict[str, Any]] = None, 136 | example: bool = False, 137 | openai: bool = True, 138 | ) -> Union[str, List[Dict[str, Any]]]: 139 | """Return the prompt for the behavior.""" 140 | behaviors = { 141 | "goal_prediction": self._goal_prediction, 142 | "task_planning": self._task_planning, 143 | "hypothesis_generation": self._hypothesis_generation, 144 | "geometric_reasoning": self._geometric_reasoning, 145 | } 146 | 147 | if behavior in behaviors: 148 | behavior_kwargs = behavior_kwargs if behavior_kwargs is not None else {} 149 | query, response = behaviors[behavior](**behavior_kwargs) 150 | if example and response is None: 151 | raise ValueError( 152 | f"Example response not provided for Behavior '{behavior}'." 153 | ) 154 | else: 155 | raise ValueError(f"Behavior '{behavior}' is not supported.") 156 | 157 | if openai: 158 | assert self.role is not None 159 | if self.name_query is not None and self.name_response is not None: 160 | query_prompt = format_openai( 161 | role=self.role, content=query, name=self.name_query 162 | ) 163 | prompt = [query_prompt] 164 | if example: 165 | response_prompt = format_openai( 166 | role=self.role, content=response, name=self.name_response 167 | ) 168 | prompt.append(response_prompt) 169 | else: 170 | prompt = format_openai( 171 | role=self.role, 172 | content=query + response if example else query, 173 | name=self.name, 174 | ) 175 | prompt = [prompt] 176 | else: 177 | prompt = query + response if example else query 178 | 179 | return prompt 180 | 181 | 182 | class BehaviorPromptManager: 183 | 184 | def __init__( 185 | self, 186 | task_prompt: Dict[str, Any], 187 | system_prompts: Union[List[str], List[pathlib.Path]], 188 | example_prompts: Optional[Union[List[str], List[pathlib.Path]]] = None, 189 | event_prompts: Optional[Union[List[str], List[pathlib.Path]]] = None, 190 | openai: bool = True, 191 | ) -> None: 192 | """Initialize behavior prompt manager.""" 193 | # Instantiate prompts. 194 | self._task_prompt = BehaviorPrompt.from_config(task_prompt) 195 | system_prompts = [SystemPrompt.from_yaml(prompt) for prompt in system_prompts] 196 | self._system_prompts: Dict[str, SystemPrompt] = { 197 | prompt.behavior: prompt for prompt in system_prompts 198 | } 199 | self._example_prompts: List[BehaviorPrompt] = ( 200 | [BehaviorPrompt.from_yaml(prompt) for prompt in example_prompts] 201 | if example_prompts is not None 202 | else [] 203 | ) 204 | self._event_prompts: List[BehaviorPrompt] = ( 205 | [BehaviorPrompt.from_yaml(prompt) for prompt in event_prompts] 206 | if event_prompts is not None 207 | else [] 208 | ) 209 | self._openai = openai 210 | 211 | @classmethod 212 | def from_config(cls, config: Dict[str, Any]) -> "BehaviorPromptManager": 213 | """Create a behavior prompt manager from a config.""" 214 | return cls(**config) 215 | 216 | @classmethod 217 | def from_yaml(cls, file: Union[str, pathlib.Path]) -> "BehaviorPromptManager": 218 | """Create a behavior prompt manager from a YAML file.""" 219 | if isinstance(file, str): 220 | file = pathlib.Path(file) 221 | 222 | with file.open("r") as f: 223 | config = yaml.safe_load(f) 224 | 225 | assert config["prompt"] == "BehaviorPromptManager" 226 | config = config["prompt_kwargs"] 227 | 228 | return cls.from_config(config) 229 | 230 | @property 231 | def task_prompt(self) -> BehaviorPrompt: 232 | """Get the task prompt.""" 233 | return self._task_prompt 234 | 235 | @property 236 | def event_prompts(self) -> List[BehaviorPrompt]: 237 | """Get the event prompts.""" 238 | return self._event_prompts 239 | 240 | @event_prompts.setter 241 | def event_prompts(self, event_prompts: List[BehaviorPrompt]) -> None: 242 | """Set the event prompts.""" 243 | self._event_prompts = event_prompts 244 | 245 | @property 246 | def behaviors(self) -> Set[str]: 247 | """Return the behaviors.""" 248 | return set(self._system_prompts.keys()) 249 | 250 | def generate_prompt( 251 | self, 252 | behavior: str, 253 | use_examples: bool = True, 254 | use_events: bool = False, 255 | openai: Optional[bool] = None, 256 | ) -> Union[str, List[Dict[str, Any]]]: 257 | """Generate a prompt for the behavior.""" 258 | openai = self._openai if openai is None else openai 259 | prompts = [] 260 | 261 | # System prompt. 262 | if behavior in self._system_prompts: 263 | system_prompt = self._system_prompts[behavior] 264 | prompts.append(system_prompt.prompt(openai=openai)) 265 | else: 266 | raise ValueError(f"Behavior '{behavior}' is not supported.") 267 | 268 | # Example prompts. 269 | if use_examples: 270 | for example_prompt in self._example_prompts: 271 | prompt = example_prompt.prompt( 272 | behavior, 273 | behavior_kwargs=system_prompt.behavior_kwargs, 274 | example=True, 275 | openai=openai, 276 | ) 277 | prompts.append(prompt) 278 | 279 | # Event history prompts. 280 | if use_events: 281 | raise NotImplementedError 282 | 283 | # Current task prompt. 284 | task_prompt = self._task_prompt.prompt( 285 | behavior, behavior_kwargs=system_prompt.behavior_kwargs, openai=openai 286 | ) 287 | prompts.append(task_prompt) 288 | 289 | if openai: 290 | temp = [] 291 | for p in prompts: 292 | if isinstance(p, list): 293 | temp.extend(p) 294 | else: 295 | temp.append(p) 296 | prompts = temp 297 | else: 298 | prompts = "\n\n".join(prompts) 299 | 300 | return prompts 301 | 302 | 303 | class Llama2Prompt: 304 | 305 | SYSTEM_PROMPT = """[INST] <> 306 | {SYSTEM_MESSAGE} 307 | <> 308 | 309 | """ 310 | 311 | def __init__(self, system_message: str): 312 | """Initialize Llama 2 prompt.""" 313 | self._system_message = system_message 314 | self._messages = [] 315 | self._responses = [] 316 | 317 | def add_message(self, message: str): 318 | """Add a message to the prompt.""" 319 | self._messages.append(message) 320 | 321 | def add_response(self, response: str): 322 | """Add a response to the prompt.""" 323 | self._responses.append(response) 324 | 325 | def __str__(self) -> str: 326 | """Return the prompt as a string.""" 327 | prompt = self.SYSTEM_PROMPT.replace("{SYSTEM_MESSAGE}", self._system_message) 328 | for i in range(len(self._messages + self._responses)): 329 | if i % 2 == 0: 330 | if i == 0: 331 | prompt += f"{self._messages[i]} [/INST]" 332 | else: 333 | prompt += f" {self._messages[i]} [/INST]" 334 | else: 335 | prompt += f" {self._responses[i]} [INST]" 336 | 337 | return prompt 338 | -------------------------------------------------------------------------------- /LLM/fm_planning/models/pretrained/utils.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Dict, Union, Optional 2 | 3 | import pathlib 4 | 5 | from LLM.fm_planning import models 6 | from LLM.fm_planning.models.pretrained import PretrainedModel 7 | from LLM.fm_planning.models.pretrained.generative import * 8 | from LLM.fm_planning.models.pretrained.embeddings import * 9 | from LLM.fm_planning.utils import configs 10 | 11 | 12 | class PretrainedModelFactory(configs.Factory[PretrainedModel]): 13 | """Pretrained model factory.""" 14 | 15 | def __init__( 16 | self, 17 | config: Union[str, pathlib.Path, Dict[str, Any]], 18 | api_key: Optional[str] = None, 19 | device: str = "auto", 20 | ): 21 | """Creates the pretrained model factory from a config. 22 | 23 | Args: 24 | config: Config path or dict. 25 | api_key: Pretrained model API key. 26 | device: Torch device. 27 | """ 28 | super().__init__(config, "model", models) 29 | 30 | if issubclass(self.cls, OpenAIGenerativeModel): 31 | if "api_key" not in self.kwargs: 32 | assert api_key is not None, "OpenAI API key must be provided." 33 | self.kwargs["api_key"] = api_key 34 | 35 | elif issubclass(self.cls, LlamaGenerativeModel): 36 | self.kwargs["device"] = device 37 | 38 | elif issubclass(self.cls, OpenAIEmbeddingModel): 39 | if "api_key" not in self.kwargs: 40 | assert api_key is not None, "OpenAI API key must be provided." 41 | self.kwargs["api_key"] = api_key 42 | 43 | elif issubclass(self.cls, HuggingFaceEmbeddingModel): 44 | self.kwargs["device"] = device 45 | 46 | else: 47 | raise ValueError(f"Invalid model type: {self.cls}") 48 | 49 | 50 | def load( 51 | config: Union[str, pathlib.Path, Dict[str, Any]], 52 | device: str = "auto", 53 | **kwargs, 54 | ) -> PretrainedModel: 55 | """Loads the pretrained model factory from a config. 56 | 57 | Args: 58 | config: Config path or dict. 59 | device: Torch device. 60 | **kwargs: Optional model constructor kwargs. 61 | 62 | Returns: 63 | Pretrained model instance. 64 | """ 65 | pretrained_model_factory = PretrainedModelFactory( 66 | config=config, 67 | device=device, 68 | ) 69 | return pretrained_model_factory(**kwargs) 70 | 71 | 72 | def load_config(path: Union[str, pathlib.Path]) -> Dict[str, Any]: 73 | """Loads a pretrained model config from path. 74 | 75 | Args: 76 | path: Path to the config, config directory. 77 | 78 | Returns: 79 | Pretrained model config dict. 80 | """ 81 | return configs.load_config(path, "model") 82 | -------------------------------------------------------------------------------- /LLM/fm_planning/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from . import configs 2 | from . import nest 3 | from . import random 4 | from . import tensors 5 | from . import timing 6 | from . import typing 7 | -------------------------------------------------------------------------------- /LLM/fm_planning/utils/configs.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | import subprocess 3 | import types 4 | from typing import Any, Callable, Dict, Generic, List, Optional, Type, TypeVar, Union 5 | 6 | import yaml # type: ignore 7 | 8 | 9 | T = TypeVar("T") 10 | 11 | 12 | def save_git_hash(path: pathlib.Path) -> None: 13 | """Saves the current git hash to the given path. 14 | 15 | Args: 16 | path: Path to save git hash. 17 | """ 18 | process = subprocess.Popen( 19 | ["git", "rev-parse", "HEAD"], shell=False, stdout=subprocess.PIPE 20 | ) 21 | git_head_hash = process.communicate()[0].strip() 22 | with open(path / "git_hash.txt", "wb") as f: 23 | f.write(git_head_hash) 24 | 25 | 26 | def get_class(classname: Union[str, Type[T]], module: types.ModuleType) -> Type[T]: 27 | """Gets the class from the given module. 28 | 29 | Returns classname directly if it is already a class. 30 | 31 | Args: 32 | classname: Class name with '.' denoting submodules. 33 | module: Python module to search. 34 | 35 | Returns: 36 | Class. 37 | """ 38 | 39 | def _get_submodule(module, submodules: List[str]) -> Type[T]: 40 | if not submodules: 41 | return module 42 | return _get_submodule(vars(module)[submodules[0]], submodules[1:]) 43 | 44 | if isinstance(classname, str): 45 | submodules = classname.split(".") 46 | 47 | try: 48 | return _get_submodule(module, submodules) 49 | except KeyError as e: 50 | raise KeyError(f"Cannot find {classname} in {module.__name__}:\n{e}") 51 | else: 52 | return classname 53 | 54 | 55 | def get_instance( 56 | classname: Union[str, T], kwargs: Dict[str, Any], module: types.ModuleType 57 | ) -> T: 58 | """Creates an instance of the given class with kwargs. 59 | 60 | Returns classname directly if it is already an instance. 61 | 62 | Args: 63 | classname: Class name with '.' denoting submodules. 64 | kwargs: Class constructor kwargs . 65 | module: Python module to search. 66 | 67 | Returns: 68 | Class instance. 69 | """ 70 | if isinstance(classname, str): 71 | cls: Type[T] = get_class(classname, module) 72 | return cls(**kwargs) 73 | else: 74 | return classname 75 | 76 | 77 | def parse_class(config: Dict[str, Any], key: str, module: types.ModuleType) -> Type: 78 | """Parses the class from a config. 79 | 80 | Args: 81 | config: Config dict. 82 | key: Dict key containing class name as its value. 83 | module: Python module to search. 84 | 85 | Returns: 86 | Class. 87 | """ 88 | if key not in config: 89 | raise KeyError(f"{key} missing from config") 90 | return get_class(config[key], module) 91 | 92 | 93 | def parse_kwargs(config: Dict[str, Any], key: str) -> Dict: 94 | """Parses the kwargs from a config. 95 | 96 | Args: 97 | config: Config dict. 98 | key: Dict key containing kwargs as its value. 99 | 100 | Returns: 101 | Kwargs or empty dict. 102 | """ 103 | try: 104 | kwargs = config[key] 105 | except KeyError: 106 | return {} 107 | return {} if kwargs is None else kwargs 108 | 109 | 110 | def load_config( 111 | path: Union[str, pathlib.Path], config_prefix: Optional[str] = None 112 | ) -> Dict[str, Any]: 113 | """Loads a config from path. 114 | 115 | Args: 116 | path: Path to the config, config directory, or checkpoint. 117 | config_prefix: Prefix of config file to search: "{config_prefix}_config.yaml". 118 | 119 | Returns: 120 | Config dict. 121 | """ 122 | if isinstance(path, str): 123 | path = pathlib.Path(path) 124 | 125 | if path.suffix == ".yaml": 126 | config_path = path 127 | else: 128 | if path.suffix == ".pt": 129 | path = path.parent 130 | 131 | config_name = "config.yaml" 132 | if config_prefix is not None: 133 | config_name = f"{config_prefix}_{config_name}" 134 | 135 | config_path = path / config_name 136 | 137 | with open(config_path, "r") as f: 138 | config = yaml.safe_load(f) 139 | 140 | return config 141 | 142 | 143 | class Factory(Generic[T]): 144 | """Base factory class.""" 145 | 146 | def __init__( 147 | self, 148 | config: Union[str, pathlib.Path, Dict[str, Any]], 149 | key: str, 150 | module: types.ModuleType, 151 | ): 152 | """Parses the config. 153 | 154 | Args: 155 | config: Config path or dict. 156 | key: Key of class definition in the config dict. 157 | module: Python module of class. 158 | """ 159 | if not isinstance(config, dict): 160 | with open(config, "r") as f: 161 | config = yaml.safe_load(f) 162 | assert isinstance(config, dict) 163 | 164 | self._config = config 165 | self._cls = parse_class(config, key, module) 166 | self._kwargs = dict(parse_kwargs(config, f"{key}_kwargs")) 167 | self._key = key 168 | self._last_instance: Optional[T] = None 169 | self._post_hooks: List[Callable[[T], None]] = [] 170 | 171 | @property 172 | def config(self) -> Dict[str, Any]: 173 | """Loaded config dict.""" 174 | return self._config 175 | 176 | @property 177 | def cls(self) -> Type: 178 | """Parsed class name.""" 179 | return self._cls 180 | 181 | @property 182 | def kwargs(self) -> Dict[str, Any]: 183 | """Parsed class kwargs.""" 184 | return self._kwargs 185 | 186 | @property 187 | def last_instance(self) -> Optional[T]: 188 | """Last created instance.""" 189 | return self._last_instance 190 | 191 | def save_config(self, path: Union[str, pathlib.Path]) -> None: 192 | """Saves the config to path. 193 | 194 | Args: 195 | path: Directory where config will be saved. 196 | """ 197 | path = pathlib.Path(path) 198 | with open(path / f"{self._key}_config.yaml", "w") as f: 199 | yaml.dump(self.config, f) 200 | 201 | def get_instance(self, *args, **kwargs) -> T: 202 | """Gets the last created instance or creates a new one with the given args. 203 | 204 | Args: 205 | *args: Constructor args. 206 | **kwargs: Constructor kwargs. 207 | 208 | Returns: 209 | Last created instance. 210 | """ 211 | if self.last_instance is None: 212 | self.__call__(*args, **kwargs) 213 | assert self.last_instance is not None 214 | return self.last_instance 215 | 216 | def add_post_hook(self, post_hook: Callable[[T], Any]) -> None: 217 | """Adds a callback function to call when this factory is called. 218 | 219 | Args: 220 | post_hook: Function to call. 221 | """ 222 | self._post_hooks.append(post_hook) 223 | 224 | def run_post_hooks(self, instance: T) -> None: 225 | """Runs the post hooks. 226 | 227 | Args: 228 | instance: Instance to pass to the post hooks. 229 | """ 230 | self._last_instance = instance 231 | for post_hook in self._post_hooks: 232 | post_hook(instance) 233 | 234 | def __call__(self, *args, **kwargs) -> T: 235 | """Creates an instance of the class. 236 | 237 | Args: 238 | *args: Constructor args. 239 | **kwargs: Constructor kwargs. 240 | 241 | Returns: 242 | Class instance. 243 | """ 244 | merged_kwargs = dict(self.kwargs) 245 | merged_kwargs.update(kwargs) 246 | instance = self.cls(*args, **merged_kwargs) 247 | 248 | self.run_post_hooks(instance) 249 | 250 | return instance 251 | -------------------------------------------------------------------------------- /LLM/fm_planning/utils/nest.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Generator, Iterator, Optional, Tuple, Type, Union 2 | 3 | import numpy as np 4 | import torch 5 | 6 | from LLM.fm_planning.utils import typing 7 | 8 | 9 | def map_structure( 10 | func: Callable, 11 | *args, 12 | atom_type: Union[Type, Tuple[Type, ...]] = ( 13 | torch.Tensor, 14 | np.ndarray, 15 | *typing.scalars, 16 | type(None), 17 | ), 18 | skip_type: Optional[Union[Type, Tuple[Type, ...]]] = None, 19 | ): 20 | """Applies the function over the nested structure atoms. 21 | 22 | Works like tensorflow.nest.map_structure(): 23 | https://www.tensorflow.org/api_docs/python/tf/nest/map_structure 24 | 25 | Args: 26 | func: Function applied to the atoms of *args. 27 | *args: Nested structure arguments of `func`. 28 | atom_type: Types considered to be atoms in the nested structure. 29 | skip_type: Types to be skipped and returned as-is in the nested structure. 30 | 31 | Returns: 32 | Results of func(*args_atoms) in the same nested structure as *args. 33 | """ 34 | arg_0 = args[0] 35 | if isinstance(arg_0, atom_type): 36 | return func(*args) 37 | elif skip_type is not None and isinstance(arg_0, skip_type): 38 | return arg_0 if len(args) == 1 else args 39 | elif isinstance(arg_0, dict): 40 | return { 41 | key: map_structure( 42 | func, 43 | *(arg[key] for arg in args), 44 | atom_type=atom_type, 45 | skip_type=skip_type, 46 | ) 47 | for key in arg_0 48 | } 49 | elif hasattr(arg_0, "__iter__"): 50 | iterable_class = type(arg_0) 51 | return iterable_class( 52 | map_structure(func, *args_i, atom_type=atom_type, skip_type=skip_type) 53 | for args_i in zip(*args) 54 | ) 55 | else: 56 | return arg_0 if len(args) == 1 else args 57 | 58 | 59 | def structure_iterator( 60 | structure, 61 | atom_type: Union[Type, Tuple[Type, ...]] = ( 62 | torch.Tensor, 63 | np.ndarray, 64 | *typing.scalars, 65 | type(None), 66 | ), 67 | skip_type: Optional[Union[Type, Tuple[Type, ...]]] = None, 68 | ) -> Iterator: 69 | """Provides an iterator over the atom values in the flattened nested structure. 70 | 71 | Args: 72 | structure: Nested structure. 73 | atom_type: Types considered to be atoms in the nested structure. 74 | skip_type: Types to be skipped and returned as-is in the nested structure. 75 | 76 | Returns: 77 | Iterator over the atom values in the flattened nested structure. 78 | """ 79 | 80 | def iterate_structure( 81 | structure, 82 | ) -> Generator: 83 | if isinstance(structure, atom_type): 84 | yield structure 85 | elif skip_type is not None and isinstance(structure, skip_type): 86 | pass 87 | elif isinstance(structure, dict): 88 | for val in structure.values(): 89 | for elem in iterate_structure(val): 90 | yield elem 91 | elif hasattr(structure, "__iter__"): 92 | for val in structure: 93 | for elem in iterate_structure(val): 94 | yield elem 95 | else: 96 | pass 97 | 98 | return iter(iterate_structure(structure)) 99 | -------------------------------------------------------------------------------- /LLM/fm_planning/utils/random.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | import random 4 | import numpy as np 5 | import torch 6 | 7 | 8 | def seed(n: Optional[int]) -> None: 9 | """Sets the random seed. 10 | 11 | Args: 12 | n: Optional seed. If None, no seed is set. 13 | """ 14 | if n is None: 15 | return 16 | 17 | torch.manual_seed(n) 18 | np.random.seed(n) 19 | random.seed(n) 20 | -------------------------------------------------------------------------------- /LLM/fm_planning/utils/tensors.py: -------------------------------------------------------------------------------- 1 | import math 2 | from typing import Any, Callable, Iterator, List, Optional, Sequence, Tuple, Type, Union 3 | 4 | import numpy as np 5 | import torch 6 | 7 | from LLM.fm_planning.utils import nest, typing 8 | 9 | 10 | def device(device: Union[str, torch.device] = "auto") -> torch.device: 11 | """Gets the torch device. 12 | 13 | Args: 14 | device: "cpu", "gpu", or "auto". 15 | 16 | Returns: 17 | Torch device. 18 | """ 19 | if isinstance(device, torch.device): 20 | return device 21 | 22 | if device == "auto": 23 | device = "cuda" if torch.cuda.is_available() else "cpu" 24 | return torch.device(device) 25 | 26 | 27 | def to_tensor( 28 | x: Union[torch.Tensor, np.ndarray, Sequence[float], Sequence[int], typing.Scalar] 29 | ) -> torch.Tensor: 30 | """Converts the scalar or array to a tensor. 31 | 32 | Args: 33 | x: Scalar or array. 34 | 35 | Returns: 36 | Tensor. 37 | """ 38 | if isinstance(x, torch.Tensor): 39 | return x 40 | elif isinstance(x, np.ndarray): 41 | return torch.from_numpy(x) 42 | else: 43 | return torch.tensor(x) 44 | 45 | 46 | def dim( 47 | x: Union[torch.Tensor, np.ndarray, Sequence[float], Sequence[int], typing.Scalar] 48 | ) -> int: 49 | """Gets the number of dimensions of x. 50 | 51 | Args: 52 | x: Scalar or array. 53 | 54 | Returns: 55 | Number of dimensions. 56 | """ 57 | if isinstance(x, torch.Tensor): 58 | return x.dim() 59 | elif isinstance(x, np.ndarray): 60 | return x.ndim 61 | elif isinstance(x, (float, int)): 62 | return 0 63 | else: 64 | return 1 65 | 66 | 67 | def map_structure( 68 | func: Callable, 69 | *args, 70 | atom_type: Union[Type, Tuple[Type, ...]] = (torch.Tensor, np.ndarray), 71 | ): 72 | """Maps the function over the structure containing either Torch tensors or Numpy 73 | arrays. 74 | 75 | Args: 76 | func: Function to be mapped. 77 | *args: Nested structure arguments of `func`. 78 | atom_type: Type to which the function should be applied. 79 | """ 80 | return nest.map_structure( 81 | func, 82 | *args, 83 | atom_type=atom_type, 84 | skip_type=(np.ndarray, torch.Tensor, *typing.scalars, str, type(None)), 85 | ) 86 | 87 | 88 | def structure_iterator( 89 | structure, atom_type: Union[Type, Tuple[Type, ...]] = (torch.Tensor, np.ndarray) 90 | ) -> Iterator: 91 | """Provides an iterator over the Torch tensors or Numpy arrays in the nested 92 | structure. 93 | 94 | Args: 95 | structure: Nested structure 96 | atom_type: Types considered to be atoms in the nested structure. 97 | 98 | Returns: 99 | Iterator over the atom values in the flattened nested structure. 100 | """ 101 | return nest.structure_iterator( 102 | structure, 103 | atom_type=atom_type, 104 | skip_type=(np.ndarray, torch.Tensor, *typing.scalars, str, type(None)), 105 | ) 106 | 107 | 108 | def to(structure, device: torch.device): 109 | """Moves the nested structure to the given device. 110 | 111 | Numpy arrays are converted to Torch tensors first. 112 | 113 | Args: 114 | structure: Nested structure. 115 | device: Torch device. 116 | 117 | Returns: 118 | Transferred structure. 119 | """ 120 | 121 | def _to(x): 122 | if isinstance(x, torch.Tensor): 123 | return x.to(device) 124 | 125 | try: 126 | return torch.from_numpy(x).to(device) 127 | except TypeError: 128 | return x 129 | 130 | return map_structure(_to, structure) 131 | 132 | 133 | def numpy(structure): 134 | """Converts the nested structure to Numpy arrays. 135 | 136 | Args: 137 | structure: Nested structure. 138 | 139 | Returns: 140 | Numpy structure. 141 | """ 142 | return map_structure( 143 | lambda x: x.cpu().detach().numpy(), structure, atom_type=torch.Tensor 144 | ) 145 | 146 | 147 | def from_numpy(structure, device: Optional[torch.device] = None): 148 | """Converts the nested structure to Torch tensors. 149 | 150 | Args: 151 | structure: Nested structure. 152 | 153 | Returns: 154 | Tensor structure. 155 | """ 156 | if device is None: 157 | return map_structure( 158 | lambda x: torch.from_numpy(x), structure, atom_type=np.ndarray 159 | ) 160 | return map_structure( 161 | lambda x: torch.from_numpy(x).to(device), structure, atom_type=np.ndarray 162 | ) 163 | 164 | 165 | def unsqueeze(structure, dim: int): 166 | def _unsqueeze( 167 | x: Union[torch.Tensor, np.ndarray] 168 | ) -> Union[torch.Tensor, np.ndarray]: 169 | if isinstance(x, np.ndarray): 170 | return np.expand_dims(x, dim) 171 | elif isinstance(x, torch.Tensor): 172 | return x.unsqueeze(dim) 173 | return x 174 | 175 | return map_structure(_unsqueeze, structure) 176 | 177 | 178 | # TODO: Handle device. 179 | def numpy_wrap(func: Callable) -> Callable: 180 | """Decorator that creates a wrapper around Torch functions to be compatible 181 | with Numpy inputs and outputs. 182 | 183 | Args: 184 | func: Torch function. 185 | 186 | Returns: 187 | Function compatible with Torch or Numpy. 188 | """ 189 | 190 | def numpy_func(*args, **kwargs): 191 | try: 192 | next(structure_iterator((args, kwargs), atom_type=np.ndarray)) 193 | except StopIteration: 194 | is_numpy = False 195 | else: 196 | is_numpy = True 197 | args, kwargs = from_numpy((args, kwargs)) 198 | 199 | result = func(*args, **kwargs) 200 | 201 | if is_numpy: 202 | result = numpy(result) 203 | 204 | return result 205 | 206 | return numpy_func 207 | 208 | 209 | def torch_wrap(func: Callable) -> Callable: 210 | """Decorator that creates a wrapper around Numpy functions to be compatible 211 | with Torch inputs and outputs. 212 | 213 | Args: 214 | func: Numpy function. 215 | 216 | Returns: 217 | Function compatible with Torch or Numpy. 218 | """ 219 | 220 | def torch_func(*args, **kwargs): 221 | try: 222 | tensor = next(structure_iterator((args, kwargs))) 223 | except StopIteration: 224 | device = None 225 | else: 226 | device = tensor.device 227 | args, kwargs = numpy((args, kwargs)) 228 | 229 | result = func(*args, **kwargs) 230 | 231 | if device is not None: 232 | result = to(result, device) 233 | 234 | return result 235 | 236 | return torch_func 237 | 238 | 239 | def vmap(dims: int) -> Callable: 240 | """Decorator that vectorizes functions. 241 | 242 | Args: 243 | dims: Number of dimensions of the first tensor function input. 244 | func: Function to vectorize. 245 | 246 | Returns: 247 | Vectorized function. 248 | """ 249 | 250 | def append(x: Any, xs: List[Any]) -> None: 251 | xs.append(x) 252 | 253 | def stack(x: Any, xs: List[Any]) -> Union[torch.Tensor, np.ndarray]: 254 | if isinstance(x, torch.Tensor): 255 | return torch.stack(xs, dim=0) 256 | else: 257 | return np.array(xs) 258 | 259 | def _vmap(func: Callable) -> Callable: 260 | def vectorized_func(*args, **kwargs): 261 | try: 262 | arr = next(structure_iterator((args, kwargs))) 263 | except StopIteration: 264 | is_batch = False 265 | else: 266 | arr_dim = arr.dim() if isinstance(arr, torch.Tensor) else arr.ndim 267 | is_batch = arr_dim != dims 268 | 269 | if not is_batch: 270 | return func(*args, **kwargs) 271 | 272 | # Flatten batch dims. 273 | batch_shape = arr.shape[:-dims] if dims > 0 else arr.shape 274 | batch_dims = len(batch_shape) 275 | batch_size = np.prod(batch_shape) 276 | args, kwargs = map_structure( 277 | lambda x: x.reshape(batch_size, *x.shape[batch_dims:]), (args, kwargs) 278 | ) 279 | 280 | # Call func for each batch element. 281 | results = None 282 | for i in range(batch_size): 283 | args_i, kwargs_i = map_structure(lambda x: x[i], (args, kwargs)) 284 | result = func(*args_i, **kwargs_i) 285 | if i == 0: 286 | results = map_structure(lambda x: [x], result) 287 | else: 288 | map_structure(append, result, results) 289 | results = map_structure(stack, result, results) 290 | 291 | # Restore batch dims. 292 | results = map_structure( 293 | lambda x: x.reshape(*batch_shape, *x.shape[1:]), results 294 | ) 295 | 296 | return results 297 | 298 | return vectorized_func 299 | 300 | return _vmap 301 | 302 | 303 | def batch(dims: int) -> Callable: 304 | """Decorator that ensures function inputs have at least one batch dimension. 305 | 306 | If original arguments are not batched, returned results will also not have a batch. 307 | 308 | Args: 309 | dims: Number of dimensions of the first tensor function input. 310 | func: Function to vectorize. 311 | 312 | Returns: 313 | Flexible batch function. 314 | """ 315 | 316 | def _vmap(func: Callable) -> Callable: 317 | def batched_func(*args, **kwargs): 318 | try: 319 | arr = next(structure_iterator((args, kwargs))) 320 | except StopIteration: 321 | is_batch = False 322 | else: 323 | arr_dim = arr.dim() if isinstance(arr, torch.Tensor) else arr.ndim 324 | is_batch = arr_dim != dims 325 | 326 | if is_batch: 327 | return func(*args, **kwargs) 328 | 329 | args, kwargs = map_structure(lambda x: x.unsqueeze(0), (args, kwargs)) 330 | results = func(*args, **kwargs) 331 | results = map_structure(lambda x: x.squeeze(0), results) 332 | 333 | return results 334 | 335 | return batched_func 336 | 337 | return _vmap 338 | 339 | 340 | def rgb_to_cnn(img_rgb: torch.Tensor, contiguous: bool = False) -> torch.Tensor: 341 | if contiguous: 342 | return img_rgb.moveaxis(-1, -3).contiguous().float() / 255 343 | else: 344 | return img_rgb.moveaxis(-1, -3).float() / 255 345 | 346 | 347 | def cnn_to_rgb(img_cnn: torch.Tensor, contiguous: bool = False) -> torch.Tensor: 348 | img_rgb = (255 * img_cnn.clip(0, 1).moveaxis(-3, -1) + 0.5).to(torch.uint8) 349 | if contiguous: 350 | return img_rgb.contiguous() 351 | else: 352 | return img_rgb 353 | 354 | 355 | def get_num_free_bytes() -> int: 356 | cuda_device = torch.cuda.current_device() 357 | num_unreserved_bytes: int = torch.cuda.mem_get_info(cuda_device)[0] # type: ignore 358 | num_reserved_bytes = torch.cuda.memory_reserved(cuda_device) 359 | num_allocated_bytes = torch.cuda.memory_allocated(cuda_device) 360 | num_free_bytes = num_unreserved_bytes + num_reserved_bytes - num_allocated_bytes 361 | 362 | return num_free_bytes 363 | 364 | 365 | def compute_minibatch(batch_size: int, element_size: int) -> Tuple[int, int]: 366 | num_free_bytes = get_num_free_bytes() 367 | max_minibatch_size = int(num_free_bytes / (2 * element_size)) 368 | 369 | # Redistribute batch size equally across all iterations. 370 | num_batches = int(math.ceil(batch_size / max_minibatch_size) + 0.5) 371 | minibatch_size = int(math.ceil(batch_size / num_batches) + 0.5) 372 | 373 | return minibatch_size, num_batches 374 | -------------------------------------------------------------------------------- /LLM/fm_planning/utils/timing.py: -------------------------------------------------------------------------------- 1 | import collections 2 | import time 3 | from typing import Dict, List, Sequence 4 | 5 | import numpy as np 6 | 7 | 8 | class Timer: 9 | """Timer to keep track of timing intervals for different keys.""" 10 | 11 | def __init__(self): 12 | self._tics = {} 13 | 14 | def keys(self) -> Sequence[str]: 15 | """Timer keys.""" 16 | return self._tics.keys() 17 | 18 | def tic(self, key: str) -> float: 19 | """Starts timing for the given key. 20 | 21 | Args: 22 | key: Time interval key. 23 | 24 | Returns: 25 | Current time. 26 | """ 27 | self._tics[key] = time.time() 28 | return self._tics[key] 29 | 30 | def toc(self, key: str, set_tic: bool = False) -> float: 31 | """Returns the time elapsed since the last tic for the given key. 32 | 33 | Args: 34 | key: Time interval key. 35 | set_tic: Reset the tic to the current time. 36 | 37 | Returns: 38 | Time elapsed since the last tic. 39 | """ 40 | toc = time.time() 41 | tic = self._tics[key] 42 | if set_tic: 43 | self._tics[key] = toc 44 | return toc - tic 45 | 46 | 47 | class Profiler(Timer): 48 | """Profiler to keep track of average time interval for different keys.""" 49 | 50 | class ProfilerContext: 51 | """Context manager for timing code inside a `with` block.""" 52 | 53 | def __init__(self, profiler: "Profiler", key: str): 54 | self.profiler = profiler 55 | self.key = key 56 | 57 | def __enter__(self) -> float: 58 | return self.profiler.tic(self.key) 59 | 60 | def __exit__(self, type, value, traceback) -> None: 61 | self.profiler.toc(self.key) 62 | 63 | def __init__(self, disabled: bool = False): 64 | """Initializes the profiler with the given status. 65 | 66 | Args: 67 | disabled: Disable the profiler. 68 | """ 69 | super().__init__() 70 | self._disabled = disabled 71 | self._tictocs: Dict[str, List[float]] = collections.defaultdict(list) 72 | 73 | def disable(self) -> None: 74 | """Disables the profiler so that tic and toc do nothing.""" 75 | self._disabled = True 76 | 77 | def enable(self) -> None: 78 | """Enables the profiler.""" 79 | self._disabled = False 80 | 81 | def tic(self, key: str) -> float: 82 | """Starts timing for the given key. 83 | 84 | Args: 85 | key: Time interval key. 86 | 87 | Returns: 88 | Current time. 89 | """ 90 | if self._disabled: 91 | return 0.0 92 | return super().tic(key) 93 | 94 | def toc(self, key: str, set_tic: bool = False) -> float: 95 | """Returns the time elapsed since the last tic for the given key. 96 | 97 | Args: 98 | key: Time interval key. 99 | set_tic: Reset the tic to the current time. 100 | 101 | Returns: 102 | Time elapsed since the last tic. 103 | """ 104 | if self._disabled: 105 | return 0.0 106 | tictoc = super().toc(key, set_tic) 107 | self._tictocs[key].append(tictoc) 108 | return tictoc 109 | 110 | def profile(self, key: str) -> ProfilerContext: 111 | """Times the code inside a `with` block for the given key. 112 | 113 | Args: 114 | key: Time interval key. 115 | 116 | Returns: 117 | Profiler context. 118 | """ 119 | return Profiler.ProfilerContext(self, key) 120 | 121 | def compute_average(self, key: str, reset: bool = False) -> float: 122 | """Computes the average time interval for the given key. 123 | 124 | Args: 125 | key: Time interval key. 126 | reset: Reset the collected time intervals. 127 | 128 | Returns: 129 | Average time interval. 130 | """ 131 | mean = float(np.mean(self._tictocs[key])) 132 | if reset: 133 | self._tictocs[key] = [] 134 | return mean 135 | 136 | def compute_sum(self, key: str, reset: bool = False) -> float: 137 | """Computes the total time interval for the given key. 138 | 139 | Args: 140 | key: Time interval key. 141 | reset: Reset the collected time intervals. 142 | 143 | Returns: 144 | Total time interval. 145 | """ 146 | sum = float(np.sum(self._tictocs[key])) 147 | if reset: 148 | self._tictocs[key] = [] 149 | return sum 150 | 151 | def collect_profiles(self) -> Dict[str, float]: 152 | """Collects and resets the average time intervals for all keys. 153 | 154 | Returns: 155 | Dict mapping from key to average time interval. 156 | """ 157 | return { 158 | key: self.compute_average(key, reset=True) 159 | for key, tictoc in self._tictocs.items() 160 | if len(tictoc) > 0 161 | } 162 | -------------------------------------------------------------------------------- /LLM/fm_planning/utils/typing.py: -------------------------------------------------------------------------------- 1 | from typing import TypeVar, Union, Mapping 2 | 3 | import numpy as np 4 | import torch 5 | 6 | 7 | Scalar = Union[float, int, bool] 8 | scalars = (float, int, bool) 9 | Tensor = Union[np.ndarray, torch.Tensor] 10 | ArrayType = TypeVar("ArrayType", np.ndarray, torch.Tensor) 11 | StateType = TypeVar("StateType") 12 | ObsType = TypeVar("ObsType") 13 | ActType = TypeVar("ActType") 14 | ModelBatchType = TypeVar("ModelBatchType") 15 | DatasetBatchType = TypeVar("DatasetBatchType", bound=Mapping) 16 | -------------------------------------------------------------------------------- /LLM/package.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | fm_planning 4 | 0.0.0 5 | Planning with Foundation Models (FMs): Language Models and Vision-Language Models 6 | 7 | 8 | 9 | 10 | Christopher Agia 11 | 12 | 13 | 14 | 15 | 16 | MIT 17 | 18 | 19 | 20 | 21 | 22 | https://github.com/agiachris/fm-planning 23 | 24 | 25 | 26 | 27 | 28 | Christopher Agia 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | catkin 52 | 53 | rospy 54 | std_msgs 55 | message_generation 56 | 57 | rospy 58 | std_msgs 59 | 60 | rospy 61 | std_msgs 62 | message_runtime 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | -------------------------------------------------------------------------------- /LLM/pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "fm-planning" 3 | version = "0.1.0" 4 | description = "Planning with Foundation Models (FMs): Language Models and Vision-Language Models" 5 | authors = [ 6 | { name = "Christopher Agia", email = "cagia@cs.stanford.edu" }, 7 | ] 8 | license = { file = "LICENSE" } 9 | readme = "README.md" 10 | requires-python = ">=3.8" 11 | classifiers = [ 12 | "Programming Language :: Python :: 3", 13 | "License :: OSI Approved :: MIT License", 14 | "Operating System :: OS Independent", 15 | ] 16 | dependencies = [ 17 | "numpy", 18 | "torch", 19 | "matplotlib", 20 | "torchvision", 21 | "hydra-core", 22 | "openai", 23 | "black", 24 | "mypy", 25 | "pyyaml", 26 | "transformers", 27 | "pysymbolic", 28 | ] 29 | 30 | [tool.setuptools.packages.find] 31 | include = ["fm_planning*"] -------------------------------------------------------------------------------- /LLM/scripts/demos/llm_planner.py: -------------------------------------------------------------------------------- 1 | from typing import Union, Optional 2 | 3 | import ast 4 | import argparse 5 | import pathlib 6 | import sys 7 | import os 8 | sys.path.append(os.getcwd()) 9 | from LLM.fm_planning import models, utils 10 | 11 | 12 | def demo( 13 | model_config: Union[str, pathlib.Path], 14 | prompt_config: Union[str, pathlib.Path], 15 | api_key: Optional[str] = None, 16 | device: str = "auto", 17 | seed: int = 0, 18 | ) -> None: 19 | """Run a demo.""" 20 | # Set seed. 21 | utils.random.seed(seed) 22 | 23 | # Load model. 24 | model_factory = models.PretrainedModelFactory( 25 | model_config, api_key=api_key, device=device 26 | ) 27 | model = model_factory() 28 | 29 | # Load prompt. 30 | prompt_manager = models.BehaviorPromptManager.from_yaml(prompt_config) 31 | 32 | # Predict goals. 33 | goal_prediction_prompt = prompt_manager.generate_prompt( 34 | behavior="goal_prediction", 35 | use_examples=True, 36 | ) 37 | response = model.forward(goal_prediction_prompt) 38 | predicted_goals = ast.literal_eval(response["choices"][0]["message"]["content"]) 39 | 40 | # Update prompts with predicted goals. 41 | prompt_manager.task_prompt.goals = predicted_goals 42 | 43 | # Predict plans. 44 | task_planning_prompt = prompt_manager.generate_prompt( 45 | behavior="task_planning", 46 | use_examples=True, 47 | ) 48 | response = model.forward(task_planning_prompt) 49 | predicted_plans = ast.literal_eval(response["choices"][0]["message"]["content"]) 50 | 51 | print(f"Predicted goals: {predicted_goals}") 52 | print(f"Predicted plans: {predicted_plans}") 53 | 54 | 55 | def main(args: argparse.Namespace) -> None: 56 | demo(**vars(args)) 57 | 58 | 59 | if __name__ == "__main__": 60 | parser = argparse.ArgumentParser() 61 | parser.add_argument("--model-config", type=str, required=True) 62 | parser.add_argument("--prompt-config", type=str, required=True) 63 | parser.add_argument("--api-key", type=str) 64 | parser.add_argument("--device", type=str, default="auto") 65 | parser.add_argument("--seed", type=int, default=0) 66 | main(parser.parse_args()) 67 | -------------------------------------------------------------------------------- /LLM/scripts/llm_planner.py: -------------------------------------------------------------------------------- 1 | from typing import Union, Optional 2 | 3 | import ast 4 | import argparse 5 | import pathlib 6 | import sys 7 | import os 8 | sys.path.append(os.getcwd()) 9 | from LLM.fm_planning import models, utils 10 | 11 | 12 | def demo( 13 | model_config: Union[str, pathlib.Path], 14 | prompt_config: Union[str, pathlib.Path], 15 | api_key: Optional[str] = None, 16 | device: str = "auto", 17 | seed: int = 0, 18 | ) -> None: 19 | """Run a demo.""" 20 | # Set seed. 21 | utils.random.seed(seed) 22 | 23 | # Load model. 24 | model_factory = models.PretrainedModelFactory( 25 | model_config, api_key=api_key, device=device 26 | ) 27 | model = model_factory() 28 | 29 | # Load prompt. 30 | prompt_manager = models.BehaviorPromptManager.from_yaml(prompt_config) 31 | 32 | # Predict goals. 33 | goal_prediction_prompt = prompt_manager.generate_prompt( 34 | behavior="goal_prediction", 35 | use_examples=True, 36 | ) 37 | response = model.forward(goal_prediction_prompt) 38 | predicted_goals = ast.literal_eval(response["choices"][0]["message"]["content"]) 39 | 40 | # Update prompts with predicted goals. 41 | prompt_manager.task_prompt.goals = predicted_goals 42 | 43 | # Predict plans. 44 | task_planning_prompt = prompt_manager.generate_prompt( 45 | behavior="task_planning", 46 | use_examples=True, 47 | ) 48 | response = model.forward(task_planning_prompt) 49 | predicted_plans = ast.literal_eval(response["choices"][0]["message"]["content"]) 50 | 51 | print(f"Predicted goals: {predicted_goals}") 52 | print(f"Predicted plans: {predicted_plans}") 53 | 54 | 55 | def main(args: argparse.Namespace) -> None: 56 | demo(**vars(args)) 57 | 58 | 59 | if __name__ == "__main__": 60 | parser = argparse.ArgumentParser() 61 | parser.add_argument("--model-config", type=str, required=True) 62 | parser.add_argument("--prompt-config", type=str, required=True) 63 | parser.add_argument("--api-key", type=str) 64 | parser.add_argument("--device", type=str, default="auto") 65 | parser.add_argument("--seed", type=int, default=0) 66 | main(parser.parse_args()) 67 | -------------------------------------------------------------------------------- /LLM/setup.cfg: -------------------------------------------------------------------------------- 1 | # Dummy setup.cfg for editable installs until setuptools supports PEP 660. 2 | 3 | [options] 4 | packages = fm_planning -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Points2Plans: From Point Clouds to Long-Horizon Plans with Composable Relational Dynamics 2 | Code to accompany our ICRA 2025 paper: _Points2Plans: From Point Clouds to Long-Horizon Plans with Composable Relational Dynamics_. [[PDF]](https://arxiv.org/pdf/2408.14769) [[Website]](https://sites.google.com/stanford.edu/points2plans) 3 | 4 | ## Approach Overview 5 | ![Overview figure](images/teaser_v1.png) 6 | 7 | This repository includes: 8 | 9 | * :hammer_and_wrench: A relational dynamics model that excels at long-horizon prediction of point cloud states without the need to train on multi-step data 10 | * :rocket: A latent-geometric space dynamics rollout strategy that significantly increases the horizons over which predicted point cloud states are reliable for planning 11 | * 🦾 A task planning and goal prediction module using Large Language Models (LLMs) 12 | 13 | ## Setup 14 | ### System Requirements 15 | This codebase is primarily tested on Ubuntu 20.04, an NVIDIA GeForce RTX 3090 Ti, and CUDA 11.7. 16 | 17 | ### Virtual Env Installation 18 | ``` 19 | conda env create -f conda_env.yml 20 | ``` 21 | 22 | ## Task Planning and Goal Prediction Module with LLMs 23 | ```bash 24 | python LLM/scripts/llm_planner.py \ 25 | --model-config LLM/configs/models/pretrained/generative/$Model \ 26 | --prompt-config LLM/configs/prompts/evaluation/p1/$Task \ 27 | --api-key $YourAPIKey 28 | ``` 29 | 30 | ## Relational Dynamics 31 | ### Quick Start with Pretrained Models 32 | - Download pretrained models from [this link](https://huggingface.co/datasets/ll4ma-lab/Points2Plans/tree/main/pretrained_model) 33 | - Download test data for [constrained packing task](https://huggingface.co/datasets/ll4ma-lab/Points2Plans/tree/main/test) 34 | 35 | ```bash 36 | python relational_dynamics/main.py \ 37 | --result_dir $PretrainedModelDir \ 38 | --checkpoint_path $PretrainedModelDir/checkpoint/pretrained.pth \ 39 | --test_dir $TestDataDir \ 40 | --test_max_size $TestSize 41 | ``` 42 | 43 | ### Training 44 | 45 | - Download [training datasets](https://huggingface.co/datasets/ll4ma-lab/Points2Plans/tree/main/train) 46 | 47 | To generate your own data, please refer to our simulation repository using [[isaacgym]](https://bitbucket.org/robot-learning/ll4ma_isaac/src/CoRL_2024/). 48 | 49 | ```bash 50 | python relational_dynamics/main.py \ 51 | --result_dir $YourResultDir \ 52 | --train_dir $TrainingDataDir \ 53 | --batch_size $BatchSize \ 54 | --num_epochs $TrainingEpochs \ 55 | --max_size $TrainingSize 56 | ``` 57 | 58 | ## Baseline: [eRDTransformer](https://sites.google.com/view/erelationaldynamics) 59 | 60 | ### Training 61 | ```bash 62 | python relational_dynamics/main.py \ 63 | --result_dir $YourResultDir \ 64 | --train_dir $TrainingDataDir \ 65 | --batch_size $BatchSize \ 66 | --num_epochs $TrainingEpochs \ 67 | --delta_forward False \ 68 | --latent_forward True \ 69 | --max_size $TrainingSize 70 | ``` 71 | 72 | ### Test 73 | - Download pretrained models from [this link](https://huggingface.co/datasets/ll4ma-lab/Points2Plans/tree/main/baseline_pretrained) 74 | 75 | ```bash 76 | python relational_dynamics/main.py \ 77 | --result_dir $PretrainedModelDir \ 78 | --checkpoint_path $PretrainedModelDir/checkpoint/baseline_pretrained.pth \ 79 | --test_dir $TestDataDir \ 80 | --delta_forward False \ 81 | --latent_forward True \ 82 | --test_max_size $TestSize 83 | ``` 84 | 85 | 86 | ## Citation 87 | If you find our work useful in your research, please cite: 88 | ``` 89 | @InProceedings{huang-icra2025-p2p, 90 | title = {{Points2Plans: From Point Clouds to Long-Horizon Plans with Composable Relational Dynamics}}, 91 | author = {Yixuan Huang and Christopher Agia and Jimmy Wu and Tucker Hermans and Jeannette Bohg}, 92 | booktitle = {IEEE International Conference on Robotics and Automation (ICRA)}, 93 | year = {2025}, 94 | url = {https://sites.google.com/stanford.edu/points2plans} 95 | } 96 | ``` -------------------------------------------------------------------------------- /conda_env.yml: -------------------------------------------------------------------------------- 1 | name: points2plans 2 | channels: 3 | - pytorch 4 | - pyg 5 | - nvidia 6 | - conda-forge 7 | - defaults 8 | dependencies: 9 | - _libgcc_mutex=0.1=conda_forge 10 | - _openmp_mutex=4.5=2_gnu 11 | - absl-py=1.3.0=py38h06a4308_0 12 | - aiohttp=3.8.3=py38h5eee18b_0 13 | - aiosignal=1.2.0=pyhd3eb1b0_0 14 | - alsa-lib=1.2.8=h166bdaf_0 15 | - anyio=3.5.0=py38h06a4308_0 16 | - aom=3.5.0=h27087fc_0 17 | - argon2-cffi=21.3.0=pyhd3eb1b0_0 18 | - argon2-cffi-bindings=21.2.0=py38h7f8727e_0 19 | - asttokens=2.0.5=pyhd3eb1b0_0 20 | - async-timeout=4.0.2=py38h06a4308_0 21 | - attr=2.5.1=h166bdaf_1 22 | - attrs=22.1.0=py38h06a4308_0 23 | - babel=2.9.1=pyhd3eb1b0_0 24 | - backcall=0.2.0=pyhd3eb1b0_0 25 | - beautifulsoup4=4.11.1=py38h06a4308_0 26 | - blas=1.0=mkl 27 | - bleach=4.1.0=pyhd3eb1b0_0 28 | - blinker=1.4=py38h06a4308_0 29 | - boost-cpp=1.78.0=h75c5d50_1 30 | - bottleneck=1.3.5=py38h7deecbd_0 31 | - brotli=1.0.9=h5eee18b_7 32 | - brotli-bin=1.0.9=h5eee18b_7 33 | - brotlipy=0.7.0=py38h0a891b7_1005 34 | - bzip2=1.0.8=h7f98852_4 35 | - c-ares=1.18.1=h7f8727e_0 36 | - ca-certificates=2022.12.7=ha878542_0 37 | - cached-property=1.5.2=py_0 38 | - cachetools=4.2.2=pyhd3eb1b0_0 39 | - catkin_pkg=0.5.2=pyhd8ed1ab_0 40 | - catkin_tools=0.9.3=pyhd8ed1ab_0 41 | - certifi=2022.12.7=pyhd8ed1ab_0 42 | - cffi=1.15.1=py38h4a40e3a_3 43 | - chainer=7.8.0=pyhd3eb1b0_0 44 | - charset-normalizer=2.1.1=pyhd8ed1ab_0 45 | - click=8.0.4=py38h06a4308_0 46 | - cmake=3.16.3=h28c56e5_0 47 | - coreutils=9.2=h0b41bf4_0 48 | - cuda=11.7.1=0 49 | - cuda-cccl=11.7.91=0 50 | - cuda-command-line-tools=11.7.1=0 51 | - cuda-compiler=11.7.1=0 52 | - cuda-cudart=11.7.99=0 53 | - cuda-cudart-dev=11.7.99=0 54 | - cuda-cuobjdump=11.7.91=0 55 | - cuda-cupti=11.7.101=0 56 | - cuda-cuxxfilt=11.7.91=0 57 | - cuda-demo-suite=12.0.76=0 58 | - cuda-documentation=12.0.76=0 59 | - cuda-driver-dev=11.7.99=0 60 | - cuda-gdb=12.0.90=0 61 | - cuda-libraries=11.7.1=0 62 | - cuda-libraries-dev=11.7.1=0 63 | - cuda-memcheck=11.8.86=0 64 | - cuda-nsight=12.0.78=0 65 | - cuda-nsight-compute=12.0.0=0 66 | - cuda-nvcc=11.7.99=0 67 | - cuda-nvdisasm=12.0.76=0 68 | - cuda-nvml-dev=11.7.91=0 69 | - cuda-nvprof=12.0.90=0 70 | - cuda-nvprune=11.7.91=0 71 | - cuda-nvrtc=11.7.99=0 72 | - cuda-nvrtc-dev=11.7.99=0 73 | - cuda-nvtx=11.7.91=0 74 | - cuda-nvvp=12.0.90=0 75 | - cuda-runtime=11.7.1=0 76 | - cuda-sanitizer-api=12.0.90=0 77 | - cuda-toolkit=11.7.1=0 78 | - cuda-tools=11.7.1=0 79 | - cuda-visual-tools=11.7.1=0 80 | - curl=7.88.1=hdc1c0ab_1 81 | - cycler=0.11.0=pyhd3eb1b0_0 82 | - dbus=1.13.18=hb2f20db_0 83 | - debugpy=1.5.1=py38h295c915_0 84 | - decorator=5.1.1=pyhd3eb1b0_0 85 | - defusedxml=0.7.1=pyhd3eb1b0_0 86 | - docutils=0.19=py38h578d9bd_1 87 | - double-conversion=3.2.0=h27087fc_1 88 | - eigen=3.3.7=hd09550d_1 89 | - entrypoints=0.4=py38h06a4308_0 90 | - executing=0.8.3=pyhd3eb1b0_0 91 | - expat=2.5.0=h27087fc_0 92 | - ffmpeg=5.1.2=gpl_h8dda1f0_105 93 | - fftw=3.3.10=nompi_hf0379b8_106 94 | - filelock=3.6.0=pyhd3eb1b0_0 95 | - flann=1.9.1=he05ef13_1011 96 | - font-ttf-dejavu-sans-mono=2.37=hd3eb1b0_0 97 | - font-ttf-inconsolata=2.001=hcb22688_0 98 | - font-ttf-source-code-pro=2.030=hd3eb1b0_0 99 | - font-ttf-ubuntu=0.83=h8b1ccd4_0 100 | - fontconfig=2.14.1=hc2a2eb6_0 101 | - fonts-anaconda=1=h8fa9717_0 102 | - fonts-conda-ecosystem=1=hd3eb1b0_0 103 | - fonttools=4.25.0=pyhd3eb1b0_0 104 | - freetype=2.12.1=hca18f0e_1 105 | - frozenlist=1.3.3=py38h5eee18b_0 106 | - gds-tools=1.5.0.59=0 107 | - gettext=0.21.1=h27087fc_0 108 | - giflib=5.2.1=h7b6447c_0 109 | - gl2ps=1.4.2=h70c0345_1 110 | - glew=2.1.0=h295c915_3 111 | - glib=2.74.1=h6239696_1 112 | - glib-tools=2.74.1=h6239696_1 113 | - gmp=6.2.1=h58526e2_0 114 | - gnutls=3.7.8=hf3e180e_0 115 | - google-auth=2.6.0=pyhd3eb1b0_0 116 | - google-auth-oauthlib=0.4.4=pyhd3eb1b0_0 117 | - grpcio=1.52.1=py38h8dc9893_1 118 | - gst-plugins-base=1.21.3=h4243ec0_1 119 | - gstreamer=1.21.3=h25f0c4b_1 120 | - gstreamer-orc=0.4.33=h166bdaf_0 121 | - h5py=3.7.0=nompi_py38h7927eab_102 122 | - hdf4=4.2.15=h9772cbc_5 123 | - hdf5=1.12.2=nompi_h4df4325_101 124 | - icu=70.1=h27087fc_0 125 | - idna=3.4=pyhd8ed1ab_0 126 | - importlib-metadata=4.11.3=py38h06a4308_0 127 | - importlib_resources=5.2.0=pyhd3eb1b0_1 128 | - intel-openmp=2022.1.0=h9e868ea_3769 129 | - ipdb=0.13.9=pyhd8ed1ab_0 130 | - ipykernel=6.15.2=py38h06a4308_0 131 | - ipython=8.7.0=py38h06a4308_0 132 | - ipython_genutils=0.2.0=pyhd3eb1b0_1 133 | - ipywidgets=7.6.5=pyhd3eb1b0_1 134 | - jack=1.9.21=h583fa2b_2 135 | - jedi=0.18.1=py38h06a4308_1 136 | - jinja2=3.1.2=py38h06a4308_0 137 | - joblib=1.1.1=py38h06a4308_0 138 | - jpeg=9e=h166bdaf_2 139 | - json5=0.9.6=pyhd3eb1b0_0 140 | - jsoncpp=1.9.5=h4bd325d_1 141 | - jsonschema=4.16.0=py38h06a4308_0 142 | - jupyter=1.0.0=py38h06a4308_8 143 | - jupyter_client=7.4.8=py38h06a4308_0 144 | - jupyter_console=6.4.4=py38h06a4308_0 145 | - jupyter_core=4.11.2=py38h06a4308_0 146 | - jupyter_server=1.23.4=py38h06a4308_0 147 | - jupyterlab=3.5.0=py38h06a4308_0 148 | - jupyterlab_pygments=0.1.2=py_0 149 | - jupyterlab_server=2.16.3=py38h06a4308_0 150 | - jupyterlab_widgets=1.0.0=pyhd3eb1b0_1 151 | - keyutils=1.6.1=h166bdaf_0 152 | - kiwisolver=1.4.2=py38h295c915_0 153 | - krb5=1.20.1=h81ceb04_0 154 | - lame=3.100=h166bdaf_1003 155 | - lcms2=2.12=h3be6417_0 156 | - ld_impl_linux-64=2.39=hcc3a1bd_1 157 | - lerc=4.0.0=h27087fc_0 158 | - libabseil=20230125.0=cxx17_hcb278e6_1 159 | - libaec=1.0.6=h9c3ff4c_0 160 | - libblas=3.9.0=16_linux64_mkl 161 | - libbrotlicommon=1.0.9=h5eee18b_7 162 | - libbrotlidec=1.0.9=h5eee18b_7 163 | - libbrotlienc=1.0.9=h5eee18b_7 164 | - libcap=2.66=ha37c62d_0 165 | - libcblas=3.9.0=16_linux64_mkl 166 | - libclang=15.0.6=default_h2e3cab8_0 167 | - libclang13=15.0.6=default_h3a83d3e_0 168 | - libcublas=11.10.3.66=0 169 | - libcublas-dev=11.10.3.66=0 170 | - libcufft=10.7.2.124=h4fbf590_0 171 | - libcufft-dev=10.7.2.124=h98a8f43_0 172 | - libcufile=1.5.0.59=0 173 | - libcufile-dev=1.5.0.59=0 174 | - libcups=2.3.3=h36d4200_3 175 | - libcurand=10.3.1.50=0 176 | - libcurand-dev=10.3.1.50=0 177 | - libcurl=7.88.1=hdc1c0ab_1 178 | - libcusolver=11.4.0.1=0 179 | - libcusolver-dev=11.4.0.1=0 180 | - libcusparse=11.7.4.91=0 181 | - libcusparse-dev=11.7.4.91=0 182 | - libdb=6.2.32=h6a678d5_1 183 | - libdeflate=1.14=h166bdaf_0 184 | - libdrm=2.4.114=h166bdaf_0 185 | - libedit=3.1.20221030=h5eee18b_0 186 | - libev=4.33=h7f8727e_1 187 | - libevent=2.1.10=h28343ad_4 188 | - libffi=3.4.2=h7f98852_5 189 | - libflac=1.4.2=h27087fc_0 190 | - libgcc-ng=12.2.0=h65d4601_19 191 | - libgcrypt=1.10.1=h166bdaf_0 192 | - libgfortran-ng=11.2.0=h00389a5_1 193 | - libgfortran5=11.2.0=h1234567_1 194 | - libglib=2.74.1=h606061b_1 195 | - libglu=9.0.0=hf484d3e_1 196 | - libgomp=12.2.0=h65d4601_19 197 | - libgpg-error=1.45=hc0c96e0_0 198 | - libgrpc=1.52.1=hcf146ea_1 199 | - libiconv=1.17=h166bdaf_0 200 | - libidn2=2.3.2=h7f8727e_0 201 | - liblapack=3.9.0=16_linux64_mkl 202 | - libllvm15=15.0.6=h63197d8_0 203 | - libnetcdf=4.8.1=nompi_h261ec11_106 204 | - libnghttp2=1.52.0=h61bc06f_0 205 | - libnpp=11.7.4.75=0 206 | - libnpp-dev=11.7.4.75=0 207 | - libnsl=2.0.0=h7f98852_0 208 | - libnvjpeg=11.8.0.2=0 209 | - libnvjpeg-dev=11.8.0.2=0 210 | - libogg=1.3.5=h27cfd23_1 211 | - libopus=1.3.1=h7b6447c_0 212 | - libpciaccess=0.17=h166bdaf_0 213 | - libpng=1.6.39=h753d276_0 214 | - libpq=15.2=hb675445_0 215 | - libprotobuf=3.21.12=h3eb15da_0 216 | - libsndfile=1.1.0=hcb278e6_1 217 | - libsodium=1.0.18=h7b6447c_0 218 | - libsqlite=3.40.0=h753d276_0 219 | - libssh2=1.10.0=hf14f497_3 220 | - libstdcxx-ng=12.2.0=h46fd767_19 221 | - libsystemd0=252=h2a991cd_0 222 | - libtasn1=4.19.0=h166bdaf_0 223 | - libtheora=1.1.1=h7f8727e_3 224 | - libtiff=4.4.0=h82bc61c_5 225 | - libtool=2.4.6=h6a678d5_1009 226 | - libudev1=252=h166bdaf_0 227 | - libunistring=0.9.10=h27cfd23_0 228 | - libuuid=2.32.1=h7f98852_1000 229 | - libuv=1.40.0=h7b6447c_0 230 | - libva=2.16.0=h166bdaf_0 231 | - libvorbis=1.3.7=h7b6447c_0 232 | - libvpx=1.11.0=h295c915_0 233 | - libwebp=1.2.4=h11a3e52_0 234 | - libwebp-base=1.2.4=h166bdaf_0 235 | - libxcb=1.13=h7f98852_1004 236 | - libxkbcommon=1.0.3=he3ba5ed_0 237 | - libxml2=2.10.3=h7463322_0 238 | - libxslt=1.1.37=h873f0b0_0 239 | - libzip=1.9.2=hc929e4a_1 240 | - libzlib=1.2.13=h166bdaf_4 241 | - loguru=0.5.3=py38h06a4308_3 242 | - lxml=4.9.2=py38h215a2d7_0 243 | - lz4-c=1.9.4=h6a678d5_0 244 | - make=4.3=hd18ef5c_1 245 | - markdown=3.4.1=py38h06a4308_0 246 | - markupsafe=2.1.1=py38h7f8727e_0 247 | - matplotlib=3.5.1=py38h06a4308_1 248 | - matplotlib-base=3.5.1=py38ha18d171_1 249 | - matplotlib-inline=0.1.6=py38h06a4308_0 250 | - mistune=0.8.4=py38h7b6447c_1000 251 | - mkl=2022.1.0=hc2b9512_224 252 | - mpg123=1.31.1=h27087fc_0 253 | - multidict=6.0.2=py38h5eee18b_0 254 | - munkres=1.1.4=py_0 255 | - mysql-common=8.0.32=ha901b37_1 256 | - mysql-libs=8.0.32=hd7da12d_1 257 | - nbclassic=0.4.8=py38h06a4308_0 258 | - nbclient=0.5.13=py38h06a4308_0 259 | - nbconvert=6.5.4=py38h06a4308_0 260 | - ncurses=6.3=h27087fc_1 261 | - nest-asyncio=1.5.5=py38h06a4308_0 262 | - nettle=3.8.1=hc379101_1 263 | - nlohmann_json=3.10.5=h295c915_0 264 | - notebook=6.5.2=py38h06a4308_0 265 | - notebook-shim=0.2.2=py38h06a4308_0 266 | - nsight-compute=2022.4.0.15=0 267 | - nspr=4.35=h27087fc_0 268 | - nss=3.82=he02c5a1_0 269 | - numexpr=2.7.3=py38h43a58ef_1 270 | - numpy=1.24.0=py38hab0fcb9_0 271 | - oauthlib=3.2.1=py38h06a4308_0 272 | - openh264=2.3.1=h27087fc_1 273 | - openjpeg=2.4.0=h3ad879b_0 274 | - openssl=3.1.0=h0b41bf4_0 275 | - osrf_pycommon=0.2.1=pyhd8ed1ab_0 276 | - p11-kit=0.24.1=hc5aa10d_0 277 | - packaging=22.0=py38h06a4308_0 278 | - pandas=1.5.2=py38h417a72b_0 279 | - pandocfilters=1.5.0=pyhd3eb1b0_0 280 | - parso=0.8.3=pyhd3eb1b0_0 281 | - pcl=1.13.0=h4f734b3_0 282 | - pcre2=10.40=hc3806b6_0 283 | - pexpect=4.8.0=pyhd3eb1b0_3 284 | - pickleshare=0.7.5=pyhd3eb1b0_1003 285 | - pillow=9.3.0=py38hace64e9_1 286 | - pip=22.3.1=pyhd8ed1ab_0 287 | - pkgutil-resolve-name=1.3.10=py38h06a4308_0 288 | - ply=3.11=py38_0 289 | - proj=9.1.0=h93bde94_0 290 | - prometheus_client=0.14.1=py38h06a4308_0 291 | - prompt-toolkit=3.0.20=pyhd3eb1b0_0 292 | - prompt_toolkit=3.0.20=hd3eb1b0_0 293 | - protobuf=4.21.12=py38h8dc9893_0 294 | - psutil=5.9.0=py38h5eee18b_0 295 | - pthread-stubs=0.4=h36c2ea0_1001 296 | - ptyprocess=0.7.0=pyhd3eb1b0_2 297 | - pugixml=1.11.4=h295c915_1 298 | - pulseaudio=16.1=h126f2b6_0 299 | - pure_eval=0.2.2=pyhd3eb1b0_0 300 | - pyasn1=0.4.8=pyhd3eb1b0_0 301 | - pyasn1-modules=0.2.8=py_0 302 | - pycparser=2.21=pyhd8ed1ab_0 303 | - pyg=2.2.0=py38_torch_1.13.0_cu117 304 | - pygments=2.11.2=pyhd3eb1b0_0 305 | - pyjwt=2.4.0=py38h06a4308_0 306 | - pyopenssl=22.1.0=pyhd8ed1ab_0 307 | - pyparsing=3.0.9=py38h06a4308_0 308 | - pyqt=5.15.7=py38h7492b6b_2 309 | - pyqt5-sip=12.11.0=py38hfa26641_2 310 | - pyrsistent=0.18.0=py38heee7806_0 311 | - pysocks=1.7.1=py38h578d9bd_5 312 | - python=3.8.16=he550d4f_1_cpython 313 | - python-dateutil=2.8.2=pyhd3eb1b0_0 314 | - python-fastjsonschema=2.16.2=py38h06a4308_0 315 | - python_abi=3.8=2_cp38 316 | - pytorch=1.13.1=py3.8_cuda11.7_cudnn8.5.0_0 317 | - pytorch-cluster=1.6.0=py38_torch_1.13.0_cu117 318 | - pytorch-cuda=11.7=h67b0de4_1 319 | - pytorch-mutex=1.0=cuda 320 | - pytorch-scatter=2.1.0=py38_torch_1.13.0_cu117 321 | - pytorch-sparse=0.6.16=py38_torch_1.13.0_cu117 322 | - pytz=2022.7=py38h06a4308_0 323 | - pyyaml=6.0=py38h5eee18b_1 324 | - pyzmq=23.2.0=py38h6a678d5_0 325 | - qhull=2020.2=hdb19cb5_2 326 | - qt-main=5.15.6=hf6cd601_5 327 | - qtconsole=5.3.2=py38h06a4308_0 328 | - qtpy=2.2.0=py38h06a4308_0 329 | - re2=2023.02.02=hcb278e6_0 330 | - readline=8.1.2=h0f457ee_0 331 | - requests=2.28.1=pyhd8ed1ab_1 332 | - requests-oauthlib=1.3.0=py_0 333 | - rhash=1.4.3=h166bdaf_0 334 | - rsa=4.7.2=pyhd3eb1b0_1 335 | - scikit-learn=1.1.3=py38h6a678d5_0 336 | - scipy=1.9.3=py38h8ce737c_2 337 | - seaborn=0.12.1=py38h06a4308_0 338 | - send2trash=1.8.0=pyhd3eb1b0_1 339 | - setuptools=65.6.3=pyhd8ed1ab_0 340 | - sip=6.7.5=py38hfa26641_0 341 | - six=1.16.0=pyhd3eb1b0_1 342 | - sniffio=1.2.0=py38h06a4308_1 343 | - soupsieve=2.3.2.post1=py38h06a4308_0 344 | - sqlite=3.40.0=h5082296_0 345 | - stack_data=0.2.0=pyhd3eb1b0_0 346 | - svt-av1=1.4.1=hcb278e6_0 347 | - tbb=2021.6.0=hdb19cb5_0 348 | - tbb-devel=2021.6.0=hdb19cb5_0 349 | - tensorboard=2.10.0=py38h06a4308_0 350 | - tensorboard-data-server=0.6.1=py38h52d8a92_0 351 | - tensorboard-plugin-wit=1.8.1=py38h06a4308_0 352 | - tensorboardx=2.2=pyhd3eb1b0_0 353 | - terminado=0.13.1=py38h06a4308_0 354 | - threadpoolctl=2.2.0=pyh0d69192_0 355 | - tinycss2=1.2.1=py38h06a4308_0 356 | - tk=8.6.12=h27826a3_0 357 | - toml=0.10.2=pyhd3eb1b0_0 358 | - tomli=2.0.1=py38h06a4308_0 359 | - torchaudio=0.13.1=py38_cu117 360 | - torchvision=0.14.1=py38_cu117 361 | - tornado=6.2=py38h5eee18b_0 362 | - tqdm=4.64.1=py38h06a4308_0 363 | - traitlets=5.7.1=py38h06a4308_0 364 | - urllib3=1.26.13=pyhd8ed1ab_0 365 | - utfcpp=3.2.1=h06a4308_0 366 | - vtk=9.2.2=qt_py38he961636_204 367 | - wcwidth=0.2.5=pyhd3eb1b0_0 368 | - webencodings=0.5.1=py38_1 369 | - websocket-client=0.58.0=py38h06a4308_4 370 | - werkzeug=2.2.2=py38h06a4308_0 371 | - wheel=0.38.4=pyhd8ed1ab_0 372 | - widgetsnbextension=3.5.2=py38h06a4308_0 373 | - wslink=1.10.0=pyhd8ed1ab_0 374 | - x264=1!164.3095=h166bdaf_2 375 | - x265=3.5=h924138e_3 376 | - xcb-util=0.4.0=h516909a_0 377 | - xcb-util-image=0.4.0=h166bdaf_0 378 | - xcb-util-keysyms=0.4.0=h516909a_0 379 | - xcb-util-renderutil=0.3.9=h166bdaf_0 380 | - xcb-util-wm=0.4.1=h516909a_0 381 | - xorg-fixesproto=5.0=h7f98852_1002 382 | - xorg-kbproto=1.0.7=h7f98852_1002 383 | - xorg-libice=1.0.10=h7f98852_0 384 | - xorg-libsm=1.2.3=hd9c2040_1000 385 | - xorg-libx11=1.7.2=h7f98852_0 386 | - xorg-libxau=1.0.9=h7f98852_0 387 | - xorg-libxdmcp=1.1.3=h7f98852_0 388 | - xorg-libxext=1.3.4=h7f98852_1 389 | - xorg-libxfixes=5.0.3=h7f98852_1004 390 | - xorg-libxt=1.2.1=h7f98852_2 391 | - xorg-xextproto=7.3.0=h7f98852_1002 392 | - xorg-xproto=7.0.31=h27cfd23_1007 393 | - xz=5.2.6=h166bdaf_0 394 | - yaml=0.2.5=h7b6447c_0 395 | - yarl=1.8.1=py38h5eee18b_0 396 | - zeromq=4.3.4=h2531618_0 397 | - zipp=3.8.0=py38h06a4308_0 398 | - zlib=1.2.13=h166bdaf_4 399 | - zstd=1.5.2=h6239696_4 400 | - pip: 401 | - addict==2.4.0 402 | - annotated-types==0.7.0 403 | - astroid==2.15.0 404 | - configargparse==1.5.3 405 | - dash==2.7.1 406 | - dash-core-components==2.0.0 407 | - dash-html-components==2.0.0 408 | - dash-table==5.0.0 409 | - dgl==0.9.1 410 | - dill==0.3.6 411 | - distro==1.9.0 412 | - flask==2.2.2 413 | - freetype-py==2.3.0 414 | - h11==0.14.0 415 | - httpcore==1.0.5 416 | - httpx==0.27.0 417 | - imageio==2.23.0 418 | - isort==5.12.0 419 | - itsdangerous==2.1.2 420 | - lazy-object-proxy==1.9.0 421 | - nbformat==5.5.0 422 | - networkx==2.2 423 | - open3d==0.16.0 424 | - openai==1.36.1 425 | - opencv-python==4.10.0.84 426 | - plotly==5.11.0 427 | - pycollada==0.6 428 | - pydantic==2.8.2 429 | - pydantic-core==2.20.1 430 | - pyglet==2.0.2.1 431 | - pylint==2.17.0 432 | - pyopengl==3.1.0 433 | - pyquaternion==0.9.9 434 | - pyrender==0.1.45 435 | - tenacity==8.1.0 436 | - tomlkit==0.11.6 437 | - trimesh==3.17.1 438 | - typing-extensions==4.12.2 439 | - urdfpy==0.0.22 440 | - wrapt==1.15.0 441 | -------------------------------------------------------------------------------- /images/teaser_v1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yixuanhuang98/Points2Plans/78c0b0f0a23d2a352022584d369398741c553772/images/teaser_v1.png -------------------------------------------------------------------------------- /relational_dynamics/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yixuanhuang98/Points2Plans/78c0b0f0a23d2a352022584d369398741c553772/relational_dynamics/.DS_Store -------------------------------------------------------------------------------- /relational_dynamics/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yixuanhuang98/Points2Plans/78c0b0f0a23d2a352022584d369398741c553772/relational_dynamics/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /relational_dynamics/__pycache__/base_RD.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yixuanhuang98/Points2Plans/78c0b0f0a23d2a352022584d369398741c553772/relational_dynamics/__pycache__/base_RD.cpython-38.pyc -------------------------------------------------------------------------------- /relational_dynamics/config/__pycache__/base_config.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yixuanhuang98/Points2Plans/78c0b0f0a23d2a352022584d369398741c553772/relational_dynamics/config/__pycache__/base_config.cpython-38.pyc -------------------------------------------------------------------------------- /relational_dynamics/config/base_config.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | import torch 4 | import os 5 | 6 | class BaseConfig(object): 7 | def __init__(self, args, dtype=torch.FloatTensor, **kwargs): 8 | self.args = args 9 | self.result_dir = args.result_dir 10 | self.dtype = dtype 11 | 12 | def get_device(self): 13 | if self.dtype == torch.FloatTensor: 14 | return torch.device("cpu") 15 | else: 16 | return torch.device("cuda") 17 | 18 | def get_logger_dir(self): 19 | return os.path.join(self.result_dir, 'logs') 20 | 21 | def get_test_logger_dir(self): 22 | return os.path.join(self.result_dir, 'test_logs') 23 | 24 | def get_model_checkpoint_dir(self): 25 | return os.path.join(self.result_dir, 'checkpoint') -------------------------------------------------------------------------------- /relational_dynamics/dataloader/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yixuanhuang98/Points2Plans/78c0b0f0a23d2a352022584d369398741c553772/relational_dynamics/dataloader/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /relational_dynamics/dataloader/__pycache__/dataloader.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yixuanhuang98/Points2Plans/78c0b0f0a23d2a352022584d369398741c553772/relational_dynamics/dataloader/__pycache__/dataloader.cpython-38.pyc -------------------------------------------------------------------------------- /relational_dynamics/dataloader/__pycache__/farthest_point_sampling.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yixuanhuang98/Points2Plans/78c0b0f0a23d2a352022584d369398741c553772/relational_dynamics/dataloader/__pycache__/farthest_point_sampling.cpython-38.pyc -------------------------------------------------------------------------------- /relational_dynamics/dataloader/__pycache__/real_robot_dataloader.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yixuanhuang98/Points2Plans/78c0b0f0a23d2a352022584d369398741c553772/relational_dynamics/dataloader/__pycache__/real_robot_dataloader.cpython-38.pyc -------------------------------------------------------------------------------- /relational_dynamics/main.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import argparse 3 | import pickle 4 | import sys 5 | import os 6 | import pprint 7 | import torch 8 | import torch.nn as nn 9 | import torch.optim as optim 10 | import time 11 | import json 12 | import copy 13 | import random 14 | import torch.nn.functional as F 15 | 16 | sys.path.append(os.getcwd()) 17 | from relational_dynamics.utils.colors import bcolors 18 | from relational_dynamics.utils.other_util import create_log_dirs 19 | from relational_dynamics.utils import parse_util 20 | from relational_dynamics.config.base_config import BaseConfig 21 | from relational_dynamics.base_RD import RelationalDynamics 22 | 23 | def main(args): 24 | dtype = torch.FloatTensor 25 | if args.cuda: 26 | dtype = torch.cuda.FloatTensor 27 | 28 | config = BaseConfig(args, dtype=dtype) 29 | create_log_dirs(config) 30 | 31 | if len(args.checkpoint_path) > 0: 32 | planner = RelationalDynamics(config) 33 | planner.load_checkpoint(args.checkpoint_path) 34 | result_dict = planner.RD(train=False, threshold = 0) 35 | else: 36 | trainer = RelationalDynamics(config) 37 | result_dict = trainer.RD() 38 | 39 | if __name__ == '__main__': 40 | parser = parse_util.get_parser() 41 | args = parser.parse_args() 42 | np.set_printoptions(precision=4, linewidth=120) 43 | if args.set_random_seed: 44 | seed = args.seed 45 | random.seed(args.seed) 46 | np.random.seed(seed) 47 | torch.manual_seed(seed) 48 | torch.cuda.manual_seed_all(seed) 49 | torch.backends.cudnn.deterministic = True 50 | torch.backends.cudnn.benchmark = False 51 | 52 | main(args) 53 | -------------------------------------------------------------------------------- /relational_dynamics/model/__pycache__/GNN_pytorch_geometry.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yixuanhuang98/Points2Plans/78c0b0f0a23d2a352022584d369398741c553772/relational_dynamics/model/__pycache__/GNN_pytorch_geometry.cpython-38.pyc -------------------------------------------------------------------------------- /relational_dynamics/model/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yixuanhuang98/Points2Plans/78c0b0f0a23d2a352022584d369398741c553772/relational_dynamics/model/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /relational_dynamics/model/__pycache__/contact_model.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yixuanhuang98/Points2Plans/78c0b0f0a23d2a352022584d369398741c553772/relational_dynamics/model/__pycache__/contact_model.cpython-38.pyc -------------------------------------------------------------------------------- /relational_dynamics/model/__pycache__/encoder_decoder.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yixuanhuang98/Points2Plans/78c0b0f0a23d2a352022584d369398741c553772/relational_dynamics/model/__pycache__/encoder_decoder.cpython-38.pyc -------------------------------------------------------------------------------- /relational_dynamics/model/__pycache__/losses.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yixuanhuang98/Points2Plans/78c0b0f0a23d2a352022584d369398741c553772/relational_dynamics/model/__pycache__/losses.cpython-38.pyc -------------------------------------------------------------------------------- /relational_dynamics/model/__pycache__/models.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yixuanhuang98/Points2Plans/78c0b0f0a23d2a352022584d369398741c553772/relational_dynamics/model/__pycache__/models.cpython-38.pyc -------------------------------------------------------------------------------- /relational_dynamics/model/__pycache__/pointconv_util_groupnorm.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yixuanhuang98/Points2Plans/78c0b0f0a23d2a352022584d369398741c553772/relational_dynamics/model/__pycache__/pointconv_util_groupnorm.cpython-38.pyc -------------------------------------------------------------------------------- /relational_dynamics/model/models.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | import torch.nn as nn 4 | import torch.nn.functional as F 5 | 6 | from torch.nn import TransformerEncoder, TransformerEncoderLayer 7 | from relational_dynamics.model.pointconv_util_groupnorm import PointConvDensitySetAbstraction 8 | 9 | class EmbeddingNetTorch(nn.Module): 10 | def __init__( 11 | self, n_objects: int, 12 | width: int, layers: int, heads: int, input_feature_num: int, 13 | d_hidden: int, n_unary: int, n_binary: int, 14 | dim_feedforward: int, 15 | use_discrete_place = False, 16 | latent_discrete_continuous = False, 17 | seperate_discrete_continuous = False, 18 | simple_encoding = False, 19 | transformer_dynamics = False, 20 | torch_embedding = False, 21 | complicated_pre_dynamics = False, 22 | train_env_identity = False, 23 | total_env_identity = 1, 24 | one_bit_env = False, 25 | direct_transformer = False, 26 | enable_high_push = False, 27 | enable_place_inside = False, 28 | seperate_place = False, 29 | use_seperate_latent_embedding = True, 30 | seperate_action_emb = False, 31 | use_mlp_encoder = False, 32 | ): 33 | super(EmbeddingNetTorch, self).__init__() 34 | d_input = width 35 | 36 | self.use_seperate_latent_embedding = use_seperate_latent_embedding 37 | 38 | self.train_env_identity = train_env_identity 39 | 40 | self.direct_transformer = direct_transformer 41 | 42 | self.enable_high_push = enable_high_push 43 | 44 | self.enable_place_inside = enable_place_inside 45 | 46 | self.seperate_place = seperate_place 47 | 48 | self.seperate_action_emb = seperate_action_emb 49 | 50 | self.seperate_discrete_continuous = seperate_discrete_continuous 51 | 52 | self.simple_encoding = simple_encoding 53 | 54 | self.transformer_dynamics = transformer_dynamics 55 | 56 | self.torch_embedding = torch_embedding 57 | 58 | self.complicated_pre_dynamics = complicated_pre_dynamics 59 | 60 | encoder_layers = TransformerEncoderLayer(width, heads, batch_first = True, dim_feedforward = dim_feedforward) 61 | 62 | self.transformer = TransformerEncoder(encoder_layers, layers) 63 | 64 | self.one_hot_encoding_dim = (int)(width/2) 65 | 66 | self.one_hot_encoding_embed = nn.Sequential( 67 | nn.Embedding(n_objects, self.one_hot_encoding_dim) 68 | ) 69 | 70 | self.continuous_action_emb = nn.Sequential( 71 | nn.Linear(2, self.one_hot_encoding_dim), 72 | nn.ReLU(), 73 | nn.Linear(self.one_hot_encoding_dim, self.one_hot_encoding_dim) 74 | ) 75 | self.continuous_action_emb_1 = nn.Sequential( 76 | nn.Linear(2, self.one_hot_encoding_dim), 77 | nn.ReLU(), 78 | nn.Linear(self.one_hot_encoding_dim, self.one_hot_encoding_dim) 79 | ) 80 | 81 | encoder_layers_1 = TransformerEncoderLayer(width, heads, batch_first = True, dim_feedforward = dim_feedforward) 82 | self.graph_dynamics_0 = TransformerEncoder(encoder_layers_1, layers) 83 | encoder_layers_2 = TransformerEncoderLayer(width, heads, batch_first = True, dim_feedforward = dim_feedforward) 84 | self.graph_dynamics_1 = TransformerEncoder(encoder_layers_2, layers) 85 | 86 | self.n_unary = n_unary 87 | self.n_binary = n_binary 88 | self.d_hidden = d_hidden 89 | 90 | self.f_unary = self.get_head_unary(d_input, d_hidden, 1, n_unary) 91 | self.f_binary = self.get_head(d_input, d_hidden, 2, n_binary) 92 | 93 | 94 | self.ln_post = nn.LayerNorm(width) 95 | 96 | def get_head(self, d_input, d_hidden, n_args, n_binary): 97 | if d_hidden > 1: 98 | head = nn.Sequential( 99 | nn.Linear(d_input * n_args, d_hidden), 100 | nn.ReLU(), 101 | nn.Linear(d_hidden, d_hidden), 102 | nn.ReLU(), 103 | nn.Linear(d_hidden, n_binary), 104 | nn.Sigmoid() 105 | ) 106 | else: 107 | head = nn.Sequential( 108 | nn.Linear(d_input * n_args, d_hidden), 109 | nn.Sigmoid() 110 | ) 111 | return head 112 | 113 | def get_head_unary(self, d_input, d_hidden, n_args, n_unary): 114 | if d_hidden > 1: 115 | head = nn.Sequential( 116 | nn.Linear(d_input * n_args, d_hidden), 117 | nn.ReLU(), 118 | nn.Linear(d_hidden, n_unary) 119 | ) 120 | else: 121 | head = nn.Sequential( 122 | nn.Linear(d_input * n_args, d_hidden) 123 | ) 124 | return head 125 | 126 | class QuickReadoutNet(nn.Module): 127 | def __init__( 128 | self, n_objects: int, 129 | width: int, layers: int, heads: int, input_feature_num: int, 130 | d_hidden: int, n_unary: int, n_binary: int, 131 | dim_feedforward: int, 132 | pose_num: int, 133 | train_env_identity = False, total_env_identity = 2, 134 | train_grasp_identity = False, 135 | train_inside_feasibility = False, 136 | binary_grasp = False, 137 | open_close_drawer = False, 138 | softmax_identity = False, 139 | seperate_identity = False, 140 | train_obj_boundary = False, 141 | train_obj_move = False, 142 | one_bit_env = False, 143 | pe = False, 144 | transformer_decoder = False, 145 | remove_orientation = False, 146 | pose_trans_decoder = False, 147 | ): 148 | super(QuickReadoutNet, self).__init__() 149 | d_input = width 150 | 151 | self.train_env_identity = train_env_identity 152 | self.train_grasp_identity = train_grasp_identity 153 | self.train_inside_feasibility = train_inside_feasibility 154 | self.binary_grasp = binary_grasp 155 | 156 | self.open_close_drawer = open_close_drawer 157 | self.softmax_identity = softmax_identity 158 | self.seperate_identity = seperate_identity 159 | self.train_obj_boundary = train_obj_boundary 160 | self.train_obj_move = train_obj_move 161 | self.pe = pe 162 | self.transformer_decoder = transformer_decoder 163 | self.pose_trans_decoder = pose_trans_decoder 164 | self.remove_orientation = remove_orientation 165 | 166 | total_env_identity = 3 167 | 168 | 169 | self.one_hot_encoding_dim = (int)(width/2) 170 | 171 | self.one_hot_encoding_embed = nn.Sequential( 172 | nn.Linear(n_objects, self.one_hot_encoding_dim), 173 | nn.ReLU(inplace=True), 174 | nn.Linear(self.one_hot_encoding_dim, self.one_hot_encoding_dim) 175 | ) 176 | 177 | self.action_emb = nn.Sequential( 178 | nn.Linear(input_feature_num, width), 179 | nn.ReLU(), 180 | nn.Linear(width, width) 181 | ) 182 | 183 | self.env_output_identity = nn.Sequential( 184 | nn.Linear(d_input, d_hidden), 185 | nn.ReLU(), 186 | nn.Linear(d_hidden, total_env_identity), 187 | nn.Sigmoid() 188 | ) 189 | 190 | 191 | self.grasp_output_identity = self.get_head(d_input, d_hidden, 2, 2) 192 | 193 | 194 | self.pose_estimation = nn.Sequential( 195 | nn.Linear(d_input, d_hidden), 196 | nn.ReLU(), 197 | nn.Linear(d_hidden, pose_num) 198 | ) 199 | 200 | self.n_unary = n_unary 201 | self.n_binary = n_binary 202 | self.d_hidden = d_hidden 203 | 204 | self.f_unary = self.get_head_unary(d_input, d_hidden, 1, n_unary) 205 | self.f_binary = self.get_head(d_input, d_hidden, 2, n_binary) 206 | self.ln_post = nn.LayerNorm(width) 207 | 208 | def get_head(self, d_input, d_hidden, n_args, n_binary): 209 | if d_hidden > 1: 210 | head = nn.Sequential( 211 | nn.Linear(d_input * n_args, d_hidden), 212 | nn.ReLU(), 213 | nn.Linear(d_hidden, d_hidden), 214 | nn.ReLU(), 215 | nn.Linear(d_hidden, n_binary), 216 | nn.Sigmoid() 217 | ) 218 | else: 219 | head = nn.Sequential( 220 | nn.Linear(d_input * n_args, d_hidden), 221 | nn.Sigmoid() 222 | ) 223 | return head 224 | 225 | def get_head_unary(self, d_input, d_hidden, n_args, n_unary): 226 | if d_hidden > 1: 227 | head = nn.Sequential( 228 | nn.Linear(d_input * n_args, d_hidden), 229 | nn.ReLU(), 230 | nn.Linear(d_hidden, n_unary) 231 | ) 232 | else: 233 | head = nn.Sequential( 234 | nn.Linear(d_input * n_args, d_hidden) 235 | ) 236 | return head 237 | 238 | 239 | def forward(self, objs, edge_index): 240 | 241 | batch_size, n_obj = objs.shape[:2] # # shape = [*, n_obj, width] 242 | 243 | x = objs # x = (pointconv feature + one hot encoding(128)) * 244 | 245 | 246 | z = self.f_unary(x) 247 | 248 | self.env_identity = self.env_output_identity(x) 249 | 250 | self.predicted_pose = self.pose_estimation(x) 251 | 252 | x1 = x[:, edge_index[0, :], :] 253 | 254 | x2 = x[:, edge_index[1, :], :] 255 | 256 | concat_x = torch.cat([x1, x2], dim=-1) 257 | y = self.f_binary(concat_x) 258 | 259 | self.binary_grasp_identity = self.grasp_output_identity(concat_x) 260 | 261 | return_dict = {'pred': z, 262 | 'current_embed': x, 263 | 'pred_sigmoid': y, 264 | 'env_identity': self.env_identity, 265 | 'grasp_identity': self.binary_grasp_identity, 266 | 'predicted_pose': self.predicted_pose} 267 | 268 | return return_dict 269 | 270 | 271 | class PointConv(nn.Module): 272 | def __init__(self, normal_channel=False, use_rgb=False, output_dim = 128): 273 | super(PointConv, self).__init__() 274 | 275 | if normal_channel: 276 | additional_channel = 3 277 | else: 278 | additional_channel = 0 279 | 280 | if use_rgb: 281 | rgb_channel = 3 282 | else: 283 | rgb_channel = 0 284 | self.output_dim = output_dim 285 | self.normal_channel = normal_channel 286 | 287 | self.sa1 = PointConvDensitySetAbstraction(npoint=128, nsample=8, in_channel=6+additional_channel+rgb_channel, mlp=[32], bandwidth = 0.1, group_all=False) 288 | self.sa2 = PointConvDensitySetAbstraction(npoint=64, nsample=16, in_channel= 32 + 3, mlp=[64], bandwidth = 0.2, group_all=False) 289 | self.sa3 = PointConvDensitySetAbstraction(npoint=1, nsample=None, in_channel= 64 + 3, mlp=[output_dim], bandwidth = 0.4, group_all=True) 290 | 291 | def forward(self, xyz): 292 | # Set Abstraction layers 293 | B,C,N = xyz.shape 294 | if self.normal_channel: 295 | l0_points = xyz 296 | l0_xyz = xyz[:,:3,:] 297 | else: 298 | l0_points = xyz 299 | l0_xyz = xyz[:, :3, :] 300 | 301 | l1_xyz, l1_points = self.sa1(l0_xyz, l0_points) 302 | l2_xyz, l2_points = self.sa2(l1_xyz, l1_points) 303 | l3_xyz, l3_points = self.sa3(l2_xyz, l2_points) 304 | x = l3_points.view(B, self.output_dim) 305 | 306 | return x 307 | -------------------------------------------------------------------------------- /relational_dynamics/model/pointconv_util_groupnorm.py: -------------------------------------------------------------------------------- 1 | """ 2 | Utility function for PointConv 3 | Originally from : https://github.com/yanx27/Pointnet_Pointnet2_pytorch/blob/master/utils.py 4 | Modify by Wenxuan Wu 5 | Date: September 2019 6 | """ 7 | import torch 8 | import torch.nn as nn 9 | import torch.nn.functional as F 10 | from time import time 11 | import numpy as np 12 | # from sklearn.neighbors.kde import KernelDensity 13 | 14 | def timeit(tag, t): 15 | print("{}: {}s".format(tag, time() - t)) 16 | return time() 17 | 18 | def square_distance(src, dst): 19 | """ 20 | Calculate Euclid distance between each two points. 21 | src^T * dst = xn * xm + yn * ym + zn * zm； 22 | sum(src^2, dim=-1) = xn*xn + yn*yn + zn*zn; 23 | sum(dst^2, dim=-1) = xm*xm + ym*ym + zm*zm; 24 | dist = (xn - xm)^2 + (yn - ym)^2 + (zn - zm)^2 25 | = sum(src**2,dim=-1)+sum(dst**2,dim=-1)-2*src^T*dst 26 | Input: 27 | src: source points, [B, N, C] 28 | dst: target points, [B, M, C] 29 | Output: 30 | dist: per-point square distance, [B, N, M] 31 | """ 32 | B, N, _ = src.shape 33 | _, M, _ = dst.shape 34 | dist = -2 * torch.matmul(src, dst.permute(0, 2, 1)) 35 | dist += torch.sum(src ** 2, -1).view(B, N, 1) 36 | dist += torch.sum(dst ** 2, -1).view(B, 1, M) 37 | return dist 38 | 39 | def index_points(points, idx): 40 | """ 41 | Input: 42 | points: input points data, [B, N, C] 43 | idx: sample index data, [B, S] 44 | Return: 45 | new_points:, indexed points data, [B, S, C] 46 | """ 47 | device = points.device 48 | B = points.shape[0] 49 | view_shape = list(idx.shape) 50 | view_shape[1:] = [1] * (len(view_shape) - 1) 51 | repeat_shape = list(idx.shape) 52 | repeat_shape[0] = 1 53 | batch_indices = torch.arange(B, dtype=torch.long).to(device).view(view_shape).repeat(repeat_shape) 54 | new_points = points[batch_indices, idx, :] 55 | return new_points 56 | 57 | def farthest_point_sample(xyz, npoint): 58 | """ 59 | Input: 60 | xyz: pointcloud data, [B, N, C] 61 | npoint: number of samples 62 | Return: 63 | centroids: sampled pointcloud index, [B, npoint] 64 | """ 65 | #import ipdb; ipdb.set_trace() 66 | device = xyz.device 67 | B, N, C = xyz.shape 68 | centroids = torch.zeros(B, npoint, dtype=torch.long).to(device) 69 | distance = torch.ones(B, N).to(device) * 1e10 70 | #farthest = torch.randint(0, N, (B,), dtype=torch.long).to(device) 71 | farthest = torch.zeros(B, dtype=torch.long).to(device) 72 | batch_indices = torch.arange(B, dtype=torch.long).to(device) 73 | for i in range(npoint): 74 | centroids[:, i] = farthest 75 | centroid = xyz[batch_indices, farthest, :].view(B, 1, 3) 76 | dist = torch.sum((xyz - centroid) ** 2, -1) 77 | mask = dist < distance 78 | distance[mask] = dist[mask] 79 | farthest = torch.max(distance, -1)[1] 80 | return centroids 81 | 82 | def query_ball_point(radius, nsample, xyz, new_xyz): 83 | """ 84 | Input: 85 | radius: local region radius 86 | nsample: max sample number in local region 87 | xyz: all points, [B, N, C] 88 | new_xyz: query points, [B, S, C] 89 | Return: 90 | group_idx: grouped points index, [B, S, nsample] 91 | """ 92 | device = xyz.device 93 | B, N, C = xyz.shape 94 | _, S, _ = new_xyz.shape 95 | group_idx = torch.arange(N, dtype=torch.long).to(device).view(1, 1, N).repeat([B, S, 1]) 96 | sqrdists = square_distance(new_xyz, xyz) 97 | group_idx[sqrdists > radius ** 2] = N 98 | group_idx = group_idx.sort(dim=-1)[0][:, :, :nsample] 99 | group_first = group_idx[:, :, 0].view(B, S, 1).repeat([1, 1, nsample]) 100 | mask = group_idx == N 101 | group_idx[mask] = group_first[mask] 102 | return group_idx 103 | 104 | def knn_point(nsample, xyz, new_xyz): 105 | """ 106 | Input: 107 | nsample: max sample number in local region 108 | xyz: all points, [B, N, C] 109 | new_xyz: query points, [B, S, C] 110 | Return: 111 | group_idx: grouped points index, [B, S, nsample] 112 | """ 113 | sqrdists = square_distance(new_xyz, xyz) 114 | _, group_idx = torch.topk(sqrdists, nsample, dim = -1, largest=False, sorted=False) 115 | return group_idx 116 | 117 | def sample_and_group(npoint, nsample, xyz, points, density_scale = None): 118 | """ 119 | Input: 120 | npoint: 121 | nsample: 122 | xyz: input points position data, [B, N, C] 123 | points: input points data, [B, N, D] 124 | Return: 125 | new_xyz: sampled points position data, [B, 1, C] 126 | new_points: sampled points data, [B, 1, N, C+D] 127 | """ 128 | B, N, C = xyz.shape 129 | S = npoint 130 | fps_idx = farthest_point_sample(xyz, npoint) # [B, npoint, C] 131 | new_xyz = index_points(xyz, fps_idx) 132 | idx = knn_point(nsample, xyz, new_xyz) 133 | grouped_xyz = index_points(xyz, idx) # [B, npoint, nsample, C] 134 | grouped_xyz_norm = grouped_xyz - new_xyz.view(B, S, 1, C) 135 | if points is not None: 136 | grouped_points = index_points(points, idx) 137 | new_points = torch.cat([grouped_xyz_norm, grouped_points], dim=-1) # [B, npoint, nsample, C+D] 138 | else: 139 | new_points = grouped_xyz_norm 140 | 141 | if density_scale is None: 142 | return new_xyz, new_points, grouped_xyz_norm, idx 143 | else: 144 | grouped_density = index_points(density_scale, idx) 145 | return new_xyz, new_points, grouped_xyz_norm, idx, grouped_density 146 | 147 | def sample_and_group_all(xyz, points, density_scale = None): 148 | """ 149 | Input: 150 | xyz: input points position data, [B, N, C] 151 | points: input points data, [B, N, D] 152 | Return: 153 | new_xyz: sampled points position data, [B, 1, C] 154 | new_points: sampled points data, [B, 1, N, C+D] 155 | """ 156 | device = xyz.device 157 | B, N, C = xyz.shape 158 | #new_xyz = torch.zeros(B, 1, C).to(device) 159 | new_xyz = xyz.mean(dim = 1, keepdim = True) 160 | grouped_xyz = xyz.view(B, 1, N, C) - new_xyz.view(B, 1, 1, C) 161 | if points is not None: 162 | new_points = torch.cat([grouped_xyz, points.view(B, 1, N, -1)], dim=-1) 163 | else: 164 | new_points = grouped_xyz 165 | if density_scale is None: 166 | return new_xyz, new_points, grouped_xyz 167 | else: 168 | grouped_density = density_scale.view(B, 1, N, 1) 169 | return new_xyz, new_points, grouped_xyz, grouped_density 170 | 171 | def group(nsample, xyz, points): 172 | """ 173 | Input: 174 | npoint: 175 | nsample: 176 | xyz: input points position data, [B, N, C] 177 | points: input points data, [B, N, D] 178 | Return: 179 | new_xyz: sampled points position data, [B, 1, C] 180 | new_points: sampled points data, [B, 1, N, C+D] 181 | """ 182 | B, N, C = xyz.shape 183 | S = N 184 | new_xyz = xyz 185 | idx = knn_point(nsample, xyz, new_xyz) 186 | grouped_xyz = index_points(xyz, idx) # [B, npoint, nsample, C] 187 | grouped_xyz_norm = grouped_xyz - new_xyz.view(B, S, 1, C) 188 | if points is not None: 189 | grouped_points = index_points(points, idx) 190 | new_points = torch.cat([grouped_xyz_norm, grouped_points], dim=-1) # [B, npoint, nsample, C+D] 191 | else: 192 | new_points = grouped_xyz_norm 193 | 194 | return new_points, grouped_xyz_norm 195 | 196 | def compute_density(xyz, bandwidth): 197 | ''' 198 | xyz: input points position data, [B, N, C] 199 | ''' 200 | #import ipdb; ipdb.set_trace() 201 | B, N, C = xyz.shape 202 | sqrdists = square_distance(xyz, xyz) 203 | gaussion_density = torch.exp(- sqrdists / (2.0 * bandwidth * bandwidth)) / (2.5 * bandwidth) 204 | xyz_density = gaussion_density.mean(dim = -1) 205 | 206 | return xyz_density 207 | 208 | class DensityNet(nn.Module): 209 | def __init__(self, hidden_unit = [16, 8]): 210 | super(DensityNet, self).__init__() 211 | self.mlp_convs = nn.ModuleList() 212 | self.mlp_bns = nn.ModuleList() 213 | 214 | self.mlp_convs.append(nn.Conv2d(1, hidden_unit[0], 1)) 215 | self.mlp_bns.append(nn.GroupNorm(1, hidden_unit[0])) 216 | for i in range(1, len(hidden_unit)): 217 | self.mlp_convs.append(nn.Conv2d(hidden_unit[i - 1], hidden_unit[i], 1)) 218 | self.mlp_bns.append(nn.GroupNorm(1, hidden_unit[i])) 219 | self.mlp_convs.append(nn.Conv2d(hidden_unit[-1], 1, 1)) 220 | self.mlp_bns.append(nn.GroupNorm(1, 1)) 221 | 222 | def forward(self, density_scale): 223 | for i, conv in enumerate(self.mlp_convs): 224 | bn = self.mlp_bns[i] 225 | density_scale = bn(conv(density_scale)) 226 | if i == len(self.mlp_convs): 227 | density_scale = F.sigmoid(density_scale) 228 | else: 229 | density_scale = F.relu(density_scale) 230 | 231 | return density_scale 232 | 233 | class WeightNet(nn.Module): 234 | 235 | def __init__(self, in_channel, out_channel, hidden_unit = [8, 8]): 236 | super(WeightNet, self).__init__() 237 | 238 | self.mlp_convs = nn.ModuleList() 239 | self.mlp_bns = nn.ModuleList() 240 | if hidden_unit is None or len(hidden_unit) == 0: 241 | self.mlp_convs.append(nn.Conv2d(in_channel, out_channel, 1)) 242 | self.mlp_bns.append(nn.GroupNorm(1, out_channel)) 243 | else: 244 | self.mlp_convs.append(nn.Conv2d(in_channel, hidden_unit[0], 1)) 245 | self.mlp_bns.append(nn.GroupNorm(1, hidden_unit[0])) 246 | for i in range(1, len(hidden_unit)): 247 | self.mlp_convs.append(nn.Conv2d(hidden_unit[i - 1], hidden_unit[i], 1)) 248 | self.mlp_bns.append(nn.GroupNorm(1, hidden_unit[i])) 249 | self.mlp_convs.append(nn.Conv2d(hidden_unit[-1], out_channel, 1)) 250 | self.mlp_bns.append(nn.GroupNorm(1, out_channel)) 251 | 252 | def forward(self, localized_xyz): 253 | #xyz : BxCxKxN 254 | 255 | weights = localized_xyz 256 | for i, conv in enumerate(self.mlp_convs): 257 | bn = self.mlp_bns[i] 258 | weights = F.relu(bn(conv(weights))) 259 | 260 | return weights 261 | 262 | class PointConvSetAbstraction(nn.Module): 263 | def __init__(self, npoint, nsample, in_channel, mlp, group_all): 264 | super(PointConvSetAbstraction, self).__init__() 265 | self.npoint = npoint 266 | self.nsample = nsample 267 | self.mlp_convs = nn.ModuleList() 268 | self.mlp_bns = nn.ModuleList() 269 | last_channel = in_channel 270 | for out_channel in mlp: 271 | self.mlp_convs.append(nn.Conv2d(last_channel, out_channel, 1)) 272 | self.mlp_bns.append(nn.GroupNorm(1, out_channel)) 273 | last_channel = out_channel 274 | 275 | self.weightnet = WeightNet(3, 16) 276 | self.linear = nn.Linear(16 * mlp[-1], mlp[-1]) 277 | self.bn_linear = nn.GroupNorm(1, mlp[-1]) 278 | self.group_all = group_all 279 | 280 | def forward(self, xyz, points): 281 | """ 282 | Input: 283 | xyz: input points position data, [B, C, N] 284 | points: input points data, [B, D, N] 285 | Return: 286 | new_xyz: sampled points position data, [B, C, S] 287 | new_points_concat: sample points feature data, [B, D', S] 288 | """ 289 | B = xyz.shape[0] 290 | xyz = xyz.permute(0, 2, 1) 291 | if points is not None: 292 | points = points.permute(0, 2, 1) 293 | 294 | if self.group_all: 295 | new_xyz, new_points, grouped_xyz_norm = sample_and_group_all(xyz, points) 296 | else: 297 | new_xyz, new_points, grouped_xyz_norm, _ = sample_and_group(self.npoint, self.nsample, xyz, points) 298 | # new_xyz: sampled points position data, [B, npoint, C] 299 | # new_points: sampled points data, [B, npoint, nsample, C+D] 300 | new_points = new_points.permute(0, 3, 2, 1) # [B, C+D, nsample,npoint] 301 | for i, conv in enumerate(self.mlp_convs): 302 | bn = self.mlp_bns[i] 303 | new_points = F.relu(bn(conv(new_points))) 304 | 305 | grouped_xyz = grouped_xyz_norm.permute(0, 3, 2, 1) 306 | weights = self.weightnet(grouped_xyz) 307 | new_points = torch.matmul(input=new_points.permute(0, 3, 1, 2), other = weights.permute(0, 3, 2, 1)).view(B, self.npoint, -1) 308 | new_points = self.linear(new_points) 309 | new_points = self.bn_linear(new_points.permute(0, 2, 1)) 310 | new_points = F.relu(new_points) 311 | new_xyz = new_xyz.permute(0, 2, 1) 312 | 313 | return new_xyz, new_points 314 | 315 | class PointConvDensitySetAbstraction(nn.Module): 316 | def __init__(self, npoint, nsample, in_channel, mlp, bandwidth, group_all): 317 | super(PointConvDensitySetAbstraction, self).__init__() 318 | self.npoint = npoint 319 | self.nsample = nsample 320 | self.mlp_convs = nn.ModuleList() 321 | self.mlp_bns = nn.ModuleList() 322 | last_channel = in_channel 323 | for out_channel in mlp: 324 | self.mlp_convs.append(nn.Conv2d(last_channel, out_channel, 1)) 325 | self.mlp_bns.append(nn.GroupNorm(1, out_channel)) 326 | last_channel = out_channel 327 | 328 | self.weightnet = WeightNet(3, 16) 329 | self.linear = nn.Linear(16 * mlp[-1], mlp[-1]) 330 | self.bn_linear = nn.GroupNorm(1, mlp[-1]) 331 | self.densitynet = DensityNet() 332 | self.group_all = group_all 333 | self.bandwidth = bandwidth 334 | 335 | def forward(self, xyz, points): 336 | """ 337 | Input: 338 | xyz: input points position data, [B, C, N] 339 | points: input points data, [B, D, N] 340 | Return: 341 | new_xyz: sampled points position data, [B, C, S] 342 | new_points_concat: sample points feature data, [B, D', S] 343 | """ 344 | B = xyz.shape[0] 345 | N = xyz.shape[2] 346 | xyz = xyz.permute(0, 2, 1) 347 | if points is not None: 348 | points = points.permute(0, 2, 1) 349 | 350 | xyz_density = compute_density(xyz, self.bandwidth) 351 | inverse_density = 1.0 / xyz_density 352 | 353 | if self.group_all: 354 | new_xyz, new_points, grouped_xyz_norm, grouped_density = sample_and_group_all(xyz, points, inverse_density.view(B, N, 1)) 355 | else: 356 | new_xyz, new_points, grouped_xyz_norm, _, grouped_density = sample_and_group(self.npoint, self.nsample, xyz, points, inverse_density.view(B, N, 1)) 357 | # new_xyz: sampled points position data, [B, npoint, C] 358 | # new_points: sampled points data, [B, npoint, nsample, C+D] 359 | new_points = new_points.permute(0, 3, 2, 1) # [B, C+D, nsample,npoint] 360 | for i, conv in enumerate(self.mlp_convs): 361 | bn = self.mlp_bns[i] 362 | new_points = F.relu(bn(conv(new_points))) 363 | 364 | inverse_max_density = grouped_density.max(dim = 2, keepdim=True)[0] 365 | density_scale = grouped_density / inverse_max_density 366 | density_scale = self.densitynet(density_scale.permute(0, 3, 2, 1)) 367 | new_points = new_points * density_scale 368 | 369 | grouped_xyz = grouped_xyz_norm.permute(0, 3, 2, 1) 370 | weights = self.weightnet(grouped_xyz) 371 | new_points = torch.matmul(input=new_points.permute(0, 3, 1, 2), other = weights.permute(0, 3, 2, 1)).view(B, self.npoint, -1) 372 | new_points = self.linear(new_points) 373 | new_points = self.bn_linear(new_points.permute(0, 2, 1)) 374 | new_points = F.relu(new_points) 375 | new_xyz = new_xyz.permute(0, 2, 1) 376 | 377 | return new_xyz, new_points -------------------------------------------------------------------------------- /relational_dynamics/utils/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yixuanhuang98/Points2Plans/78c0b0f0a23d2a352022584d369398741c553772/relational_dynamics/utils/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /relational_dynamics/utils/__pycache__/colors.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yixuanhuang98/Points2Plans/78c0b0f0a23d2a352022584d369398741c553772/relational_dynamics/utils/__pycache__/colors.cpython-38.pyc -------------------------------------------------------------------------------- /relational_dynamics/utils/__pycache__/data_utils.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yixuanhuang98/Points2Plans/78c0b0f0a23d2a352022584d369398741c553772/relational_dynamics/utils/__pycache__/data_utils.cpython-38.pyc -------------------------------------------------------------------------------- /relational_dynamics/utils/__pycache__/math_util.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yixuanhuang98/Points2Plans/78c0b0f0a23d2a352022584d369398741c553772/relational_dynamics/utils/__pycache__/math_util.cpython-38.pyc -------------------------------------------------------------------------------- /relational_dynamics/utils/__pycache__/other_util.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yixuanhuang98/Points2Plans/78c0b0f0a23d2a352022584d369398741c553772/relational_dynamics/utils/__pycache__/other_util.cpython-38.pyc -------------------------------------------------------------------------------- /relational_dynamics/utils/__pycache__/parse_util.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yixuanhuang98/Points2Plans/78c0b0f0a23d2a352022584d369398741c553772/relational_dynamics/utils/__pycache__/parse_util.cpython-38.pyc -------------------------------------------------------------------------------- /relational_dynamics/utils/__pycache__/tensorboardx_logger.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yixuanhuang98/Points2Plans/78c0b0f0a23d2a352022584d369398741c553772/relational_dynamics/utils/__pycache__/tensorboardx_logger.cpython-38.pyc -------------------------------------------------------------------------------- /relational_dynamics/utils/__pycache__/torch_util.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yixuanhuang98/Points2Plans/78c0b0f0a23d2a352022584d369398741c553772/relational_dynamics/utils/__pycache__/torch_util.cpython-38.pyc -------------------------------------------------------------------------------- /relational_dynamics/utils/__pycache__/torch_utils.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yixuanhuang98/Points2Plans/78c0b0f0a23d2a352022584d369398741c553772/relational_dynamics/utils/__pycache__/torch_utils.cpython-38.pyc -------------------------------------------------------------------------------- /relational_dynamics/utils/colors.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | class bcolors: 4 | HEADER = '\033[95m' 5 | OKBLUE = '\033[94m' 6 | OKGREEN = '\033[92m' 7 | WARNING = '\033[93m' 8 | FAIL = '\033[91m' 9 | ENDC = '\033[0m' 10 | 11 | Black = '\033[30m' 12 | Red = '\033[31m' 13 | Green = '\033[32m' 14 | Yellow = '\033[33m' 15 | Blue = '\033[34m' 16 | Purple = '\033[35m' 17 | Cyan = '\033[36m' 18 | LightGray = '\033[37m' 19 | DarkGray = '\033[30m' 20 | LightRed = '\033[31m' 21 | LightGreen = '\033[32m' 22 | LightYellow = '\033[93m' 23 | LightBlue = '\033[34m' 24 | LightPurple = '\033[35m' 25 | LightCyan = '\033[36m' 26 | White = '\033[97m' 27 | 28 | BckgrDefault = '\033[49m' 29 | BckgrBlack = '\033[40m' 30 | BckgrRed = '\033[41m' 31 | BckgrGreen = '\033[42m' 32 | BckgrYellow = '\033[43m' 33 | BckgrBlue = '\033[44m' 34 | BckgrPurple = '\033[45m' 35 | BckgrCyan = '\033[46m' 36 | BckgrLightGray = '\033[47m' 37 | BckgrDarkGray = '\033[100m' 38 | BckgrLightRed = '\033[101m' 39 | BckgrLightGreen = '\033[102m' 40 | BckgrLightYellow = '\033[103m' 41 | BckgrLightBlue = '\033[104m' 42 | BckgrLightPurple = '\033[105m' 43 | 44 | 45 | @staticmethod 46 | def header(msg): 47 | return bcolors.HEADER + msg + bcolors.ENDC 48 | 49 | @staticmethod 50 | def okblue(msg): 51 | return bcolors.OKBLUE + msg + bcolors.ENDC 52 | 53 | @staticmethod 54 | def okgreen(msg): 55 | return bcolors.OKGREEN + msg + bcolors.ENDC 56 | 57 | @staticmethod 58 | def warning(msg): 59 | return bcolors.WARNING + msg + bcolors.ENDC 60 | 61 | @staticmethod 62 | def fail(msg): 63 | return bcolors.FAIL + msg + bcolors.ENDC 64 | 65 | @staticmethod 66 | def c_cyan(msg): 67 | return bcolors.Cyan + msg + bcolors.ENDC 68 | 69 | @staticmethod 70 | def c_red(msg): 71 | return bcolors.Red + msg + bcolors.ENDC 72 | 73 | @staticmethod 74 | def c_yellow(msg): 75 | return bcolors.Yellow + msg + bcolors.ENDC 76 | 77 | @staticmethod 78 | def c_blue(msg): 79 | return bcolors.Blue + msg + bcolors.ENDC 80 | 81 | @staticmethod 82 | def c_purple(msg): 83 | return bcolors.Purple + msg + bcolors.ENDC 84 | 85 | @staticmethod 86 | def c_green(msg): 87 | return bcolors.Green + msg + bcolors.ENDC 88 | -------------------------------------------------------------------------------- /relational_dynamics/utils/data_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | import os 4 | import pickle 5 | import time 6 | import h5py 7 | 8 | import torch.nn as nn 9 | 10 | def scale_min_max(x, x_min, x_max, desired_min, desired_max): 11 | """ 12 | Applies min/max scaling on one data instance. 13 | 14 | Args: 15 | x (ndarray): Data to be scaled. 16 | x_min (flt): Minimum value of data (over full dataset). 17 | x_max (flt): Maximum value of data (over full dataset). 18 | desired_min (flt): Desired minimum value. 19 | desired_max (flt): Desired maximum value. 20 | """ 21 | return ((desired_max - desired_min) * (x - x_min) / (x_max - x_min)) + desired_min 22 | 23 | def rotate_2d(x, theta): 24 | """ Rotate x by theta degrees (counter clockwise) 25 | 26 | @param x: a 2D vector (numpy array with shape 2) 27 | @param theta: a scalar representing radians 28 | """ 29 | 30 | rot_mat = np.array([ 31 | [np.cos(theta), -np.sin(theta)], 32 | [np.sin(theta), np.cos(theta)] 33 | ], dtype=np.float32) 34 | 35 | return rot_mat.dot(x) 36 | 37 | 38 | def save_emb_data_to_h5(result_dir, result_dict): 39 | emb_h5_path = os.path.join(result_dir, 'train_result_emb.h5') 40 | emb_h5_f = h5py.File(emb_h5_path, 'w') 41 | 42 | # Create a new dictionary so that each scene emb which can have different 43 | # number of objects as compared to other scenes will be stored separately. 44 | result_h5_dict = {'emb': {}} 45 | for k, v in result_dict['emb'].items(): 46 | if k != 'train_img_emb' and k != 'test_img_emb': 47 | result_h5_dict['emb'][k] = v 48 | else: 49 | assert type(v) is list 50 | result_h5_dict['emb'][k] = dict() 51 | for scene_i, scene_emb in enumerate(v): 52 | result_h5_dict['emb'][k][f'{scene_i:05d}'] = np.copy(scene_emb) 53 | 54 | result_h5_dict = {'emb': result_h5_dict['emb'] } 55 | recursively_save_dict_contents_to_group(emb_h5_f, '/', result_h5_dict) 56 | emb_h5_f.flush() 57 | emb_h5_f.close() 58 | pkl_path = os.path.join(result_dir, 'train_result_info.pkl') 59 | with open(pkl_path, 'wb') as pkl_f: 60 | pkl_output_dict = {'output': result_dict['output']} 61 | pickle.dump(pkl_output_dict, pkl_f, protocol=2) 62 | 63 | print(bcolors.c_blue("Did save emb data: {}".format(emb_h5_path))) 64 | 65 | 66 | def get_activation(activation): 67 | """ 68 | Returns specified activation function. Note this is preferable to functional 69 | interface since these can be added to ModuleList. 70 | """ 71 | if activation == 'relu': 72 | return nn.ReLU() 73 | elif activation == 'leaky_relu': 74 | return nn.LeakyReLU(0.2, inplace=True) 75 | elif activation == 'prelu': 76 | return nn.PReLU() 77 | elif activation == 'sigmoid': 78 | return nn.Sigmoid() 79 | elif activation == 'tanh': 80 | return nn.Tanh() 81 | elif activation == 'softplus': 82 | return nn.Softplus() 83 | else: 84 | raise ValueError(f"Unknown activation type: {activation}") 85 | 86 | 87 | def get_norm(norm, out_size): 88 | if norm == 'layer2d': 89 | # Using 1 group is equivalent to layer norm, but seems you need 90 | # to do it this way with groupnorm for convolution layers 91 | return nn.GroupNorm(1, out_size) 92 | elif norm == 'layer': 93 | return nn.LayerNorm((out_size,)) 94 | elif norm == 'batch': 95 | return nn.BatchNorm2d(num_features=out_size) 96 | else: 97 | raise ValueError(f"Unknown normalization type: {norm}") 98 | 99 | 100 | def get_delta_pose_for_data_info(info): 101 | before_pose = info['before']['other_pos'] + info['before']['other_q'] 102 | after_pose = info['after']['other_pos'] + info['after']['other_q'] 103 | delta_pose = [after_pose[i]-before_pose[i] for i in range(len(before_pose))] 104 | return delta_pose 105 | 106 | 107 | def get_euclid_dist_matrix_for_data(data_arr, squared=True): 108 | '''Get euclidean distance matrix for data.''' 109 | data_dot_prod = np.dot(data_arr, data_arr.T) 110 | sq_norm = np.diagonal(data_dot_prod) 111 | dist = sq_norm[None, :] - 2*data_dot_prod + sq_norm[:, None] 112 | dist = np.maximum(dist, 1e-8) 113 | if not squared: 114 | dist = np.sqrt(dist) 115 | return dist 116 | 117 | 118 | def get_dist_matrix_for_data(data_list, 119 | save_inter_scene_dist_dir=None, 120 | save_inter_scene_dist_file_prefix='', 121 | top_K=100, 122 | bottom_K=100): 123 | inter_scene_dist_by_path_dict = {'path': {}, 'idx': {}} 124 | 125 | get_dist_start_time = time.time() 126 | pos_list = [] 127 | 128 | has_info = data_list[0].get('info') is not None 129 | for i in range(len(data_list)): 130 | data_i = data_list[i] 131 | if has_info: 132 | pos_list.append(get_delta_pose_for_data_info(data_i['info'])) 133 | else: 134 | pos_list.append(data_i['delta_pose']) 135 | 136 | pos_arr = np.array(pos_list)[:, :3] 137 | for i in range(len(data_list)): 138 | data_i = data_list[i] 139 | if has_info: 140 | data_i_pos = np.array( 141 | get_delta_pose_for_data_info(data_i['info']))[:3] 142 | else: 143 | data_i_pos = np.array(data_i['delta_pose'])[:3] 144 | data_i_dist = np.linalg.norm(pos_arr - data_i_pos, axis=1) 145 | 146 | top_k_idxs = np.argpartition(data_i_dist, top_K + 1)[:top_K] 147 | bottom_k_idxs = np.argpartition(-data_i_dist, bottom_K)[:bottom_K] 148 | 149 | assert len(top_k_idxs) == top_K 150 | assert len(bottom_k_idxs) == bottom_K 151 | 152 | inter_scene_dist_by_path_dict['path'][data_i['path']] = { 153 | 'top': [(data_i_dist[idx], idx, data_list[idx]['path']) 154 | for idx in top_k_idxs], 155 | 'bottom': [(data_i_dist[idx], idx, data_list[idx]['path']) 156 | for idx in bottom_k_idxs], 157 | } 158 | inter_scene_dist_by_path_dict['idx'][i] = \ 159 | inter_scene_dist_by_path_dict['path'][data_i['path']] 160 | 161 | get_dist_end_time = time.time() 162 | print("Get dist time: {:.4f}".format(get_dist_end_time - get_dist_start_time)) 163 | 164 | # Save the file only if required. 165 | if save_inter_scene_dist_dir is not None: 166 | if save_inter_scene_dist_file_prefix is not None: 167 | if len(save_inter_scene_dist_file_prefix) > 0: 168 | file_name = '{}_inter_scene_dist.pkl'.format( 169 | save_inter_scene_dist_file_prefix) 170 | else: 171 | file_name = 'inter_scene_dist.pkl' 172 | pkl_path = os.path.join(save_inter_scene_dist_dir, file_name) 173 | with open(pkl_path, 'wb') as pkl_f: 174 | pickle.dump(inter_scene_dist_by_path_dict, pkl_f, protocol=2) 175 | print("Did save inter_scene_dist: {}".format(pkl_path)) 176 | 177 | return inter_scene_dist_by_path_dict 178 | 179 | 180 | def str2bool(v): 181 | if isinstance(v, bool): 182 | return v 183 | if v.lower() in ('yes', 'true', 't', 'y', '1'): 184 | return True 185 | elif v.lower() in ('no', 'false', 'f', 'n', '0'): 186 | return False 187 | else: 188 | raise argparse.ArgumentTypeError('Boolean value expected.') 189 | 190 | 191 | def recursively_get_dict_from_group(group_or_data): 192 | d = {} 193 | if type(group_or_data) == h5py.Dataset: 194 | return np.array(group_or_data) 195 | 196 | # Else it's still a group 197 | for k in group_or_data.keys(): 198 | v = recursively_get_dict_from_group(group_or_data[k]) 199 | d[k] = v 200 | return d 201 | 202 | def convert_list_of_array_to_dict_of_array_for_hdf5(arr_list): 203 | arr_dict = {} 204 | for i, a in enumerate(arr_list): 205 | arr_dict[str(i)] = a 206 | return arr_dict 207 | 208 | def recursively_save_dict_contents_to_group(h5file, path, dic): 209 | """ 210 | Take an already open HDF5 file and insert the contents of a dictionary 211 | at the current path location. Can call itself recursively to fill 212 | out HDF5 files with the contents of a dictionary. 213 | """ 214 | assert type(dic) is type({}), "must provide a dictionary" 215 | assert type(path) is type(''), "path must be a string" 216 | assert type(h5file) is h5py._hl.files.File, "must be an open h5py file" 217 | 218 | for key in dic: 219 | assert type(key) is type(''), 'dict keys must be strings to save to hdf5' 220 | did_save_key = False 221 | 222 | if type(dic[key]) in (np.int64, np.float64, type(''), int, float): 223 | h5file[path + key] = dic[key] 224 | did_save_key = True 225 | assert h5file[path + key].value == dic[key], \ 226 | 'The data representation in the HDF5 file does not match the ' \ 227 | 'original dict.' 228 | if type(dic[key]) is type([]): 229 | h5file[path + key] = np.array(dic[key]) 230 | did_save_key = True 231 | if type(dic[key]) is np.ndarray: 232 | h5file[path + key] = dic[key] 233 | did_save_key = True 234 | # assert np.array_equal(h5file[path + key].value, dic[key]), \ 235 | # 'The data representation in the HDF5 file does not match the ' \ 236 | # 'original dict.' 237 | if type(dic[key]) is type({}): 238 | recursively_save_dict_contents_to_group(h5file, 239 | path + key + '/', 240 | dic[key]) 241 | did_save_key = True 242 | if not did_save_key: 243 | print("Dropping key from h5 file: {}".format(path + key)) -------------------------------------------------------------------------------- /relational_dynamics/utils/math_util.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | from pyquaternion import Quaternion 4 | 5 | 6 | def rotation_to_quat(R): 7 | T = np.eye(4) 8 | T[:3,:3] = R 9 | return homogeneous_to_quat(T) 10 | 11 | 12 | def homogeneous_to_position(T): 13 | return T[:3, 3] 14 | 15 | 16 | def homogeneous_to_rotation(T): 17 | return T[:3, :3] 18 | 19 | 20 | def homogeneous_to_quat(T): 21 | """ 22 | Converts rotation matrix from homogeneous TF matrix to quaternion. 23 | """ 24 | q = Quaternion(matrix=T) # w, x, y, z 25 | return np.array([q.x, q.y, q.z, q.w]) # Need to switch to x, y, z, w 26 | 27 | 28 | def homogeneous_to_pose(T): 29 | """ 30 | Converts homogeneous TF matrix to a 3D position and quaternion. 31 | """ 32 | return homogeneous_to_position(T), homogeneous_to_quat(T) 33 | 34 | 35 | def quat_to_homogeneous(q): 36 | """ 37 | Converts quaternion to homogeneous TF matrix. Assumes (x, y, z, w) quaternion input. 38 | """ 39 | return Quaternion(q[3], q[0], q[1], q[2]).transformation_matrix # Quaternion is (w, x, y, z) 40 | 41 | 42 | def quat_to_rotation(q): 43 | """ 44 | Converts quaternion to rotation matrix. Assumes (x, y, z, w) quaternion input. 45 | """ 46 | return quat_to_homogeneous(q)[:3,:3] 47 | 48 | 49 | def pose_to_homogeneous(p, q): 50 | """ 51 | Converts position and quaternion to a homogeneous TF matrix. 52 | """ 53 | T = quat_to_homogeneous(q) 54 | T[:3, 3] = p 55 | return T 56 | 57 | 58 | def homogeneous_transpose(T): 59 | """ 60 | Converts TF to TF inverse, also use np.matmul instead of np.dot 61 | """ 62 | new_T = np.eye(4) 63 | new_T[:3,:3] = T[:3,:3].T 64 | new_T[:3,3] = -np.matmul(new_T[:3,:3], T[:3,3]) 65 | return new_T 66 | 67 | 68 | def random_quat(): 69 | q = Quaternion.random() 70 | return np.array([q.x, q.y, q.z, q.w]) 71 | 72 | 73 | def random_rotation(): 74 | return quat_to_rotation(random_quat()) 75 | 76 | 77 | def get_x_axis_rotation(theta): 78 | return np.array([[1, 0, 0], 79 | [0, np.cos(theta), -np.sin(theta)], 80 | [0, np.sin(theta), np.cos(theta)]]) 81 | 82 | 83 | def get_y_axis_rotation(theta): 84 | return np.array([[ np.cos(theta), 0, np.sin(theta)], 85 | [ 0, 1, 0], 86 | [-np.sin(theta), 0, np.cos(theta)]]) 87 | 88 | 89 | def get_z_axis_rotation(theta): 90 | return np.array([[np.cos(theta), -np.sin(theta), 0], 91 | [np.sin(theta), np.cos(theta), 0], 92 | [ 0, 0, 1]]) 93 | 94 | 95 | def quat_mul(a, b): 96 | assert a.shape == b.shape 97 | shape = a.shape 98 | a = a.reshape(-1, 4) 99 | b = b.reshape(-1, 4) 100 | 101 | x1, y1, z1, w1 = a[:, 0], a[:, 1], a[:, 2], a[:, 3] 102 | x2, y2, z2, w2 = b[:, 0], b[:, 1], b[:, 2], b[:, 3] 103 | ww = (z1 + x1) * (x2 + y2) 104 | yy = (w1 - y1) * (w2 + z2) 105 | zz = (w1 + y1) * (w2 - z2) 106 | xx = ww + yy + zz 107 | qq = 0.5 * (xx + (z1 - x1) * (x2 - y2)) 108 | w = qq - ww + (z1 - y1) * (y2 - z2) 109 | x = qq - xx + (x1 + w1) * (x2 + w2) 110 | y = qq - yy + (w1 - x1) * (y2 + z2) 111 | z = qq - zz + (z1 + y1) * (w2 - x2) 112 | 113 | quat = torch.stack([x, y, z, w], dim=-1).view(shape) 114 | 115 | return quat 116 | 117 | 118 | def quat_conjugate(a): 119 | shape = a.shape 120 | a = a.reshape(-1, 4) 121 | return torch.cat((-a[:, :3], a[:, -1:]), dim=-1).view(shape) 122 | 123 | 124 | def quaternion_error(desired, current, square=False, numpy=False, flatten=False): 125 | q_c = quat_conjugate(current) 126 | q_r = quat_mul(desired, q_c) 127 | error = q_r[:, 0:3] * torch.sign(q_r[:, 3]).unsqueeze(-1) 128 | if square: 129 | error = error**2 130 | if numpy: 131 | error = error.cpu().numpy() 132 | if flatten: 133 | error = error.flatten() 134 | return error 135 | 136 | 137 | def position_error(desired, current, square=False, numpy=False, flatten=False): 138 | error = desired - current 139 | if square: 140 | error = error**2 141 | if numpy: 142 | error = error.cpu().numpy() 143 | if flatten: 144 | error = error.flatten() 145 | return error 146 | 147 | 148 | def geodesic_error(R1, R2): 149 | return np.arccos(np.clip((np.trace(np.dot(R1, R2.T)) - 1.) / 2., -1, 1)) 150 | 151 | 152 | 153 | if __name__ == '__main__': 154 | R1 = random_rotation() 155 | R2 = random_rotation() 156 | 157 | print(geodesic_error(R1, R2)) 158 | -------------------------------------------------------------------------------- /relational_dynamics/utils/other_util.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | import torch.nn as nn 4 | import os 5 | 6 | from relational_dynamics.utils import torch_util 7 | from relational_dynamics.utils.data_utils import get_norm, get_activation, scale_min_max 8 | 9 | def rotate_2d(x, theta): 10 | """ Rotate x by theta degrees (counter clockwise) 11 | 12 | @param x: a 2D vector (numpy array with shape 2) 13 | @param theta: a scalar representing radians 14 | """ 15 | 16 | rot_mat = np.array([ 17 | [np.cos(theta), -np.sin(theta)], 18 | [np.sin(theta), np.cos(theta)] 19 | ], dtype=np.float32) 20 | 21 | return rot_mat.dot(x) 22 | 23 | def create_log_dirs(config): 24 | args = config.args 25 | # Create logger directory if required 26 | if not os.path.exists(args.result_dir): 27 | os.makedirs(args.result_dir) 28 | if not os.path.exists(config.get_logger_dir()): 29 | os.makedirs(config.get_logger_dir()) 30 | if not os.path.exists(config.get_model_checkpoint_dir()): 31 | os.makedirs(config.get_model_checkpoint_dir()) 32 | 33 | class LinearBlock(nn.Module): 34 | 35 | def __init__(self, in_size, out_size, activation=None, norm=None): 36 | super(LinearBlock, self).__init__() 37 | 38 | self.layers = nn.ModuleList() 39 | self.layers.append(nn.Linear(in_size, out_size)) 40 | if activation: 41 | self.layers.append(get_activation(activation)) 42 | if norm is not None: 43 | self.layers.append(get_norm(norm, out_size)) 44 | 45 | def forward(self, x): 46 | for layer in self.layers: 47 | x = layer(x) 48 | return x 49 | 50 | class MLP(nn.Module): 51 | 52 | def __init__(self, in_size, out_size, hidden_sizes=[], activation='relu', 53 | norm=None, out_activation=None, out_norm=None, vae=False): 54 | """ 55 | Multi-layer perception module. Stacks linear layers with customizable sizes, 56 | activations, and normalization. 57 | 58 | Args: 59 | in_size (int): Size of input tensor 60 | out_size (int): Size of output tensor 61 | hidden_sizes (List[int]): Output sizes of each hidden layer 62 | activation (str): Activations to apply to each hidden layer 63 | layer_norms (bool): Apply layer norm to hidden layers if true 64 | out_activation (str): Activation to apply to output layer (default is none) 65 | out_layer_norm (bool): Apply layer norm to output layer if true 66 | vae (bool): Sample output as VAE 67 | Returns: 68 | Output of last layer; if vae=True, returns sample of output as well as mean and std 69 | 70 | TODO right now only layer norm supported, can make more general 71 | """ 72 | super(MLP, self).__init__() 73 | 74 | self.vae = vae 75 | if self.vae: 76 | out_size *= 2 77 | 78 | self.layers = nn.ModuleList() 79 | prev_size = in_size 80 | for size in hidden_sizes: 81 | self.layers.append(LinearBlock(prev_size, size, activation, norm)) 82 | prev_size = size 83 | self.layers.append(LinearBlock(prev_size, out_size, out_activation, out_norm)) 84 | 85 | def forward(self, x): 86 | for layer in self.layers: 87 | x = layer(x) 88 | 89 | if self.vae: 90 | mu, log_std = torch.chunk(x, 2, dim=-1) 91 | std = torch.exp(log_std) 92 | noise = torch.randn_like(mu) 93 | if self.training: 94 | return mu + std * noise 95 | else: 96 | return mu 97 | else: 98 | return x 99 | -------------------------------------------------------------------------------- /relational_dynamics/utils/torch_util.py: -------------------------------------------------------------------------------- 1 | from math import pi 2 | 3 | import torch 4 | import torch.nn.functional as F 5 | 6 | def convert_to_float_tensors(tensor_dict, keys=[]): 7 | keys = keys if keys else tensor_dict.keys() 8 | for k in keys: 9 | if torch.is_tensor(tensor_dict[k]): 10 | tensor_dict[k] = tensor_dict[k].float() 11 | else: 12 | tensor_dict[k] = torch.FloatTensor(tensor_dict[k]) 13 | 14 | 15 | def convert_to_long_tensors(tensor_dict, keys=[]): 16 | keys = keys if keys else tensor_dict.keys() 17 | for k in keys: 18 | if torch.is_tensor(tensor_dict[k]): 19 | tensor_dict[k] = tensor_dict[k].long() 20 | else: 21 | tensor_dict[k] = torch.LongTensor(tensor_dict[k]) 22 | 23 | 24 | def make_batch(x, n_batch=1): 25 | """ 26 | Batchifies a tensor by adding a batch dim and repeating it over that 27 | dimension n_batch times. 28 | """ 29 | if not isinstance(x, torch.Tensor): 30 | x = torch.tensor(x) 31 | ndim = x.dim() 32 | x = x.unsqueeze(0) 33 | if n_batch > 1: 34 | x = x.repeat(n_batch, *[1]*ndim) # unrolls list of 1's of length ndim 35 | return x 36 | 37 | 38 | def move_batch_to_device(batch_dict, device): 39 | """ 40 | Recursive function that moves a (nested) dictionary of tensors to the specified device. 41 | """ 42 | for k, v in batch_dict.items(): 43 | if isinstance(v, torch.Tensor): 44 | batch_dict[k] = v.to(device) 45 | elif isinstance(v, dict): 46 | move_batch_to_device(v, device) 47 | 48 | 49 | def move_models_to_device(models_dict, device): 50 | """ 51 | Assuming flat dictionary where values are all type torch.nn.Module. 52 | """ 53 | for k, v in models_dict.items(): 54 | models_dict[k] = v.to(device) 55 | 56 | 57 | def set_models_to_train(models_dict): 58 | for v in models_dict.values(): 59 | v.train() 60 | 61 | 62 | def set_models_to_eval(models_dict): 63 | for v in models_dict.values(): 64 | v.eval() 65 | 66 | 67 | def load_state_dicts(models_dict, state_dicts): 68 | for k, v in models_dict.items(): 69 | if k not in state_dicts: 70 | print(f"Model {k} does not have state to load") 71 | #ui_util.print_warning(f"Model {k} does not have state to load") 72 | continue 73 | v.load_state_dict(state_dicts[k]) 74 | 75 | 76 | def accumulate_parameters(models_dict): 77 | params = [] 78 | for v in models_dict.values(): 79 | params += list(v.parameters()) 80 | return params 81 | 82 | 83 | 84 | 85 | # torch quaternion functions from NVIDIA: 86 | 87 | def quat_mul(a, b): 88 | assert a.shape == b.shape 89 | shape = a.shape 90 | a = a.reshape(-1, 4) 91 | b = b.reshape(-1, 4) 92 | 93 | x1, y1, z1, w1 = a[:, 0], a[:, 1], a[:, 2], a[:, 3] 94 | x2, y2, z2, w2 = b[:, 0], b[:, 1], b[:, 2], b[:, 3] 95 | ww = (z1 + x1) * (x2 + y2) 96 | yy = (w1 - y1) * (w2 + z2) 97 | zz = (w1 + y1) * (w2 - z2) 98 | xx = ww + yy + zz 99 | qq = 0.5 * (xx + (z1 - x1) * (x2 - y2)) 100 | w = qq - ww + (z1 - y1) * (y2 - z2) 101 | x = qq - xx + (x1 + w1) * (x2 + w2) 102 | y = qq - yy + (w1 - x1) * (y2 + z2) 103 | z = qq - zz + (z1 + y1) * (w2 - x2) 104 | 105 | quat = torch.stack([x, y, z, w], dim=-1).view(shape) 106 | 107 | return quat 108 | 109 | 110 | def quat_conjugate(a): 111 | shape = a.shape 112 | a = a.reshape(-1, 4) 113 | return torch.cat((-a[:, :3], a[:, -1:]), dim=-1).view(shape) 114 | 115 | 116 | def quaternion_error(desired, current, square=False, numpy=False, flatten=False): 117 | q_c = quat_conjugate(current) 118 | q_r = quat_mul(desired, q_c) 119 | error = q_r[:, 0:3] * torch.sign(q_r[:, 3]).unsqueeze(-1) 120 | if square: 121 | error = error**2 122 | if numpy: 123 | error = error.cpu().numpy() 124 | if flatten: 125 | error = error.flatten() 126 | return error 127 | 128 | 129 | def position_error(desired, current, square=False, numpy=False, flatten=False): 130 | error = desired - current 131 | if square: 132 | error = error**2 133 | if numpy: 134 | error = error.cpu().numpy() 135 | if flatten: 136 | error = error.flatten() 137 | return error 138 | 139 | def random_quat(batch_size=1, device='cuda'): 140 | """ 141 | Computes a random quaternion sampled uniformly from the unit sphere. 142 | 143 | Note this is primarily implemented so that it uses torch random generator 144 | instead of numpy to avoid issues with how torch/np random generators 145 | interact when training with randomization: 146 | https://tanelp.github.io/posts/a-bug-that-plagues-thousands-of-open-source-ml-projects/ 147 | 148 | See: https://github.com/KieranWynn/pyquaternion/blob/master/pyquaternion/quaternion.py#L261 149 | """ 150 | r1 = torch.rand(batch_size).to(device).view(-1, 1) 151 | r2 = torch.rand(batch_size).to(device).view(-1, 1) 152 | r3 = torch.rand(batch_size).to(device).view(-1, 1) 153 | 154 | w = torch.sqrt(1.0 - r1) * (torch.sin(2.0 * pi * r2)) 155 | x = torch.sqrt(1.0 - r1) * (torch.cos(2.0 * pi * r2)) 156 | y = torch.sqrt(r1) * (torch.sin(2.0 * pi * r3)) 157 | z = torch.sqrt(r1) * (torch.cos(2.0 * pi * r3)) 158 | # Normalize just to be sure since there can be slight numerical differences from pure unit 159 | return F.normalize(torch.cat([x, y, z, w], dim=-1)) 160 | 161 | 162 | def random_rotation(batch_size=1, device='cuda'): 163 | return quat_to_rotation(random_quat(batch_size, device)) 164 | 165 | 166 | def quat_to_rotation(q): 167 | batch = q.size(0) 168 | qx = q[:,0].view(batch, 1) 169 | qy = q[:,1].view(batch, 1) 170 | qz = q[:,2].view(batch, 1) 171 | qw = q[:,3].view(batch, 1) 172 | 173 | # Unit quaternion rotation matrices computatation 174 | xx = qx*qx 175 | yy = qy*qy 176 | zz = qz*qz 177 | xy = qx*qy 178 | xz = qx*qz 179 | yz = qy*qz 180 | xw = qx*qw 181 | yw = qy*qw 182 | zw = qz*qw 183 | 184 | row0 = torch.cat((1-2*yy-2*zz, 2*xy - 2*zw, 2*xz + 2*yw), 1) 185 | row1 = torch.cat((2*xy+ 2*zw, 1-2*xx-2*zz, 2*yz-2*xw ), 1) 186 | row2 = torch.cat((2*xz-2*yw, 2*yz+2*xw, 1-2*xx-2*yy), 1) 187 | 188 | matrix = torch.cat((row0.view(batch,1,3), row1.view(batch,1,3), row2.view(batch,1,3)),1) 189 | return matrix 190 | 191 | 192 | if __name__ == '__main__': 193 | # print(random_rotation(1)) 194 | 195 | a = torch.tensor([[1,2,3], 196 | [4,5,6], 197 | [7,8,9]]) 198 | print(make_batch(a, 11).shape) 199 | --------------------------------------------------------------------------------